common_utils.cpp

/*--------------------------------------------------------------------------*/
/*--------------------------- common_utils.cpp -----------------------------*/
/*--------------------------------------------------------------------------*/
/** @file
 * Implementation of some common utilities for SMS++ tools whose API is
 * defined in common_utils.h.
 *
 * \author Antonio Frangioni \n
 *         Dipartimento di Informatica \n
 *         Universita' di Pisa \n
 *
 * \author Niccolo' Iardella \n
 *         Dipartimento di Informatica \n
 *         Universita' di Pisa \n
 *
 * \author Donato Meoli \n
 *         Dipartimento di Informatica \n
 *         Universita' di Pisa \n
 *
 * \copyright © by Antonio Frangioni, Niccolo' Iardella, Donato Meoli
 */
/*--------------------------------------------------------------------------*/
/*------------------------------ INCLUDES ----------------------------------*/
/*--------------------------------------------------------------------------*/

#include <algorithm>
#include <fstream>
#include <iostream>
#include <limits>
#include <set>
#include <sstream>

#if defined( __APPLE__ )
 #include <mach-o/dyld.h>  // for _NSGetExecutablePath()
#endif

#include <Block.h>
#include <BlockSolverConfig.h>
#include <Solution.h>

#include "common_utils.h"

/*--------------------------------------------------------------------------*/
/*----------------------------- VERSIONS -----------------------------------*/
/*--------------------------------------------------------------------------*/
/* The header is generated by CMake, hence the guard: a makefile build simply
 * reports no version. */

#if __has_include( <SMS++_toolsConfig.h> )
 #include <SMS++_toolsConfig.h>
#endif

/*--------------------------------------------------------------------------*/
/*-------------------------------- USING -----------------------------------*/
/*--------------------------------------------------------------------------*/

using namespace SMSpp_di_unipi_it;

/*--------------------------------------------------------------------------*/
/*------------------------ MPI SAFE ENVIRONMENT ----------------------------*/
/*--------------------------------------------------------------------------*/
/* Some tools (those using SDDPBlock or InvestmentBlock) call MPI_Init().
 * On systems where Open MPI / UCX are installed but no usable transport is
 * available (no IB, missing UCX vfs.sock, ...), or where the hwloc GL
 * component hangs probing the GPU topology via XOpenDisplay(), the MPI
 * runtime can hang on startup spinning on futex / X11 sockets.
 *
 * To make every tool work out-of-the-box, we pre-seed safe defaults with
 * setenv(..., 0): the third argument is "overwrite = false", so any user
 * who has already exported UCX_TLS / OMPI_MCA_* / HWLOC_COMPONENTS (e.g.
 * on an HPC cluster with a real fabric) keeps full control. This is
 * executed before main() via a static initializer; it is the same
 * mechanism used by tests/common_utils.cpp.                              */

namespace {

void set_default_env( const char * name , const char * value ) {
#ifdef _WIN32
 if( std::getenv( name ) == nullptr )
  _putenv_s( name , value );
#else
 setenv( name , value , 0 );
#endif
}

struct SmsppMpiSafeEnvInit {
 SmsppMpiSafeEnvInit() {
  set_default_env( "UCX_TLS"     , "tcp,self" );
  set_default_env( "OMPI_MCA_btl", "tcp,self" );
  set_default_env( "OMPI_MCA_pml", "ob1"      );
  // the hwloc GL component probes the GPU topology via XOpenDisplay(),
  // which may hang inside MPI_Init(); no SMS++ target has a use for it
  set_default_env( "HWLOC_COMPONENTS", "-gl"  );
  }
 };

static SmsppMpiSafeEnvInit smspp_mpi_safe_env_init_;

}  // anonymous namespace

/*--------------------------------------------------------------------------*/
/*------------------------------- GLOBALS ----------------------------------*/
/*--------------------------------------------------------------------------*/
/** @name Global variables used by every tool
 *  @{ */

std::string docopt_desc {};     ///< tool description
std::string docopt_args {};     ///< description of the <file> argument
std::string docopt_examples {}; ///< usage examples printed by --help

std::string filename {};        ///< input filename
std::string bconf_file {};      ///< BlockConfig filename
std::string sconf_file {};      ///< BlockSolverConfig filename

std::string default_bconf_name = "BCfg.txt";   ///< default -B filename
std::string default_sconf_name = "BSCfg.txt";  ///< default -S filename
std::string state_in_file {};   ///< State to be loaded into the Solver
std::string state_out_file {};  ///< final State of the Solver
std::string block_prefix {};    ///< prefix for all Block files
std::string conf_prefix {};     ///< prefix for all Configuration files
std::string exe {};             ///< name of the executable file
std::string sol_input {};       ///< filename of input Solution
std::string sol_output {};      ///< filename of output Solution
std::string sol_cfg_file {};    ///< filename of output Solution Configuration

bool output_solution = false;   ///< true if solution has be output
bool sol_verbose = false;       ///< if the Solver should be verbose
bool writeprob = false;         ///< if the problem should be written back
std::string prob_file {};       ///< filename of the problem written back (-n)
char input_format = 0;          ///< native format of the input file (-f)
bool dryrun = false;            ///< if compute() need not really ba called

int verbosity_level = 0;        ///< verbosity (0 = silent, > 0 = verbose)

/// default short command-line options
std::string short_opts = "a:B:b:p:S:c:on:I:O:C:Dv::hV";

/// default long command-line options
std::vector< option > long_opts = {
 { "help"            , no_argument       , nullptr , 'h' } ,
 { "save-state"      , required_argument , nullptr , 'a' } ,
 { "blockcfg"        , required_argument , nullptr , 'B' } ,
 { "load-state"      , required_argument , nullptr , 'b' } ,
 { "prefix"          , required_argument , nullptr , 'p' } ,
 { "solvercfg"       , required_argument , nullptr , 'S' } ,
 { "configdir"       , required_argument , nullptr , 'c' } ,
 { "output-solution" , no_argument       , nullptr , 'o' } ,
 { "nc4problem"      , required_argument , nullptr , 'n' } ,
 { "inputsol"        , required_argument , nullptr , 'I' } ,
 { "outputsol"       , required_argument , nullptr , 'O' } ,
 { "outsolcfg"       , required_argument , nullptr , 'C' } ,
 { "dryrun"          , no_argument       , nullptr , 'D' } ,
 { "verbose"         , optional_argument , nullptr , 'v' } ,
 { "version"         , no_argument       , nullptr , 'V' } ,
 { nullptr           , no_argument       , nullptr , 0 }
};

/// default command-line options help string
std::string help =
 "  -h, --help                      print this help\n"
 "  -V, --version                   print the SMS++ tools version and exit\n"
 "  -a, --save-state <file>         save State of the Solver\n"
 "  -B, --blockcfg <file>           Block Configuration\n"
 "  -b, --load-state <file>         load State for the Solver\n"
 "  -p, --prefix <path>             the prefix for all Block filenames\n"
 "  -S, --solvercfg <file>          Solver Configuration\n"
 "  -c, --configdir <path>          the prefix for all Config filenames\n"
 "  -I, --inputsol <file>           input Solution\n"
 "  -O, --outputsol <file>          output Solution\n"
 "  -C, --outsolcfg <file>          output Solution Configuration\n"
 "  -o, --output-solution           output the solutions\n"
 "  -n, --nc4problem <file>         write nc4 problem on file\n"
 "  -D, --dryrun                    skip the compute() call\n"
 "  -v[N], --verbose[=N]            verbose output (0 = silent, 1 = basic,\n"
 "                                  2 = debug, with the Solver logs and the\n"
 "                                  parameters of the Solvers of the Block,\n"
 "                                  3 = those of the sub-Blocks' Solvers"
 " too);\n"
 "                                  N is attached, as in -v2 or --verbose=2:\n"
 "                                  -v 2, with the space, gives level 1\n";

/** @} ---------------------------------------------------------------------*/
/*------------------------------ FUNCTIONS ---------------------------------*/
/*--------------------------------------------------------------------------*/
/** @name Utility functions
 *  @{ */

/*--------------------------------------------------------------------------*/

int read_open_netCDF( netCDF::NcFile & f , std::string fn )
{
 fn = resolve_with_prefix( block_prefix , fn );

 try {
  f.open( fn , netCDF::NcFile::read );
  }
 catch( netCDF::exceptions::NcException & e ) {
  std::cerr << exe << ": cannot open nc4 file " << fn << std::endl;
  exit( 1 );
  }

 netCDF::NcGroupAtt gtype = f.getAtt( "SMS++_file_type" );
 if( gtype.isNull() ) {
  std::cerr << exe << ": " << fn << " is not an SMS++ nc4 file" << std::endl;
  exit( 1 );
  }

 int type;
 gtype.getValues( &type );

 if( ( type != eProbFile ) && ( type != eBlockFile ) ) {
  std::cerr << exe << ": " << fn << " is not a valid SMS++ file" << std::endl;
  exit( 1 );
  }

 return( type );
 }

/*--------------------------------------------------------------------------*/

namespace {

std::string tool_conf_prefix {};   ///< the -c prefix set by the tool, if any
bool conf_prefix_given = false;    ///< true if -c is on the command line
bool bconf_given = false;          ///< true if -B is on the command line
bool sconf_given = false;          ///< true if -S is on the command line

/// true if any of the Configuration files is found with conf_prefix
bool config_found( void )
{
 auto found = []( const std::string & name ) {
  return( ( ! name.empty() ) &&
	  std::filesystem::exists(
		       resolve_with_prefix( conf_prefix , name ) ) );
  };

 return( found( bconf_file ) || found( sconf_file ) || found( sol_cfg_file ) ||
	 ( ( ! bconf_given ) && found( default_bconf_name ) ) ||
	 ( ( ! sconf_given ) && found( default_sconf_name ) ) );
 }

}  // anonymous namespace

/*--------------------------------------------------------------------------*/

void docopt( void )
{
 // the layout of GNU --help, which help2man turns into a man page
 std::cout << "Usage: " << exe << " [options] <file>" << std::endl
           << "  or:  " << exe << " -h | --help" << std::endl
           << "  or:  " << exe << " -V | --version" << std::endl << std::endl;
 // a block of text, possibly under a title, followed by an empty line
 auto section = []( const std::string & title , const std::string & text ) {
  if( text.empty() )
   return;
  if( ! title.empty() )
   std::cout << title << std::endl;
  std::cout << text;
  if( text.back() != '\n' )
   std::cout << std::endl;
  std::cout << std::endl;
  };

 section( "" , docopt_desc );
 section( "Arguments:" , docopt_args );
 section( "Options:" , help );

 // how the Configuration files are looked up, the same for every tool
 std::cout << "Configuration files:" << std::endl
	   << "  The files given to -B, -S and -C, and those included by "
	   << "them, are looked" << std::endl
	   << "  up under the -c prefix ["
	   << ( tool_conf_prefix.empty() ? "current directory"
		                         : tool_conf_prefix )
	   << "]." << std::endl;

 std::string defaults;
 if( ! default_bconf_name.empty() )
  defaults = "-B " + default_bconf_name;
 if( ! default_sconf_name.empty() )
  defaults += ( defaults.empty() ? "-S " : ", -S " ) + default_sconf_name;
 if( ! defaults.empty() )
  std::cout << "  Defaults, used if they exist: " << defaults << std::endl;
 std::cout << "  An empty name, as in -B '', means no file." << std::endl;

 if( auto dir = installed_config_dir() ; ! dir.empty() )
  std::cout << "  Without -c, if none of these files is found, the "
	    << "configuration installed" << std::endl
	    << "  with the tool is used instead:" << std::endl
	    << "    " << dir << std::endl;
 std::cout << std::endl;

 section( "Examples:" , docopt_examples );

 std::cout << "Exit status:" << std::endl
	   << "  0 if the run completes, whatever the status of the Solvers,"
	   << " nonzero if" << std::endl
	   << "  the input or a configuration cannot be used." << std::endl;
 }

/*--------------------------------------------------------------------------*/

void drop_standard_option( char opt )
{
 // the long options, but the final nullptr record
 long_opts.erase( std::remove_if( long_opts.begin() ,
				  std::prev( long_opts.end() ) ,
				  [ opt ]( const option & o ) {
				   return( o.val == opt ); } ) ,
		  std::prev( long_opts.end() ) );

 // the line of the help
 const std::string head = std::string( "  -" ) + opt + ",";
 for( std::size_t pos = 0 ; pos < help.size() ; ) {
  auto end = help.find( '\n' , pos );
  end = ( end == std::string::npos ) ? help.size() : end + 1;
  if( help.compare( pos , head.size() , head ) == 0 )
   help.erase( pos , end - pos );
  else
   pos = end;
  }
 }

/*--------------------------------------------------------------------------*/

void add_format_option( const std::string & formats )
{
 short_opts += "f:";
 long_opts.insert( std::prev( long_opts.end() ) ,
                   { "format" , required_argument , nullptr , 'f' } );
 help += "  -f, --format <c>                native format of the input file, "
         "if not\n"
         "                                  netCDF, among:\n" + formats;
 }

/*--------------------------------------------------------------------------*/

bool process_standard_arg( int opt )
{
 switch( opt ) {
  case 'a': state_out_file = std::string( optarg ); break;
  case 'B': bconf_file = std::string( optarg );
            bconf_given = true;
            break;
  case 'b': state_in_file = std::string( optarg ); break;
  case 'p' : {
   block_prefix = normalize_prefix( std::string( optarg ) );
   Block::set_filename_prefix( std::string( block_prefix ) );
   break;
  }
  case 'S': sconf_file = std::string( optarg );
            sconf_given = true;
            break;
  case 'c': conf_prefix = normalize_prefix( std::string( optarg ) );
            conf_prefix_given = true;
            break;
  case 'o': output_solution = true; break;
  case 'I': sol_input = std::string( optarg ); break;
  case 'O': sol_output = std::string( optarg ); break;
  case 'C': sol_cfg_file = std::string( optarg ); break;
  case 'n': writeprob = true;
            prob_file = std::string( optarg );
            break;
  case 'D': dryrun = true; break;
  case 'f': input_format = optarg[ 0 ]; break;
  case 'v': {
   sol_verbose = true;
   verbosity_level = optarg ? std::atoi( optarg ) : 1;
   break;
  }
  case 'h': docopt(); exit( 0 );
  case 'V': {
   #ifdef SMSpp_tools_VERSION
    std::cout << "SMS++ tools version " << SMSpp_tools_VERSION << std::endl;
   #else
    std::cout << "SMS++ tools version not available" << std::endl;
   #endif
   exit( 0 );
   }
  case '?':
  default:  return( false );
  }
 return( true );
 }

/*--------------------------------------------------------------------------*/

void process_args( int argc , char ** argv ,
                   bool ( *custom_arg )( int opt ) )
{
 exe = get_filename( argv[ 0 ] );
 tool_conf_prefix = conf_prefix;
 if( argc < 2 ) {
  std::cerr << exe << ": no input file" << std::endl
	    << "Try '" << exe << " --help' for more information" << std::endl;
  exit( 1 );
  }

 while( true ) {  // options
  const auto opt = getopt_long( argc , argv , short_opts.data() ,
				long_opts.data() , nullptr );
  if( opt == -1 ) break;

  // tool-specific options are processed first: a tool that re-defines one
  // of the standard letters (e.g. the -n of sddp_solver) means its own
  if( custom_arg && custom_arg( opt ) )  // tool-specific option
   continue;                             // next

  if( process_standard_arg( opt ) )  // if it is a standard one
   continue;                         // next

  std::cerr << "Try '" << exe << " --help' for more information"
	    << std::endl;
  exit( 1 );
  }

 if( optind < argc )  // last argument == [Block] filename
  filename = std::string( argv[ optind ] );
 else {
  std::cerr << exe << ": no input file" << std::endl
            << "Try '" << exe << " --help' for more information" << std::endl;
  exit( 1 );
  }

 // without -c, a run that finds none of its Configuration files with the
 // prefix of the tool takes the whole configuration from the directory
 // installed with the tool, if any: a single prefix serves all the files, so
 // that the files included by the installed ones are found among them
 if( ( ! conf_prefix_given ) && ( ! config_found() ) )
  if( auto dir = installed_config_dir() ; ! dir.empty() )
   conf_prefix = dir;

 // if -B / -S was not given on the command line, fall back to the
 // conventional default file name, but only when the file is actually
 // reachable through the -c prefix: a run that points -c at the config/
 // directory then needs no -B / -S, while a missing default is silently
 // ignored (so it produces no spurious "cannot open" message and -B stays
 // genuinely optional). See default_config_file() for the lookup rules
 if( ! bconf_given )
  bconf_file = default_config_file( default_bconf_name );
 if( ! sconf_given )
  sconf_file = default_config_file( default_sconf_name );

 // hand the (final) -c prefix to Configuration, so that it is applied not only
 // to the top-level -B / -S files but also to every filename referenced from
 // *inside* a Configuration file (the "*filename" includes and the strInnerBSC
 // meta-config chain), which are opened by SMS++/Solver code that only knows
 // this executable-wide prefix. This mirrors Block::set_filename_prefix(),
 // set from the -p prefix for the Block "filename" references. Because the
 // prefix is now applied here, the -B / -S / Solution-config filenames are
 // passed to Configuration::deserialize() *bare* (no resolve_with_prefix at
 // the point of use), so it is prepended exactly once
 Configuration::set_filename_prefix( std::string( conf_prefix ) );

 // report which Configuration files are in use, so a run with defaults makes
 // clear which files it picked up (mirrors the "X is a Block file" message)
 report_config_file( "BlockConfig (-B)" , bconf_file );
 report_config_file( "BlockSolverConfig (-S)" , sconf_file );
 }  // end( process_args )

/*--------------------------------------------------------------------------*/

void process_args( int argc , char ** argv )
{
 process_args( argc , argv , nullptr );
 }

/*--------------------------------------------------------------------------*/

void smspp_terminate( void ) {
 std::cerr << "Uncaught exception in executing SMS++:\n";
 try {
  std::rethrow_exception( std::current_exception() );
 }
 catch( const std::exception & e ) {
  std::cerr << "\tException type: " << typeid( e ).name() << "\n";
  std::cerr << "\tException message: " << e.what() << "\n";
 } catch( ... ) {
  std::cerr << "\tUnknown exception" << std::endl;
 }
 std::abort(); // or exit(1)
 }

/*--------------------------------------------------------------------------*/

Block * get_Block( const std::string & b_file )
{
 auto block = Block::deserialize( b_file );
 if( ! block ) {
  std::cerr << "Error: " << b_file << " does not contain a valid Block"
	    << std::endl;
  exit( 1 );
  }

 return( block );
 }

/*--------------------------------------------------------------------------*/

Block * get_Block( const std::string & b_file , const std::string & classname ,
                   char frmt )
{
 // the "<file>[i]" form only exists for netCDF files
 if( ( ! b_file.empty() ) && ( b_file.back() == ']' ) )
  return( get_Block( b_file ) );

 // a netCDF file starts with "CDF" (classic) or "\x89HDF" (netCDF-4)
 const auto fn = resolve_with_prefix( block_prefix , b_file );
 std::ifstream in( fn , std::ios::binary );
 if( ! in ) {
  std::cerr << exe << ": cannot open " << fn << std::endl;
  exit( 1 );
  }
 char magic[ 4 ] = { 0 , 0 , 0 , 0 };
 in.read( magic , 4 );
 in.close();
 if( ( std::string( magic , 3 ) == "CDF" ) ||
     ( std::string( magic + 1 , 3 ) == "HDF" ) )
  return( get_Block( b_file ) );

 // a native format: the Block is created and load()-ed from the file
 if( classname.empty() ) {
  std::cerr << exe << ": " << fn << " is not an SMS++ netCDF file"
            << std::endl;
  exit( 1 );
  }
 auto block = Block::new_Block( classname );
 if( ! block ) {
  std::cerr << exe << ": no " << classname << " in this executable"
            << std::endl;
  exit( 1 );
  }
 block->load( fn , frmt );
 return( block );
 }

/*--------------------------------------------------------------------------*/

Block * get_Block( const netCDF::NcGroup & group )
{
 auto block = Block::new_Block( group );
 if( ! block ) {
  std::cerr << "Error: group does not contain a valid Block" << std::endl;
  exit( 1 );
  }

 return( block );
 }

/*--------------------------------------------------------------------------*/

Configuration * get_config( const std::string & conf_file )
{
 if( conf_file.empty() )
  return( nullptr );

 // the -c prefix is applied by Configuration itself (see set_filename_prefix()
 // in process_args), so the bare name is passed here to avoid double-prefixing
 auto cfg = Configuration::deserialize( conf_file );
 return( cfg );
 }

/*--------------------------------------------------------------------------*/

BlockConfig * get_blockconfig( const std::string & conf_file )
{
 auto cfg = get_config( conf_file );
 auto bcfg = dynamic_cast< BlockConfig * >( cfg );
 if( ! bcfg )
  delete cfg;
 return( bcfg );
 }

/*--------------------------------------------------------------------------*/

BlockSolverConfig * get_blocksolverconfig( const std::string & conf_file )
{
 auto cfg = get_config( conf_file );
 auto bscfg = dynamic_cast< BlockSolverConfig * >( cfg );
 if( ! bscfg )
  delete cfg;
 return( bscfg );
 }

/*--------------------------------------------------------------------------*/

std::string get_str_par( const ComputeConfig * compute_config ,
                         const std::string & par_name )
{
 for( const auto & pair : compute_config->str_pars )
  if( pair.first == par_name )
   return( pair.second );
 return( "" );
 }

/*--------------------------------------------------------------------------*/

int get_int_par( const ComputeConfig * compute_config ,
                 const std::string & par_name )
{
 for( const auto & pair : compute_config->int_pars )
  if( pair.first == par_name )
   return( pair.second );
 return( Inf< int >() );
 }

/*--------------------------------------------------------------------------*/

void erase_str_par( ComputeConfig * compute_config ,
                    const std::string & par_name )
{
 auto & str_pars = compute_config->str_pars;
 str_pars.erase( std::remove_if( str_pars.begin() , str_pars.end() ,
                                 [ & ]( const auto & pair ) {
                                  return( pair.first == par_name ); } ) ,
                 str_pars.end() );
 }

/*--------------------------------------------------------------------------*/

void report_config_file( const std::string & what , const std::string & file )
{
 if( verbosity_level < 1 )
  return;
 if( file.empty() )
  std::cout << "no " << what << std::endl;
 else
  std::cout << resolve_with_prefix( conf_prefix , file ) << " is the "
            << what << std::endl;
 }

/*--------------------------------------------------------------------------*/

std::string default_config_file( const std::string & name )
{
 // look for the file only at the conf_prefix location (the -c prefix, which
 // a plain run points at the config/ directory); a missing default is
 // reported as an empty string, so the caller can keep the file genuinely
 // optional. The bare \p name is returned, not the resolved path, since
 // every consumer re-resolves it against conf_prefix at the point of use
 if( std::filesystem::exists( resolve_with_prefix( conf_prefix , name ) ) )
  return( name );
 return( std::string{} );
 }

/*--------------------------------------------------------------------------*/

std::string installed_config_dir( void )
{
 #ifdef SMSPP_TOOL_CONFIG_DIR
  static const std::string dir = []() -> std::string {
   // the path of the running executable
   std::error_code ec;
   std::filesystem::path self;
   #if defined( _WIN32 )
    char * p = nullptr;
    if( ( _get_pgmptr( &p ) == 0 ) && p )
     self = p;
   #elif defined( __APPLE__ )
    uint32_t size = 0;
    _NSGetExecutablePath( nullptr , &size );
    std::string buf( size , '\0' );
    if( _NSGetExecutablePath( buf.data() , &size ) == 0 )
     self = buf.c_str();
   #else
    self = std::filesystem::read_symlink( "/proc/self/exe" , ec );
   #endif
   if( self.empty() )
    return( std::string() );

   // resolve the links first, so that a link to the executable (as the ones
   // a package manager puts in its bin/) leads to the real installed tree
   self = std::filesystem::weakly_canonical( self , ec );
   if( ec )
    return( std::string() );

   auto cfg = self.parent_path() / SMSPP_TOOL_CONFIG_DIR;
   cfg = cfg.lexically_normal();
   if( ! std::filesystem::is_directory( cfg , ec ) )
    return( std::string() );

   return( normalize_prefix( cfg.string() ) );
   }();

  return( dir );
 #else
  return( std::string() );
 #endif
 }

/*--------------------------------------------------------------------------*/

using MapConfig = SimpleConfiguration< std::map< std::string ,
						 Configuration * > >;

/*--------------------------------------------------------------------------*/

void config_Block( Block * block ,
		   Configuration * b_config , Configuration * s_config )
{
 if( b_config ) {
  // handle the special case of a "meta" BlockConfig: BlockConfig-ure all
  // Block whose classname() matches, falling back to the "*" entry (if any)
  // for the non-matching ones
  if( auto mb = dynamic_cast< MapConfig * >( b_config ) )
   for_each_by_classname( block , mb->f_value ,
			  []( Block * b , Configuration * c ) {
			   if( auto bc = dynamic_cast< BlockConfig * >( c ) ) {
			    auto cbc = bc->clone();
			    cbc->apply( b );
			    delete cbc;
			    }
			   } );
  else  // must be an "ordinary" BlockConfig, just apply() it
   if( auto * bc = dynamic_cast< BlockConfig * >( b_config ) )
    bc->apply( block );
   else
    throw( std::invalid_argument( "config_Block: b_config is not a valid "
				  "[meta]BlockConfig" ) );
  }

 // the initial Solution, if any, is written once the BlockConfig is in
 // place and before the Solvers are registered, since a Solver may read it
 // when it is attached (say, LagrangianDualSolver takes its first
 // multipliers from the dual values of the relaxed constraints); the
 // abstract representation it is written into is therefore generated here,
 // and solve_all() writes it again before each compute()
 if( s_config && ( ! sol_input.empty() ) ) {
  std::function< void( Block * ) > generate = [ & ]( Block * b ) {
   b->generate_abstract_variables();
   b->generate_abstract_constraints();
   b->generate_objective();
   for( auto sb : b->get_nested_Blocks() )
    generate( sb );
   };
  generate( block );
  get_initial_Solution( block );
  }

 if( s_config ) {
  // handle the special case of a "meta" BlockSolverConfig: the same, father
  // first, as the Solver of a Block finds those of its sub-Block when it is
  // compute()-d, not when it is registered
  if( auto mb = dynamic_cast< MapConfig * >( s_config ) ) {
   for_each_by_classname( block , mb->f_value ,
			  []( Block * b , Configuration * c ) {
			   if( auto bsc =
			       dynamic_cast< BlockSolverConfig * >( c ) )
			    bsc->apply( b );
			   } );

   // finally, clear() all the BlockSolverConfig for final cleanup
   mb->clear();
   }
  else {  // must be an "ordinary" BlockSolverConfig, just apply() it
   if( auto * sc = dynamic_cast< BlockSolverConfig * >( s_config ) ) {
    sc->apply( block );
    sc->clear();
    }
   else
    throw( std::invalid_argument( "config_Block: s_config is not a valid "
				  "[meta]BlockSolverConfig" ) );
   }
  }
 }

/*--------------------------------------------------------------------------*/

void cleanup_bsc( Block * block , Configuration * s_config )
{
 if( ! s_config )
  return;

 // handle the special case of a "meta" BlockSolverConfig: apply the
 // clear()-ed BlockSolverConfig to all Block whose classname() matches,
 // or that of the "*" entry (if any) to the non-matching ones
 if( auto mb = dynamic_cast< MapConfig * >( s_config ) )
  for_each_by_classname( block , mb->f_value ,
			 []( Block * b , Configuration * c ) {
			  if( auto bsc =
			      dynamic_cast< BlockSolverConfig * >( c ) )
			   bsc->apply( b );
			  } );
 else  // it *must* be a BlockSolverConfig, it has been checked before
  static_cast< BlockSolverConfig * >( s_config )->apply( block );
 }

/*--------------------------------------------------------------------------*/

void require_solver_config( const std::string & bsc_file )
{
 if( bsc_file.empty() )
  throw( std::invalid_argument(
   "a BlockSolverConfig must be provided (did you forget the -S option?)" ) );
 }

/*--------------------------------------------------------------------------*/

void get_all( const std::string & b_file , const std::string & bc_file ,
	      const std::string & bsc_file , Block * & block ,
	      Configuration * & s_config )
{
 // a generic Block is solved as deserialized unless a BlockConfig is given:
 // only the BlockSolverConfig (-S) is mandatory here, the BlockConfig (-B)
 // stays optional (a plain MCFBlock / BinaryKnapsackBlock needs no
 // formulation choice)
 require_solver_config( bsc_file );
 block = get_Block( b_file );
 auto b_config = get_config( bc_file );
 s_config = get_config( bsc_file );
 config_Block( block , b_config , s_config );
 delete b_config;
 }

/*--------------------------------------------------------------------------*/

void get_all( const netCDF::NcGroup & group , const std::string & bc_file ,
	      const std::string & bsc_file , Block * & block ,
	      Configuration * & s_config )
{
 require_solver_config( bsc_file );  // -S mandatory, -B optional (see above)
 block = get_Block( group );
 auto b_config = get_config( bc_file );
 s_config = get_config( bsc_file );
 config_Block( block , b_config , s_config );
 delete b_config;
 }

/*--------------------------------------------------------------------------*/
/// get Block, BlockConfig and BlockSolverConfig from group

void get_all( const netCDF::NcGroup & group , Block * & block ,
	      Configuration * & s_config )
{
 // deserialize Block
 block = Block::new_Block( group.getGroup( "Block" ) );
 if( ! block ) {
  std::cerr << "Error: group does not contain a valid Block" << std::endl;
  exit( 1 );
  }

 // deserialize BlockConfig
 auto b_config = BlockConfig::new_Configuration(
					  group.getGroup( "BlockConfig" ) );

 // deserialize BlockSolverConfig
 s_config = BlockSolverConfig::new_Configuration(
					   group.getGroup( "BlockSolver" ) );

 config_Block( block , b_config , s_config );
 delete b_config;
 }

/*--------------------------------------------------------------------------*/

void print_status( int status )
{
 std::cout << "Status = " << status << " (";

 // note: the description is printed without the closing ")" and the
 // std::endl, which are added below, so that a status that none of the
 // cases catches (a :Solver-specific one extending the set of return
 // codes after kLastSolverError) still produces a complete line
 switch( status ) {
  case Solver::kStillRunning: std::cout << "Still running";           break;
  case Solver::kUnEval:       std::cout << "Not evaluated";           break;
  case Solver::kOK:           std::cout << "Success";                 break;
  case Solver::kError:        std::cout << "Error";                   break;
  case Solver::kInfeasible:   std::cout << "Infeasible";              break;
  case Solver::kUnbounded:    std::cout << "Unbounded";               break;
  case Solver::kStopTime:     std::cout << "Stopped for time limit";  break;
  case Solver::kStopIter:     std::cout << "Stopped for iteration limit";
                                                                      break;
  case Solver::kBlockLocked:  std::cout << "Block could not be locked";
                                                                      break;
  case Solver::kLowPrecision: std::cout << "Solution found, but not provably "
                                           "optimal";                 break;
  default:                    std::cout << "Solver-specific status";
  }

 std::cout << ")" << std::endl;
 }

/*--------------------------------------------------------------------------*/

void get_initial_Solution( Block * block )
{
 if( sol_input.empty() )
  return;

 if( auto initsol = Solution::deserialize( sol_input ) ) {
  initsol->write( block );
  delete initsol;
  }
 else
  std::cout << "Warning: input Solution " << sol_input << " invalid"
	    << std::endl;
 }

/*--------------------------------------------------------------------------*/

void get_initial_State( Solver * solver )
{
 if( state_in_file.empty() )
  return;

 try {
  auto state = State::deserialize( state_in_file );
  solver->put_State( *state );
  delete( state );
  }
 catch( netCDF::exceptions::NcException & e ) {
  std::cout << "Warning: State file " << state_in_file
	    << " could not be loaded" << std::endl;
  }
 catch( const std::exception & e ) {
  std::cout << "Warning: error " << e.what()
	    << " occurred while loading the Solver State" << std::endl;
  }
 }

/*--------------------------------------------------------------------------*/

void write_final_Solution( Block * block , Configuration * cfg ,
			   bool replace )
{
 if( sol_output.empty() )
  return;

 // use provided Configuration if any, otherwise (possibly) load one
 Configuration * outsolcfg = cfg;
 if( ( ! outsolcfg ) && ( ! sol_cfg_file.empty() ) )
  if( ! ( outsolcfg = Configuration::deserialize( sol_cfg_file ) ) )
   std::cout << "Warning: output Solution Configuration "
             << sol_cfg_file << " invalid" << std::endl;

 if( auto sol = block->get_Solution( outsolcfg , false ) ) {
  sol->serialize( sol_output , replace );
  delete sol;
  }
 else
  std::cout << "Warning: output Solution empty" << std::endl;

 // if using a "local" Configuration, release it
 if( ! cfg )
  delete outsolcfg;
 }

/*--------------------------------------------------------------------------*/

void write_final_State( Solver * solver , bool replace )
{
 if( state_out_file.empty() )
  return;

 try {
  solver->serialize_State( state_out_file , replace );
  }
 catch( netCDF::exceptions::NcException & e ) {
  std::cout << "Warning: State file " << state_out_file
	    << " could not be opened" << std::endl;
  }
 catch( const std::exception & e ) {
  std::cout << "Warning: error " << e.what()
	    << " occurred while saving the Solver State" << std::endl;
  }
 }

/*--------------------------------------------------------------------------*/

void print_solver_parameters( const std::vector< Block * > & roots )
{
 if( verbosity_level < 2 )
  return;

 // one entry per distinct ( Solver class , Block class , parameters ), in
 // the order they are first met, with the number of Solvers sharing it
 struct Entry {
  std::string solver , owner , pars;
  std::size_t count;
  };
 std::vector< Entry > entries;
 std::set< Block * > visited;

 std::function< void( Block * ) > visit = [ & ]( Block * b ) {
  if( ( ! b ) || ( ! visited.insert( b ).second ) )
   return;
  for( auto solver : b->get_registered_solvers() ) {
   std::ostringstream pars;
   solver->print_parameters( pars );
   Entry e{ solver->classname() , b->classname() , pars.str() , 1 };
   auto it = std::find_if( entries.begin() , entries.end() ,
                           [ & ]( const Entry & o ) {
                            return( ( o.solver == e.solver ) &&
                                    ( o.owner == e.owner ) &&
                                    ( o.pars == e.pars ) ); } );
   if( it == entries.end() )
    entries.push_back( std::move( e ) );
   else
    ++it->count;
   }
  if( verbosity_level >= 3 )
   for( auto sb : b->get_nested_Blocks() )
    visit( sb );
  };
 for( auto b : roots )
  visit( b );

 for( const auto & e : entries ) {
  std::cout << std::endl << "--- parameters of " << e.solver << " on "
            << e.owner;
  if( e.count > 1 )
   std::cout << " (" << e.count << " Solvers)";
  std::cout << std::endl << e.pars;
  }
 std::cout << std::endl;
 }

/*--------------------------------------------------------------------------*/

int solve_all( Block * block ,
               const std::function< bool( void ) > & is_writer ,
               bool write_solution )
{
 // load initial Solution, if provided - - - - - - - - - - - - - - - - - - - -
 int retval = 0;
 Solution * initsol = nullptr;
 if( ! sol_input.empty() )
  if( ! ( initsol = Solution::deserialize( sol_input ) ) )
   std::cout << "Warning: input Solution " << sol_input << " invalid"
	      << std::endl;

 // prepare file and Configuration for final Solution(s) - - - - - - - - - - -
 Configuration * outsolcfg = nullptr;
 if( ( ! sol_output.empty() ) && ( ! sol_cfg_file.empty() ) )
  if( ! ( outsolcfg = Configuration::deserialize( sol_cfg_file ) ) )
   std::cout << "Warning: output Solution Configuration "
	     << sol_cfg_file << " invalid" << std::endl;

 // a Solver must be explicitly provided: if no Solver is registered to the
 // Block (typically because no valid BlockSolverConfig was given via -S),
 // there is nothing to solve and silently doing nothing would hide the
 // configuration mistake. The only legitimate no-Solver use is writing the
 // problem back (-n), which has already happened by now, so just return.
 if( block->get_registered_solvers().empty() ) {
  if( writeprob )
   return( 0 );
  throw( std::invalid_argument(
   "no Solver registered to the Block: a BlockSolverConfig must be provided "
   "(did you forget the -S option?)" ) );
  }

 print_solver_parameters( block );

 // for each of the registered Solver- - - - - - - - - - - - - - - - - - - - -
 //- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 for( auto solver : block->get_registered_solvers() ) {
  std::cout << "Solver: " << solver->classname() << std::endl;

  if( initsol )  // set the initial Solution, if provided- - - - - - - - - - -
   initsol->write( block );

  // load the initial State, if provided - - - - - - - - - - - - - - - - - - -
  // note: this is bound to fail if there are multiple Solver, since the
  // State is supposed to be Solver-specific, unless all Solver but at
  // most one ignore the State
  get_initial_State( solver );

  if( ! dryrun ) {  // compute() - - - - - - - - - - - - - - - - - - - - - - -
   std::chrono::time_point< std::chrono::system_clock > start , end;
   start = std::chrono::system_clock::now();
   auto status = solver->compute();
   end = std::chrono::system_clock::now();
   std::chrono::duration< double > compute_time = end - start;
   std::cout << "Elapsed time: " << compute_time.count() << " s" << std::endl;

   if( status != Solver::kOK ) retval = 1;

   auto ub = solver->get_ub();
   auto lb = solver->get_lb();
   print_status( status );
   // all the digits of a double, which whoever reads the bounds needs
   const auto prec = std::cout.precision(
                      std::numeric_limits< double >::max_digits10 );
   std::cout << "Upper bound = " << ub << std::endl;
   std::cout << "Lower bound = " << lb << std::endl;
   std::cout.precision( prec );

   if( ! sol_output.empty() ) {
    if( solver->has_var_solution() )
     solver->get_var_solution();
    else
     std::cout << "Warning: var solution required but not available"
	       << std::endl;

    if( auto cdas = dynamic_cast< CDASolver * >( solver ) )
     if( cdas->has_dual_solution() )
      cdas->get_dual_solution();
    }
   }

  // only the writer writes, once all have retrieved the solution- - - - - -
  const bool writer = ( ! is_writer ) || is_writer();

  // write final Solution, if required - - - - - - - - - - - - - - - - - - - -
  if( writer && write_solution )
   write_final_Solution( block , outsolcfg );

  // write final State, if required- - - - - - - - - - - - - - - - - - - - - -
  if( writer )
   write_final_State( solver );

  }  // end( for( each Solver ) )- - - - - - - - - - - - - - - - - - - - - - -
     //- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

 delete outsolcfg;
 delete initsol;

 return( retval );

 }  // end( solve_all )

/*--------------------------------------------------------------------------*/

void write_nc4problem( Block * block ,
		       Configuration * b_config , Configuration * s_config )
{
 std::string nc4_file = prob_file;
 if( nc4_file.empty() ) {
  std::size_t found = filename.find_last_of( '.' );
  nc4_file = filename.substr( 0 , found ) + "_problem.nc4";
  }

 netCDF::NcFile outfile;
 try {
  outfile.open( nc4_file , netCDF::NcFile::replace );
  }
 catch( netCDF::exceptions::NcException & e ) {
  std::cerr << exe << ": cannot open nc4 file " << nc4_file << std::endl;
  exit( 1 );
  }

 outfile.putAtt( "SMS++_file_type" , netCDF::NcInt() , eProbFile );

 block->Block::serialize( outfile , eProbFile );
 netCDF::NcGroup g = outfile.getGroup( "Prob_0" );

 auto new_bc = g.addGroup( "BlockConfig" );
 if( b_config )
  b_config->serialize( new_bc );

 auto new_bsc = g.addGroup( "BlockSolver" );
 if( s_config )
  s_config->serialize( new_bsc );
 outfile.close();
 }

/*--------------------------------------------------------------------------*/
/*------------------------- end common_utils.cpp ---------------------------*/
/*--------------------------------------------------------------------------*/