main.cpp

Go to the documentation of this file.

#include <string>
#include <vector>
#include <math.h>
#include <sstream>
#include <regex>
 
#include <boost/algorithm/string.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/ip/host_name.hpp>
#include <boost/program_options.hpp>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result"
#include <boost/process/child.hpp>
#pragma GCC diagnostic pop
#include <boost/process/env.hpp>
#include <boost/process/system.hpp>
#include <boost/process/io.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/fstream.hpp>
#include <fc/crypto/private_key.hpp>
#include <fc/crypto/public_key.hpp>
#include <fc/io/json.hpp>
#include <fc/network/ip.hpp>
#include <fc/reflect/variant.hpp>
#include <fc/log/logger_config.hpp>
#include <ifaddrs.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <net/if.h>
#include <sysio/chain/genesis_state.hpp>
 
#include "config.hpp"
 
using namespace std;
namespace bfs = boost::filesystem;
namespace bp = boost::process;
namespace bpo = boost::program_options;
using boost::asio::ip::tcp;
using boost::asio::ip::host_name;
using bpo::options_description;
using bpo::variables_map;
using public_key_type = fc::crypto::public_key;
using private_key_type = fc::crypto::private_key;
using namespace sysio::launcher::config;
 
const string block_dir = "blocks";
const string shared_mem_dir = "state";
 
struct local_identity {
  vector <fc::ip::address> addrs;
  vector <string> names;
 
  void initialize () {
    names.push_back ("localhost");
    names.push_back ("127.0.0.1");
 
    boost::system::error_code ec;
    string hn = host_name (ec);
    if (ec.value() != boost::system::errc::success) {
      cerr << "unable to retrieve host name: " << ec.message() << endl;
    }
    else {
      names.push_back (hn);
      if (hn.find ('.') != string::npos) {
        names.push_back (hn.substr (0,hn.find('.')));
      }
    }
 
    ifaddrs *ifap = 0;
    if (::getifaddrs (&ifap) == 0) {
      for (ifaddrs *p_if = ifap; p_if != 0; p_if = p_if->ifa_next) {
        if (p_if->ifa_addr != 0 &&
            p_if->ifa_addr->sa_family == AF_INET &&
            (p_if->ifa_flags & IFF_UP) == IFF_UP) {
          sockaddr_in *ifaddr = reinterpret_cast<sockaddr_in *>(p_if->ifa_addr);
          int32_t in_addr = ntohl(ifaddr->sin_addr.s_addr);
 
          if (in_addr != 0) {
            fc::ip::address ifa(in_addr);
            addrs.push_back (ifa);
          }
        }
      }
      ::freeifaddrs (ifap);
    }
    else {
      cerr << "unable to query local ip interfaces" << endl;
      addrs.push_back (fc::ip::address("127.0.0.1"));
    }
  }
 
  bool contains (const string &name) const {
    try {
      fc::ip::address test(name);
      for (const auto &a : addrs) {
        if (a == test)
          return true;
      }
    }
    catch (...) {
      // not an ip address
      for (const auto& n : names) {
        if (n == name)
          return true;
      }
    }
    return false;
  }
 
} local_id;
 
class eosd_def;
 
class host_def {
public:
  host_def ()
    : genesis("genesis.json"),
      ssh_identity (""),
      ssh_args (""),
      sysio_home(),
      host_name("127.0.0.1"),
      public_name("localhost"),
      listen_addr("0.0.0.0"),
      base_p2p_port(9876),
      base_http_port(8888),
      def_file_size(8192),
      instances(),
      p2p_count(0),
      http_count(0),
      dot_label_str()
  {}
 
  string           genesis;
  string           ssh_identity;
  string           ssh_args;
  string           sysio_home;
  string           host_name;
  string           public_name;
  string           listen_addr;
  uint16_t         base_p2p_port;
  uint16_t         base_http_port;
  uint16_t         def_file_size;
  vector<eosd_def> instances;
 
  uint16_t p2p_port() {
    return base_p2p_port + p2p_count++;
  }
 
  uint16_t http_port() {
    return base_http_port + http_count++;
  }
 
  uint16_t p2p_bios_port() {
    return base_p2p_port - 100;
  }
 
  uint16_t http_bios_port() {
     return base_http_port - 100;
  }
 
  bool is_local( ) const {
    return local_id.contains( host_name );
  }
 
  const string &dot_label () {
    if (dot_label_str.empty() ) {
      mk_dot_label();
    }
    return dot_label_str;
  }
 
 
private:
  uint16_t p2p_count;
  uint16_t http_count;
  string   dot_label_str;
 
protected:
  void mk_dot_label() {
    if (public_name.empty()) {
      dot_label_str = host_name;
    }
    else if (boost::iequals(public_name,host_name)) {
      dot_label_str = public_name;
    }
    else
      dot_label_str = public_name + "/" + host_name;
  }
};
 
class tn_node_def;
 
class eosd_def {
public:
  eosd_def()
    : config_dir_name (),
      data_dir_name (),
      p2p_port(),
      http_port(),
      file_size(),
      name(),
      node(),
      host(),
      p2p_endpoint() {}
 
 
 
  string       config_dir_name;
  string       data_dir_name;
  uint16_t     p2p_port;
  uint16_t     http_port;
  uint16_t     file_size;
  string       name;
  tn_node_def* node;
  string       host;
  string       p2p_endpoint;
 
  void set_host (host_def* h, bool is_bios);
  void mk_dot_label ();
  const string &dot_label () {
    if (dot_label_str.empty() ) {
      mk_dot_label();
    }
    return dot_label_str;
  }
 
  string get_node_num() const {
     return name.substr( name.length() - 2 );
  }
private:
  string dot_label_str;
};
 
class tn_node_def {
public:
  string          name;
  vector<private_key_type> keys;
  vector<string>  peers;
  vector<string>  producers;
  eosd_def*       instance;
  string          gelf_endpoint;
  bool            dont_start = false;
};
 
void
eosd_def::mk_dot_label () {
  dot_label_str = name + "\\nprod=";
  if (node == 0 || node->producers.empty()) {
    dot_label_str += "<none>";
  }
  else {
    bool docomma = false;
    for (auto &prod: node->producers) {
      if (docomma)
        dot_label_str += ",";
      else
        docomma = true;
      dot_label_str += prod;
    }
  }
}
 
void
eosd_def::set_host( host_def* h, bool is_bios ) {
  host = h->host_name;
  p2p_port = is_bios ? h->p2p_bios_port() : h->p2p_port();
  http_port = is_bios ? h->http_bios_port() : h->http_port();
  file_size = h->def_file_size;
  p2p_endpoint = h->public_name + ":" + boost::lexical_cast<string, uint16_t>(p2p_port);
}
 
struct remote_deploy {
  string ssh_cmd = "/usr/bin/ssh";
  string scp_cmd = "/usr/bin/scp";
  string ssh_identity;
  string ssh_args;
  bfs::path local_config_file = "temp_config";
};
 
struct testnet_def {
  string name;
  remote_deploy ssh_helper;
  map <string,tn_node_def> nodes;
};
 
struct prodkey_def {
  string producer_name;
  public_key_type block_signing_key;
};
 
struct producer_set_def {
  vector<prodkey_def> schedule;
};
 
struct server_name_def {
  string ipaddr;
  string name;
  bool has_bios;
  string sysio_home;
  uint16_t instances;
  server_name_def () : ipaddr(), name(), has_bios(false), sysio_home(), instances(1) {}
};
 
struct server_identities {
  vector<server_name_def> producer;
  vector<server_name_def> nonprod;
  vector<string> db;
  string default_sysio_home;
  remote_deploy ssh;
};
 
struct node_rt_info {
  bool remote;
  string pid_file;
  string kill_cmd;
};
 
struct last_run_def {
  vector <node_rt_info> running_nodes;
};
 
enum launch_modes {
  LM_NONE,
  LM_LOCAL,
  LM_REMOTE,
  LM_NAMED,
  LM_ALL,
  LM_VERIFY
};
 
enum allowed_connection : char {
  PC_NONE = 0,
  PC_PRODUCERS = 1 << 0,
  PC_SPECIFIED = 1 << 1,
  PC_ANY = 1 << 2
};
 
class producer_names {
public:
   static string producer_name(unsigned int producer_number, bool shared_producer = false);
private:
   static const int total_chars = 12;
   static const char slot_chars[];
   static const char valid_char_range;
};
 
const char producer_names::slot_chars[] = "abcdefghijklmnopqrstuvwxyz";
const char producer_names::valid_char_range = sizeof(producer_names::slot_chars) - 1;
 
// for 26 or fewer total producers create "defproducera" .. "defproducerz"
// above 26 produce  "defproducera" .. "defproducerz",  "defproduceaa" .. "defproducerb", etc.
string producer_names::producer_name(unsigned int producer_number, bool shared_producer) {
   // keeping legacy "defproducer[a-z]", but if greater than valid_char_range, will use "defpraaaaaaa"
   // shared_producer will appear in all nodes' config
   char prod_name[] = "defproducera";
   if (producer_number > valid_char_range) {
      for (int current_char_loc = 5; current_char_loc < total_chars; ++current_char_loc) {
         prod_name[current_char_loc] = slot_chars[0];
      }
   }
 
   prod_name[total_chars] = '\0';
   for (int current_char_loc = total_chars - 1; current_char_loc >= 0; --current_char_loc) {
      const unsigned int slot_value = static_cast<char>(producer_number % valid_char_range);
      producer_number /= valid_char_range;
      prod_name[current_char_loc] = slot_chars[slot_value];
      if (!producer_number)
         break;
   }
 
   // make sure we haven't cycled back to the first 26 names (some time after 26^6)
   if (string(prod_name) == "defproducera" && producer_number != 0)
      throw std::runtime_error( "launcher not designed to handle numbers this large " );
 
   if (shared_producer) {
      prod_name[0] = 's';
      prod_name[1] = 'h';
      prod_name[2] = 'r';
   }
   return prod_name;
}
 
struct launcher_def {
   bool force_overwrite;
   size_t total_nodes;
   size_t unstarted_nodes;
   size_t prod_nodes;
   size_t producers;
   size_t shared_producers;
   size_t next_node;
   string shape;
   allowed_connection allowed_connections = PC_NONE;
   bfs::path genesis;
   bfs::path output;
   bfs::path host_map_file;
   bfs::path server_ident_file;
   bfs::path stage;
 
   string erd;
   bfs::path config_dir_base;
   bfs::path data_dir_base;
   bool skip_transaction_signatures = false;
   string eosd_extra_args;
   std::map<uint,string> specific_nodeop_args;
   std::map<uint,string> specific_nodeop_installation_paths;
   testnet_def network;
   string gelf_endpoint;
   vector <string> aliases;
   vector <host_def> bindings;
   int per_host = 0;
   last_run_def last_run;
   int start_delay = 0;
   bool gelf_enabled;
   bool nogen;
   bool boot;
   bool add_enable_stale_production = false;
   string launch_name;
   string launch_time;
   server_identities servers;
   producer_set_def producer_set;
   string start_temp;
   string start_script;
   std::optional<uint32_t> max_block_cpu_usage;
   std::optional<uint32_t> max_transaction_cpu_usage;
   sysio::chain::genesis_state genesis_from_file;
 
   void assign_name (eosd_def &node, bool is_bios);
 
   void set_options (bpo::options_description &cli);
   void initialize (const variables_map &vmap);
   void init_genesis ();
   void load_servers ();
   bool generate ();
   void define_network ();
   void bind_nodes ();
   host_def *find_host (const string &name);
   host_def *find_host_by_name_or_address (const string &name);
   host_def *deploy_config_files (tn_node_def &node);
   string compose_scp_command (const host_def &host, const bfs::path &source,
                               const bfs::path &destination);
   void write_config_file (tn_node_def &node);
   void write_logging_config_file (tn_node_def &node);
   void write_genesis_file (tn_node_def &node);
   void write_setprods_file ();
   void write_bios_boot ();
 
   bool   is_bios_ndx (size_t ndx);
   size_t start_ndx();
   bool   next_ndx(size_t &ndx);
   size_t skip_ndx (size_t from, size_t offset);
 
   void make_line (bool make_ring = true);
   void make_star ();
   void make_mesh ();
   void make_custom ();
   void write_dot_file ();
   void format_ssh (const string &cmd, const string &host_name, string &ssh_cmd_line);
   void do_command(const host_def& host, const string& name, vector<pair<string, string>> env_pairs, const string& cmd);
   bool do_ssh (const string &cmd, const string &host_name);
   void prep_remote_config_dir (eosd_def &node, host_def *host);
   void launch (eosd_def &node, string &gts);
   void kill (launch_modes mode, string sig_opt);
   static string get_node_num(uint16_t node_num);
   pair<host_def, eosd_def> find_node(uint16_t node_num);
   vector<pair<host_def, eosd_def>> get_nodes(const string& node_number_list);
   void bounce (const string& node_numbers);
   void down (const string& node_numbers);
   void roll (const string& host_names);
   void start_all (string &gts, launch_modes mode);
   void ignite ();
   string find_and_remove_arg(string str, string substr);
};
 
void
launcher_def::set_options (bpo::options_description &cfg) {
  cfg.add_options()
    ("force,f", bpo::bool_switch(&force_overwrite)->default_value(false), "Force overwrite of existing configuration files and erase blockchain")
    ("nodes,n",bpo::value<size_t>(&total_nodes)->default_value(1),"total number of nodes to configure and launch")
    ("unstarted-nodes",bpo::value<size_t>(&unstarted_nodes)->default_value(0),"total number of nodes to configure, but not launch")
    ("pnodes,p",bpo::value<size_t>(&prod_nodes)->default_value(1),"number of nodes that contain one or more producers")
    ("producers",bpo::value<size_t>(&producers)->default_value(21),"total number of non-bios and non-shared producer instances in this network")
    ("shared-producers",bpo::value<size_t>(&shared_producers)->default_value(0),"total number of shared producers on each non-bios nodes")
    ("mode,m",bpo::value<vector<string>>()->multitoken()->default_value({"any"}, "any"),"connection mode, combination of \"any\", \"producers\", \"specified\", \"none\"")
    ("shape,s",bpo::value<string>(&shape)->default_value("star"),"network topology, use \"star\", \"mesh\", \"ring\", \"line\" or give a filename for custom")
    ("genesis,g",bpo::value<string>()->default_value("./genesis.json"),"set the path to genesis.json")
    ("skip-signature", bpo::bool_switch(&skip_transaction_signatures)->default_value(false), (string(node_executable_name) + " does not require transaction signatures.").c_str())
    (node_executable_name, bpo::value<string>(&eosd_extra_args), ("forward " + string(node_executable_name) + " command line argument(s) to each instance of " + string(node_executable_name) + ", enclose arg(s) in quotes").c_str())
    ("specific-num", bpo::value<vector<uint>>()->composing(), ("forward " + string(node_executable_name) + " command line argument(s) (using \"--specific-" + string(node_executable_name) + "\" flag) to this specific instance of " + string(node_executable_name) + ". This parameter can be entered multiple times and requires a paired \"--specific-" + string(node_executable_name) +"\" flag each time it is used").c_str())
    (("specific-" + string(node_executable_name)).c_str(), bpo::value<vector<string>>()->composing(), ("forward " + string(node_executable_name) + " command line argument(s) to its paired specific instance of " + string(node_executable_name) + "(using \"--specific-num\"), enclose arg(s) in quotes").c_str())
    ("spcfc-inst-num", bpo::value<vector<uint>>()->composing(), ("Specify a specific version installation path (using \"--spcfc-inst-"+ string(node_executable_name) + "\" flag) for launching this specific instance of " + string(node_executable_name) + ". This parameter can be entered multiple times and requires a paired \"--spcfc-inst-" + string(node_executable_name) + "\" flag each time it is used").c_str())
    (("spcfc-inst-" + string(node_executable_name)).c_str(), bpo::value<vector<string>>()->composing(), ("Provide a specific version installation path to its paired specific instance of " + string(node_executable_name) + "(using \"--spcfc-inst-num\")").c_str())
    ("delay,d",bpo::value<int>(&start_delay)->default_value(0),"seconds delay before starting each node after the first")
    ("boot",bpo::bool_switch(&boot)->default_value(false),"After deploying the nodes and generating a boot script, invoke it.")
    ("nogen",bpo::bool_switch(&nogen)->default_value(false),"launch nodes without writing new config files")
    ("host-map",bpo::value<string>(),"a file containing mapping specific nodes to hosts. Used to enhance the custom shape argument")
    ("servers",bpo::value<string>(),"a file containing ip addresses and names of individual servers to deploy as producers or non-producers ")
    ("per-host",bpo::value<int>(&per_host)->default_value(0),("specifies how many " + string(node_executable_name) + " instances will run on a single host. Use 0 to indicate all on one.").c_str())
    ("network-name",bpo::value<string>(&network.name)->default_value("testnet_"),"network name prefix used in GELF logging source")
    ("enable-gelf-logging",bpo::value<bool>(&gelf_enabled)->default_value(false),"enable gelf logging appender in logging configuration file")
    ("gelf-endpoint",bpo::value<string>(&gelf_endpoint)->default_value("10.160.11.21:12201"),"hostname:port or ip:port of GELF endpoint")
    ("template",bpo::value<string>(&start_temp)->default_value("testnet.template"),"the startup script template")
    ("script",bpo::value<string>(&start_script)->default_value("bios_boot.sh"),"the generated startup script name")
    ("max-block-cpu-usage",bpo::value<uint32_t>(),"Provide the \"max-block-cpu-usage\" value to use in the genesis.json file")
    ("max-transaction-cpu-usage",bpo::value<uint32_t>(),"Provide the \"max-transaction-cpu-usage\" value to use in the genesis.json file")
        ;
}
 
template<class enum_type, class=typename std::enable_if<std::is_enum<enum_type>::value>::type>
inline enum_type& operator|=(enum_type&lhs, const enum_type& rhs)
{
  using T = std::underlying_type_t <enum_type>;
  return lhs = static_cast<enum_type>(static_cast<T>(lhs) | static_cast<T>(rhs));
}
 
template <typename T>
void retrieve_paired_array_parameters (const variables_map &vmap, const std::string& num_selector, const std::string& paired_selector, std::map<uint,T>& selector_map) {
   if (vmap.count(num_selector)) {
     const auto specific_nums = vmap[num_selector].as<vector<uint>>();
     const auto specific_args = vmap[paired_selector].as<vector<string>>();
     if (specific_nums.size() != specific_args.size()) {
       cerr << "ERROR: every " << num_selector << " argument must be paired with a " << paired_selector << " argument" << endl;
       exit (-1);
     }
     const auto total_nodes = vmap["nodes"].as<size_t>();
     for(uint i = 0; i < specific_nums.size(); ++i)
     {
       const auto& num = specific_nums[i];
       if (num >= total_nodes) {
         cerr << "\"--" << num_selector << "\" provided value= " << num << " is higher than \"--nodes\" provided value=" << total_nodes << endl;
         exit (-1);
       }
       selector_map[num] = specific_args[i];
     }
   }
}
 
void
launcher_def::initialize (const variables_map &vmap) {
  if (vmap.count("mode")) {
    const vector<string> modes = vmap["mode"].as<vector<string>>();
    for(const string&m : modes)
    {
      if (boost::iequals(m, "any"))
        allowed_connections |= PC_ANY;
      else if (boost::iequals(m, "producers"))
        allowed_connections |= PC_PRODUCERS;
      else if (boost::iequals(m, "specified"))
        allowed_connections |= PC_SPECIFIED;
      else if (boost::iequals(m, "none"))
        allowed_connections = PC_NONE;
      else {
        cerr << "unrecognized connection mode: " << m << endl;
        exit (-1);
      }
    }
  }
 
  if (vmap.count("max-block-cpu-usage")) {
     max_block_cpu_usage = vmap["max-block-cpu-usage"].as<uint32_t>();
  }
 
  if (vmap.count("max-transaction-cpu-usage")) {
     max_transaction_cpu_usage = vmap["max-transaction-cpu-usage"].as<uint32_t>();
  }
 
  genesis = vmap["genesis"].as<string>();
  if (vmap.count("host-map")) {
     host_map_file = vmap["host-map"].as<string>();
  }
  if (vmap.count("servers")) {
     server_ident_file = vmap["servers"].as<string>();
  }
 
  retrieve_paired_array_parameters(vmap, "specific-num", "specific-" + string(node_executable_name), specific_nodeop_args);
  retrieve_paired_array_parameters(vmap, "spcfc-inst-num", "spcfc-inst-" + string(node_executable_name), specific_nodeop_installation_paths);
 
  using namespace std::chrono;
  system_clock::time_point now = system_clock::now();
  std::time_t now_c = system_clock::to_time_t(now);
  ostringstream dstrm;
  dstrm << std::put_time(std::localtime(&now_c), "%Y_%m_%d_%H_%M_%S");
  launch_time = dstrm.str();
 
  if ( ! (shape.empty() ||
          boost::iequals( shape, "ring" ) ||
          boost::iequals( shape, "line" ) ||
          boost::iequals( shape, "star" ) ||
          boost::iequals( shape, "mesh" )) &&
       host_map_file.empty()) {
    bfs::path src = shape;
    host_map_file = src.stem().string() + "_hosts.json";
  }
 
  if( !host_map_file.empty() ) {
    try {
      fc::json::from_file(host_map_file).as<vector<host_def>>(bindings);
      for (auto &binding : bindings) {
        for (auto &eosd : binding.instances) {
          eosd.host = binding.host_name;
          eosd.p2p_endpoint = binding.public_name + ":" + boost::lexical_cast<string,uint16_t>(eosd.p2p_port);
 
          aliases.push_back (eosd.name);
        }
      }
    } catch (...) { // this is an optional feature, so an exception is OK
    }
  }
 
  config_dir_base = "etc/sysio";
  data_dir_base = "var/lib";
  next_node = 0;
  ++prod_nodes; // add one for the bios node
  ++total_nodes;
 
  load_servers ();
 
  if (prod_nodes > (producers + 1))
    prod_nodes = producers;
  if (prod_nodes > total_nodes)
    total_nodes = prod_nodes + unstarted_nodes;
  else if (total_nodes < prod_nodes + unstarted_nodes) {
    cerr << "ERROR: if provided, \"--nodes\" must be equal or greater than the number of nodes indicated by \"--pnodes\" and \"--unstarted-nodes\"." << endl;
    exit (-1);
  }
 
  if (vmap.count("specific-num")) {
    const auto specific_nums = vmap["specific-num"].as<vector<uint>>();
    const auto specific_args = vmap["specific-" + string(node_executable_name)].as<vector<string>>();
    if (specific_nums.size() != specific_args.size()) {
      cerr << "ERROR: every specific-num argument must be paired with a specific-" << node_executable_name << " argument" << endl;
      exit (-1);
    }
    // don't include bios
    const auto allowed_nums = total_nodes - 1;
    for(uint i = 0; i < specific_nums.size(); ++i)
    {
      const auto& num = specific_nums[i];
      if (num >= allowed_nums) {
        cerr << "\"--specific-num\" provided value= " << num << " is higher than \"--nodes\" provided value=" << total_nodes << endl;
        exit (-1);
      }
      specific_nodeop_args[num] = specific_args[i];
    }
  }
 
  char* erd_env_var = getenv ("SYSIO_HOME");
  if (erd_env_var == nullptr || std::string(erd_env_var).empty()) {
     erd_env_var = getenv ("PWD");
  }
 
  if (erd_env_var != nullptr) {
     erd = erd_env_var;
  } else {
     erd.clear();
  }
 
  stage = bfs::path(erd);
  if (!bfs::exists(stage)) {
    cerr << "\"" << erd << "\" is not a valid path. Please ensure environment variable SYSIO_HOME is set to the build path." << endl;
    exit (-1);
  }
  stage /= bfs::path("staging");
  bfs::create_directories (stage);
  if (bindings.empty()) {
    define_network ();
  }
 
}
 
void
launcher_def::load_servers () {
  if (!server_ident_file.empty()) {
    try {
      fc::json::from_file(server_ident_file).as<server_identities>(servers);
      prod_nodes = 0;
      for (auto &s : servers.producer) {
         prod_nodes += s.instances;
      }
 
      total_nodes = prod_nodes;
      for (auto &s : servers.nonprod) {
         total_nodes += s.instances;
      }
 
      per_host = 1;
      network.ssh_helper = servers.ssh;
    }
    catch (...) {
      cerr << "unable to load server identity file " << server_ident_file << endl;
      exit (-1);
    }
  }
}
 
 
void
launcher_def::assign_name (eosd_def &node, bool is_bios) {
   string node_cfg_name;
 
   if (is_bios) {
      node.name = "bios";
      node_cfg_name = "node_bios";
   }
   else {
      string dex = next_node < 10 ? "0":"";
      dex += boost::lexical_cast<string,int>(next_node++);
      node.name = network.name + dex;
      node_cfg_name = "node_" + dex;
   }
  node.config_dir_name = (config_dir_base / node_cfg_name).string();
  node.data_dir_name = (data_dir_base / node_cfg_name).string();
}
 
bool
launcher_def::generate () {
 
  if (boost::iequals (shape,"ring")) {
    make_line ();
  }
  else if (boost::iequals (shape,"line")) {
    make_line(false);
  }
  else if (boost::iequals (shape, "star")) {
    make_star ();
  }
  else if (boost::iequals (shape, "mesh")) {
    make_mesh ();
  }
  else {
    make_custom ();
  }
 
  if( !nogen ) {
     write_setprods_file();
     write_bios_boot();
     init_genesis();
     for (auto &node : network.nodes) {
        write_config_file(node.second);
        write_logging_config_file(node.second);
        write_genesis_file(node.second);
     }
  }
  write_dot_file ();
 
  if (!output.empty()) {
    bfs::path savefile = output;
    {
      bfs::ofstream sf (savefile);
      sf << fc::json::to_pretty_string (network) << endl;
      sf.close();
    }
    if (host_map_file.empty()) {
      savefile = bfs::path (output.stem().string() + "_hosts.json");
    }
    else {
      savefile = bfs::path (host_map_file);
    }
 
    {
      bfs::ofstream sf (savefile);
 
      sf << fc::json::to_pretty_string (bindings) << endl;
      sf.close();
    }
     return false;
  }
 
  return true;
}
 
void
launcher_def::write_dot_file () {
  bfs::ofstream df ("testnet.dot");
  df << "digraph G\n{\nlayout=\"circo\";\n";
  for (auto &node : network.nodes) {
    for (const auto &p : node.second.peers) {
      string pname=network.nodes.find(p)->second.instance->dot_label();
      df << "\"" << node.second.instance->dot_label ()
         << "\"->\"" << pname
         << "\" [dir=\"forward\"];" << std::endl;
    }
  }
  df << "}\n";
}
 
void
launcher_def::define_network () {
 
  if (per_host == 0) {
    host_def local_host;
    local_host.sysio_home = erd;
    local_host.genesis = genesis.string();
    for (size_t i = 0; i < (total_nodes); i++) {
      eosd_def eosd;
      assign_name(eosd, i == 0);
      aliases.push_back(eosd.name);
      eosd.set_host (&local_host, i == 0);
      local_host.instances.emplace_back(move(eosd));
    }
    bindings.emplace_back(move(local_host));
  }
  else {
    int ph_count = 0;
    host_def *lhost = nullptr;
    size_t host_ndx = 0;
    size_t num_prod_addr = servers.producer.size();
    size_t num_nonprod_addr = servers.nonprod.size();
    for (size_t i = total_nodes; i > 0; i--) {
       bool do_bios = false;
      if (ph_count == 0) {
        if (lhost) {
          bindings.emplace_back(move(*lhost));
          delete lhost;
        }
        lhost = new host_def;
        lhost->genesis = genesis.string();
        if (host_ndx < num_prod_addr ) {
           do_bios = servers.producer[host_ndx].has_bios;
          lhost->host_name = servers.producer[host_ndx].ipaddr;
          lhost->public_name = servers.producer[host_ndx].name;
          ph_count = servers.producer[host_ndx].instances;
        }
        else if (host_ndx - num_prod_addr < num_nonprod_addr) {
          size_t ondx = host_ndx - num_prod_addr;
           do_bios = servers.nonprod[ondx].has_bios;
          lhost->host_name = servers.nonprod[ondx].ipaddr;
          lhost->public_name = servers.nonprod[ondx].name;
          ph_count = servers.nonprod[ondx].instances;
        }
        else {
          string ext = host_ndx < 10 ? "0" : "";
          ext += boost::lexical_cast<string,int>(host_ndx);
          lhost->host_name = "pseudo_" + ext;
          lhost->public_name = lhost->host_name;
          ph_count = 1;
        }
        lhost->sysio_home =
          (local_id.contains (lhost->host_name) || servers.default_sysio_home.empty()) ?
          erd : servers.default_sysio_home;
        host_ndx++;
      } // ph_count == 0
 
      eosd_def eosd;
      assign_name(eosd, do_bios);
 
      aliases.push_back(eosd.name);
      eosd.set_host (lhost, do_bios);
      do_bios = false;
      lhost->instances.emplace_back(move(eosd));
      --ph_count;
    } // for i
    bindings.emplace_back( move(*lhost) );
    delete lhost;
  }
}
 
 
void
launcher_def::bind_nodes () {
   if (prod_nodes < 2) {
      cerr << "Unable to allocate producers due to insufficient prod_nodes = " << prod_nodes << "\n";
      exit (10);
   }
   size_t non_bios = prod_nodes - 1;
   int per_node = producers / non_bios;
   int extra = producers % non_bios;
   unsigned int i = 0;
   unsigned int producer_number = 0;
   const auto to_not_start_node = total_nodes - unstarted_nodes - 1;
   for (auto &h : bindings) {
      for (auto &inst : h.instances) {
         bool is_bios = inst.name == "bios";
         tn_node_def node;
         node.name = inst.name;
         node.instance = &inst;
         auto kp = is_bios ?
            private_key_type(string("5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3")) :
            private_key_type::generate();
         auto pubkey = kp.get_public_key();
         node.keys.emplace_back (move(kp));
         if (is_bios) {
            string prodname = "sysio";
            node.producers.push_back(prodname);
            producer_set.schedule.push_back({prodname,pubkey});
         }
        else {
           if (i < non_bios) {
              int count = per_node;
              if (extra ) {
                 ++count;
                 --extra;
              }
              while (count--) {
                 const auto prodname = producer_names::producer_name(producer_number);
                 node.producers.push_back(prodname);
                 producer_set.schedule.push_back({prodname,pubkey});
                 ++producer_number;
              }
              for (unsigned j = 0; j < shared_producers; ++j) {
                 const auto prodname = producer_names::producer_name(j, true);
                 node.producers.push_back(prodname);
                 producer_set.schedule.push_back({prodname,pubkey});                 
              }
           }
           node.dont_start = i >= to_not_start_node;
        }
        node.gelf_endpoint = gelf_endpoint;
        network.nodes[node.name] = move(node);
        inst.node = &network.nodes[inst.name];
        if (!is_bios) i++;
      }
  }
}
 
host_def *
launcher_def::find_host (const string &name)
{
  host_def *host = nullptr;
  for (auto &h : bindings) {
    if (h.host_name == name) {
      host = &h;
      break;
    }
  }
  if (host == 0) {
    cerr << "could not find host for " << name << endl;
    exit(-1);
  }
  return host;
}
 
host_def *
launcher_def::find_host_by_name_or_address (const string &host_id)
{
  host_def *host = nullptr;
  for (auto &h : bindings) {
    if ((h.host_name == host_id) || (h.public_name == host_id)) {
      host = &h;
      break;
    }
  }
  if (host == 0) {
    cerr << "could not find host for " << host_id << endl;
    exit(-1);
  }
  return host;
}
 
host_def *
launcher_def::deploy_config_files (tn_node_def &node) {
  boost::system::error_code ec;
  eosd_def &instance = *node.instance;
  host_def *host = find_host (instance.host);
 
  bfs::path source = stage / instance.config_dir_name / "config.ini";
  bfs::path logging_source = stage / instance.config_dir_name / "logging.json";
  bfs::path genesis_source = stage / instance.config_dir_name / "genesis.json";
 
  if (host->is_local()) {
    bfs::path cfgdir = bfs::path(host->sysio_home) / instance.config_dir_name;
    bfs::path dd = bfs::path(host->sysio_home) / instance.data_dir_name;
 
    if (!bfs::exists (cfgdir)) {
       if (!bfs::create_directories (cfgdir, ec) && ec.value()) {
          cerr << "could not create new directory: " << instance.config_dir_name
               << " errno " << ec.value() << " " << strerror(ec.value()) << endl;
          exit (-1);
       }
    }
    else if (bfs::exists (cfgdir / "config.ini") && !force_overwrite) {
       cerr << cfgdir / "config.ini" << " exists. Use -f|--force to overwrite configuration\n";
       exit (-1);
    }
 
    if (!bfs::exists (dd)) {
       if (!bfs::create_directories (dd, ec) && ec.value()) {
          cerr << "could not create new directory: " << instance.config_dir_name
               << " errno " << ec.value() << " " << strerror(ec.value()) << endl;
          exit (-1);
       }
    }
    else if (force_overwrite) {
       int64_t count =  bfs::remove_all (dd / block_dir, ec);
       if (ec.value() != 0) {
          cerr << "count = " << count << " could not remove old directory: " << dd
               << " " << strerror(ec.value()) << endl;
          exit (-1);
        }
        count = bfs::remove_all (dd / shared_mem_dir, ec);
        if (ec.value() != 0) {
          cerr << "count = " << count << " could not remove old directory: " << dd
               << " " << strerror(ec.value()) << endl;
          exit (-1);
        }
    }
    else if (bfs::exists (dd/ block_dir) || bfs::exists (dd / shared_mem_dir)) {
       cerr << "either " << block_dir << " or " << shared_mem_dir << " exist in \n";
       cerr << dd << ". Use -f|--force to erase blockchain data" << endl;
        exit (-1);
    }
 
    bfs::copy_file (genesis_source, cfgdir / "genesis.json", bfs::copy_option::overwrite_if_exists);
    bfs::copy_file (logging_source, cfgdir / "logging.json", bfs::copy_option::overwrite_if_exists);
    bfs::copy_file (source, cfgdir / "config.ini", bfs::copy_option::overwrite_if_exists);
  }
  else {
    prep_remote_config_dir (instance, host);
 
    bfs::path rfile = bfs::path (host->sysio_home) / instance.config_dir_name / "config.ini";
    auto scp_cmd_line = compose_scp_command(*host, source, rfile);
 
    cerr << "cmdline = " << scp_cmd_line << endl;
    int res = boost::process::system (scp_cmd_line);
    if (res != 0) {
      cerr << "unable to scp config file to host " << host->host_name << endl;
      exit(-1);
    }
 
    rfile = bfs::path (host->sysio_home) / instance.config_dir_name / "logging.json";
 
    scp_cmd_line = compose_scp_command(*host, logging_source, rfile);
 
    res = boost::process::system (scp_cmd_line);
    if (res != 0) {
      cerr << "unable to scp logging config file to host " << host->host_name << endl;
      exit(-1);
    }
 
    rfile = bfs::path (host->sysio_home) / instance.config_dir_name / "genesis.json";
 
    scp_cmd_line = compose_scp_command(*host, genesis_source, rfile);
 
    res = boost::process::system (scp_cmd_line);
    if (res != 0) {
       cerr << "unable to scp genesis.json file to host " << host->host_name << endl;
       exit(-1);
    }
  }
  return host;
}
 
string
launcher_def::compose_scp_command (const host_def& host, const bfs::path& source, const bfs::path& destination) {
  string scp_cmd_line = network.ssh_helper.scp_cmd + " ";
  const string &args = host.ssh_args.length() ? host.ssh_args : network.ssh_helper.ssh_args;
  if (args.length()) {
    scp_cmd_line += args + " ";
  }
  scp_cmd_line += source.string() + " ";
 
  const string &uid = host.ssh_identity.length() ? host.ssh_identity : network.ssh_helper.ssh_identity;
  if (uid.length()) {
    scp_cmd_line += uid + "@";
  }
 
  scp_cmd_line += host.host_name + ":" + destination.string();
 
  return scp_cmd_line;
}
 
void
launcher_def::write_config_file (tn_node_def &node) {
   bool is_bios = (node.name == "bios");
   bfs::path filename;
   eosd_def &instance = *node.instance;
   host_def *host = find_host (instance.host);
 
   bfs::path dd = stage / instance.config_dir_name;
   if (!bfs::exists(dd)) {
      try {
         bfs::create_directories (dd);
      } catch (const bfs::filesystem_error &ex) {
         cerr << "write_config_files threw " << ex.what() << endl;
         exit (-1);
      }
   }
 
   filename = dd / "config.ini";
 
   bfs::ofstream cfg(filename);
   if (!cfg.good()) {
      cerr << "unable to open " << filename << " " << strerror(errno) << "\n";
      exit (-1);
   }
 
   cfg << "blocks-dir = " << block_dir << "\n";
   cfg << "http-server-address = " << host->host_name << ":" << instance.http_port << "\n";
   cfg << "http-validate-host = false\n";
   cfg << "p2p-listen-endpoint = " << host->listen_addr << ":" << instance.p2p_port << "\n";
   cfg << "p2p-server-address = " << host->public_name << ":" << instance.p2p_port << "\n";
 
 
   if (is_bios) {
    cfg << "enable-stale-production = true\n";
  }
  if (allowed_connections & PC_ANY) {
    cfg << "allowed-connection = any\n";
  }
  else if (allowed_connections == PC_NONE) {
    cfg << "allowed-connection = none\n";
  }
  else
  {
    if (allowed_connections & PC_PRODUCERS) {
      cfg << "allowed-connection = producers\n";
    }
    if (allowed_connections & PC_SPECIFIED) {
      cfg << "allowed-connection = specified\n";
      cfg << "peer-key = \"" << node.keys.begin()->get_public_key().to_string() << "\"\n";
      cfg << "peer-private-key = [\"" << node.keys.begin()->get_public_key().to_string()
          << "\",\"" << node.keys.begin()->to_string() << "\"]\n";
    }
  }
 
  if(!is_bios) {
     auto &bios_node = network.nodes["bios"];
     cfg << "p2p-peer-address = " << bios_node.instance->p2p_endpoint<< "\n";
  }
  for (const auto &p : node.peers) {
     cfg << "p2p-peer-address = " << network.nodes.find(p)->second.instance->p2p_endpoint << "\n";
  }
  if (node.producers.size()) {
    for (const auto &kp : node.keys ) {
       cfg << "private-key = [\"" << kp.get_public_key().to_string()
           << "\",\"" << kp.to_string() << "\"]\n";
    }
    for (auto &p : node.producers) {
      cfg << "producer-name = " << p << "\n";
    }
    cfg << "plugin = sysio::producer_plugin\n";
  }
  cfg << "plugin = sysio::net_plugin\n";
  cfg << "plugin = sysio::chain_api_plugin\n";
  cfg.close();
}
 
void
launcher_def::write_logging_config_file(tn_node_def &node) {
  bfs::path filename;
  eosd_def &instance = *node.instance;
 
  bfs::path dd = stage / instance.config_dir_name;
  if (!bfs::exists(dd)) {
    bfs::create_directories(dd);
  }
 
  filename = dd / "logging.json";
 
  bfs::ofstream cfg(filename);
  if (!cfg.good()) {
    cerr << "unable to open " << filename << " " << strerror(errno) << "\n";
    exit (9);
  }
 
  auto log_config = fc::logging_config::default_config();
  if(gelf_enabled) {
     log_config.appenders.push_back(
           fc::appender_config( "net", "gelf",
                           fc::mutable_variant_object()
                                 ( "endpoint", node.gelf_endpoint )
                                 ( "host", instance.name )
           ) );
     log_config.loggers.front().appenders.push_back( "net" );
  }
 
  fc::logger_config p2p( "net_plugin_impl" );
  p2p.level = fc::log_level::debug;
  p2p.appenders.push_back( "stderr" );
  if( gelf_enabled ) p2p.appenders.push_back( "net" );
  log_config.loggers.emplace_back( p2p );
 
  fc::logger_config http( "http_plugin" );
  http.level = fc::log_level::debug;
  http.appenders.push_back( "stderr" );
  if( gelf_enabled ) http.appenders.push_back( "net" );
  log_config.loggers.emplace_back( http );
 
  fc::logger_config pp( "producer_plugin" );
  pp.level = fc::log_level::debug;
  pp.appenders.push_back( "stderr" );
  if( gelf_enabled ) pp.appenders.push_back( "net" );
  log_config.loggers.emplace_back( pp );
 
  fc::logger_config tt( "transaction_success_tracing" );
  tt.level = fc::log_level::debug;
  tt.appenders.push_back( "stderr" );
  if( gelf_enabled ) tt.appenders.push_back( "net" );
  log_config.loggers.emplace_back( tt );
 
  fc::logger_config tft( "transaction_failure_tracing" );
  tft.level = fc::log_level::debug;
  tft.appenders.push_back( "stderr" );
  if( gelf_enabled ) tft.appenders.push_back( "net" );
  log_config.loggers.emplace_back( tft );
 
  fc::logger_config tts( "transaction_trace_success" );
  tts.level = fc::log_level::debug;
  tts.appenders.push_back( "stderr" );
  if( gelf_enabled ) tts.appenders.push_back( "net" );
  log_config.loggers.emplace_back( tts );
 
   fc::logger_config ttf( "transaction_trace_failure" );
   ttf.level = fc::log_level::debug;
   ttf.appenders.push_back( "stderr" );
   if( gelf_enabled ) ttf.appenders.push_back( "net" );
   log_config.loggers.emplace_back( ttf );
 
  fc::logger_config ta( "trace_api" );
  ta.level = fc::log_level::debug;
  ta.appenders.push_back( "stderr" );
  if( gelf_enabled ) ta.appenders.push_back( "net" );
  log_config.loggers.emplace_back( ta );
 
  auto str = fc::json::to_pretty_string( log_config, fc::time_point::maximum(), fc::json::output_formatting::stringify_large_ints_and_doubles );
  cfg.write( str.c_str(), str.size() );
  cfg.close();
}
 
void
launcher_def::init_genesis () {
   const bfs::path genesis_path = genesis.is_complete() ? genesis : bfs::current_path() / genesis;
   if (!bfs::exists(genesis_path)) {
      cout << "generating default genesis file " << genesis_path << endl;
      sysio::chain::genesis_state default_genesis;
      fc::json::save_to_file( default_genesis, genesis_path, true );
   }
   string bioskey = network.nodes["bios"].keys[0].get_public_key().to_string();
 
   fc::json::from_file(genesis_path).as<sysio::chain::genesis_state>(genesis_from_file);
   genesis_from_file.initial_key = public_key_type(bioskey);
   if (max_block_cpu_usage)
      genesis_from_file.initial_configuration.max_block_cpu_usage = *max_block_cpu_usage;
   if (max_transaction_cpu_usage)
      genesis_from_file.initial_configuration.max_transaction_cpu_usage = *max_transaction_cpu_usage;
}
 
void
launcher_def::write_genesis_file(tn_node_def &node) {
  bfs::path filename;
  eosd_def &instance = *node.instance;
 
  bfs::path dd = stage / instance.config_dir_name;
  if (!bfs::exists(dd)) {
    bfs::create_directories(dd);
  }
 
  filename = dd / "genesis.json";
  fc::json::save_to_file( genesis_from_file, dd / "genesis.json", true );
}
 
void
launcher_def::write_setprods_file() {
   bfs::path filename = bfs::current_path() / "setprods.json";
   bfs::ofstream psfile (filename);
   if(!psfile.good()) {
      cerr << "unable to open " << filename << " " << strerror(errno) << "\n";
    exit (9);
  }
   producer_set_def no_bios;
   for (auto &p : producer_set.schedule) {
      if (p.producer_name != "sysio")
         no_bios.schedule.push_back(p);
   }
  auto str = fc::json::to_pretty_string( no_bios, fc::time_point::maximum(), fc::json::output_formatting::stringify_large_ints_and_doubles );
  psfile.write( str.c_str(), str.size() );
  psfile.close();
}
 
void
launcher_def::write_bios_boot () {
   bfs::ifstream src(bfs::path(config_dir_base) / "launcher" / start_temp);
   if(!src.good()) {
      cerr << "unable to open " << config_dir_base << "launcher/" << start_temp << " " << strerror(errno) << "\n";
    exit (9);
  }
 
   bfs::ofstream brb (bfs::current_path() / start_script);
   if(!brb.good()) {
      cerr << "unable to open " << bfs::current_path() << "/" << start_script << " " << strerror(errno) << "\n";
    exit (9);
  }
 
   auto &bios_node = network.nodes["bios"];
   uint16_t biosport = bios_node.instance->http_port;
   string bhost = bios_node.instance->host;
   string line;
   string prefix = "###INSERT ";
   size_t len = prefix.length();
   while (getline(src,line)) {
      if (line.substr(0,len) == prefix) {
         string key = line.substr(len);
         if (key == "envars") {
            brb << "bioshost=" << bhost << "\nbiosport=" << biosport << "\n";
         }
         else if (key == "prodkeys" ) {
            for (auto &node : network.nodes) {
               brb << "wcmd import -n ignition --private-key " << node.second.keys[0].to_string() << "\n";
            }
         }
         else if (key == "cacmd") {
            for (auto &p : producer_set.schedule) {
               if (p.producer_name == "sysio") {
                  continue;
               }
               brb << "cacmd " << p.producer_name
                   << " " << p.block_signing_key.to_string() << " " << p.block_signing_key.to_string() << "\n";
            }
         }
      }
      brb << line << "\n";
   }
   src.close();
   brb.close();
}
 
bool launcher_def::is_bios_ndx (size_t ndx) {
   return aliases[ndx] == "bios";
}
 
size_t launcher_def::start_ndx() {
   return is_bios_ndx(0) ? 1 : 0;
}
 
bool launcher_def::next_ndx(size_t &ndx) {
   ++ndx;
   bool loop = ndx == total_nodes;
   if (loop)
      ndx = start_ndx();
   else
      if (is_bios_ndx(ndx)) {
         loop = next_ndx(ndx);
      }
   return loop;
}
 
size_t launcher_def::skip_ndx (size_t from, size_t offset) {
   size_t ndx = (from + offset) % total_nodes;
   if (total_nodes > 2) {
      size_t attempts = total_nodes - 1;
      while (--attempts && (is_bios_ndx(ndx) || ndx == from)) {
         next_ndx(ndx);
      }
   }
   return ndx;
}
 
void
launcher_def::make_line (bool make_ring) {
  bind_nodes();
  size_t non_bios = total_nodes - 1;
  if (non_bios > 2) {
     // since we have at least 3 indexes, every next_ndx(i) call is guaranteed to not be the end of the loop
     bool end_of_loop = false;
     for (size_t i = start_ndx(); !end_of_loop; next_ndx(i)) {
        size_t front = i;
        end_of_loop = next_ndx (front);
        // if this is the end of the loop and this is not a ring, then don't connect the end to the beginning to make a ring
        if (end_of_loop && !make_ring) {
           break;
        }
        network.nodes.find(aliases[i])->second.peers.push_back (aliases[front]);
     }
  }
  else if (non_bios == 2) {
     size_t n0 = start_ndx();
     size_t n1 = n0;
     next_ndx(n1);
     network.nodes.find(aliases[n0])->second.peers.push_back (aliases[n1]);
     if (make_ring) {
        network.nodes.find(aliases[n1])->second.peers.push_back (aliases[n0]);
     }
     // since there are only 2 nodes, this really just affects startup and that only index 0 will initiate the connection if !make_ring
  }
}
 
void
launcher_def::make_star () {
  size_t non_bios = total_nodes - 1;
  if (non_bios < 4) {
    make_line ();
    return;
  }
  bind_nodes();
 
  size_t links = 3;
  if (non_bios > 12) {
    links = static_cast<size_t>(sqrt(non_bios)) + 2;
  }
  size_t gap = non_bios > 6 ? 3 : (non_bios - links)/2 +1;
  while (non_bios % gap == 0) {
    ++gap;
  }
  // use to prevent duplicates since all connections are bidirectional
  std::map <string, std::set<string>> peers_to_from;
  bool loop = false;
  for (size_t i = start_ndx(); !loop; loop = next_ndx(i)) {
     const auto& iter = network.nodes.find(aliases[i]);
    auto &current = iter->second;
    const auto& current_name = iter->first;
    size_t ndx = i;
    for (size_t l = 1; l <= links; l++) {
       ndx = skip_ndx(ndx, l * gap);
       auto &peer = aliases[ndx];
      for (bool found = true; found; ) {
        found = false;
        for (auto &p : current.peers) {
          if (p == peer) {
             next_ndx(ndx);
             if (ndx == i) {
                next_ndx(ndx);
             }
            peer = aliases[ndx];
            found = true;
            break;
          }
        }
      }
      // if already established, don't add to list
      if (peers_to_from[peer].count(current_name) < 2) {
        current.peers.push_back(peer); // current_name -> peer
        // keep track of bidirectional relationships to prevent duplicates
        peers_to_from[current_name].insert(peer);
      }
    }
  }
}
 
void
launcher_def::make_mesh () {
  size_t non_bios = total_nodes - 1;
  bind_nodes();
  // use to prevent duplicates since all connections are bidirectional
  std::map <string, std::set<string>> peers_to_from;
  bool loop = false;
  for (size_t i = start_ndx();!loop; loop = next_ndx(i)) {
    const auto& iter = network.nodes.find(aliases[i]);
    auto &current = iter->second;
    const auto& current_name = iter->first;
 
    for (size_t j = 1; j < non_bios; j++) {
       size_t ndx = skip_ndx(i,j);
      const auto& peer = aliases[ndx];
      // if already established, don't add to list
      if (peers_to_from[peer].count(current_name) < 2) {
        current.peers.push_back (peer);
        // keep track of bidirectional relationships to prevent duplicates
        peers_to_from[current_name].insert(peer);
      }
    }
  }
}
 
void
launcher_def::make_custom () {
  bfs::path source = shape;
  fc::json::from_file(source).as<testnet_def>(network);
  for (auto &h : bindings) {
    for (auto &inst : h.instances) {
      tn_node_def *node = &network.nodes[inst.name];
      for (auto &p : node->producers) {
         producer_set.schedule.push_back({p,node->keys[0].get_public_key()});
      }
      node->instance = &inst;
      inst.node = node;
    }
  }
}
 
void
launcher_def::format_ssh (const string &cmd,
                          const string &host_name,
                          string & ssh_cmd_line) {
 
  ssh_cmd_line = network.ssh_helper.ssh_cmd + " ";
  if (network.ssh_helper.ssh_args.length()) {
    ssh_cmd_line += network.ssh_helper.ssh_args + " ";
  }
  if (network.ssh_helper.ssh_identity.length()) {
    ssh_cmd_line += network.ssh_helper.ssh_identity + "@";
  }
  ssh_cmd_line += host_name + " \"" + cmd + "\"";
  cerr << "cmdline = " << ssh_cmd_line << endl;
}
 
bool
launcher_def::do_ssh (const string &cmd, const string &host_name) {
  string ssh_cmd_line;
  format_ssh (cmd, host_name, ssh_cmd_line);
  int res = boost::process::system (ssh_cmd_line);
  return (res == 0);
}
 
void
launcher_def::prep_remote_config_dir (eosd_def &node, host_def *host) {
  bfs::path abs_config_dir = bfs::path(host->sysio_home) / node.config_dir_name;
  bfs::path abs_data_dir = bfs::path(host->sysio_home) / node.data_dir_name;
 
  string acd = abs_config_dir.string();
  string add = abs_data_dir.string();
  string cmd = "cd " + host->sysio_home;
 
  cmd = "cd " + host->sysio_home;
  if (!do_ssh(cmd, host->host_name)) {
    cerr << "Unable to switch to path " << host->sysio_home
         << " on host " <<  host->host_name << endl;
    exit (-1);
  }
 
  cmd = "cd " + acd;
  if (!do_ssh(cmd,host->host_name)) {
     cmd = "mkdir -p " + acd;
     if (!do_ssh (cmd, host->host_name)) {
        cerr << "Unable to invoke " << cmd << " on host " << host->host_name << endl;
        exit (01);
     }
  }
  cmd = "cd " + add;
  if (do_ssh(cmd,host->host_name)) {
    if(force_overwrite) {
      cmd = "rm -rf " + add + "/" + block_dir + " ;"
          + "rm -rf " + add + "/" + shared_mem_dir;
      if (!do_ssh (cmd, host->host_name)) {
        cerr << "Unable to remove old data directories on host "
             << host->host_name << endl;
        exit (-1);
      }
    }
    else {
      cerr << add << " already exists on host " << host->host_name << ".  Use -f/--force to overwrite configuration and erase blockchain" << endl;
      exit (-1);
    }
  }
  else {
    cmd = "mkdir -p " + add;
    if (!do_ssh (cmd, host->host_name)) {
      cerr << "Unable to invoke " << cmd << " on host "
           << host->host_name << endl;
      exit (-1);
    }
  }
}
 
string
launcher_def::find_and_remove_arg(string args, string arg ){
   if (args.empty() || arg.empty())
      return args;
 
   string left, middle, right;
   size_t found = args.find(arg);
   if (found != std::string::npos){
      left = args.substr(0, found);
      middle = args.substr(found + 2); //skip --
      found = middle.find("--");
      if (found != std::string::npos){
         right = middle.substr(found);
      }
      return left + right;
   } else {
      return args;
   }
}
 
void
launcher_def::launch (eosd_def &instance, string &gts) {
  bfs::path dd = instance.data_dir_name;
  bfs::path reout = dd / "stdout.txt";
  bfs::path reerr_sl = dd / "stderr.txt";
  bfs::path reerr_base = bfs::path("stderr." + launch_time + ".txt");
  bfs::path reerr = dd / reerr_base;
  bfs::path pidf  = dd / bfs::path(string(node_executable_name) + ".pid");
  host_def* host;
  try {
     host = deploy_config_files (*instance.node);
  } catch (const bfs::filesystem_error &ex) {
     cerr << "deploy_config_files threw " << ex.what() << endl;
     exit (-1);
  }
 
  node_rt_info info;
  info.remote = !host->is_local();
 
  string install_path;
  if (instance.name != "bios" && !specific_nodeop_installation_paths.empty()) {
     const auto node_num = boost::lexical_cast<uint16_t,string>(instance.get_node_num());
     if (specific_nodeop_installation_paths.count(node_num)) {
        install_path = specific_nodeop_installation_paths[node_num] + "/";
     }
  }
  string eosdcmd = install_path + "programs/nodeop/" + string(node_executable_name) + " ";
  if (skip_transaction_signatures) {
    eosdcmd += "--skip-transaction-signatures ";
  }
  if (!eosd_extra_args.empty()) {
    eosdcmd += eosd_extra_args + " ";
  }
  if (instance.name != "bios" && !specific_nodeop_args.empty()) {
     const auto node_num = boost::lexical_cast<uint16_t,string>(instance.get_node_num());
     if (specific_nodeop_args.count(node_num)) {
        eosdcmd += specific_nodeop_args[node_num] + " ";
     }
  }
 
  if( add_enable_stale_production ) {
    eosdcmd += "--enable-stale-production true ";
    add_enable_stale_production = false;
  }
 
  eosdcmd += " --config-dir " + instance.config_dir_name + " --data-dir " + instance.data_dir_name;
  eosdcmd += " --genesis-json " + instance.config_dir_name + "/genesis.json";
  if (gts.length()) {
    eosdcmd += " --genesis-timestamp " + gts;
  }
 
  if (eosdcmd.find("sysio::history_api_plugin") != string::npos && eosdcmd.find("sysio::trace_api_plugin") != string::npos){
    // remove trace_api_plugin from old version nodes in multiversion test
    eosdcmd = find_and_remove_arg(eosdcmd, "--plugin sysio::trace_api_plugin");
    eosdcmd = find_and_remove_arg(eosdcmd, "--trace-no-abis");
    eosdcmd = find_and_remove_arg(eosdcmd, "--trace-rpc-abi");
  }
 
  if (!host->is_local()) {
    if (instance.node->dont_start) {
      cerr << "Unable to use \"unstarted-nodes\" with a remote hose" << endl;
      exit (-1);
    }
    string cmdl ("cd ");
    cmdl += host->sysio_home + "; nohup " + eosdcmd + " > "
      + reout.string() + " 2> " + reerr.string() + "& echo $! > " + pidf.string()
      + "; rm -f " + reerr_sl.string()
      + "; ln -s " + reerr_base.string() + " " + reerr_sl.string();
    if (!do_ssh (cmdl, host->host_name)){
      cerr << "Unable to invoke " << cmdl
           << " on host " << host->host_name << endl;
      exit (-1);
    }
 
    string cmd = "cd " + host->sysio_home + "; kill -15 $(cat " + pidf.string() + ")";
    format_ssh (cmd, host->host_name, info.kill_cmd);
  }
  else if (!instance.node->dont_start) {
    cerr << "spawning child, " << eosdcmd << endl;
 
    bp::child c(eosdcmd, bp::std_out > reout, bp::std_err > reerr );
    bfs::remove(reerr_sl);
    bfs::create_symlink (reerr_base, reerr_sl);
 
    bfs::ofstream pidout (pidf);
    pidout << c.id() << flush;
    pidout.close();
 
    info.pid_file = pidf.string();
    info.kill_cmd = "";
 
    if(!c.running()) {
      cerr << "child not running after spawn " << eosdcmd << endl;
      for (int i = 0; i > 0; i++) {
        if (c.running () ) break;
      }
    }
    c.detach();
  }
  else {
    cerr << "not spawning child, " << eosdcmd << endl;
 
    const bfs::path dd = instance.data_dir_name;
    const bfs::path start_file  = dd / "start.cmd";
    bfs::ofstream sf (start_file);
 
    sf << eosdcmd << endl;
    sf.close();
  }
  last_run.running_nodes.emplace_back (move(info));
}
 
#if 0
void
launcher_def::kill_instance(eosd_def, string sig_opt) {
}
#endif
 
void
launcher_def::kill (launch_modes mode, string sig_opt) {
  switch (mode) {
  case LM_NONE:
    return;
  case LM_VERIFY:
    // no-op
    return;
  case LM_NAMED: {
    cerr << "feature not yet implemented " << endl;
    #if 0
      auto node = network.nodes.find(launch_name);
      kill_instance (node.second.instance, sig_opt);
      #endif
    break;
  }
  case LM_ALL:
  case LM_LOCAL:
  case LM_REMOTE : {
    bfs::path source = "last_run.json";
    try {
       fc::json::from_file( source ).as<last_run_def>( last_run );
       for( auto& info : last_run.running_nodes ) {
          if( mode == LM_ALL || (info.remote && mode == LM_REMOTE) ||
              (!info.remote && mode == LM_LOCAL) ) {
             try {
                if( info.pid_file.length() ) {
                   string pid;
                   fc::json::from_file( info.pid_file ).as<string>( pid );
                   string kill_cmd = "kill " + sig_opt + " " + pid;
                   boost::process::system( kill_cmd );
                } else {
                   boost::process::system( info.kill_cmd );
                }
             } catch( fc::exception& fce ) {
                cerr << "unable to kill fc::exception=" << fce.to_detail_string() << endl;
             } catch( std::exception& stde ) {
                cerr << "unable to kill std::exception=" << stde.what() << endl;
             } catch( ... ) {
                cerr << "Unable to kill" << endl;
             }
          }
       }
    } catch( fc::exception& fce ) {
       cerr << "unable to open " << source << " fc::exception=" << fce.to_detail_string() << endl;
    } catch( std::exception& stde ) {
       cerr << "unable to open " << source << " std::exception=" << stde.what() << endl;
    } catch( ... ) {
       cerr << "Unable to open " << source << endl;
    }
  }
  }
}
 
string
launcher_def::get_node_num(uint16_t node_num) {
   string node_num_str = node_num < 10 ? "0":"";
   node_num_str += boost::lexical_cast<string,uint16_t>(node_num);
   return node_num_str;
}
 
pair<host_def, eosd_def>
launcher_def::find_node(uint16_t node_num) {
   const string node_name = network.name + get_node_num(node_num);
   for (const auto& host: bindings) {
      for (const auto& node: host.instances) {
         if (node_name == node.name) {
            return make_pair(host, node);
         }
      }
   }
   cerr << "Unable to find node " << node_num << endl;
   exit (-1);
}
 
vector<pair<host_def, eosd_def>>
launcher_def::get_nodes(const string& node_number_list) {
   vector<pair<host_def, eosd_def>> node_list;
   if (fc::to_lower(node_number_list) == "all") {
      for (auto host: bindings) {
         for (auto node: host.instances) {
            cout << "host=" << host.host_name << ", node=" << node.name << endl;
            node_list.push_back(make_pair(host, node));
         }
      }
   }
   else {
      vector<string> nodes;
      boost::split(nodes, node_number_list, boost::is_any_of(","));
      for (string node_number: nodes) {
         uint16_t node = -1;
         try {
            node = boost::lexical_cast<uint16_t,string>(node_number);
         }
         catch(boost::bad_lexical_cast &) {
            // This exception will be handled below
         }
         if (node < 0 || node > 99) {
            cerr << "Bad node number found in node number list: " << node_number << endl;
            exit(-1);
         }
         node_list.push_back(find_node(node));
      }
   }
   return node_list;
}
 
void
launcher_def::do_command(const host_def& host, const string& name,
                         vector<pair<string, string>> env_pairs, const string& cmd) {
   if (!host.is_local()) {
      string rcmd = "cd " + host.sysio_home + "; ";
      for (auto& env_pair : env_pairs) {
         rcmd += "export " + env_pair.first + "=" + env_pair.second + "; ";
      }
      rcmd += cmd;
      if (!do_ssh(rcmd, host.host_name)) {
         cerr << "Remote command failed for " << name << endl;
         exit (-1);
      }
   }
   else {
      bp::environment e;
      for (auto& env_pair : env_pairs) {
         e.emplace(env_pair.first, env_pair.second);
      }
      bp::child c(cmd, e);
      c.wait();
   }
}
 
void
launcher_def::bounce (const string& node_numbers) {
   auto node_list = get_nodes(node_numbers);
   for (auto node_pair: node_list) {
      const host_def& host = node_pair.first;
      const eosd_def& node = node_pair.second;
      const string node_num = node.get_node_num();
      cout << "Bouncing " << node.name << endl;
      string cmd = "./scripts/sysio-tn_bounce.sh " + eosd_extra_args;
      if (node_num != "bios" && !specific_nodeop_args.empty()) {
         const auto node_num_i = boost::lexical_cast<uint16_t,string>(node_num);
         if (specific_nodeop_args.count(node_num_i)) {
            cmd += " " + specific_nodeop_args[node_num_i];
         }
      }
 
      do_command(host, node.name, { { "SYSIO_HOME", host.sysio_home }, { "SYSIO_NODE", node_num } }, cmd);
   }
}
 
void
launcher_def::down (const string& node_numbers) {
   auto node_list = get_nodes(node_numbers);
   for (auto node_pair: node_list) {
      const host_def& host = node_pair.first;
      const eosd_def& node = node_pair.second;
      const string node_num = node.get_node_num();
      cout << "Taking down " << node.name << endl;
      string cmd = "./scripts/sysio-tn_down.sh ";
      do_command(host, node.name,
                 { { "SYSIO_HOME", host.sysio_home }, { "SYSIO_NODE", node_num }, { "SYSIO_TN_RESTART_CONFIG_DIR", node.config_dir_name } },
                 cmd);
   }
}
 
void
launcher_def::roll (const string& host_names) {
   vector<string> hosts;
   boost::split(hosts, host_names, boost::is_any_of(","));
   for (string host_name: hosts) {
      cout << "Rolling " << host_name << endl;
      auto host = find_host_by_name_or_address(host_name);
      string cmd = "./scripts/sysio-tn_roll.sh ";
      do_command(*host, host_name, { { "SYSIO_HOME", host->sysio_home } }, cmd);
   }
}
 
void
launcher_def::ignite() {
   if (boot) {
      cerr << "Invoking the blockchain boot script, " << start_script << "\n";
      string script("bash " + start_script);
      bp::child c(script);
      try {
         cerr << "waiting for script completion\n";
         c.wait();
      } catch (bfs::filesystem_error &ex) {
         cerr << "wait threw error " << ex.what() << "\n";
      }
      catch (...) {
         // when script dies wait throws an exception but that is ok
      }
   } else {
      cerr << "**********************************************************************\n"
           << "run 'bash " << start_script << "' to kick off delegated block production\n"
           << "**********************************************************************\n";
   }
 
 }
 
void
launcher_def::start_all (string &gts, launch_modes mode) {
  switch (mode) {
  case LM_NONE:
    return;
  case LM_VERIFY:
    //validate configuration, report findings, exit
    return;
  case LM_NAMED : {
    try {
      add_enable_stale_production = false;
      auto node = network.nodes.find(launch_name);
      launch(*node->second.instance, gts);
    } catch (fc::exception& fce) {
       cerr << "unable to launch " << launch_name << " fc::exception=" << fce.to_detail_string() << endl;
    } catch (std::exception& stde) {
       cerr << "unable to launch " << launch_name << " std::exception=" << stde.what() << endl;
    } catch (...) {
      cerr << "Unable to launch " << launch_name << endl;
      exit (-1);
    }
    break;
  }
  case LM_ALL:
  case LM_REMOTE:
  case LM_LOCAL: {
 
    for (auto &h : bindings ) {
      if (mode == LM_ALL ||
          (h.is_local() ? mode == LM_LOCAL : mode == LM_REMOTE)) {
        for (auto &inst : h.instances) {
          try {
             cerr << "launching " << inst.name << endl;
             launch (inst, gts);
          } catch (fc::exception& fce) {
             cerr << "unable to launch " << inst.name << " fc::exception=" << fce.to_detail_string() << endl;
          } catch (std::exception& stde) {
             cerr << "unable to launch " << inst.name << " std::exception=" << stde.what() << endl;
          } catch (...) {
            cerr << "unable to launch " << inst.name << endl;
          }
          sleep (start_delay);
        }
      }
    }
    break;
  }
  }
  bfs::path savefile = "last_run.json";
  bfs::ofstream sf (savefile);
 
  sf << fc::json::to_pretty_string (last_run) << endl;
  sf.close();
}
 
//------------------------------------------------------------
 
void write_default_config(const bfs::path& cfg_file, const options_description &cfg ) {
   bfs::path parent = cfg_file.parent_path();
   if (parent.empty()) {
      parent = ".";
   }
   if(!bfs::exists(parent)) {
      try {
         bfs::create_directories(parent);
      } catch (bfs::filesystem_error &ex) {
         cerr << "could not create new directory: " << cfg_file.parent_path()
              << " caught " << ex.what() << endl;
         exit (-1);
      }
   }
 
   std::ofstream out_cfg( bfs::path(cfg_file).make_preferred().string());
   for(const boost::shared_ptr<bpo::option_description>& od : cfg.options())
   {
      if(!od->description().empty()) {
         out_cfg << "# " << od->description() << std::endl;
      }
      boost::any store;
      if(!od->semantic()->apply_default(store))
         out_cfg << "# " << od->long_name() << " = " << std::endl;
      else {
         auto example = od->format_parameter();
         if(example.empty())
            // This is a boolean switch
            out_cfg << od->long_name() << " = " << "false" << std::endl;
         else {
            // The string is formatted "arg (=<interesting part>)"
            example.erase(0, 6);
            example.erase(example.length()-1);
            out_cfg << od->long_name() << " = " << example << std::endl;
         }
      }
      out_cfg << std::endl;
   }
   out_cfg.close();
}
 
 
int main (int argc, char *argv[]) {
 
  variables_map vmap;
  options_description cfg ("Testnet launcher config options");
  options_description cli ("launcher command line options");
  launcher_def top;
  string gts;
  launch_modes mode;
  string kill_arg;
  string bounce_nodes;
  string down_nodes;
  string roll_nodes;
  bfs::path config_dir;
  bfs::path config_file;
 
  local_id.initialize();
  top.set_options(cfg);
 
  cli.add_options()
    ("timestamp,i",bpo::value<string>(&gts),"set the timestamp for the first block. Use \"now\" to indicate the current time")
    ("launch,l",bpo::value<string>(), "select a subset of nodes to launch. Currently may be \"all\", \"none\", or \"local\". If not set, the default is to launch all unless an output file is named, in which case it starts none.")
    ("output,o",bpo::value<bfs::path>(&top.output),"save a copy of the generated topology in this file")
    ("kill,k", bpo::value<string>(&kill_arg),"The launcher retrieves the previously started process ids and issues a kill to each.")
    ("down", bpo::value<string>(&down_nodes),"comma-separated list of node numbers that will be taken down using the sysio-tn_down.sh script")
    ("bounce", bpo::value<string>(&bounce_nodes),"comma-separated list of node numbers that will be restarted using the sysio-tn_bounce.sh script")
    ("roll", bpo::value<string>(&roll_nodes),"comma-separated list of host names where the nodes should be rolled to a new version using the sysio-tn_roll.sh script")
    ("version,v", "print version information")
    ("help,h","print this list")
    ("config-dir", bpo::value<bfs::path>(), "Directory containing configuration files such as config.ini")
    ("config,c", bpo::value<bfs::path>()->default_value( "config.ini" ), "Configuration file name relative to config-dir");
 
  cli.add(cfg);
 
  try {
    bpo::store(bpo::parse_command_line(argc, argv, cli), vmap);
    bpo::notify(vmap);
 
    top.initialize(vmap);
 
    if (vmap.count("help") > 0) {
      cli.print(cerr);
      return 0;
    }
    if (vmap.count("version") > 0) {
      cout << sysio::launcher::config::version_str << endl;
      return 0;
    }
 
    if( vmap.count( "config-dir" ) ) {
      config_dir = vmap["config-dir"].as<bfs::path>();
      if( config_dir.is_relative() )
         config_dir = bfs::current_path() / config_dir;
    }
 
   bfs::path config_file_name = config_dir / "config.ini";
   if( vmap.count( "config" ) ) {
      config_file_name = vmap["config"].as<bfs::path>();
      if( config_file_name.is_relative() )
         config_file_name = config_dir / config_file_name;
   }
 
   if(!bfs::exists(config_file_name)) {
      if(config_file_name.compare(config_dir / "config.ini") != 0)
      {
         cout << "Config file " << config_file_name << " missing." << std::endl;
         return -1;
      }
      write_default_config(config_file_name, cfg);
   }
 
 
   bpo::store(bpo::parse_config_file<char>(config_file_name.make_preferred().string().c_str(),
                                           cfg, true), vmap);
 
 
 
    if (vmap.count("launch")) {
      string l = vmap["launch"].as<string>();
      if (boost::iequals(l,"all"))
        mode = LM_ALL;
      else if (boost::iequals(l,"local"))
        mode = LM_LOCAL;
      else if (boost::iequals(l,"remote"))
        mode = LM_REMOTE;
      else if (boost::iequals(l,"none"))
        mode = LM_NONE;
      else if (boost::iequals(l,"verify"))
        mode = LM_VERIFY;
      else {
        mode = LM_NAMED;
        top.launch_name = l;
      }
    }
    else {
      mode = !kill_arg.empty() || top.output.empty() ? LM_ALL : LM_NONE;
    }
 
    if (!kill_arg.empty()) {
      cout << "killing" << std::endl;
      if (kill_arg[0] != '-') {
        kill_arg = "-" + kill_arg;
      }
      top.kill (mode, kill_arg);
    }
    else if (!bounce_nodes.empty()) {
       top.bounce(bounce_nodes);
    }
    else if (!down_nodes.empty()) {
       top.down(down_nodes);
    }
    else if (!roll_nodes.empty()) {
       top.roll(roll_nodes);
    }
    else {
      top.generate();
      top.start_all(gts, mode);
      top.ignite();
    }
  } catch (bpo::unknown_option &ex) {
    cerr << ex.what() << endl;
    cli.print (cerr);
  }
  return 0;
}
 
 
//-------------------------------------------------------------
// @ignore local_config_file
FC_REFLECT( remote_deploy,
            (ssh_cmd)(scp_cmd)(ssh_identity)(ssh_args) )
 
FC_REFLECT( prodkey_def,
            (producer_name)(block_signing_key))
 
FC_REFLECT( producer_set_def,
            (schedule))
 
// @ignore listen_addr, p2p_count, http_count, dot_label_str
FC_REFLECT( host_def,
            (genesis)(ssh_identity)(ssh_args)(sysio_home)
            (host_name)(public_name)
            (base_p2p_port)(base_http_port)(def_file_size)
            (instances) )
 
// @ignore node, dot_label_str
FC_REFLECT( eosd_def,
            (config_dir_name)(data_dir_name)(p2p_port)
            (http_port)(file_size)(name)(host)
            (p2p_endpoint) )
 
// @ignore instance, gelf_endpoint
FC_REFLECT( tn_node_def, (name)(keys)(peers)(producers)(dont_start) )
 
FC_REFLECT( testnet_def, (name)(ssh_helper)(nodes) )
 
FC_REFLECT( server_name_def, (ipaddr) (name) (has_bios) (sysio_home) (instances) )
 
FC_REFLECT( server_identities, (producer) (nonprod) (db) (default_sysio_home) (ssh) )
 
FC_REFLECT( node_rt_info, (remote)(pid_file)(kill_cmd) )
 
FC_REFLECT( last_run_def, (running_nodes) )