sysio::chain::detail::block_log_impl Class Reference

Collaboration diagram for sysio::chain::detail::block_log_impl:

Public Member Functions
	block_log_impl (std::optional< block_log_prune_config > prune_conf)
void	check_open_files ()
void	reopen ()
void	try_exit_vacuum ()
void	close ()
template<typename T >
void	reset (const T &t, const signed_block_ptr &genesis_block, uint32_t first_block_num)
void	write (const genesis_state &gs)
void	write (const chain_id_type &chain_id)
void	flush ()
void	append (const signed_block_ptr &b)
void	prune (const fc::log_level &loglevel)
void	vacuum ()
size_t	convert_existing_header_to_vacuumed ()
uint64_t	get_block_pos (uint32_t block_num)

Static Public Member Functions
template<typename ChainContext , typename Lambda >
static std::optional< ChainContext >	extract_chain_context (const fc::path &data_dir, Lambda &&lambda)

Public Attributes
signed_block_ptr	head
block_id_type	head_id
fc::cfile	block_file
fc::cfile	index_file
bool	open_files = false
bool	genesis_written_to_block_log = false
uint32_t	version = 0
uint32_t	first_block_num = 0
uint32_t	index_first_block_num = 0
std::optional< block_log_prune_config >	prune_config

Detailed Description

Definition at line 40 of file block_log.cpp.

Constructor & Destructor Documentation

block_log_impl()

sysio::chain::detail::block_log_impl::block_log_impl ( std::optional< block_log_prune_config > prune_conf )

inline

Definition at line 53 of file block_log.cpp.

                                                                           :
              prune_config(prune_conf) {
               if(prune_config) {
                  SYS_ASSERT(prune_config->prune_blocks, block_log_exception, "block log prune configuration requires at least one block");
                  SYS_ASSERT(__builtin_popcount(prune_config->prune_threshold) == 1, block_log_exception, "block log prune threshold must be power of 2");
                  //switch this over to the mask that will be used
                  prune_config->prune_threshold = ~(prune_config->prune_threshold-1);
               }
            }

Member Function Documentation

append()

void sysio::chain::detail::block_log_impl::append

(

const signed_block_ptr &

b

)

Definition at line 362 of file block_log.cpp.

                                                              {
      try {
         SYS_ASSERT( genesis_written_to_block_log, block_log_append_fail, "Cannot append to block log until the genesis is first written" );
 
         check_open_files();
 
         block_file.seek_end(0);
         index_file.seek_end(0);
         //if pruned log, rewind over count trailer if any block is already present
         if(prune_config && head)
            block_file.skip(-sizeof(uint32_t));
         uint64_t pos = block_file.tellp();
 
         SYS_ASSERT(index_file.tellp() == sizeof(uint64_t) * (b->block_num() - index_first_block_num),
                   block_log_append_fail,
                   "Append to index file occuring at wrong position.",
                   ("position", (uint64_t) index_file.tellp())
                   ("expected", (b->block_num() - index_first_block_num) * sizeof(uint64_t)));
         auto data = fc::raw::pack(*b);
         block_file.write(data.data(), data.size());
         block_file.write((char*)&pos, sizeof(pos));
         const uint64_t end = block_file.tellp();
         index_file.write((char*)&pos, sizeof(pos));
         head = b;
         head_id = b->calculate_id();
 
         if(prune_config) {
            if((pos&prune_config->prune_threshold) != (end&prune_config->prune_threshold))
               prune(fc::log_level::debug);
 
            const uint32_t num_blocks_in_log = chain::block_header::num_from_id(head_id) - first_block_num + 1;
            fc::raw::pack(block_file, num_blocks_in_log);
         }
 
         flush();
      }
      FC_LOG_AND_RETHROW()
   }

Here is the call graph for this function:

check_open_files()

void sysio::chain::detail::block_log_impl::check_open_files ( )

inline

Definition at line 63 of file block_log.cpp.

                                           {
               if( !open_files ) {
                  reopen();
               }
            }

Here is the call graph for this function:

Here is the caller graph for this function:

close()

void sysio::chain::detail::block_log_impl::close ( )

inline

Definition at line 91 of file block_log.cpp.

                         {
               if( block_file.is_open() )
                  block_file.close();
               if( index_file.is_open() )
                  index_file.close();
               open_files = false;
            }

Here is the call graph for this function:

convert_existing_header_to_vacuumed()

size_t sysio::chain::detail::block_log_impl::convert_existing_header_to_vacuumed

(

)

Definition at line 435 of file block_log.cpp.

                                                                    {
      uint32_t old_version;
      uint32_t old_first_block_num;
      const auto totem = block_log::npos;
 
      block_file.seek(0);
      fc::raw::unpack(block_file, old_version);
      fc::raw::unpack(block_file, old_first_block_num);
      SYS_ASSERT(is_pruned_log_and_mask_version(old_version), block_log_exception, "Trying to vacuumed a non-pruned block log");
 
      if(block_log::contains_genesis_state(old_version, old_first_block_num)) {
         //we'll always write a v3 log, but need to possibly mutate the genesis_state to a chainid should we have pruned a log starting with a genesis_state
         genesis_state gs;
         auto ds = block_file.create_datastream();
         fc::raw::unpack(ds, gs);
 
         block_file.seek(0);
         fc::raw::pack(block_file, block_log::max_supported_version);
         fc::raw::pack(block_file, first_block_num);
         if(first_block_num == 1) {
            SYS_ASSERT(old_first_block_num == 1, block_log_exception, "expected an old first blocknum of 1");
            fc::raw::pack(block_file, gs);
         }
         else
            fc::raw::pack(block_file, gs.compute_chain_id());
         fc::raw::pack(block_file, totem);
      }
      else {
         //read in the existing chainid, to parrot back out
         fc::sha256 chainid;
         fc::raw::unpack(block_file, chainid);
 
         block_file.seek(0);
         fc::raw::pack(block_file, block_log::max_supported_version);
         fc::raw::pack(block_file, first_block_num);
         fc::raw::pack(block_file, chainid);
         fc::raw::pack(block_file, totem);
      }
 
      return block_file.tellp();
   }

Here is the call graph for this function:

extract_chain_context()

template<typename ChainContext , typename Lambda >

std::optional< ChainContext > sysio::chain::detail::block_log_impl::extract_chain_context ( const fc::path & data_dir, Lambda && lambda )

static

Definition at line 942 of file block_log.cpp.

                                                                                                                  {
      SYS_ASSERT( fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), block_log_not_found,
                  "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir)          );
 
      std::fstream  block_stream;
      block_stream.open( (data_dir / "blocks.log").generic_string().c_str(), LOG_READ );
 
      uint32_t version = 0;
      block_stream.read( (char*)&version, sizeof(version) );
      is_pruned_log_and_mask_version(version);
      SYS_ASSERT( version >= block_log::min_supported_version && version <= block_log::max_supported_version, block_log_unsupported_version,
                  "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]",
                  ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) );
 
      uint32_t first_block_num = 1;
      if (version != 1) {
         block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) );
      }
 
      return lambda(block_stream, version, first_block_num);
   }

Here is the call graph for this function:

Here is the caller graph for this function:

flush()

void sysio::chain::detail::block_log_impl::flush

(

)

Definition at line 430 of file block_log.cpp.

                                    {
      block_file.flush();
      index_file.flush();
   }

Here is the caller graph for this function:

get_block_pos()

uint64_t sysio::chain::detail::block_log_impl::get_block_pos

(

uint32_t

block_num

)

Definition at line 657 of file block_log.cpp.

                                                                  {
      check_open_files();
      if (!(head && block_num <= block_header::num_from_id(head_id) && block_num >= first_block_num))
         return block_log::npos;
      index_file.seek(sizeof(uint64_t) * (block_num - index_first_block_num));
      uint64_t pos;
      index_file.read((char*)&pos, sizeof(pos));
      return pos;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

prune()

void sysio::chain::detail::block_log_impl::prune

(

const fc::log_level &

loglevel

)

Definition at line 401 of file block_log.cpp.

                                                               {
      if(!head)
         return;
      const uint32_t head_num = chain::block_header::num_from_id(head_id);
      if(head_num - first_block_num < prune_config->prune_blocks)
         return;
 
      const uint32_t prune_to_num = head_num - prune_config->prune_blocks + 1;
 
      static_assert( block_log::max_supported_version == 3, "Code was written to support version 3 format, need to update this code for latest format." );
      const genesis_state gs;
      const size_t max_header_size_v1  = sizeof(uint32_t) + fc::raw::pack_size(gs) + sizeof(uint64_t);
      const size_t max_header_size_v23 = sizeof(uint32_t) + sizeof(uint32_t) + sizeof(chain_id_type) + sizeof(uint64_t);
      const auto max_header_size = std::max(max_header_size_v1, max_header_size_v23);
 
      block_file.punch_hole(max_header_size, get_block_pos(prune_to_num));
 
      first_block_num = prune_to_num;
      block_file.flush();
 
      if(auto l = fc::logger::get(); l.is_enabled(loglevel))
         l.log(fc::log_message(fc::log_context(loglevel, __FILE__, __LINE__, __func__),
                               "blocks.log pruned to blocks ${b}-${e}", fc::mutable_variant_object()("b", first_block_num)("e", head_num)));
   }

Here is the call graph for this function:

Here is the caller graph for this function:

reopen()

void sysio::chain::detail::detail::block_log_impl::reopen

(

)

Definition at line 130 of file block_log.cpp.

                                        {
         close();
 
         // open to create files if they don't exist
         //ilog("Opening block log at ${path}", ("path", my->block_file.generic_string()));
         block_file.open( LOG_WRITE_C );
         index_file.open( LOG_WRITE_C );
 
         close();
 
         block_file.open( LOG_RW_C );
         index_file.open( LOG_RW_C );
 
         open_files = true;
      }

Here is the call graph for this function:

Here is the caller graph for this function:

reset()

template<typename T >

void sysio::chain::detail::block_log_impl::reset	(	const T &	t,
		const signed_block_ptr &	genesis_block,
		uint32_t	first_block_num )

Definition at line 548 of file block_log.cpp.

                                                                                                          {
      close();
 
      fc::remove_all( block_file.get_file_path() );
      fc::remove_all( index_file.get_file_path() );
 
      reopen();
 
      version = 0; // version of 0 is invalid; it indicates that subsequent data was not properly written to the block log
      index_first_block_num = first_block_num = first_bnum;
 
      block_file.seek_end(0);
      block_file.write((char*)&version, sizeof(version));
      block_file.write((char*)&first_block_num, sizeof(first_block_num));
 
      write(t);
      genesis_written_to_block_log = true;
 
      // append a totem to indicate the division between blocks and header
      auto totem = block_log::npos;
      block_file.write((char*)&totem, sizeof(totem));
 
      if (first_block) {
         append(first_block);
      } else {
         head.reset();
         head_id = {};
         if(prune_config)
            fc::raw::pack(block_file, (uint32_t)0);
      }
 
      auto pos = block_file.tellp();
 
      static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" );
 
      // going back to write correct version to indicate that all block log header data writes completed successfully
      version = block_log::max_supported_version;
      if(prune_config)
         version |= pruned_version_flag;
      block_file.seek( 0 );
      block_file.write( (char*)&version, sizeof(version) );
      block_file.seek( pos );
      flush();
   }

Here is the call graph for this function:

try_exit_vacuum()

void sysio::chain::detail::block_log_impl::try_exit_vacuum ( )

inline

Definition at line 72 of file block_log.cpp.

                                   {
               //for a pruned log that has at least one block, see if we should vacuum it
               if( block_file.is_open() && index_file.is_open() && prune_config && prune_config->vacuum_on_close) {
                  if(!head) {
                     //disregard vacuum_on_close size if there isn't even a block and just do it silently anyways
                     vacuum();
                  }
                  else {
                     const size_t first_data_pos = get_block_pos(first_block_num);
                     block_file.seek_end(-sizeof(uint32_t));
                     const size_t last_data_pos = block_file.tellp();
                     if(last_data_pos - first_data_pos < prune_config->vacuum_on_close) {
                        ilog("Vacuuming pruned block log");
                        vacuum();
                     }
                  }
               }
            }

Here is the call graph for this function:

vacuum()

void sysio::chain::detail::block_log_impl::vacuum

(

)

Definition at line 477 of file block_log.cpp.

                                     {
      //go ahead and write a new valid header now. if the vacuum fails midway, at least this means maybe the
      // block recovery can get through some blocks.
      size_t copy_to_pos = convert_existing_header_to_vacuumed();
 
      version = block_log::max_supported_version;
      prune_config.reset();
 
      //if there is no head block though, bail now, otherwise first_block_num won't actually be available
      // and it'll mess this all up. Be sure to still remove the 4 byte trailer though.
      if(!head) {
         block_file.flush();
         fc::resize_file(block_file.get_file_path(), fc::file_size(block_file.get_file_path()) - sizeof(uint32_t));
         return;
      }
 
      size_t copy_from_pos = get_block_pos(first_block_num);
      block_file.seek_end(-sizeof(uint32_t));
      size_t copy_sz = block_file.tellp() - copy_from_pos;
      const uint32_t num_blocks_in_log = chain::block_header::num_from_id(head_id) - first_block_num + 1;
 
      const size_t offset_bytes = copy_from_pos - copy_to_pos;
      const size_t offset_blocks = first_block_num - index_first_block_num;
 
      std::vector<char> buff;
      buff.resize(4*1024*1024);
 
      auto tick = std::chrono::time_point_cast<std::chrono::seconds>(std::chrono::system_clock::now());
      while(copy_sz) {
         const size_t copy_this_round = std::min(buff.size(), copy_sz);
         block_file.seek(copy_from_pos);
         block_file.read(buff.data(), copy_this_round);
         block_file.punch_hole(copy_to_pos, copy_from_pos+copy_this_round);
         block_file.seek(copy_to_pos);
         block_file.write(buff.data(), copy_this_round);
 
         copy_from_pos += copy_this_round;
         copy_to_pos += copy_this_round;
         copy_sz -= copy_this_round;
 
         const auto tock = std::chrono::time_point_cast<std::chrono::seconds>(std::chrono::system_clock::now());
         if(tick < tock - std::chrono::seconds(5)) {
            ilog("Vacuuming pruned block log, ${b} bytes remaining", ("b", copy_sz));
            tick = tock;
         }
      }
      block_file.flush();
      fc::resize_file(block_file.get_file_path(), block_file.tellp());
 
      index_file.flush();
      {
         boost::interprocess::mapped_region index_mapped(index_file, boost::interprocess::read_write);
         uint64_t* index_ptr = (uint64_t*)index_mapped.get_address();
 
         for(uint32_t new_block_num = 0; new_block_num < num_blocks_in_log; ++new_block_num) {
            const uint64_t new_pos = index_ptr[new_block_num + offset_blocks] - offset_bytes;
            index_ptr[new_block_num] = new_pos;
 
            if(new_block_num + 1 != num_blocks_in_log)
               block_file.seek(index_ptr[new_block_num + offset_blocks + 1] - offset_bytes - sizeof(uint64_t));
            else
               block_file.seek_end(-sizeof(uint64_t));
            block_file.write((char*)&new_pos, sizeof(new_pos));
         }
      }
      fc::resize_file(index_file.get_file_path(), num_blocks_in_log*sizeof(uint64_t));
 
      index_first_block_num = first_block_num;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

write() [1/2]

void sysio::chain::detail::block_log_impl::write

(

const chain_id_type &

chain_id

)

Definition at line 608 of file block_log.cpp.

                                                                   {
      block_file << chain_id;
   }

write() [2/2]

void sysio::chain::detail::block_log_impl::write

(

const genesis_state &

gs

)

Definition at line 603 of file block_log.cpp.

                                                             {
      auto data = fc::raw::pack(gs);
      block_file.write(data.data(), data.size());
   }

Here is the call graph for this function:

Member Data Documentation

block_file

fc::cfile sysio::chain::detail::block_log_impl::block_file

Definition at line 44 of file block_log.cpp.

first_block_num

uint32_t sysio::chain::detail::block_log_impl::first_block_num = 0

Definition at line 49 of file block_log.cpp.

genesis_written_to_block_log

bool sysio::chain::detail::block_log_impl::genesis_written_to_block_log = false

Definition at line 47 of file block_log.cpp.

head

signed_block_ptr sysio::chain::detail::block_log_impl::head

Definition at line 42 of file block_log.cpp.

head_id

block_id_type sysio::chain::detail::block_log_impl::head_id

Definition at line 43 of file block_log.cpp.

index_file

fc::cfile sysio::chain::detail::block_log_impl::index_file

Definition at line 45 of file block_log.cpp.

index_first_block_num

uint32_t sysio::chain::detail::block_log_impl::index_first_block_num = 0

Definition at line 50 of file block_log.cpp.

open_files

bool sysio::chain::detail::block_log_impl::open_files = false

Definition at line 46 of file block_log.cpp.

prune_config

std::optional<block_log_prune_config> sysio::chain::detail::block_log_impl::prune_config

Definition at line 51 of file block_log.cpp.

version

uint32_t sysio::chain::detail::block_log_impl::version = 0

Definition at line 48 of file block_log.cpp.

---

The documentation for this class was generated from the following file:

• libraries/chain/block_log.cpp