sysio::trace_api::slice_directory Class Reference

#include <store_provider.hpp>

Classes
struct	index_header

Public Types
enum class	open_state { read , write }

Public Member Functions
	slice_directory (const boost::filesystem::path &slice_dir, uint32_t width, std::optional< uint32_t > minimum_irreversible_history_blocks, std::optional< uint32_t > minimum_uncompressed_irreversible_history_blocks, size_t compression_seek_point_stride)
uint32_t	slice_number (uint32_t block_height) const
bool	find_or_create_index_slice (uint32_t slice_number, open_state state, fc::cfile &index_file) const
bool	find_index_slice (uint32_t slice_number, open_state state, fc::cfile &index_file, bool open_file=true) const
bool	find_or_create_trace_slice (uint32_t slice_number, open_state state, fc::cfile &trace_file) const
bool	find_trace_slice (uint32_t slice_number, open_state state, fc::cfile &trace_file, bool open_file=true) const
std::optional< compressed_file >	find_compressed_trace_slice (uint32_t slice_number, bool open_file=true) const
void	find_or_create_slice_pair (uint32_t slice_number, open_state state, fc::cfile &trace, fc::cfile &index)
bool	find_or_create_trx_id_slice (uint32_t slice_number, open_state state, fc::cfile &trx_id_file) const
bool	find_trx_id_slice (uint32_t slice_number, open_state state, fc::cfile &trx_id_file, bool open_file=true) const
void	set_lib (uint32_t lib)
void	start_maintenance_thread (log_handler log)
void	stop_maintenance_thread ()
void	run_maintenance_tasks (uint32_t lib, const log_handler &log)

Detailed Description

Provides access to the slice directory. It is only intended to be used by store_provider and unit tests.

Definition at line 96 of file store_provider.hpp.

Member Enumeration Documentation

open_state

enum class sysio::trace_api::slice_directory::open_state

strong

Enumerator
read
write

Definition at line 102 of file store_provider.hpp.

{ read /*read from front to back*/, write /*write to end of file*/ };

Constructor & Destructor Documentation

slice_directory()

sysio::trace_api::slice_directory::slice_directory	(	const boost::filesystem::path &	slice_dir,
		uint32_t	width,
		std::optional< uint32_t >	minimum_irreversible_history_blocks,
		std::optional< uint32_t >	minimum_uncompressed_irreversible_history_blocks,
		size_t	compression_seek_point_stride )

Definition at line 153 of file store_provider.cpp.

   : _slice_dir(slice_dir)
   , _width(width)
   , _minimum_irreversible_history_blocks(minimum_irreversible_history_blocks)
   , _minimum_uncompressed_irreversible_history_blocks(minimum_uncompressed_irreversible_history_blocks)
   , _compression_seek_point_stride(compression_seek_point_stride)
   , _best_known_lib(0) {
      if (!exists(_slice_dir)) {
         bfs::create_directories(slice_dir);
      }
   }

Here is the call graph for this function:

Member Function Documentation

find_compressed_trace_slice()

std::optional< compressed_file > sysio::trace_api::slice_directory::find_compressed_trace_slice	(	uint32_t	slice_number,
		bool	open_file = true ) const

Find the read-only compressed trace file associated with the indicated slice_number

Parameters

slice_number	: slice number of the requested slice file
open_file	: indicate if the file should be opened (if found) or not

Returns

if file was found (i.e. already existed) returns an optional containing a compressed_file which is open (or not) depending on the open_file paraneter, Otherwise, the returned optional is empty

Definition at line 227 of file store_provider.cpp.

                                                                                                                         {
      auto filename = make_filename(_trace_prefix, _compressed_trace_ext, slice_number, _width);
      const path slice_path = _slice_dir / filename;
      const bool file_exists = exists(slice_path);
 
      if (file_exists) {
         auto result = compressed_file(slice_path);
         if (open_file) {
            result.open();
         }
 
         return std::move(result);
      } else {
         return {};
      }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

find_index_slice()

bool sysio::trace_api::slice_directory::find_index_slice	(	uint32_t	slice_number,
		open_state	state,
		fc::cfile &	index_file,
		bool	open_file = true ) const

Find the index file associated with the indicated slice_number

Parameters

slice_number	: slice number of the requested slice file
state	: indicate if the file is going to be written to (appended) or read
index_file	: the cfile that will be set to the appropriate slice filename (always) and opened to that file (if it was found)
open_file	: indicate if the file should be opened (if found) or not

Returns

the true if file was found (i.e. already existed), if not found index_file is set to the appropriate file, but not open

Definition at line 173 of file store_provider.cpp.

                                                                                                                            {
      const bool found = find_slice(_trace_index_prefix, slice_number, index_file, open_file);
      if( !found || !open_file ) {
         return found;
      }
 
      validate_existing_index_slice_file(index_file, state);
      return true;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

find_or_create_index_slice()

bool sysio::trace_api::slice_directory::find_or_create_index_slice	(	uint32_t	slice_number,
		open_state	state,
		fc::cfile &	index_file ) const

Find or create the index file associated with the indicated slice_number

Parameters

slice_number	: slice number of the requested slice file
state	: indicate if the file is going to be written to (appended) or read
index_file	: the cfile that will be set to the appropriate slice filename and opened to that file

Returns

the true if file was found (i.e. already existed)

Definition at line 165 of file store_provider.cpp.

                                                                                                                      {
      const bool found = find_index_slice(slice_number, state, index_file);
      if( !found ) {
         create_new_index_slice_file(index_file);
      }
      return found;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

find_or_create_slice_pair()

void sysio::trace_api::slice_directory::find_or_create_slice_pair	(	uint32_t	slice_number,
		open_state	state,
		fc::cfile &	trace,
		fc::cfile &	index )

Find or create a trace and index file pair

Parameters

slice_number	: slice number of the requested slice file
state	: indicate if the file is going to be written to (appended) or read
trace	: the cfile that will be set to the appropriate slice filename and opened to that file

Definition at line 262 of file store_provider.cpp.

                                                                                                                          {
      const bool trace_found = find_or_create_trace_slice(slice_number, state, trace);
      const bool index_found = find_or_create_index_slice(slice_number, state, index);
      if (trace_found != index_found) {
         const std::string trace_status = trace_found ? "existing" : "new";
         const std::string index_status = index_found ? "existing" : "new";
         elog("Trace file is ${ts}, but it's metadata file is ${is}. This means the files are not consistent.", ("ts", trace_status)("is", index_status));
      }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

find_or_create_trace_slice()

bool sysio::trace_api::slice_directory::find_or_create_trace_slice	(	uint32_t	slice_number,
		open_state	state,
		fc::cfile &	trace_file ) const

Find or create the trace file associated with the indicated slice_number

Parameters

slice_number	: slice number of the requested slice file
state	: indicate if the file is going to be written to (appended) or read
trace_file	: the cfile that will be set to the appropriate slice filename and opened to that file

Returns

the true if file was found (i.e. already existed)

Definition at line 201 of file store_provider.cpp.

                                                                                                                      {
      const bool found = find_trace_slice(slice_number, state, trace_file);
 
      if( !found ) {
         trace_file.open(fc::cfile::create_or_update_rw_mode);
      }
 
      return found;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

find_or_create_trx_id_slice()

bool sysio::trace_api::slice_directory::find_or_create_trx_id_slice	(	uint32_t	slice_number,
		open_state	state,
		fc::cfile &	trx_id_file ) const

Find or create a trx id file that contains all the transaction ids and associated block numbers

Parameters

slice_number	: slice number of the requested slice file
state	: indicate if the file is going to be written to (appended) or read
trx_id_file	: the cfile

Returns

true if file was found (i.e. already existed)

Definition at line 272 of file store_provider.cpp.

                                                                                                                        {
       const bool found = find_trx_id_slice(slice_number, state, trx_id_file);
       if( !found ) {
           trx_id_file.open(fc::cfile::create_or_update_rw_mode);
       }
       return found;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

find_trace_slice()

bool sysio::trace_api::slice_directory::find_trace_slice	(	uint32_t	slice_number,
		open_state	state,
		fc::cfile &	trace_file,
		bool	open_file = true ) const

Find the trace file associated with the indicated slice_number

Parameters

slice_number	: slice number of the requested slice file
state	: indicate if the file is going to be written to (appended) or read
trace_file	: the cfile that will be set to the appropriate slice filename (always) and opened to that file (if it was found)
open_file	: indicate if the file should be opened (if found) or not

Returns

the true if file was found (i.e. already existed), if not found index_file is set to the appropriate file, but not open

Definition at line 211 of file store_provider.cpp.

                                                                                                                            {
      const bool found = find_slice(_trace_prefix, slice_number, trace_file, open_file);
 
      if( !found || !open_file ) {
         return found;
      }
 
      if( state == open_state::write ) {
         trace_file.seek_end(0);
      }
      else {
         trace_file.seek(0); // ensure we are at the start of the file
      }
      return true;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

find_trx_id_slice()

bool sysio::trace_api::slice_directory::find_trx_id_slice	(	uint32_t	slice_number,
		open_state	state,
		fc::cfile &	trx_id_file,
		bool	open_file = true ) const

Find the trx id file

Parameters

slice_number	: slice number of the requested slice file
state	: indicate if the file is going to be written to (appended) or read
trx_id_file	: the cfile
open_file	: indicate if the file should be opened (if found) or not

Returns

true if file was found (i.e. already existed), if not found trx_id_file is set to the appropriate file, but not opened

Definition at line 280 of file store_provider.cpp.

                                                                                                                              {
      const bool found = find_slice(_trace_trx_id_prefix, slice_number, trx_id_file, open_file);
      if( !found || !open_file ) {
         return found;
      }
      if( state == open_state::write ) {
          trx_id_file.seek_end(0);
      }
      return true;
   }

Here is the call graph for this function:

Here is the caller graph for this function:

run_maintenance_tasks()

void sysio::trace_api::slice_directory::run_maintenance_tasks	(	uint32_t	lib,
		const log_handler &	log )

Cleans up all slices that are no longer needed to maintain the minimum number of blocks past lib Compresses up all slices that can be compressed

Parameters

lib	: block number of the current lib

Definition at line 355 of file store_provider.cpp.

                                                                                   {
      if (_minimum_irreversible_history_blocks) {
         process_irreversible_slice_range(lib, *_minimum_irreversible_history_blocks, _last_cleaned_up_slice, [this, &log](uint32_t slice_to_clean){
            fc::cfile trace;
            fc::cfile index;
            fc::cfile trx_id;
 
            log(std::string("Attempting Prune of slice: ") + std::to_string(slice_to_clean));
 
            // cleanup index first to reduce the likelihood of reader finding index, but not finding trace
            const bool dont_open_file = false;
            const bool index_found = find_index_slice(slice_to_clean, open_state::read, index, dont_open_file);
            if (index_found) {
               log(std::string("Removing: ") + index.get_file_path().generic_string());
               bfs::remove(index.get_file_path());
            }
            const bool trace_found = find_trace_slice(slice_to_clean, open_state::read, trace, dont_open_file);
            if (trace_found) {
               log(std::string("Removing: ") + trace.get_file_path().generic_string());
               bfs::remove(trace.get_file_path());
            }
            const bool trx_id_found = find_trx_id_slice(slice_to_clean, open_state::read, trx_id, dont_open_file);
            if (trx_id_found) {
               log(std::string("Removing: ") + trx_id.get_file_path().generic_string());
               bfs::remove(trx_id.get_file_path());
            }
 
            auto ctrace = find_compressed_trace_slice(slice_to_clean, dont_open_file);
            if (ctrace) {
               log(std::string("Removing: ") + ctrace->get_file_path().generic_string());
               bfs::remove(ctrace->get_file_path());
            }
         });
      }
 
      // Only process compression if its configured AND there is a range of irreversible blocks which would not also
      // be deleted
      if (_minimum_uncompressed_irreversible_history_blocks &&
          (!_minimum_irreversible_history_blocks || *_minimum_uncompressed_irreversible_history_blocks < *_minimum_irreversible_history_blocks) )
      {
         process_irreversible_slice_range(lib, *_minimum_uncompressed_irreversible_history_blocks, _last_compressed_slice, [this, &log](uint32_t slice_to_compress){
            fc::cfile trace;
            const bool dont_open_file = false;
            const bool trace_found = find_trace_slice(slice_to_compress, open_state::read, trace, dont_open_file);
 
            log(std::string("Attempting compression of slice: ") + std::to_string(slice_to_compress));
 
            if (trace_found) {
               auto compressed_path = trace.get_file_path();
               compressed_path.replace_extension(_compressed_trace_ext);
 
               log(std::string("Compressing: ") + trace.get_file_path().generic_string());
               compressed_file::process(trace.get_file_path(), compressed_path.generic_string(), _compression_seek_point_stride);
 
               // after compression is complete, delete the old uncompressed file
               log(std::string("Removing: ") + trace.get_file_path().generic_string());
               bfs::remove(trace.get_file_path());
            }
         });
      }
   }

Here is the call graph for this function:

Here is the caller graph for this function:

set_lib()

void sysio::trace_api::slice_directory::set_lib

(

uint32_t

lib

)

set the LIB for maintenance

Parameters

lib

Definition at line 291 of file store_provider.cpp.

                                             {
      {
         std::scoped_lock lock(_maintenance_mtx);
         _best_known_lib = lib;
      }
      _maintenance_condition.notify_one();
   }

Here is the call graph for this function:

Here is the caller graph for this function:

slice_number()

uint32_t sysio::trace_api::slice_directory::slice_number ( uint32_t block_height ) const

inline

Return the slice number that would include the passed in block_height

Parameters

block_height

: height of the requested data

Returns

the slice number for the block_height

Definition at line 112 of file store_provider.hpp.

                                                         {
         return block_height / _width;
      }

Here is the caller graph for this function:

start_maintenance_thread()

void sysio::trace_api::slice_directory::start_maintenance_thread

(

log_handler

log

)

Start a thread which does background maintenance

Definition at line 299 of file store_provider.cpp.

                                                                 {
      _maintenance_thread = std::thread([this, log=std::move(log)](){
         fc::set_os_thread_name( "trace-mx" );
         uint32_t last_lib = 0;
 
         while(true) {
            std::unique_lock<std::mutex> lock(_maintenance_mtx);
            while ( last_lib >= _best_known_lib && !_maintenance_shutdown ) {
               _maintenance_condition.wait(lock);
            }
 
            if (_maintenance_shutdown) {
               break;
            }
 
            uint32_t best_known_lib = _best_known_lib;
            lock.unlock();
 
            log(std::string("Waking up to handle lib: ") + std::to_string(best_known_lib));
 
            if (last_lib < best_known_lib) {
               try {
                  run_maintenance_tasks(best_known_lib, log);
                  last_lib = best_known_lib;
               } FC_LOG_AND_DROP();
            }
         }
      });
   }

Here is the call graph for this function:

Here is the caller graph for this function:

stop_maintenance_thread()

void sysio::trace_api::slice_directory::stop_maintenance_thread

(

)

Stop and join the thread doing background maintenance

Definition at line 329 of file store_provider.cpp.

                                                 {
      {
         std::scoped_lock lock(_maintenance_mtx);
         _maintenance_shutdown = true;
      }
      _maintenance_condition.notify_one();
      _maintenance_thread.join();
   }

Here is the call graph for this function:

Here is the caller graph for this function:

---

The documentation for this class was generated from the following files:

• plugins/trace_api_plugin/include/sysio/trace_api/store_provider.hpp
• plugins/trace_api_plugin/store_provider.cpp