Wire Sysio Wire Sysion 1.0.0
Loading...
Searching...
No Matches
chainbase.hpp
Go to the documentation of this file.
1#pragma once
2
3#include <boost/interprocess/managed_mapped_file.hpp>
4#include <boost/interprocess/containers/map.hpp>
5#include <boost/interprocess/containers/set.hpp>
6#include <boost/interprocess/containers/flat_map.hpp>
7#include <boost/interprocess/containers/deque.hpp>
8#include <boost/interprocess/containers/string.hpp>
9#include <boost/interprocess/allocators/allocator.hpp>
10#include <boost/interprocess/sync/interprocess_sharable_mutex.hpp>
11#include <boost/interprocess/sync/sharable_lock.hpp>
12#include <boost/core/demangle.hpp>
13
14#include <boost/multi_index_container.hpp>
15
16#include <boost/chrono.hpp>
17#include <boost/config.hpp>
18#include <boost/filesystem.hpp>
19#include <boost/lexical_cast.hpp>
20#include <boost/throw_exception.hpp>
21
22#include <array>
23#include <atomic>
24#include <fstream>
25#include <iostream>
26#include <stdexcept>
27#include <typeindex>
28#include <typeinfo>
29#include <set>
30
35
36#ifndef CHAINBASE_NUM_RW_LOCKS
37 #define CHAINBASE_NUM_RW_LOCKS 10
38#endif
39
40#ifdef CHAINBASE_CHECK_LOCKING
41 #define CHAINBASE_REQUIRE_READ_LOCK(m, t) require_read_lock(m, typeid(t).name())
42 #define CHAINBASE_REQUIRE_WRITE_LOCK(m, t) require_write_lock(m, typeid(t).name())
43#else
44 #define CHAINBASE_REQUIRE_READ_LOCK(m, t)
45 #define CHAINBASE_REQUIRE_WRITE_LOCK(m, t)
46#endif
47
48namespace chainbase {
49
50 namespace bip = boost::interprocess;
51 namespace bfs = boost::filesystem;
52 using std::unique_ptr;
53 using std::vector;
54
55 template<typename T>
56 using allocator = bip::allocator<T, pinnable_mapped_file::segment_manager>;
57
58 template<typename T>
60
62
63 typedef boost::interprocess::interprocess_sharable_mutex read_write_mutex;
64 typedef boost::interprocess::sharable_lock< read_write_mutex > read_lock;
65
69 template<typename T>
70 class oid {
71 public:
72 oid( int64_t i = 0 ):_id(i){}
73
74 oid& operator++() { ++_id; return *this; }
75
76 friend bool operator < ( const oid& a, const oid& b ) { return a._id < b._id; }
77 friend bool operator > ( const oid& a, const oid& b ) { return a._id > b._id; }
78 friend bool operator <= ( const oid& a, const oid& b ) { return a._id <= b._id; }
79 friend bool operator >= ( const oid& a, const oid& b ) { return a._id >= b._id; }
80 friend bool operator == ( const oid& a, const oid& b ) { return a._id == b._id; }
81 friend bool operator != ( const oid& a, const oid& b ) { return a._id != b._id; }
82 friend std::ostream& operator<<(std::ostream& s, const oid& id) {
83 s << boost::core::demangle(typeid(oid<T>).name()) << '(' << id._id << ')'; return s;
84 }
85
87 };
88
89 template<uint16_t TypeNumber, typename Derived>
90 struct object
91 {
93 static const uint16_t type_id = TypeNumber;
94 };
95
99 template<typename T>
100 struct get_index_type {};
101
105 #define CHAINBASE_SET_INDEX_TYPE( OBJECT_TYPE, INDEX_TYPE ) \
106 namespace chainbase { template<> struct get_index_type<OBJECT_TYPE> { typedef INDEX_TYPE type; }; }
107
108 #define CHAINBASE_DEFAULT_CONSTRUCTOR( OBJECT_TYPE ) \
109 template<typename Constructor, typename Allocator> \
110 OBJECT_TYPE( Constructor&& c, Allocator&& ) { c(*this); }
111
121 {
122 public:
123 int_incrementer( int32_t& target ) : _target(target)
124 { ++_target; }
126 { --_target; }
127
129 { return _target; }
130
131 private:
132 int32_t& _target;
133 };
134
135 template<typename MultiIndexType>
137
139 public:
140 virtual ~abstract_session(){};
141 virtual void push() = 0;
142 virtual void squash() = 0;
143 virtual void undo() = 0;
144 };
145
146 template<typename SessionType>
148 {
149 public:
150 session_impl( SessionType&& s ):_session( std::move( s ) ){}
151
152 virtual void push() override { _session.push(); }
153 virtual void squash() override{ _session.squash(); }
154 virtual void undo() override { _session.undo(); }
155 private:
156 SessionType _session;
157 };
158
160 {
161 public:
162 abstract_index( void* i ):_idx_ptr(i){}
163 virtual ~abstract_index(){}
164 virtual void set_revision( uint64_t revision ) = 0;
165 virtual unique_ptr<abstract_session> start_undo_session( bool enabled ) = 0;
166
167 virtual int64_t revision()const = 0;
168 virtual void undo()const = 0;
169 virtual void squash()const = 0;
170 virtual void commit( int64_t revision )const = 0;
171 virtual void undo_all()const = 0;
172 virtual uint32_t type_id()const = 0;
173 virtual uint64_t row_count()const = 0;
174 virtual const std::string& type_name()const = 0;
175 virtual std::pair<int64_t, int64_t> undo_stack_revision_range()const = 0;
176
177 virtual void remove_object( int64_t id ) = 0;
178
179 void* get()const { return _idx_ptr; }
180 private:
181 void* _idx_ptr;
182 };
183
184 template<typename BaseIndex>
185 class index_impl : public abstract_index {
186 public:
187 index_impl( BaseIndex& base ):abstract_index( &base ),_base(base){}
188
189 virtual unique_ptr<abstract_session> start_undo_session( bool enabled ) override {
190 return unique_ptr<abstract_session>(new session_impl<typename BaseIndex::session>( _base.start_undo_session( enabled ) ) );
191 }
192
193 virtual void set_revision( uint64_t revision ) override { _base.set_revision( revision ); }
194 virtual int64_t revision()const override { return _base.revision(); }
195 virtual void undo()const override { _base.undo(); }
196 virtual void squash()const override { _base.squash(); }
197 virtual void commit( int64_t revision )const override { _base.commit(revision); }
198 virtual void undo_all() const override {_base.undo_all(); }
199 virtual uint32_t type_id()const override { return BaseIndex::value_type::type_id; }
200 virtual uint64_t row_count()const override { return _base.indices().size(); }
201 virtual const std::string& type_name() const override { return BaseIndex_name; }
202 virtual std::pair<int64_t, int64_t> undo_stack_revision_range()const override { return _base.undo_stack_revision_range(); }
203
204 virtual void remove_object( int64_t id ) override { return _base.remove_object( id ); }
205 private:
206 BaseIndex& _base;
207 std::string BaseIndex_name = boost::core::demangle( typeid( typename BaseIndex::value_type ).name() );
208 };
209
210 template<typename IndexType>
211 class index : public index_impl<IndexType> {
212 public:
213 index( IndexType& i ):index_impl<IndexType>( i ){}
214 };
215
216
218 {
219 public:
221 {
222 _current_lock = 0;
223 }
224
226
228 {
229 _current_lock++;
230 new( &_locks[ _current_lock % CHAINBASE_NUM_RW_LOCKS ] ) read_write_mutex();
231 }
232
234 {
235 return _locks[ _current_lock % CHAINBASE_NUM_RW_LOCKS ];
236 }
237
239 {
240 return _current_lock;
241 }
242
243 private:
244 std::array< read_write_mutex, CHAINBASE_NUM_RW_LOCKS > _locks;
245 std::atomic< uint32_t > _current_lock;
246 };
247
248
253 {
254 public:
259
260 using database_index_row_count_multiset = std::multiset<std::pair<unsigned, std::string>>;
261
262 database(const bfs::path& dir, open_flags write = read_only, uint64_t shared_file_size = 0, bool allow_dirty = false,
264 ~database();
265 database(database&&) = default;
267 bool is_read_only() const { return _read_only; }
268 void flush();
269 void set_require_locking( bool enable_require_locking );
270
271#ifdef CHAINBASE_CHECK_LOCKING
272 void require_lock_fail( const char* method, const char* lock_type, const char* tname )const;
273
274 void require_read_lock( const char* method, const char* tname )const
275 {
276 if( BOOST_UNLIKELY( _enable_require_locking & _read_only & (_read_lock_count <= 0) ) )
277 require_lock_fail(method, "read", tname);
278 }
279
280 void require_write_lock( const char* method, const char* tname )
281 {
282 if( BOOST_UNLIKELY( _enable_require_locking & (_write_lock_count <= 0) ) )
283 require_lock_fail(method, "write", tname);
284 }
285#endif
286
287 struct session {
288 public:
289 session( session&& s ):_index_sessions( std::move(s._index_sessions) ){}
290 session( vector<std::unique_ptr<abstract_session>>&& s ):_index_sessions( std::move(s) )
291 {
292 }
293
295 undo();
296 }
297
298 void push()
299 {
300 for( auto& i : _index_sessions ) i->push();
301 _index_sessions.clear();
302 }
303
304 void squash()
305 {
306 for( auto& i : _index_sessions ) i->squash();
307 _index_sessions.clear();
308 }
309
310 void undo()
311 {
312 for( auto& i : _index_sessions ) i->undo();
313 _index_sessions.clear();
314 }
315
316 private:
317 friend class database;
318 session(){}
319
320 vector< std::unique_ptr<abstract_session> > _index_sessions;
321 };
322
323 session start_undo_session( bool enabled );
324
326 if( _index_list.size() == 0 ) return -1;
327 return _index_list[0]->revision();
328 }
329
330 void undo();
331 void squash();
332 void commit( int64_t revision );
333 void undo_all();
334
335
337 {
338 CHAINBASE_REQUIRE_WRITE_LOCK( "set_revision", uint64_t );
339 for( auto i : _index_list ) i->set_revision( revision );
340 }
341
342
343 template<typename MultiIndexType>
344 void add_index() {
346 typedef generic_index<MultiIndexType> index_type;
347 typedef typename index_type::allocator_type index_alloc;
348
349 std::string type_name = boost::core::demangle( typeid( typename index_type::value_type ).name() );
350
351 if( !( _index_map.size() <= type_id || _index_map[ type_id ] == nullptr ) ) {
352 BOOST_THROW_EXCEPTION( std::logic_error( type_name + "::type_id is already in use" ) );
353 }
354
355 index_type* idx_ptr = nullptr;
356 if( _read_only )
357 idx_ptr = _db_file.get_segment_manager()->find_no_lock< index_type >( type_name.c_str() ).first;
358 else
359 idx_ptr = _db_file.get_segment_manager()->find< index_type >( type_name.c_str() ).first;
360 bool first_time_adding = false;
361 if( !idx_ptr ) {
362 if( _read_only ) {
363 BOOST_THROW_EXCEPTION( std::runtime_error( "unable to find index for " + type_name + " in read only database" ) );
364 }
365 first_time_adding = true;
366 idx_ptr = _db_file.get_segment_manager()->construct< index_type >( type_name.c_str() )( index_alloc( _db_file.get_segment_manager() ) );
367 }
368
369 idx_ptr->validate();
370
371 // Ensure the undo stack of added index is consistent with the other indices in the database
372 if( _index_list.size() > 0 ) {
373 auto expected_revision_range = _index_list.front()->undo_stack_revision_range();
374 auto added_index_revision_range = idx_ptr->undo_stack_revision_range();
375
376 if( added_index_revision_range.first != expected_revision_range.first ||
377 added_index_revision_range.second != expected_revision_range.second ) {
378
379 if( !first_time_adding ) {
380 BOOST_THROW_EXCEPTION( std::logic_error(
381 "existing index for " + type_name + " has an undo stack (revision range [" +
382 std::to_string(added_index_revision_range.first) + ", " + std::to_string(added_index_revision_range.second) +
383 "]) that is inconsistent with other indices in the database (revision range [" +
384 std::to_string(expected_revision_range.first) + ", " + std::to_string(expected_revision_range.second) +
385 "]); corrupted database?"
386 ) );
387 }
388
389 if( _read_only ) {
390 BOOST_THROW_EXCEPTION( std::logic_error(
391 "new index for " + type_name +
392 " requires an undo stack that is consistent with other indices in the database; cannot fix in read-only mode"
393 ) );
394 }
395
396 idx_ptr->set_revision( static_cast<uint64_t>(expected_revision_range.first) );
397 while( idx_ptr->revision() < expected_revision_range.second ) {
398 idx_ptr->start_undo_session(true).push();
399 }
400 }
401 }
402
403 if( type_id >= _index_map.size() )
404 _index_map.resize( type_id + 1 );
405
406 auto new_index = new index<index_type>( *idx_ptr );
407 _index_map[ type_id ].reset( new_index );
408 _index_list.push_back( new_index );
409 }
410
414
418
419 size_t get_free_memory()const
420 {
421 return _db_file.get_segment_manager()->get_free_memory();
422 }
423
424 template<typename MultiIndexType>
426 {
427 CHAINBASE_REQUIRE_READ_LOCK("get_index", typename MultiIndexType::value_type);
428 typedef generic_index<MultiIndexType> index_type;
429 typedef index_type* index_type_ptr;
430 assert( _index_map.size() > index_type::value_type::type_id );
431 assert( _index_map[index_type::value_type::type_id] );
432 return *index_type_ptr( _index_map[index_type::value_type::type_id]->get() );
433 }
434
435 template<typename MultiIndexType, typename ByIndex>
436 auto get_index()const -> decltype( ((generic_index<MultiIndexType>*)( nullptr ))->indices().template get<ByIndex>() )
437 {
438 CHAINBASE_REQUIRE_READ_LOCK("get_index", typename MultiIndexType::value_type);
439 typedef generic_index<MultiIndexType> index_type;
440 typedef index_type* index_type_ptr;
441 assert( _index_map.size() > index_type::value_type::type_id );
442 assert( _index_map[index_type::value_type::type_id] );
443 return index_type_ptr( _index_map[index_type::value_type::type_id]->get() )->indices().template get<ByIndex>();
444 }
445
446 template<typename MultiIndexType>
448 {
449 CHAINBASE_REQUIRE_WRITE_LOCK("get_mutable_index", typename MultiIndexType::value_type);
450 typedef generic_index<MultiIndexType> index_type;
451 typedef index_type* index_type_ptr;
452 assert( _index_map.size() > index_type::value_type::type_id );
453 assert( _index_map[index_type::value_type::type_id] );
454 return *index_type_ptr( _index_map[index_type::value_type::type_id]->get() );
455 }
456
457 template< typename ObjectType, typename IndexedByType, typename CompatibleKey >
458 const ObjectType* find( CompatibleKey&& key )const
459 {
460 CHAINBASE_REQUIRE_READ_LOCK("find", ObjectType);
461 typedef typename get_index_type< ObjectType >::type index_type;
462 const auto& idx = get_index< index_type >().indices().template get< IndexedByType >();
463 auto itr = idx.find( std::forward< CompatibleKey >( key ) );
464 if( itr == idx.end() ) return nullptr;
465 return &*itr;
466 }
467
468 template< typename ObjectType >
469 const ObjectType* find( oid< ObjectType > key = oid< ObjectType >() ) const
470 {
471 CHAINBASE_REQUIRE_READ_LOCK("find", ObjectType);
472 typedef typename get_index_type< ObjectType >::type index_type;
473 return get_index< index_type >().find( key );
474 }
475
476 template< typename ObjectType, typename IndexedByType, typename CompatibleKey >
477 const ObjectType& get( CompatibleKey&& key )const
478 {
479 CHAINBASE_REQUIRE_READ_LOCK("get", ObjectType);
480 auto obj = find< ObjectType, IndexedByType >( std::forward< CompatibleKey >( key ) );
481 if( !obj ) {
482 std::stringstream ss;
483 ss << "unknown key (" << boost::core::demangle( typeid( key ).name() ) << "): " << key;
484 BOOST_THROW_EXCEPTION( std::out_of_range( ss.str().c_str() ) );
485 }
486 return *obj;
487 }
488
489 template< typename ObjectType >
490 const ObjectType& get( const oid< ObjectType >& key = oid< ObjectType >() )const
491 {
492 CHAINBASE_REQUIRE_READ_LOCK("get", ObjectType);
493 auto obj = find< ObjectType >( key );
494 if( !obj ) {
495 std::stringstream ss;
496 ss << "unknown key (" << boost::core::demangle( typeid( key ).name() ) << "): " << key._id;
497 BOOST_THROW_EXCEPTION( std::out_of_range( ss.str().c_str() ) );
498 }
499 return *obj;
500 }
501
502 template<typename ObjectType, typename Modifier>
503 void modify( const ObjectType& obj, Modifier&& m )
504 {
505 CHAINBASE_REQUIRE_WRITE_LOCK("modify", ObjectType);
506 typedef typename get_index_type<ObjectType>::type index_type;
507 get_mutable_index<index_type>().modify( obj, m );
508 }
509
510 template<typename ObjectType>
511 void remove( const ObjectType& obj )
512 {
513 CHAINBASE_REQUIRE_WRITE_LOCK("remove", ObjectType);
514 typedef typename get_index_type<ObjectType>::type index_type;
515 return get_mutable_index<index_type>().remove( obj );
516 }
517
518 template<typename ObjectType, typename Constructor>
519 const ObjectType& create( Constructor&& con )
520 {
521 CHAINBASE_REQUIRE_WRITE_LOCK("create", ObjectType);
522 typedef typename get_index_type<ObjectType>::type index_type;
523 return get_mutable_index<index_type>().emplace( std::forward<Constructor>(con) );
524 }
525
528 for(const auto& ai_ptr : _index_map) {
529 if(!ai_ptr)
530 continue;
531 ret.emplace(make_pair(ai_ptr->row_count(), ai_ptr->type_name()));
532 }
533 return ret;
534 }
535
536 private:
537 pinnable_mapped_file _db_file;
538 bool _read_only = false;
539
543 vector<abstract_index*> _index_list;
544
548 vector<unique_ptr<abstract_index>> _index_map;
549
550#ifdef CHAINBASE_CHECK_LOCKING
551 int32_t _read_lock_count = 0;
552 int32_t _write_lock_count = 0;
553 bool _enable_require_locking = false;
554#endif
555 };
556
557 template<typename Object, typename... Args>
558 using shared_multi_index_container = boost::multi_index_container<Object,Args..., chainbase::node_allocator<Object> >;
559} // namepsace chainbase
std::string name
#define CHAINBASE_REQUIRE_WRITE_LOCK(m, t)
Definition chainbase.hpp:45
#define CHAINBASE_NUM_RW_LOCKS
Definition chainbase.hpp:37
#define CHAINBASE_REQUIRE_READ_LOCK(m, t)
Definition chainbase.hpp:44
virtual unique_ptr< abstract_session > start_undo_session(bool enabled)=0
virtual void commit(int64_t revision) const =0
virtual void undo_all() const =0
virtual int64_t revision() const =0
virtual void undo() const =0
virtual void squash() const =0
virtual void remove_object(int64_t id)=0
virtual std::pair< int64_t, int64_t > undo_stack_revision_range() const =0
virtual uint64_t row_count() const =0
virtual const std::string & type_name() const =0
virtual uint32_t type_id() const =0
virtual void set_revision(uint64_t revision)=0
int64_t revision() const
const ObjectType * find(oid< ObjectType > key=oid< ObjectType >()) const
void set_revision(uint64_t revision)
database & operator=(database &&)=default
const ObjectType & get(const oid< ObjectType > &key=oid< ObjectType >()) const
database(const bfs::path &dir, open_flags write=read_only, uint64_t shared_file_size=0, bool allow_dirty=false, pinnable_mapped_file::map_mode=pinnable_mapped_file::map_mode::mapped)
Definition chainbase.cpp:12
const generic_index< MultiIndexType > & get_index() const
const pinnable_mapped_file::segment_manager * get_segment_manager() const
size_t get_free_memory() const
pinnable_mapped_file::segment_manager * get_segment_manager()
std::multiset< std::pair< unsigned, std::string > > database_index_row_count_multiset
void modify(const ObjectType &obj, Modifier &&m)
const ObjectType * find(CompatibleKey &&key) const
session start_undo_session(bool enabled)
Definition chainbase.cpp:73
database(database &&)=default
bool is_read_only() const
void remove(const ObjectType &obj)
void commit(int64_t revision)
Definition chainbase.cpp:57
void set_require_locking(bool enable_require_locking)
Definition chainbase.cpp:25
const ObjectType & create(Constructor &&con)
database_index_row_count_multiset row_count_per_index() const
auto get_index() const -> decltype(((generic_index< MultiIndexType > *)(nullptr)) ->indices().template get< ByIndex >())
generic_index< MultiIndexType > & get_mutable_index()
const ObjectType & get(CompatibleKey &&key) const
virtual const std::string & type_name() const override
virtual void undo() const override
virtual uint32_t type_id() const override
virtual unique_ptr< abstract_session > start_undo_session(bool enabled) override
virtual int64_t revision() const override
virtual void commit(int64_t revision) const override
virtual std::pair< int64_t, int64_t > undo_stack_revision_range() const override
virtual void squash() const override
virtual void remove_object(int64_t id) override
virtual void undo_all() const override
index_impl(BaseIndex &base)
virtual uint64_t row_count() const override
virtual void set_revision(uint64_t revision) override
index(IndexType &i)
int_incrementer(int32_t &target)
friend bool operator>=(const oid &a, const oid &b)
Definition chainbase.hpp:79
friend bool operator!=(const oid &a, const oid &b)
Definition chainbase.hpp:81
friend bool operator==(const oid &a, const oid &b)
Definition chainbase.hpp:80
oid(int64_t i=0)
Definition chainbase.hpp:72
friend bool operator<(const oid &a, const oid &b)
Definition chainbase.hpp:76
friend bool operator<=(const oid &a, const oid &b)
Definition chainbase.hpp:78
friend std::ostream & operator<<(std::ostream &s, const oid &id)
Definition chainbase.hpp:82
friend bool operator>(const oid &a, const oid &b)
Definition chainbase.hpp:77
oid & operator++()
Definition chainbase.hpp:74
segment_manager * get_segment_manager() const
bip::managed_mapped_file::segment_manager segment_manager
read_write_mutex & current_lock()
session_impl(SessionType &&s)
virtual void push() override
virtual void squash() override
virtual void undo() override
CK_SESSION_HANDLE session
multi_index_to_undo_index< MultiIndexType > generic_index
typename multi_index_to_undo_index_impl< MultiIndexContainer >::type multi_index_to_undo_index
boost::interprocess::interprocess_sharable_mutex read_write_mutex
Definition chainbase.hpp:63
boost::multi_index_container< Object, Args..., chainbase::node_allocator< Object > > shared_multi_index_container
bip::allocator< T, pinnable_mapped_file::segment_manager > allocator
Definition chainbase.hpp:56
boost::interprocess::sharable_lock< read_write_mutex > read_lock
Definition chainbase.hpp:64
Definition name.hpp:106
const GenericPointer< typename T::ValueType > T2 T::AllocatorType & a
Definition pointer.h:1181
unsigned short uint16_t
Definition stdint.h:125
signed __int64 int64_t
Definition stdint.h:135
unsigned int uint32_t
Definition stdint.h:126
signed int int32_t
Definition stdint.h:123
unsigned __int64 uint64_t
Definition stdint.h:136
session(vector< std::unique_ptr< abstract_session > > &&s)
static const uint16_t type_id
Definition chainbase.hpp:93
oid< Derived > id_type
Definition chainbase.hpp:92
uint8_t key[16]
Definition yubico_otp.c:41
CK_RV ret
char * s
yubihsm_pkcs11_object_template template