Merge pull request #780 from steemit/steem-v0.16.2

Steem v0.16.2

CMakeLists.txt

@@ -69,6 +69,13 @@ if( LOW_MEMORY_NODE )
   SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DIS_LOW_MEM" )
 endif()
 
+OPTION( CHAINBASE_CHECK_LOCKING "Check locks in chainbase (ON or OFF)" ON )
+MESSAGE( STATUS "CHAINBASE_CHECK_LOCKING: ${CHAINBASE_CHECK_LOCKING}" )
+if( CHAINBASE_CHECK_LOCKING )
+  SET( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCHAINBASE_CHECK_LOCKING" )
+  SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DCHAINBASE_CHECK_LOCKING" )
+endif()
+
 OPTION( CLEAR_VOTES "Build source to clear old votes from memory" ON )
 if( CLEAR_VOTES )
   MESSAGE( STATUS "   CONFIGURING TO CLEAR OLD VOTES FROM MEMORY" )

Dockerfile

@@ -61,6 +61,7 @@ RUN \
         -DBUILD_STEEM_TESTNET=ON \
         -DLOW_MEMORY_NODE=OFF \
         -DCLEAR_VOTES=ON \
+        -DCHAINBASE_CHECK_LOCKING=OFF \
         .. && \
     make -j$(nproc) chain_test && \
     ./tests/chain_test && \

libraries/app/application.cpp

@@ -252,6 +252,9 @@ namespace detail {
          bool read_only = _options->count( "read-only" );
          register_builtin_apis();
 
+         if( _options->count("check-locks") )
+            _chain_db->set_require_locking( true );
+
          if( _options->count("shared-file-dir") )
             _shared_dir = fc::path( _options->at("shared-file-dir").as<string>() );
          else
@@ -936,6 +939,7 @@ void application::set_program_options(boost::program_options::options_descriptio
          ("enable-plugin", bpo::value< vector<string> >()->composing()->default_value(default_plugins, str_default_plugins), "Plugin(s) to enable, may be specified multiple times")
          ("max-block-age", bpo::value< int32_t >()->default_value(200), "Maximum age of head block when broadcasting tx via API")
          ("flush", bpo::value< uint32_t >()->default_value(100000), "Flush shared memory file to disk this many blocks")
+         ("check-locks", bpo::value< bool >()->default_value(false), "Check correctness of chainbase locking")
          ;
    command_line_options.add(configuration_file_options);
    command_line_options.add_options()

libraries/app/include/steemit/app/database_api.hpp

@@ -88,6 +88,9 @@ struct discussion_query {
    string           tag;
    uint32_t         limit = 0;
    set<string>      filter_tags;
+   set<string>      select_authors; ///< list of authors to include, posts not by this author are filtered
+   set<string>      select_tags; ///< list of tags to include, posts without these tags are filtered
+   uint32_t         truncate_body = 0; ///< the number of bytes of the post body to return, 0 for all
    optional<string> start_author;
    optional<string> start_permlink;
    optional<string> parent_author;
@@ -341,6 +344,8 @@ class database_api
       vector<discussion>   get_content_replies( string parent, string parent_permlink )const;
 
       ///@{ tags API
+      /** This API will return the top 1000 tags used by an author sorted by most frequently used */
+      vector<pair<string,uint32_t>> get_tags_used_by_author( const string& author )const;
       vector<discussion> get_discussions_by_trending( const discussion_query& query )const;
       vector<discussion> get_discussions_by_trending30( const discussion_query& query )const;
       vector<discussion> get_discussions_by_created( const discussion_query& query )const;
@@ -412,7 +417,7 @@ class database_api
    private:
       void set_pending_payout( discussion& d )const;
       void set_url( discussion& d )const;
-      discussion get_discussion( comment_id_type )const;
+      discussion get_discussion( comment_id_type, uint32_t truncate_body = 0 )const;
 
       static bool filter_default( const comment_api_obj& c ) { return false; }
       static bool exit_default( const comment_api_obj& c )   { return false; }
@@ -423,6 +428,7 @@ class database_api
                                           const string& tag,
                                           comment_id_type parent,
                                           const Index& idx, StartItr itr,
+                                          uint32_t truncate_body = 0,
                                           const std::function< bool( const comment_api_obj& ) >& filter = &database_api::filter_default,
                                           const std::function< bool( const comment_api_obj& ) >& exit   = &database_api::exit_default,
                                           const std::function< bool( const tags::tag_object& ) >& tag_exit = &database_api::tag_exit_default
@@ -442,7 +448,7 @@ FC_REFLECT( steemit::app::scheduled_hardfork, (hf_version)(live_time) );
 FC_REFLECT( steemit::app::liquidity_balance, (account)(weight) );
 FC_REFLECT( steemit::app::withdraw_route, (from_account)(to_account)(percent)(auto_vest) );
 
-FC_REFLECT( steemit::app::discussion_query, (tag)(filter_tags)(start_author)(start_permlink)(parent_author)(parent_permlink)(limit) );
+FC_REFLECT( steemit::app::discussion_query, (tag)(filter_tags)(select_tags)(select_authors)(truncate_body)(start_author)(start_permlink)(parent_author)(parent_permlink)(limit) );
 
 FC_REFLECT_ENUM( steemit::app::withdraw_route_type, (incoming)(outgoing)(all) );
 
@@ -455,6 +461,7 @@ FC_API(steemit::app::database_api,
 
    // tags
    (get_trending_tags)
+   (get_tags_used_by_author)
    (get_discussions_by_trending)
    (get_discussions_by_trending30)
    (get_discussions_by_created)

libraries/app/include/steemit/app/state.hpp

@@ -79,7 +79,9 @@ namespace steemit { namespace app {
       vector<string>              replies; ///< author/slug mapping
       share_type                  author_reputation = 0;
       asset                       promoted = asset(0, SBD_SYMBOL);
-      optional<string>            first_reblogged_by;
+      uint32_t                    body_length = 0;
+      vector<account_name_type>   reblogged_by;
+      optional<account_name_type> first_reblogged_by;
       optional<time_point_sec>    first_reblogged_on;
    };
 
@@ -99,6 +101,8 @@ namespace steemit { namespace app {
       map<uint64_t,applied_operation>         vote_history;
       map<uint64_t,applied_operation>         other_history;
       set<string>                             witness_votes;
+      vector<pair<string,uint32_t>>            tags_usage;
+      vector<pair<account_name_type,uint32_t>> guest_bloggers;
 
       optional<map<uint32_t,extended_limit_order>> open_orders;
       optional<vector<string>>                comments; /// permlinks for this user
@@ -187,7 +191,7 @@ namespace steemit { namespace app {
 FC_REFLECT_DERIVED( steemit::app::extended_account,
                    (steemit::app::account_api_obj),
                    (vesting_balance)(reputation)
-                   (transfer_history)(market_history)(post_history)(vote_history)(other_history)(witness_votes)(open_orders)(comments)(feed)(blog)(recent_replies)(blog_category)(recommended) )
+                   (transfer_history)(market_history)(post_history)(vote_history)(other_history)(witness_votes)(tags_usage)(guest_bloggers)(open_orders)(comments)(feed)(blog)(recent_replies)(blog_category)(recommended) )
 
 
 FC_REFLECT( steemit::app::vote_state, (voter)(weight)(rshares)(percent)(reputation)(time) );
@@ -196,7 +200,7 @@ FC_REFLECT( steemit::app::account_vote, (authorperm)(weight)(rshares)(percent)(t
 FC_REFLECT( steemit::app::discussion_index, (category)(trending)(trending30)(updated)(created)(responses)(active)(votes)(maturing)(best)(hot)(promoted)(cashout) )
 FC_REFLECT( steemit::app::category_index, (active)(recent)(best) )
 FC_REFLECT( steemit::app::tag_index, (trending) )
-FC_REFLECT_DERIVED( steemit::app::discussion, (steemit::app::comment_api_obj), (url)(root_title)(pending_payout_value)(total_pending_payout_value)(active_votes)(replies)(author_reputation)(promoted)(first_reblogged_by)(first_reblogged_on) )
+FC_REFLECT_DERIVED( steemit::app::discussion, (steemit::app::comment_api_obj), (url)(root_title)(pending_payout_value)(total_pending_payout_value)(active_votes)(replies)(author_reputation)(promoted)(body_length)(reblogged_by)(first_reblogged_by)(first_reblogged_on) )
 
 FC_REFLECT( steemit::app::state, (current_route)(props)(category_idx)(tag_idx)(categories)(tags)(content)(accounts)(pow_queue)(witnesses)(discussion_idx)(witness_schedule)(feed_price)(error)(market_data) )
 

libraries/chain/database.cpp

@@ -586,17 +586,16 @@ bool database::push_block(const signed_block& new_block, uint32_t skip)
    bool result;
    detail::with_skip_flags( *this, skip, [&]()
    {
-      detail::without_pending_transactions( *this, std::move(_pending_tx),
-      [&]()
+      with_write_lock( [&]()
       {
-         try
+         detail::without_pending_transactions( *this, std::move(_pending_tx), [&]()
          {
-            with_write_lock( [&]()
+            try
             {
                result = _push_block(new_block);
-            });
-         }
-         FC_CAPTURE_AND_RETHROW( (new_block) )
+            }
+            FC_CAPTURE_AND_RETHROW( (new_block) )
+         });
       });
    });
 
@@ -764,7 +763,7 @@ signed_block database::generate_block(
          result = _generate_block( when, witness_owner, block_signing_private_key );
       }
       FC_CAPTURE_AND_RETHROW( (witness_owner) )
-   } );
+   });
    return result;
 }
 
@@ -2868,9 +2867,12 @@ void database::init_genesis( uint64_t init_supply )
 
 void database::validate_transaction( const signed_transaction& trx )
 {
-   auto session = start_undo_session( true );
-   _apply_transaction( trx );
-   session.undo();
+   database::with_write_lock( [&]()
+   {
+      auto session = start_undo_session( true );
+      _apply_transaction( trx );
+      session.undo();
+   });
 }
 
 void database::notify_changed_objects()
@@ -2968,13 +2970,13 @@ void database::apply_block( const signed_block& next_block, uint32_t skip )
             x += now % span;
          }
          _next_flush_block = x;
-         ilog( "Next flush scheduled at block ${b}", ("b", x) );
+         //ilog( "Next flush scheduled at block ${b}", ("b", x) );
       }
 
       if( _next_flush_block == block_num )
       {
          _next_flush_block = 0;
-         ilog( "Flushing database shared memory at block ${b}", ("b", block_num) );
+         //ilog( "Flushing database shared memory at block ${b}", ("b", block_num) );
          chainbase::database::flush();
       }
    }

libraries/chain/include/steemit/chain/account_object.hpp

@@ -18,6 +18,8 @@ namespace steemit { namespace chain {
 
    class account_object : public object< account_object_type, account_object >
    {
+      account_object() = delete;
+
       public:
          template<typename Constructor, typename Allocator>
          account_object( Constructor&& c, allocator< Allocator > a )
@@ -114,6 +116,8 @@ namespace steemit { namespace chain {
 
    class account_authority_object : public object< account_authority_object_type, account_authority_object >
    {
+      account_authority_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          account_authority_object( Constructor&& c, allocator< Allocator > a )
@@ -155,6 +159,8 @@ namespace steemit { namespace chain {
 
    class owner_authority_history_object : public object< owner_authority_history_object_type, owner_authority_history_object >
    {
+      owner_authority_history_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          owner_authority_history_object( Constructor&& c, allocator< Allocator > a )
@@ -172,6 +178,8 @@ namespace steemit { namespace chain {
 
    class account_recovery_request_object : public object< account_recovery_request_object_type, account_recovery_request_object >
    {
+      account_recovery_request_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          account_recovery_request_object( Constructor&& c, allocator< Allocator > a )

libraries/chain/include/steemit/chain/comment_object.hpp

@@ -40,6 +40,8 @@ namespace steemit { namespace chain {
     */
    class category_object : public object< category_object_type, category_object >
    {
+      category_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          category_object( Constructor&& c, allocator< Allocator > a )
@@ -100,6 +102,8 @@ namespace steemit { namespace chain {
 
    class comment_object : public object < comment_object_type, comment_object >
    {
+      comment_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          comment_object( Constructor&& c, allocator< Allocator > a )

libraries/chain/include/steemit/chain/history_object.hpp

@@ -14,6 +14,8 @@ namespace steemit { namespace chain {
 
    class operation_object : public object< operation_object_type, operation_object >
    {
+      operation_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          operation_object( Constructor&& c, allocator< Allocator > a )

libraries/chain/include/steemit/chain/shared_authority.hpp

@@ -17,6 +17,11 @@ namespace steemit { namespace chain {
     */
    struct shared_authority
    {
+      private:
+         shared_authority() = delete;
+
+      public:
+
       template< typename Allocator >
       shared_authority( const authority& a, const Allocator& alloc ) :
          account_auths( account_pair_allocator_type( alloc.get_segment_manager() ) ),

libraries/chain/include/steemit/chain/steem_objects.hpp

@@ -70,6 +70,8 @@ namespace steemit { namespace chain {
 
    class savings_withdraw_object : public object< savings_withdraw_object_type, savings_withdraw_object >
    {
+      savings_withdraw_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          savings_withdraw_object( Constructor&& c, allocator< Allocator > a )
@@ -148,6 +150,8 @@ namespace steemit { namespace chain {
     */
    class feed_history_object  : public object< feed_history_object_type, feed_history_object >
    {
+      feed_history_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          feed_history_object( Constructor&& c, allocator< Allocator > a )

libraries/chain/include/steemit/chain/transaction_object.hpp

@@ -16,6 +16,8 @@ namespace steemit { namespace chain {
     */
    class transaction_object : public object< transaction_object_type, transaction_object >
    {
+      transaction_object() = delete;
+
       public:
          template< typename Constructor, typename Allocator >
          transaction_object( Constructor&& c, allocator< Allocator > a )

libraries/chain/include/steemit/chain/witness_objects.hpp

@@ -25,6 +25,8 @@ namespace steemit { namespace chain {
     */
    class witness_object : public object< witness_object_type, witness_object >
    {
+      witness_object() = delete;
+
       public:
          enum witness_schedule_type
          {

libraries/plugins/follow/follow_api.cpp

@@ -39,8 +39,10 @@ class follow_api_impl
 
 vector< follow_api_obj > follow_api_impl::get_followers( string following, string start_follower, follow_type type, uint16_t limit )const
 {
-   FC_ASSERT( limit <= 100 );
+   FC_ASSERT( limit <= 1000 );
    vector< follow_api_obj > result;
+   result.reserve( limit );
+
    const auto& idx = app.chain_database()->get_index<follow_index>().indices().get<by_following_follower>();
    auto itr = idx.lower_bound( std::make_tuple( following, start_follower ) );
    while( itr != idx.end() && limit && itr->following == following )
@@ -121,7 +123,11 @@ vector< feed_entry > follow_api_impl::get_feed_entries( string account, uint32_t
       entry.entry_id = itr->account_feed_id;
       if( itr->first_reblogged_by != account_name_type() )
       {
-         entry.reblog_by = itr->first_reblogged_by;
+         entry.reblog_by.reserve( itr->reblogged_by.size() );
+         for( const auto& a : itr->reblogged_by ) {
+            entry.reblog_by.push_back(a);
+         }
+         //entry.reblog_by = itr->first_reblogged_by;
          entry.reblog_on = itr->first_reblogged_on;
       }
       results.push_back( entry );
@@ -154,7 +160,11 @@ vector< comment_feed_entry > follow_api_impl::get_feed( string account, uint32_t
       entry.entry_id = itr->account_feed_id;
       if( itr->first_reblogged_by != account_name_type() )
       {
-         entry.reblog_by = itr->first_reblogged_by;
+         //entry.reblog_by = itr->first_reblogged_by;
+         entry.reblog_by.reserve( itr->reblogged_by.size() );
+         for( const auto& a : itr->reblogged_by ) {
+            entry.reblog_by.push_back( a );
+         }
          entry.reblog_on = itr->first_reblogged_on;
       }
       results.push_back( entry );
@@ -328,5 +338,33 @@ vector< account_reputation > follow_api::get_account_reputations( string lower_b
       return my->get_account_reputations( lower_bound_name, limit );
    });
 }
+vector< account_name_type > follow_api::get_reblogged_by( const string& author, const string& permlink )const {
+  auto& db = *my->app.chain_database();
+  return db.with_read_lock( [&](){
+      vector<account_name_type> result;
+      const auto& post = db.get_comment( author, permlink );
+      const auto& blog_idx = db.get_index<blog_index,by_comment>();
+      auto itr = blog_idx.lower_bound( post.id );
+      while( itr != blog_idx.end() && itr->comment == post.id && result.size() < 2000 ) {
+         result.push_back( itr->account );
+         ++itr;
+      }
+      return result;
+  });
+}
+
+vector< pair< account_name_type, uint32_t > > follow_api::get_blog_authors( const account_name_type& blog )const {
+  auto& db = *my->app.chain_database();
+  return db.with_read_lock( [&](){
+    vector< pair< account_name_type, uint32_t > > result;
+    const auto& stats_idx = db.get_index<blog_author_stats_index,by_blogger_guest_count>();
+    auto itr = stats_idx.lower_bound( boost::make_tuple( blog ) );
+    while( itr != stats_idx.end() && itr->blogger == blog && result.size() < 2000 ) {
+      result.push_back( std::make_pair( itr->guest, itr->count ) );
+      ++itr;
+    }
+    return result;
+  }); 
+}
 
 } } // steemit::follow