Merge pull request #1095 from steemit/steem-prerelease-v0.18.3

Steem v0.18.3

CMakeLists.txt

@@ -46,7 +46,8 @@ LIST(APPEND BOOST_COMPONENTS thread
                              chrono
                              unit_test_framework
                              context
-                             locale)
+                             locale
+                             coroutine)
 SET( Boost_USE_STATIC_LIBS ON CACHE STRING "ON or OFF" )
 
 OPTION( BUILD_STEEM_TESTNET "Build source for test network (ON OR OFF)" OFF )
@@ -89,14 +90,7 @@ IF( WIN32 )
   set(BOOST_ALL_DYN_LINK OFF) # force dynamic linking for all libraries
 ENDIF(WIN32)
 
-FIND_PACKAGE(Boost 1.57 REQUIRED COMPONENTS ${BOOST_COMPONENTS})
-# For Boost 1.53 on windows, coroutine was not in BOOST_LIBRARYDIR and do not need it to build,  but if boost versin >= 1.54, find coroutine otherwise will cause link errors
-IF(NOT "${Boost_VERSION}" MATCHES "1.53(.*)")
-   SET(BOOST_LIBRARIES_TEMP ${Boost_LIBRARIES})
-   FIND_PACKAGE(Boost 1.54 REQUIRED COMPONENTS coroutine)
-   LIST(APPEND BOOST_COMPONENTS coroutine)
-   SET(Boost_LIBRARIES ${BOOST_LIBRARIES_TEMP} ${Boost_LIBRARIES})
-ENDIF()
+FIND_PACKAGE(Boost 1.58 REQUIRED COMPONENTS ${BOOST_COMPONENTS})
 
 if( WIN32 )
 

Dockerfile

@@ -3,7 +3,6 @@ FROM phusion/baseimage:0.9.19
 #ARG STEEMD_BLOCKCHAIN=https://example.com/steemd-blockchain.tbz2
 
 ENV LANG=en_US.UTF-8
-ENV VERSION=0.18.1
 
 RUN \
     apt-get update && \
@@ -31,6 +30,8 @@ RUN \
         s3cmd \
         awscli \
         jq \
+        wget \
+        gdb \
     && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
@@ -93,6 +94,13 @@ RUN \
     make -j$(nproc) && \
     make install && \
     cd .. && \
+    ( /usr/local/steemd-default/bin/steemd --version \
+      | grep -o '[0-9]*\.[0-9]*\.[0-9]*' \
+      && echo '_' \
+      && git rev-parse --short HEAD ) \
+      | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n//g' \
+      > /etc/steemdversion && \
+    cat /etc/steemdversion && \
     rm -rfv build && \
     mkdir build && \
     cd build && \
@@ -205,4 +213,4 @@ RUN chmod +x /usr/local/bin/healthcheck.sh
 # AWS EB Docker requires a non-daemonized entrypoint
 ADD contrib/steemdentrypoint.sh /usr/local/bin/steemdentrypoint.sh
 RUN chmod +x /usr/local/bin/steemdentrypoint.sh
-CMD /usr/local/bin/steemdentrypoint.sh
+CMD /usr/local/bin/steemdentrypoint.sh

LICENSE.md

@@ -4,16 +4,17 @@ The following license applies to code contained within this repository that
 is created by Steemit, Inc. Other copy right holders have licensed dependencies such
 as Graphene, FC, and Boost under their own individual licenses.
 
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
+The MIT License
 
-1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-3. The currency symbols, 'STEEM' and 'SBD' are not changed and no new currency symbols are added.
-4. The STEEMIT_INIT_PUBLIC_KEY_STR is not changed from STM8GC13uCZbP44HzMLV6zPZGwVQ8Nt4Kji8PapsPiNq1BK153XTX,
-and the software is not modified in any way that would bypass the need for the corresponding private to start
-a new blockchain.
-5. The software is not used with any forks of the Steem blockchain that are not recognized by Steemit, Inc in writing.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

README.md

@@ -1,6 +1,6 @@
 # Introducing Steem (beta)
 
-Steem is an experimental Proof of Work blockchain with an unproven consensus
+Steem is an experimental Delegated Proof of Stake blockchain with an unproven consensus
 algorithm.
 
   - Currency symbol STEEM
@@ -23,7 +23,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 IN THE SOFTWARE.
 
-# Code is Documentation
+# Blockchain consensus rules
 
 Rather than attempt to describe the rules of the blockchain, it is up to
 each individual to inspect the code to understand the consensus rules.
@@ -49,12 +49,30 @@ To run a node with *all* the data (e.g. for supporting a content website)
 that uses ca. 14GB of memory and growing:
 
     docker run \
-        --env USE_WAY_TOO_MUCH_RAM=1 \
+        --env USE_WAY_TOO_MUCH_RAM=1 --env USE_FULL_WEB_NODE=1 \
         -d -p 2001:2001 -p 8090:8090 --name steemd-full \
         steemit/steem
 
     docker logs -f steemd-full
 
+# Environment variables
+
+There are quite a few environment variables that can be set to run steemd in different ways:
+
+* `USE_WAY_TOO_MUCH_RAM` - if set to true, steemd starts a 'full node'
+* `USE_FULL_WEB_NODE` - if set to true, a default config file will be used that enables a full set of API's and associated plugins.
+* `USE_NGINX_FRONTEND` - if set to true, this will enable an NGINX reverse proxy in front of steemd that proxies websocket requests to steemd. This will also enable a custom healtcheck at the path '/health' that lists how many seconds away from current blockchain time your node is. It will return a '200' if it's less than 60 seconds away from synced.
+* `USE_MULTICORE_READONLY` - if set to true, this will enable steemd in multiple reader mode to take advantage of multiple cores (if available). Read requests are handled by the read-only nodes, and write requests are forwarded back to the single 'writer' node automatically. NGINX load balances all requests to the reader nodes, 4 per available core. This setting is still considered experimental and may have trouble with some API calls until further development is completed.
+* `HOME` - set this to the path where you want steemd to store it's data files (block log, shared memory, config file, etc). By default `/var/lib/steemd` is used and exists inside the docker container. If you want to use a different mountpoint (like a ramdisk, or a different drive) then you may want to set this variable to map the volume to your docker container.
+
+# PaaS mode
+
+Steemd now supports a PaaS mode (platform as a service) that currently works with Amazon's Elastic Beanstalk service. It can be launched using the following environment variables:
+
+* `USE_PAAS` - if set to true, steemd will launch in a format that works with AWS EB. Containers will exit upon failure so that they can be relaunched automatically by ECS. This mode assumes `USE_WAY_TOO_MUCH_RAM` and `USE_FULL_WEB_NODE`, they do not need to be also set.
+* `S3_BUCKET` - set this to the name of the S3 bucket where you will store shared memory files for steemd in Amazon S3. They will be stored compressed in bz2 format with the file name `blockchain-$VERSION-latest.tar.bz2`, where $VERSION is the release number followed by the git short commit hash stored in each docker image at `/etc/steemdversion`.
+* `SYNC_TO_S3` - if set to true, the node will function to only generate shared memory files and upload them to the specified S3 bucket. This makes fast deployments and autoscaling for steemd possible.
+
 # Seed Nodes
 
 A list of some seed nodes to get you started can be found in
@@ -64,17 +82,6 @@ This same file is baked into the docker images and can be overridden by
 setting `STEEMD_SEED_NODES` in the container environment at `docker run`
 time to a whitespace delimited list of seed nodes (with port).
 
-# How to Mine
-
-The mining algorithm used by Steem requires the owner to have access to the
-private key used by the account. This means it does not favor mining pools.
-
-    ./steemd --miner=["accountname","${WIFPRIVATEKEY}"] \
-        --witness="accountname" --seed-node="52.38.66.234:2001"
-
-Make sure that your accountname is unique and not already used by someone
-else or your proof of work might not be accepted by the blockchain.
-
 # Building
 
 See [doc/building.md](doc/building.md) for detailed build instructions, including
@@ -87,9 +94,7 @@ on how to use lcov to check code test coverage.
 
 # System Requirements
 
-Minimum 8 GB RAM (16 GB Recommended).
-32 GB disk for full node or 12 GB for a consensus node. SSD is preferred.
-Any CPU with decent single core performance.
+For a full web node, you need at least 55GB of space available. Steemd uses a memory mapped file which currently holds 36GB of data and by default is set to use up to 40GB. The block log of the blockchain itself is a little over 10GB. It's highly recommended to run steemd on a fast disk such as an SSD or by placing the shared memory files in a ramdisk and using the `--shard-file-dir=/path` command line option to specify where. At least 16GB of memory is required for a full web node. Seed nodes (p2p mode) can run with as little as 4GB of memory. Any CPU with decent single core performance should be sufficient.
 
 On Linux use the following Virtual Memory configuration for the initial sync and subsequent replays. It is not needed for normal operation.
 

contrib/fullnode.config.ini

@@ -1,5 +1,5 @@
 # Set larger shared-file-size than default
-shared-file-size = 40G
+shared-file-size = 48G
 
 # Set an API to be publicly available, may be specified multiple times
 public-api = database_api login_api account_by_key_api network_broadcast_api tag_api follow_api market_history_api raw_block_api

contrib/paas-sv-run.sh

@@ -13,7 +13,14 @@ fi
 # if the writer node dies, kill runsv causing the container to exit
 STEEMD_PID=`pgrep -f p2p-endpoint`
 if [[ ! $? -eq 0 ]]; then
-  echo NOTIFYALERT! steemd has quit unexpectedly, starting a new instance..
+  echo NOTIFYALERT! steemd has quit unexpectedly, checking for core dump and then starting a new instance..
+  sleep 30
+  SAVED_PID=`cat /tmp/steemdpid`
+  if [[ -e /tmp/core.$SAVED_PID ]]; then
+    gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" /usr/local/steemd-full/bin/steemd /tmp/core.$SAVED_PID >> /tmp/stacktrace
+    STACKTRACE=`cat /tmp/stacktrace`
+    echo NOTIFYALERT! steemdsync stacktrace from coredump: $STACKTRACE
+  fi
   RUN_SV_PID=`pgrep -f /etc/service/steemd`
   kill -9 $RUN_SV_PID
 fi

contrib/startpaassteemd.sh

@@ -1,10 +1,14 @@
 #!/bin/bash
-export HOME="/var/lib/steemd"
+
+VERSION=`cat /etc/steemdversion`
 
 STEEMD="/usr/local/steemd-full/bin/steemd"
 
 chown -R steemd:steemd $HOME
 
+# clean out data dir since it may be semi-persistent block storage on the ec2 with stale data
+rm -rf $HOME/*
+
 # seed nodes come from doc/seednodes.txt which is
 # installed by docker into /etc/steemd/seednodes.txt
 SEED_NODES="$(cat /etc/steemd/seednodes.txt | awk -F' ' '{print $1}')"
@@ -38,29 +42,41 @@ mv /etc/nginx/nginx.conf /etc/nginx/nginx.original.conf
 cp /etc/nginx/steemd.nginx.conf /etc/nginx/nginx.conf
 
 # get blockchain state from an S3 bucket
-# if this url is not provieded then we might as well exit
-S3_DOWNLOAD_BUCKET=steemit-$NODE_ENV-blockchainstate
-echo steemd: beginning download and decompress of s3://$S3_DOWNLOAD_BUCKET/blockchain-$VERSION-latest.tar.bz2
-if [[ ! "$SYNC_TO_S3" ]]; then
+echo steemd: beginning download and decompress of s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2
+if [[ "$USE_RAMDISK" ]]; then
   mkdir -p /mnt/ramdisk
-  mount -t ramfs -o size=43008m ramfs /mnt/ramdisk
-  s3cmd get s3://$S3_DOWNLOAD_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x --wildcards 'blockchain/block*' -C /mnt/ramdisk 'blockchain/shared*'
-  ln -s blockchain/block_log /mnt/ramdisk/blockchain/block_log
-  ln -s blockchain/block_log.index /mnt/ramdisk/blockchain/block_log.index
+  mount -t ramfs -o size=${RAMDISK_SIZE_IN_MB:-51200}m ramfs /mnt/ramdisk
   ARGS+=" --shared-file-dir=/mnt/ramdisk/blockchain"
+  s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x --wildcards 'blockchain/block*' -C /mnt/ramdisk 'blockchain/shared*'
   chown -R steemd:steemd /mnt/ramdisk/blockchain
-else
-  s3cmd get s3://$S3_DOWNLOAD_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x
-  touch /tmp/issyncnode
-  chown www-data:www-data /tmp/issyncnode
+else  
+  s3cmd get s3://$S3_BUCKET/blockchain-$VERSION-latest.tar.bz2 - | pbzip2 -m2000dc | tar x
 fi
 if [[ $? -ne 0 ]]; then
-  echo error: unable to pull blockchain state from S3 - exitting
-  exit 1
+  if [[ ! "$SYNC_TO_S3" ]]; then
+    echo notifyalert steemd: unable to pull blockchain state from S3 - exiting
+    exit 1
+  else
+    echo notifysteemdsync steemdsync: shared memory file for $VERSION not found, creating a new one by replaying the blockchain
+    mkdir blockchain
+    aws s3 cp s3://$S3_BUCKET/block_log-latest blockchain/block_log
+    if [[ $? -ne 0 ]]; then
+      echo notifysteemdsync steemdsync: unable to pull latest block_log from S3, will sync from scratch.
+    else
+      ARGS+=" --replay-blockchain --force-validate"
+    fi
+    touch /tmp/isnewsync
+  fi
+fi
+
+cd $HOME
+
+if [[ "$SYNC_TO_S3" ]]; then
+  touch /tmp/issyncnode
+  chown www-data:www-data /tmp/issyncnode
 fi
 
-# change owner of downloaded blockchainstate to steemd user
-chown -R steemd:steemd /var/lib/steemd/*
+chown -R steemd:steemd $HOME/*
 
 # start multiple read-only instances based on the number of cores
 # attach to the local interface since a proxy will be used to loadbalance
@@ -92,7 +108,6 @@ if [[ "$USE_MULTICORE_READONLY" ]]; then
         $STEEMD \
           --rpc-endpoint=127.0.0.1:$PORT_NUM \
           --data-dir=$HOME \
-          --shared-file-dir=/mnt/ramdisk/blockchain \
           --read-forward-rpc=127.0.0.1:8091 \
           --read-only \
           2>&1 &
@@ -127,6 +142,8 @@ else
             $ARGS \
             $STEEMD_EXTRA_OPTS \
             2>&1&
+    SAVED_PID=`pgrep -f p2p-endpoint`
+    echo $SAVED_PID >> /tmp/steemdpid
     mkdir -p /etc/service/steemd
     if [[ ! "$SYNC_TO_S3" ]]; then
       cp /usr/local/bin/paas-sv-run.sh /etc/service/steemd/run

contrib/steemd.nginx.conf

@@ -10,39 +10,28 @@ events {
 http {
 
 	sendfile on;
-	#tcp_nopush on;
-	keepalive_timeout 15;
+	keepalive_timeout 0;
 	server_tokens off;
 
 	real_ip_header X-Forwarded-For;
 	set_real_ip_from 172.31.0.0/16;
 
-	# server_names_hash_bucket_size 64;
-	# server_name_in_redirect off;
-
 	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
-        # rate limiting disabled for SBDS
-        #limit_conn_zone $binary_remote_addr zone=addr:100m;
-        #limit_req_zone $binary_remote_addr zone=public:100m rate=20r/s;
-
-        #limit_conn addr 32;
-        #limit_req zone=public burst=30;
-        #limit_conn_log_level error;
-        #limit_req_log_level error;
-
-    log_format main '$remote_addr - $remote_user [$time_local]  '
+	log_format main '$remote_addr - $remote_user [$time_local]  '
                     '"$request" $status $body_bytes_sent '
                     '|"$http_referer"| "$http_user_agent"';
 
 	access_log /dev/stdout main;
 	error_log  /dev/stdout info;
 
-    log_not_found off;
+	log_not_found off;
 
 	gzip on;
 
+	proxy_ignore_client_abort on;
+
 	include /etc/nginx/conf.d/*.conf;
 	include /etc/nginx/sites-enabled/*;
 }