revert mapper changes

README.md

@@ -6,7 +6,7 @@ Ethereum Foundation maintains two public Hive instances to check for consensus,
 blockchain compatibility:
 
 - eth1 consensus, rpc, p2p tests are on <https://hivetests.ethdevops.io>
-- 'merge' testing runs on <https://hivetests2.ethdevops.io>
+- Engine API integration tests are on <https://hivetests2.ethdevops.io>
 
 **To read more about hive, please check [the documentation][doc].**
 
@@ -19,9 +19,9 @@ for validating Ethereum client implementations.
 - go-ethereum:
   - Genesis chain config couldn't handle present but empty settings: [#2790](https://github.com/ethereum/go-ethereum/pull/2790)
   - Data race between remote block import and local block mining: [#2793](https://github.com/ethereum/go-ethereum/pull/2793)
-  - Downloader didn't penalize incompatible forks hashly enough: [#2801](https://github.com/ethereum/go-ethereum/pull/2801)
+  - Downloader didn't penalize incompatible forks harshly enough: [#2801](https://github.com/ethereum/go-ethereum/pull/2801)
 - Nethermind:
-  - Bug in p2p whith bonding nodes algorithm found by Hive: [#1894](https://github.com/NethermindEth/nethermind/pull/1894)
+  - Bug in p2p with bonding nodes algorithm found by Hive: [#1894](https://github.com/NethermindEth/nethermind/pull/1894)
   - Difference in return value for 'r' parameter in getTransactionByHash: [#2372](https://github.com/NethermindEth/nethermind/issues/2372)
   - CREATE/CREATE2 behavior when account already has max nonce [#3698](https://github.com/NethermindEth/nethermind/pull/3698)
   - Blake2 performance issue with non-vectorized code [#3837](https://github.com/NethermindEth/nethermind/pull/3837)
@@ -36,7 +36,7 @@ We follow the [Collective Code Construction Contract (C4)][c4], code contributio
 as expanded and explained in [The ZeroMQ Process][zmq-process]. The core idea being that
 any patch that successfully solves an issue (bug/feature) and doesn't break any existing
 code/contracts must be optimistically merged by maintainers. Followup patches may be used
-to for additional polishes – and patches may even be outright reverted if they turn out to
+for additional polishes – and patches may even be outright reverted if they turn out to
 have a negative impact – but no change must be rejected based on personal values.
 
 ### License

clients/besu/besu.sh

@@ -99,14 +99,15 @@ if [ -d /blocks ]; then
     IMPORTFLAGS="$IMPORTFLAGS $blocks"
 fi
 
-
-# Configure mining.
-if [ "$HIVE_MINER" != "" ]; then
-    FLAGS="$FLAGS --miner-enabled --miner-coinbase=$HIVE_MINER"
-    # For clique mining, besu uses the node key as the block signing key.
-    if [ "$HIVE_CLIQUE_PRIVATEKEY" != "" ]; then
-        echo "Importing clique signing key as node key..."
-        echo "$HIVE_CLIQUE_PRIVATEKEY" > /opt/besu/key
+if [ "$HIVE_BOOTNODE" != "" ]; then
+    # Configure mining.
+    if [ "$HIVE_MINER" != "" ]; then
+        FLAGS="$FLAGS --miner-enabled --miner-coinbase=$HIVE_MINER"
+        # For clique mining, besu uses the node key as the block signing key.
+        if [ "$HIVE_CLIQUE_PRIVATEKEY" != "" ]; then
+            echo "Importing clique signing key as node key..."
+            echo "$HIVE_CLIQUE_PRIVATEKEY" > /opt/besu/key
+        fi
     fi
 fi
 if [ "$HIVE_MINER_EXTRA" != "" ]; then

clients/besu/mapper.jq

@@ -44,6 +44,7 @@ def to_int:
     "berlinBlock": env.HIVE_FORK_BERLIN|to_int,
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
     "parisBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
+    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
   }|remove_empty
 }

clients/erigon/mapper.jq

@@ -51,5 +51,6 @@ def to_bool:
     "berlinBlock": env.HIVE_FORK_BERLIN|to_int,
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
+    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
   }|remove_empty
 }

clients/ethereumjs/Dockerfile

@@ -1,5 +1,5 @@
 FROM node:16-alpine as build
-RUN apk update && apk add --no-cache bash git jq curl && rm -rf /var/cache/apk/*
+RUN apk update && apk add --no-cache bash git jq curl python3 gcc make g++ && rm -rf /var/cache/apk/*
 RUN git clone --depth 1 https://github.com/ethereumjs/ethereumjs-monorepo.git && \
     (cd ethereumjs-monorepo && npm i)
 

clients/ethereumjs/ethereumjs.sh

@@ -49,7 +49,7 @@
 set -e
 
 ethereumjs="node /ethereumjs-monorepo/packages/client/dist/bin/cli.js"
-FLAGS="--gethGenesis ./genesis.json --rpc --rpcEngine --saveReceipts --rpcEnginePort 8551 --ws --loglevel debug"
+FLAGS="--gethGenesis ./genesis.json --rpc --rpcEngine --saveReceipts --rpcEnginePort 8551 --ws --logLevel debug"
 
 
 # Configure the chain.

clients/ethereumjs/mapper.jq

@@ -55,5 +55,6 @@ def to_bool:
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
     "mergeForkBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
+    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
   }|remove_empty
 }

clients/go-ethereum/mapper.jq

@@ -54,5 +54,6 @@ def to_bool:
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
     "mergeForkBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
+    "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
   }|remove_empty
 }

clients/lodestar-bn/Dockerfile

@@ -0,0 +1,10 @@
+ARG branch=next
+
+FROM chainsafe/lodestar:$branch
+
+ADD lodestar_bn.sh /lodestar_bn.sh
+RUN chmod +x /lodestar_bn.sh
+
+RUN node /usr/app/node_modules/.bin/lodestar --version | grep "Version:.*" | grep -o "v.*" > /version.txt
+
+ENTRYPOINT ["/lodestar_bn.sh"]

clients/lodestar-bn/hive.yaml

@@ -0,0 +1,4 @@
+roles:
+  - beacon
+build_targets:
+  - mainnet

clients/lodestar-bn/lodestar_bn.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+
+# Immediately abort the script on any error encountered
+set -e
+
+if [ ! -f "/hive/input/genesis.ssz" ]; then
+    if [ -z "$HIVE_ETH2_ETH1_RPC_ADDRS" ]; then
+      echo "genesis.ssz file is missing, and no Eth1 RPC addr was provided for building genesis from scratch."
+      # TODO: alternative to start from weak-subjectivity-state
+      exit 1
+    fi
+fi
+
+mkdir -p /data/testnet_setup
+
+
+# empty bootnodes file, required for custom testnet setup, use CLI arg instead to configure it.
+
+echo "${HIVE_ETH2_CONFIG_DEPOSIT_CONTRACT_ADDRESS:-0x1111111111111111111111111111111111111111}" > /data/testnet_setup/deposit_contract.txt
+
+mkdir -p /data/beacon
+mkdir -p /data/network
+
+case "$HIVE_LOGLEVEL" in
+    0|1) LOG=error ;;
+    2)   LOG=warn  ;;
+    3)   LOG=info  ;;
+    4)   LOG=debug ;;
+    5)   LOG=silly ;;
+esac
+LOG=debug
+
+echo "bootnodes: ${HIVE_ETH2_BOOTNODE_ENRS}"
+
+CONTAINER_IP=`hostname -i | awk '{print $1;}'`
+echo Container IP: $CONTAINER_IP
+bootnodes_option=$([[ "$HIVE_ETH2_BOOTNODE_ENRS" == "" ]] && echo "" || echo "--bootnodes ${HIVE_ETH2_BOOTNODE_ENRS//,/ }")
+metrics_option=$([[ "$HIVE_ETH2_METRICS_PORT" == "" ]] && echo "" || echo "--metrics --metrics.address=$CONTAINER_IP --metrics.port=$HIVE_ETH2_METRICS_PORT")
+
+echo "bootnodes option : ${bootnodes_option}"
+
+echo -n "0x7365637265747365637265747365637265747365637265747365637265747365" > /jwtsecret
+
+echo Starting Lodestar Beacon Node
+
+node /usr/app/node_modules/.bin/lodestar \
+    beacon \
+    --logLevel="$LOG" \
+    --dataDir=/data/beacon \
+    --port="${HIVE_ETH2_P2P_TCP_PORT:-9000}" \
+    --discoveryPort="${HIVE_ETH2_P2P_UDP_PORT:-9000}" \
+    --paramsFile=/hive/input/config.yaml \
+    --genesisStateFile=/hive/input/genesis.ssz \
+    --rest \
+    --rest.namespace="*" \
+    --rest.address=0.0.0.0 \
+    --rest.port="${HIVE_ETH2_BN_API_PORT:-4000}" \
+    --eth1 \
+    --execution.urls="$HIVE_ETH2_ETH1_ENGINE_RPC_ADDRS" \
+    --eth1.depositContractDeployBlock=${HIVE_ETH2_DEPOSIT_DEPLOY_BLOCK_NUMBER:-0} \
+    --jwt-secret=/jwtsecret \
+    $metrics_option \
+    $bootnodes_option \
+    --enr.ip="${CONTAINER_IP}" \
+    --enr.tcp="${HIVE_ETH2_P2P_TCP_PORT:-9000}" \
+    --enr.udp="${HIVE_ETH2_P2P_UDP_PORT:-9000}" \
+    --network.connectToDiscv5Bootnodes=true \
+    --discv5 \
+    --subscribeAllSubnets=true \
+    --targetPeers="${HIVE_ETH2_P2P_TARGET_PEERS:-10}"

clients/lodestar-vc/Dockerfile

@@ -0,0 +1,10 @@
+ARG branch=next
+
+FROM chainsafe/lodestar:$branch
+
+ADD lodestar_vc.sh /lodestar_vc.sh
+RUN chmod +x /lodestar_vc.sh
+
+RUN node /usr/app/node_modules/.bin/lodestar --version | grep "Version:.*" | grep -o "v.*" > /version.txt
+
+ENTRYPOINT ["/lodestar_vc.sh"]

clients/lodestar-vc/hive.yaml

@@ -0,0 +1,2 @@
+roles:
+  - validator

clients/lodestar-vc/lodestar_vc.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Immediately abort the script on any error encountered
+set -e
+
+mkdir -p /data/testnet_setup
+mkdir -p /data/vc
+mkdir -p /data/validators
+mkdir -p /data/secrets
+
+cp /hive/input/config.yaml /data/testnet_setup
+
+echo "${HIVE_ETH2_CONFIG_DEPOSIT_CONTRACT_ADDRESS:-0x1111111111111111111111111111111111111111}" > /data/testnet_setup/deposit_contract.txt
+echo "${HIVE_ETH2_DEPOSIT_DEPLOY_BLOCK_NUMBER:-0}" > /data/testnet_setup/deploy_block.txt
+
+for keystore_path in /hive/input/keystores/*
+do
+  pubkey=$(basename "$keystore_path")
+  mkdir "/data/validators/$pubkey"
+  cp "/hive/input/keystores/$pubkey/keystore.json" "/data/validators/$pubkey/voting-keystore.json"
+  cp "/hive/input/secrets/$pubkey" "/data/secrets/$pubkey"
+done
+
+metrics_option=$([[ "$HIVE_ETH2_METRICS_PORT" == "" ]] && echo "" || echo "--metrics --metrics.address=$CONTAINER_IP --metrics.port=$HIVE_ETH2_METRICS_PORT")
+
+LOG=info
+case "$HIVE_LOGLEVEL" in
+    0|1) LOG=error ;;
+    2)   LOG=warn  ;;
+    3)   LOG=info  ;;
+    4)   LOG=debug ;;
+    5)   LOG=trace ;;
+esac
+
+echo Starting Lodestar Validator Client
+
+node /usr/app/node_modules/.bin/lodestar \
+    validator \
+    --logLevel="$LOG" \
+    --dataDir=/data/vc \
+    --paramsFile=/hive/input/config.yaml \
+    --keystoresDir="/data/validators" \
+    --secretsDir="/data/secrets" \
+    $metrics_option \
+    --beaconNodes="http://$HIVE_ETH2_BN_API_IP:$HIVE_ETH2_BN_API_PORT"
+

clients/nethermind/mapper.jq

@@ -136,6 +136,12 @@ def clique_engine:
     # Merge
     "MergeForkIdTransition": env.HIVE_MERGE_BLOCK_ID|to_hex,
 
+    # Shanghai
+    "eip3651TransitionTimestamp": env.HIVE_SHANGHAI_TIMESTAMP|to_hex,
+    "eip3855TransitionTimestamp": env.HIVE_SHANGHAI_TIMESTAMP|to_hex,
+    "eip3860TransitionTimestamp": env.HIVE_SHANGHAI_TIMESTAMP|to_hex,
+    "eip4895TransitionTimestamp": env.HIVE_SHANGHAI_TIMESTAMP|to_hex,
+
     # Other chain parameters
     "networkID": env.HIVE_NETWORK_ID|to_hex,
     "chainID": env.HIVE_CHAIN_ID|to_hex,

clients/nethermind/mkconfig.jq

@@ -32,7 +32,9 @@ def merge_config:
       "Merge": {
         "Enabled": true,
         "TerminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY,
-      }
+        "TerminalBlockHash": env.HIVE_TERMINAL_BLOCK_HASH,
+        "TerminalBlockNumber": env.HIVE_TERMINAL_BLOCK_NUMBER,
+      } | remove_empty
     }
   else
     {}

clients/nimbus-bn/Dockerfile

@@ -11,10 +11,10 @@ ENV NPROC=2
 
 RUN git clone --depth 1 --branch unstable https://github.com/status-im/nimbus-eth2.git \
  && cd nimbus-eth2 \
- && make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:SECONDS_PER_SLOT=6" V=1 update
+ && make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update
 
 RUN cd nimbus-eth2 && \
-    make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:SECONDS_PER_SLOT=6" nimbus_beacon_node && \
+    make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" nimbus_beacon_node && \
     mv build/nimbus_beacon_node /usr/bin/
 
 # --------------------------------- #
@@ -34,9 +34,10 @@ RUN usr/bin/nimbus_beacon_node --version > /version.txt
 
 ADD nimbus_bn.sh /nimbus_bn.sh
 RUN chmod +x /nimbus_bn.sh
+ADD nimbus_version.sh /nimbus_version.sh
+RUN chmod +x /nimbus_version.sh
 
-# TODO: output accurate client version
-RUN usr/bin/nimbus_beacon_node --version > /version.txt
+RUN /nimbus_version.sh > /version.txt
 
 EXPOSE 9000 9000/udp 4000 4000/udp
 

clients/nimbus-bn/nimbus_bn.sh

@@ -19,19 +19,15 @@ cp /hive/input/config.yaml /data/testnet_setup
 # znrt encodes the values of these items between double quotes (""), which is against the spec:
 # https://github.com/ethereum/consensus-specs/blob/v1.1.10/configs/mainnet.yaml
 # Nimbus is the only client that complains about this.
-sed -i 's/TERMINAL_TOTAL_DIFFICULTY: "\([[:digit:]]\+\)"/TERMINAL_TOTAL_DIFFICULTY: \1/' /data/testnet_setup/config.yaml
-sed -i 's/TERMINAL_BLOCK_HASH: "\(0x[[:xdigit:]]\+\)"/TERMINAL_BLOCK_HASH: \1/' /data/testnet_setup/config.yaml
-sed -i 's/GENESIS_FORK_VERSION: "\(0x[[:xdigit:]]\+\)"/GENESIS_FORK_VERSION: \1/' /data/testnet_setup/config.yaml
-sed -i 's/ALTAIR_FORK_VERSION: "\(0x[[:xdigit:]]\+\)"/ALTAIR_FORK_VERSION: \1/' /data/testnet_setup/config.yaml
-sed -i 's/BELLATRIX_FORK_VERSION: "\(0x[[:xdigit:]]\+\)"/BELLATRIX_FORK_VERSION: \1/' /data/testnet_setup/config.yaml
-sed -i 's/CAPELLA_FORK_VERSION: "\(0x[[:xdigit:]]\+\)"/CAPELLA_FORK_VERSION: \1/' /data/testnet_setup/config.yaml
-sed -i 's/SHARDING_FORK_VERSION: "\(0x[[:xdigit:]]\+\)"/SHARDING_FORK_VERSION: \1/' /data/testnet_setup/config.yaml
+sed -i 's/"//g' /data/testnet_setup/config.yaml
+sed -i '/TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH/d' /data/testnet_setup/config.yaml
 echo Using config.yaml
 cat /data/testnet_setup/config.yaml
 echo ""
 
 echo "${HIVE_ETH2_CONFIG_DEPOSIT_CONTRACT_ADDRESS:-0x1111111111111111111111111111111111111111}" > /data/testnet_setup/deposit_contract.txt
-echo "${HIVE_ETH2_DEPOSIT_DEPLOY_BLOCK_NUMBER:-0}" > /data/testnet_setup/deploy_block.txt
+echo "${HIVE_ETH2_DEPOSIT_DEPLOY_BLOCK_NUMBER:-0}" > /data/testnet_setup/deposit_contract_block.txt
+echo "${HIVE_ETH2_DEPOSIT_DEPLOY_BLOCK_HASH:-0}" > /data/testnet_setup/deposit_contract_block_hash.txt
 
 mkdir -p /data/beacon
 chmod 0700 /data/beacon

clients/nimbus-bn/nimbus_version.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Immediately abort the script on any error encountered
+set -e
+
+/usr/bin/nimbus_beacon_node --version | grep -Po ".*\bv?\d+\.\S+" | sed "s/ /_/g" | tr '\n' '/'

clients/nimbus-vc/Dockerfile

@@ -11,10 +11,10 @@ ENV NPROC=2
 
 RUN git clone --depth 1 --branch unstable https://github.com/status-im/nimbus-eth2.git \
  && cd nimbus-eth2 \
- && make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:SECONDS_PER_SLOT=6" V=1 update
+ && make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update
 
 RUN cd nimbus-eth2 && \
-    make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:SECONDS_PER_SLOT=6" nimbus_validator_client && \
+    make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" nimbus_validator_client && \
     mv build/nimbus_validator_client /usr/bin/
 
 # --------------------------------- #
@@ -30,10 +30,13 @@ RUN apt-get update \
 
 COPY --from=build /usr/bin/nimbus_validator_client /usr/bin/nimbus_validator_client
 
-RUN usr/bin/nimbus_validator_client --version > /version.txt
-
 # Add the startup script.
 ADD nimbus_vc.sh /nimbus_vc.sh
 RUN chmod +x /nimbus_vc.sh
 
+ADD nimbus_version.sh /nimbus_version.sh
+RUN chmod +x /nimbus_version.sh
+
+RUN /nimbus_version.sh > /version.txt
+
 ENTRYPOINT ["/nimbus_vc.sh"]

clients/nimbus-vc/nimbus_version.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Immediately abort the script on any error encountered
+set -e
+
+/usr/bin/nimbus_validator_client --version | grep -Po ".*\bv?\d+\.\S+" | sed "s/ /_/g" | tr '\n' '/'

clients/reth/Dockerfile

@@ -0,0 +1,26 @@
+ARG branch=main
+FROM paradigmxyz/reth:$branch
+
+# Install script tools.
+RUN apt-get update -y
+RUN apt-get install -y bash curl jq
+
+# Add genesis mapper script.
+ADD genesis.json /genesis.json
+ADD mapper.jq /mapper.jq
+
+# Add the startup script.
+ADD reth.sh /reth.sh
+RUN chmod +x /reth.sh
+
+# Add the enode URL retriever script.
+ADD enode.sh /hive-bin/enode.sh
+RUN chmod +x /hive-bin/enode.sh
+
+# Create version.txt
+RUN /usr/local/bin/reth --version | sed -e 's/reth \(.*\)/\1/' > /version.txt
+
+# Export the usual networking ports to allow outside access to the node.
+EXPOSE 8545 8546 30303 30303/udp
+
+ENTRYPOINT ["/reth.sh"]