Merge pull request #3987 from mobilecoinfoundation/nick/merge-6.0.1

.internal-ci/docker/Dockerfile.node_hw

@@ -66,5 +66,4 @@ EXPOSE 8443
 # Volumes
 VOLUME /config
 VOLUME /sealed
-VOLUME /var/run/aesmd
 VOLUME /ledger

.internal-ci/docker/entrypoints/node_hw.sh

@@ -18,10 +18,10 @@
 # MC_CLIENT_RESPONDER_ID - fully qualified name:port that fronts the client port
 #   example client1.test.mobilecoin.com:443
 # MC_MSG_SIGNER_KEY - private key for signing messages
-
-# Optional Vars consensus-service
 # MC_TX_SOURCE_URL - http url to retrieve archive (s3) blocks for node
 #   example https://s3-eu-central-1.amazonaws.com/mobilecoin.chain/node1.test.mobilecoin.com/
+
+# Optional Vars consensus-service
 # MC_PEER_LISTEN_URI
 #   default insecure-mcp://0.0.0.0:8443/
 #   tls example mcp://0.0.0.0:8443/?tls-chain=cert.pem&tls-key=key.pem
@@ -44,7 +44,6 @@
 # Required Vars ledger-distribution
 # MC_DEST - s3 path for publish ledger
 #   example s3://mobilecoin.chain/node1.test.mobilecoin.com?region=eu-central-1
-
 # AWS_ACCESS_KEY_ID - standard AWS vars
 # AWS_SECRET_ACCESS_KEY - standard AWS vars
 # AWS_REGION - standard AWS vars
@@ -66,7 +65,7 @@ set -e
 is_set()
 {
     var_name="${1}"
-    if [ -z "${!var_name}" ]
+    if [[ -z "${!var_name}" ]]
     then
         echo "${var_name} is not set."
         exit 1
@@ -107,6 +106,7 @@ then
     is_set AWS_ACCESS_KEY_ID
     is_set AWS_SECRET_ACCESS_KEY
     is_set AWS_REGION
+    is_set MC_TX_SOURCE_URL
 
     # Enable filebeat if provided with ElasticSearch target vars.
     if [[ -n "${ES_HOST}" ]]
@@ -127,51 +127,65 @@ then
     # Ledger
     echo "Bootstrapping ledger database"
 
-    # Optional Vars
+    # Required Var
     # MC_TX_SOURCE_URL - http source to retrieve block data.
 
     # Default vars
     export MC_LEDGER_PATH=${MC_LEDGER_PATH:-"/ledger"}
-    export MC_STATE_FILE=${MC_STATE_FILE:-"/ledger/.distribution-state"}
     export ORIGIN_LEDGER_PATH=${ORIGIN_LEDGER_PATH:-"/var/lib/mobilecoin/origin_data/data.mdb"}
 
+    # if the ledger exists lets make sure its up to date
     if [[ -f "${MC_LEDGER_PATH}/data.mdb" ]]
     then
         echo "Existing database found at ${MC_LEDGER_PATH}/data.mdb"
         echo "Migrating ledger to latest version"
         /usr/bin/mc-ledger-migration --ledger-db "${MC_LEDGER_PATH}"
+    fi
+
+    echo "Will attempt to restore or generate ledger from origin block or seed."
+    echo " - Check for origin block at ${MC_TX_SOURCE_URL:?}"
+    echo " - Check for origin ledger at ${ORIGIN_LEDGER_PATH}"
+    echo " - Check for seeds in INITIAL_KEYS_SEED variable"
+
+    # shellcheck disable=SC2310 # if archive_curl fails we want to do other logic not just exit
+    if archive_curl "${MC_TX_SOURCE_URL}"
+    then
+        echo "Remote archive ledger found - restore or update ledger with ledger-from-archive"
+        echo "  Note: RUST_LOG=warn so we don't get 1m+ lines of logs"
+        echo "  Please be patient"
+
+        RUST_LOG=warn /usr/bin/ledger-from-archive --ledger-db "${MC_LEDGER_PATH}"
+
+        # We just want to warm up the ledger storage.
+        # Wait here looping over ledger-from-archive until we are ready to start the node.
+        while [[ -n "${MC_LEDGER_FROM_ARCHIVE_ONLY}" ]]
+        do
+            echo "We are in ledger warm up mode - sleeping 5m before we run ledger-from-archive again."
+            echo "Stop the container and unset MC_LEDGER_FROM_ARCHIVE_ONLY to start the node."
+            sleep 300
+            RUST_LOG=info /usr/bin/ledger-from-archive --ledger-db "${MC_LEDGER_PATH}"
+        done
+
+    elif [[ -f "${ORIGIN_LEDGER_PATH}" ]]
+    then
+        echo "Found origin ledger at ${ORIGIN_LEDGER_PATH}"
+        cp "${ORIGIN_LEDGER_PATH}" "${MC_LEDGER_PATH}"
+
+    elif [[ -n "${INITIAL_KEYS_SEED}" ]]
+    then
+        echo "INITIAL_KEYS_SEED found - populating origin data"
+        export INITIALIZE_LEDGER="true"
+
+        /usr/local/bin/generate_origin_data.sh
+
+        cp /tmp/sample_data/ledger/data.mdb "${MC_LEDGER_PATH}"
+
     else
-        # Try to find origin block from s3 archive - preserve existing data, testnet/mainnet
-        if archive_curl "${MC_TX_SOURCE_URL}"
-        then
-            echo "Remote archive ledger found - restore with ledger-from-archive"
-            echo "  Note: RUST_LOG=warn so we don't get 1m+ lines of logs"
-            echo "  Please be patient"
-
-            RUST_LOG=warn /usr/bin/ledger-from-archive --ledger-db "${MC_LEDGER_PATH}"
-
-        # Copy ledger from embedded origin block
-        elif [[ -f "${ORIGIN_LEDGER_PATH}" ]]
-        then
-            echo "Found origin ledger at ${ORIGIN_LEDGER_PATH}"
-            cp "${ORIGIN_LEDGER_PATH}" "${MC_LEDGER_PATH}"
-
-        # Look for wallet keys seed - development and CD deploys
-        elif [[ -n "${INITIAL_KEYS_SEED}" ]]
-        then
-            echo "INITIAL_KEYS_SEED found - populating origin data"
-            export INITIALIZE_LEDGER="true"
-
-            /usr/local/bin/generate_origin_data.sh
-
-            cp /tmp/sample_data/ledger/data.mdb "${MC_LEDGER_PATH}"
-
-        else
-            # We ain't found nothin, bail out!
-            echo "INITIAL_KEYS_SEED not set, no remote ledger and cannot find origin ledger file"
-            exit 1
-        fi
+        # We ain't found nothin, bail out!
+        echo "no remote ledger, cannot find origin ledger file and INITIAL_KEYS_SEED not set"
+        exit 1
     fi
+
 fi
 
 # Run with docker command - probably /usr/bin/supervisord

.internal-ci/docker/support/node_hw/bin/wrapper-ledger-distribution.sh

@@ -11,18 +11,13 @@ set -e
 is_set()
 {
     var_name="${1}"
-    if [ -z "${!var_name}" ]
+    if [[ -z "${!var_name}" ]]
     then
         echo "${var_name} is not set."
         exit 1
     fi
 }
 
-archive_curl()
-{
-    /usr/bin/curl -IfsSL --retry 3 "${1}00/00/00/00/00/00/00/0000000000000000.pb" -o /dev/null
-}
-
 is_set MC_DEST
 is_set AWS_ACCESS_KEY_ID
 is_set AWS_SECRET_ACCESS_KEY
@@ -35,35 +30,6 @@ is_set MC_BRANCH
 
 # Default vars
 export MC_LEDGER_PATH=${MC_LEDGER_PATH:-"/ledger"}
-export MC_STATE_FILE=${MC_STATE_FILE:-"${MC_LEDGER_PATH}/.distribution-state"}
 export MC_SENTRY_DSN=${LEDGER_DISTRIBUTION_SENTRY_DSN}
 
-if [[ -f "${MC_STATE_FILE}" ]]
-then
-    # Check for valid state file
-    echo "mc.app:wrapper-ledger-distribution - State file found MC_START_FROM=last"
-    echo "mc.app:wrapper-ledger-distribution - Check for valid next_block"
-
-    next_block=$(jq -r .next_block "${MC_STATE_FILE}")
-    if [[ "${next_block}" -le 0 ]]
-    then
-        echo "mc.app:wrapper-ledger-distribution - Invalid next_block <= 0"
-        exit 1
-    fi
-
-    export MC_START_FROM=last
-else
-    echo "mc.app:wrapper-ledger-distribution - no state file found."
-    echo "mc.app:wrapper-ledger-distribution - checking for an existing block 0 in s3"
-
-    if archive_curl "${MC_TX_SOURCE_URL}"
-    then
-        echo "mc.app:wrapper-ledger-distribution - block 0 found in s3 MC_START_FROM=next"
-        export MC_START_FROM=next
-    else
-        echo "mc.app:wrapper-ledger-distribution - no s3 archive found MC_START_FROM=zero"
-        export MC_START_FROM=zero
-    fi
-fi
-
 /usr/bin/ledger-distribution

.internal-ci/helm/consensus-node/templates/node-configmap.yaml

@@ -9,3 +9,6 @@ data:
   MC_BLOCK_VERSION: {{ .Values.node.config.blockVersion | squote }}
   MC_CLIENT_RESPONDER_ID: "{{ .Values.node.config.clientHostname }}:443"
   MC_PEER_RESPONDER_ID: "{{ .Values.node.config.peerHostname }}:443"
+  {{- if .Values.node.config.ledgerFromArchiveOnly }}
+  MC_LEDGER_FROM_ARCHIVE_ONLY: "true"
+  {{- end }}

.internal-ci/helm/consensus-node/templates/node-deployment.yaml

@@ -88,7 +88,7 @@ spec:
             command:
             - '/usr/local/bin/grpc_health_probe'
             - '-addr=:8443'
-          failureThreshold: 120
+          failureThreshold: {{ .Values.node.startupProbe.failureThreshold }}
           periodSeconds: 30
           initialDelaySeconds: 30
         envFrom:

.internal-ci/helm/consensus-node/values.yaml

@@ -46,6 +46,10 @@ node:
 
   initContainers: []
 
+  startupProbe:
+    # wait 2 hours on startup
+    failureThreshold: 240
+
   persistence:
     enabled: true
     spec:
@@ -60,6 +64,9 @@ node:
     clientHostname: ''
     peerHostname: ''
     blockVersion: '4'
+    # When true we don't start the consensus or ledger-distribution services,
+    # just run ledger-from-archive and wait.
+    ledgerFromArchiveOnly: false
 
   ingress:
     enabled: true

.internal-ci/helm/fog-ingest/templates/fog-ingest-statefulset.yaml

@@ -227,8 +227,6 @@ spec:
       - name: supervisor-conf
         projected:
           sources:
-          - configMap:
-              name: {{ include "fog-ingest.fullname" . }}-supervisord-sgx
           - configMap:
               name: {{ include "fog-ingest.fullname" . }}-supervisord-daemon
           - configMap:

.internal-ci/helm/fog-ledger/templates/fog-ledger-fogshardrangegenerator.yaml

@@ -228,8 +228,6 @@ spec:
             - name: supervisor-conf
               projected:
                 sources:
-                - configMap:
-                    name: {{ include "fog-ledger.fullname" $ }}-supervisord-sgx
                 - configMap:
                     name: {{ include "fog-ledger.fullname" $ }}-supervisord-daemon
                 - configMap:
@@ -429,8 +427,6 @@ spec:
           - name: supervisor-conf
             projected:
               sources:
-              - configMap:
-                  name: {{ include "fog-ledger.fullname" $ }}-supervisord-sgx
               - configMap:
                   name: {{ include "fog-ledger.fullname" $ }}-supervisord-daemon
               - configMap:

.internal-ci/helm/fog-test-client/templates/fog-test-client-deployment.yaml

@@ -122,40 +122,42 @@ spec:
             port: 9090
           initialDelaySeconds: 20
           periodSeconds: 20
+      {{- if eq .Values.jaegerTracing.enabled true }}
       - name: jaeger-agent
         image: jaegertracing/jaeger-agent:latest
         imagePullPolicy: IfNotPresent
         ports:
-          - containerPort: 5775
-            name: zk-compact-trft
-            protocol: UDP
-          - containerPort: 5778
-            name: config-rest
-            protocol: TCP
-          - containerPort: 6831
-            name: jg-compact-trft
-            protocol: UDP
-          - containerPort: 6832
-            name: jg-binary-trft
-            protocol: UDP
-          - containerPort: 14271
-            name: admin-http
-            protocol: TCP
+        - containerPort: 5775
+          name: zk-compact-trft
+          protocol: UDP
+        - containerPort: 5778
+          name: config-rest
+          protocol: TCP
+        - containerPort: 6831
+          name: jg-compact-trft
+          protocol: UDP
+        - containerPort: 6832
+          name: jg-binary-trft
+          protocol: UDP
+        - containerPort: 14271
+          name: admin-http
+          protocol: TCP
         env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                apiVersion: v1
-                fieldPath: metadata.name
-          - name: HOST_IP
-            valueFrom:
-              fieldRef:
-                apiVersion: v1
-                fieldPath: status.hostIP
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        - name: HOST_IP
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: status.hostIP
         args:
-          - --reporter.grpc.host-port={{ .Values.jaegerTracing.collector }}
-          - --reporter.type=grpc
-          - --agent.tags=cluster=undefined,container.name=fog-test-client,deployment.name={{ include "chart.fullname" . }},host.ip=${HOST_IP:},pod.name=${POD_NAME:},pod.namespace={{ .Release.Namespace }}
+        - --reporter.grpc.host-port=dns:///jaeger-collector:14250
+        - --reporter.type=grpc
+        - --agent.tags=cluster=undefined,container.name=node,deployment.name={{ include "consensusNode.fullname" . }},host.ip=${HOST_IP:},pod.name=${POD_NAME:},pod.namespace={{ .Release.Namespace }}
+      {{- end }}
       nodeSelector:
         {{- toYaml .Values.fogTestClient.nodeSelector | nindent 8 }}
       affinity:

.internal-ci/helm/fog-view/templates/fog-view-fogshardrangegenerator.yaml

@@ -215,8 +215,6 @@ spec:
                 sources:
                 - configMap:
                     name: {{ include "fog-view.fullname" $ }}-supervisord-fog-view-router
-                - configMap:
-                    name: {{ include "fog-view.fullname" $ }}-supervisord-sgx
                 - configMap:
                     name: {{ include "fog-view.fullname" $ }}-supervisord-daemon
                 - configMap:
@@ -430,8 +428,6 @@ spec:
           - name: supervisor-conf
             projected:
               sources:
-              - configMap:
-                  name: {{ include "fog-view.fullname" $ }}-supervisord-sgx
               - configMap:
                   name: {{ include "fog-view.fullname" $ }}-supervisord-daemon
               - configMap: