feat(ci): create disk image after a successful full sync test (#3986)

* Create disk image after a successful full sync test * Extract full sync height and name zebrad cached state with it * Read 500 lines to extract sync height * Restrict log query to just the container output and fix regex syntax for ubuntu * Explicitly search logs in descending time Co-authored-by: Gustavo Valverde <[email protected]>
ZcashFoundation · Apr 6, 2022 · 1a4f8f6 · 1a4f8f6
1 parent b0304c5
commit 1a4f8f6
Showing 1 changed file with 48 additions and 16 deletions.
diff --git a/.github/workflows/test-full-sync.yml b/.github/workflows/test-full-sync.yml
@@ -142,6 +142,9 @@ jobs:
         with:
           short-length: 7
 
+      - name: Downcase network name for disks
+        run: |
+          echo LOWER_NET_NAME="${{ github.event.inputs.network || env.NETWORK }}" | awk '{print tolower($0)}' >> $GITHUB_ENV
       # Setup gcloud CLI
       - name: Authenticate to Google Cloud
         id: auth
@@ -153,7 +156,6 @@ jobs:
 
       # Check if our destination compute instance exists and delete it
       - name: Delete existing instance with same SHA
-        id: delete-old-instance
         run: |
           INSTANCE=$(gcloud compute instances list --filter=full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --format='value(NAME)')
           if [ -z "${INSTANCE}" ]; then
@@ -164,7 +166,6 @@ jobs:
 
       # Creates Compute Engine virtual machine instance w/ disks
       - name: Create GCP compute instance
-        id: create-instance
         run: |
           gcloud compute instances create-with-container "full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" \
           --boot-disk-size 100GB \
@@ -180,41 +181,72 @@ jobs:
           --tags zebrad \
           --zone "${{ env.ZONE }}"
 
-      # TODO: this approach is very mesy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
+      # TODO: this approach is very messy, but getting the just created container name is very error prone and GCP doesn't have a workaround for this without requiring a TTY
       # This TODO relates to the following issues:
       # https://github.com/actions/runner/issues/241
       # https://www.googlecloudcommunity.com/gc/Infrastructure-Compute-Storage/SSH-into-Compute-Container-not-easily-possible/td-p/170915
       - name: Get container name from logs
-        id: get-container-name
-        if: ${{ steps.create-instance.outcome == 'success' }}
         run: |
           INSTANCE_ID=$(gcloud compute instances describe full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} --zone ${{ env.ZONE }} --format='value(id)')
-          echo "Using instance: $INSTANCE_ID"
+          echo "INSTANCE_ID=$INSTANCE_ID" >> $GITHUB_ENV
+
+          CONTAINER_NAME=""
           while [[ ${CONTAINER_NAME} != *"full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}"* ]]; do
               CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
               echo "Using container: ${CONTAINER_NAME} from instance: ${INSTANCE_ID}"
               sleep 10
           done
-          CONTAINER_NAME=$(gcloud logging read 'log_name=projects/${{ env.PROJECT_ID }}/logs/cos_system AND jsonPayload.MESSAGE:full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}' --format='value(jsonPayload.MESSAGE)' --limit=1 | grep -o '...-full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}-....' | tr -d "'.")
-          echo "::set-output name=zebra_container::$CONTAINER_NAME"
-          sleep 60
 
-      - name: Full sync mainnet
-        id: full-sync-mainnet
-        if: ${{ steps.create-instance.outcome == 'success' }}
+          echo "CONTAINER_NAME=$CONTAINER_NAME" >> $GITHUB_ENV
+
+      - name: Get state version from logs
+        run: |
+          STATE_VERSION=""
+
+          while [[ ${STATE_VERSION} == "" ]]; do
+              STATE_VERSION=$(gcloud logging read --format='value(jsonPayload.MESSAGE)' '(resource.labels.instance_id="${{ env.INSTANCE_ID }}" AND jsonPayload.message=~".+Opened Zebra state cache.+v[0-9]+.+")' | grep -oE "v[0-9]+" || [[ $? == 1 ]] )
+              echo "STATE_VERSION: $STATE_VERSION"
+              sleep 10
+          done
+
+          echo "STATE_VERSION=$STATE_VERSION" >> $GITHUB_ENV
+
+      - name: Full sync
+        id: full-sync
         run: |
           gcloud compute ssh \
           full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
           --zone ${{ env.ZONE }} \
           --quiet \
           --ssh-flag="-o ServerAliveInterval=5" \
-          --command="docker logs --follow ${{ env.ZEBRA_CONTAINER }}"
-        env:
-          ZEBRA_CONTAINER: ${{ steps.get-container-name.outputs.zebra_container }}
+          --command="docker logs --follow ${{ env.CONTAINER_NAME }}"
+
+      - name: Get sync height from logs
+        run: |
+          SYNC_HEIGHT=""
+
+          while [[ ${SYNC_HEIGHT} == "" ]]; do
+              SYNC_HEIGHT=$(gcloud logging read --format='value(jsonPayload.MESSAGE)' --order="desc" --limit=1 '(resource.labels.instance_id="${{ env.INSTANCE_ID }}" AND jsonPayload.message=~".+finished initial sync to chain tip.+Height\([0-9]+\).+")' | grep -oE 'Height\([0-9]+\)' | grep -oE '[0-9]+' || [[ $? == 1 ]] )
+              echo "SYNC_HEIGHT: $SYNC_HEIGHT"
+              sleep 10
+          done
+
+          echo "SYNC_HEIGHT=$SYNC_HEIGHT" >> $GITHUB_ENV
+
+      # Create image from disk
+      # Force the image creation as the disk is still attached, even though is not being used by the container
+      - name: Create image from state disk
+        run: |
+          gcloud compute images create zebrad-cache-${{ env.GITHUB_SHA_SHORT }}-${{ env.STATE_VERSION }}-${{ env.NETWORK }}-${{ env.SYNC_HEIGHT }} \
+          --force \
+          --source-disk=full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }} \
+          --source-disk-zone=${{ env.ZONE }} \
+          --storage-location=us \
+          --description="Created from head branch ${{ env.GITHUB_HEAD_REF_SLUG_URL }} targeting ${{ env.GITHUB_BASE_REF_SLUG }} from PR ${{ env.GITHUB_REF_SLUG_URL }} with commit ${{ env.GITHUB_EVENT_PULL_REQUEST_HEAD_SHA }}"
 
       - name: Delete test instance
         # Do not delete the instance if the sync timeouts in GitHub
-        if: ${{ steps.full-sync-mainnet.outcome == 'success' || steps.full-sync-mainnet.outcome == 'failure' }}
+        if: ${{ steps.full-sync.outcome == 'success' || steps.full-sync.outcome == 'failure' }}
         continue-on-error: true
         run: |
           gcloud compute instances delete "full-sync-${{ env.GITHUB_REF_SLUG_URL }}-${{ env.GITHUB_SHA_SHORT }}" --zone "${{ env.ZONE }}" --delete-disks all --quiet