Migrate unit tests from Cloud Build to GitHub Workflows

.github/workflows/unittest.yaml

@@ -0,0 +1,107 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "OSV Unittests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+  merge_group:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+
+permissions: read-all
+
+jobs:
+  unittest:
+    name: Unittest the top-level code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: Install pipenv
+        run: |
+          python -m pip install --upgrade pipenv
+      - name: Run top-level unittests
+        run: |
+          bash -ex run_tests.sh
+  unittest_worker:
+    name: Unittest the worker code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: Install pipenv
+        run: |
+          python -m pip install --upgrade pipenv
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+        with:
+          install_components: 'beta,cloud-datastore-emulator'
+      - name: Run worker unittests
+        run: |
+          cd docker/worker
+          bash -ex run_tests.sh
+  unittest_importer:
+    name: Unittest the importer code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: Install pipenv
+        run: |
+          python -m pip install --upgrade pipenv
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+        with:
+          install_components: 'beta,cloud-datastore-emulator'
+      - name: Run importer unittests
+        run: |
+          cd docker/importer
+          bash -ex run_tests.sh
+  unittest_vulnfeeds:
+    name: Unittest the vulnfeeds code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.18.0'
+      - name: Run vulnfeeds unittests
+        run: |
+          cd vulnfeeds
+          bash -ex run_tests.sh

cloudbuild.yaml

@@ -16,17 +16,6 @@ steps:
 - name: 'gcr.io/cloud-builders/git'
   dir: docker/worker
   args: ['submodule', 'update', '--init']
-- name: 'gcr.io/oss-vdb/ci'
-  args: ['bash', '-ex', 'run_tests.sh']
-- name: 'gcr.io/oss-vdb/ci'
-  dir: docker/worker
-  args: ['bash', '-ex', 'run_tests.sh']
-- name: 'gcr.io/oss-vdb/ci'
-  dir: docker/importer
-  args: ['bash', '-ex', 'run_tests.sh']
-- name: 'gcr.io/oss-vdb/ci'
-  dir: vulnfeeds
-  args: ['bash', '-ex', 'run_tests.sh']
 - name: 'gcr.io/cloud-builders/gcloud'
   entrypoint: 'bash'
   args: [ '-c', "gcloud secrets versions access latest --secret=integration-test-account --format='get(payload.data)' | tr '_-' '/+' | base64 -d > /workspace/service_account.json" ]

docker/importer/importer_test.py

@@ -501,7 +501,7 @@ def test_import_override(self, mock_publish: mock.MagicMock,
 
 if __name__ == '__main__':
   os.system('pkill -f datastore')
-  ds_emulator = tests.start_datastore_emulator()
+  ds_emulator = tests.start_datastore_emulator(timeout=60)
   try:
     with ndb.Client().context() as context:
       context.set_memcache_policy(False)

docker/worker/worker_test.py

@@ -1283,7 +1283,7 @@ def test_update_bucket_cve(self):
 
 if __name__ == '__main__':
   os.system('pkill -f datastore')
-  ds_emulator = tests.start_datastore_emulator()
+  ds_emulator = tests.start_datastore_emulator(timeout=60)
   try:
     ndb_client = ndb.Client()
     with ndb_client.context() as context:

osv/tests.py

@@ -90,7 +90,7 @@ def commit(self, author_name, author_email, message='Changes'):
                              [self._repo.head.peel().oid])
 
 
-def start_datastore_emulator():
+def start_datastore_emulator(timeout=_EMULATOR_TIMEOUT):
   """Starts Datastore emulator."""
   os.environ['DATASTORE_EMULATOR_HOST'] = 'localhost:' + str(
       _DATASTORE_EMULATOR_PORT)
@@ -110,14 +110,12 @@ def start_datastore_emulator():
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
 
-  _wait_for_emulator_ready(proc, 'datastore', _DATASTORE_READY_INDICATOR)
+  _wait_for_emulator_ready(proc, 'datastore', _DATASTORE_READY_INDICATOR,
+                           timeout)
   return proc
 
 
-def _wait_for_emulator_ready(proc,
-                             emulator,
-                             indicator,
-                             timeout=_EMULATOR_TIMEOUT):
+def _wait_for_emulator_ready(proc, emulator, indicator, timeout):
   """Waits for emulator to be ready."""
 
   def _read_thread(proc, ready_event):