Trigger AWS Lambda tests on label (#2538)

Our AWS Lambda test suite currently doesn't run properly on external contributor PRs because it needs access to repo secrets, which it currently doesn't have. This PR adds a label to grant access to the secrets, which is invalidated upon any new code changes.

.github/workflows/scripts/trigger_tests_on_label.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+LABEL = "Trigger: tests using secrets"
+
+
+def _has_write(repo_id: int, username: str, *, token: str) -> bool:
+    req = Request(
+        f"https://api.github.com/repositories/{repo_id}/collaborators/{username}/permission",
+        headers={"Authorization": f"token {token}"},
+    )
+    contents = json.load(urlopen(req, timeout=10))
+
+    return contents["permission"] in {"admin", "write"}
+
+
+def _remove_label(repo_id: int, pr: int, label: str, *, token: str) -> None:
+    quoted_label = quote(label)
+    req = Request(
+        f"https://api.github.com/repositories/{repo_id}/issues/{pr}/labels/{quoted_label}",
+        method="DELETE",
+        headers={"Authorization": f"token {token}"},
+    )
+    urlopen(req)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--repo-id", type=int, required=True)
+    parser.add_argument("--pr", type=int, required=True)
+    parser.add_argument("--event", required=True)
+    parser.add_argument("--username", required=True)
+    parser.add_argument("--label-names", type=json.loads, required=True)
+    args = parser.parse_args()
+
+    token = os.environ["GITHUB_TOKEN"]
+
+    write_permission = _has_write(args.repo_id, args.username, token=token)
+
+    if (
+        not write_permission
+        # `reopened` is included here due to close => push => reopen
+        and args.event in {"synchronize", "reopened"}
+        and LABEL in args.label_names
+    ):
+        print(f"Invalidating label [{LABEL}] due to code change...")
+        _remove_label(args.repo_id, args.pr, LABEL, token=token)
+        args.label_names.remove(LABEL)
+
+    if write_permission or LABEL in args.label_names:
+        print("Permissions passed!")
+        print(f"- has write permission: {write_permission}")
+        print(f"- has [{LABEL}] label: {LABEL in args.label_names}")
+        return 0
+    else:
+        print("Permissions failed!")
+        print(f"- has write permission: {write_permission}")
+        print(f"- has [{LABEL}] label: {LABEL in args.label_names}")
+        print(f"- args.label_names: {args.label_names}")
+        print(
+            f"Please have a collaborator add the [{LABEL}] label once they "
+            f"have reviewed the code to trigger tests."
+        )
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/test-integration-aws_lambda.yml

@@ -4,22 +4,49 @@ on:
     branches:
       - master
       - release/**
-  pull_request:
+  # XXX: We are using `pull_request_target` instead of `pull_request` because we want
+  # this to run on forks with access to the secrets necessary to run the test suite.
+  # Prefer to use `pull_request` when possible.
+  pull_request_target:
+    types: [labeled, opened, reopened, synchronize]
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 permissions:
   contents: read
+  # `write` is needed to remove the `Trigger: tests using secrets` label
+  pull-requests: write
 env:
   SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.SENTRY_PYTHON_TEST_AWS_ACCESS_KEY_ID }}
   SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.SENTRY_PYTHON_TEST_AWS_SECRET_ACCESS_KEY }}
   BUILD_CACHE_KEY: ${{ github.sha }}
   CACHED_BUILD_PATHS: |
     ${{ github.workspace }}/dist-serverless
 jobs:
+  check-permissions:
+    name: permissions check
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          persist-credentials: false
+      - name: permissions
+        run: |
+          python3 -uS .github/workflows/scripts/trigger_tests_on_label.py \
+              --repo-id ${{ github.event.repository.id }} \
+              --pr ${{ github.event.number }} \
+              --event ${{ github.event.action }} \
+              --username "$ARG_USERNAME" \
+              --label-names "$ARG_LABEL_NAMES"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # these can contain special characters
+          ARG_USERNAME: ${{ github.event.pull_request.user.login }}
+          ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }}
   test-pinned:
+    needs: check-permissions
     timeout-minutes: 30
     name: aws_lambda pinned, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -34,6 +61,8 @@ jobs:
         os: [ubuntu-20.04]
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

scripts/split-tox-gh-actions/split-tox-gh-actions.py

@@ -41,6 +41,10 @@
     "aws_lambda",
 ]
 
+FRAMEWORKS_NEEDING_GITHUB_SECRETS = [
+    "aws_lambda",
+]
+
 ENV = Environment(
     loader=FileSystemLoader(TEMPLATE_DIR),
 )
@@ -152,6 +156,7 @@ def render_template(framework, py_versions_pinned, py_versions_latest):
         "needs_aws_credentials": framework in FRAMEWORKS_NEEDING_AWS,
         "needs_clickhouse": framework in FRAMEWORKS_NEEDING_CLICKHOUSE,
         "needs_postgres": framework in FRAMEWORKS_NEEDING_POSTGRES,
+        "needs_github_secrets": framework in FRAMEWORKS_NEEDING_GITHUB_SECRETS,
         "py_versions": {
             # formatted for including in the matrix
             "pinned": [f'"{v}"' for v in py_versions_pinned if v != "2.7"],

scripts/split-tox-gh-actions/templates/base.jinja

@@ -6,7 +6,15 @@ on:
       - master
       - release/**
 
+  {% if needs_github_secrets %}
+  # XXX: We are using `pull_request_target` instead of `pull_request` because we want
+  # this to run on forks with access to the secrets necessary to run the test suite.
+  # Prefer to use `pull_request` when possible.
+  pull_request_target:
+    types: [labeled, opened, reopened, synchronize]
+  {% else %}
   pull_request:
+  {% endif %}
 
 # Cancel in progress workflows on pull_requests.
 # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
@@ -16,6 +24,10 @@ concurrency:
 
 permissions:
   contents: read
+  {% if needs_github_secrets %}
+  # `write` is needed to remove the `Trigger: tests using secrets` label
+  pull-requests: write
+  {% endif %}
 
 env:
 {% if needs_aws_credentials %}
@@ -29,6 +41,10 @@ env:
     {% raw %}${{ github.workspace }}/dist-serverless{% endraw %}
 
 jobs:
+{% if needs_github_secrets %}
+{% include "check_permissions.jinja" %}
+{% endif %}
+
 {% if py_versions.pinned %}
 {% with category="pinned", versions=py_versions.pinned %}
 {% include "test.jinja" %}

scripts/split-tox-gh-actions/templates/check_permissions.jinja

@@ -0,0 +1,25 @@
+  check-permissions:
+    name: permissions check
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+        with:
+          persist-credentials: false
+
+      - name: permissions
+        run: |
+          {% raw %}
+          python3 -uS .github/workflows/scripts/trigger_tests_on_label.py \
+              --repo-id ${{ github.event.repository.id }} \
+              --pr ${{ github.event.number }} \
+              --event ${{ github.event.action }} \
+              --username "$ARG_USERNAME" \
+              --label-names "$ARG_LABEL_NAMES"
+          {% endraw %}
+        env:
+          {% raw %}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # these can contain special characters
+          ARG_USERNAME: ${{ github.event.pull_request.user.login }}
+          ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }}
+          {% endraw %}

scripts/split-tox-gh-actions/templates/test.jinja

@@ -1,4 +1,7 @@
   test-{{ category }}:
+    {% if needs_github_secrets %}
+    needs: check-permissions
+    {% endif %}
     timeout-minutes: 30
     {% if category == "py27" %}
     name: {{ framework }} {{ category }}, python 2.7
@@ -41,6 +44,12 @@
 
     steps:
       - uses: actions/checkout@v4
+      {% if needs_github_secrets %}
+      {% raw %}
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      {% endraw %}
+      {% endif %}
       {% if category != "py27" %}
       - uses: actions/setup-python@v4
         with: