pre-commit and pre-commit-ci

Add pre-commit and pre-commit-ci with the following hooks:

- pycln for cleaning up unused imports.
- isort for sorting imports.
- black for formatting code.

For more info see `docs/developers.rst`.

Also:

- Changed tox config:
  - Added environments for pre-commit.
  - Removed some redundant options.
  - Created a covreport environment for reporting coverage for all
    environments.
  - Make it possible to run a custom command instead of pytest.
  - Make it possible to run an extra command
  - Make it possible to change the mypy command
  - Use extras instead of using requirements files.
- Update setup.py
  - Remove berkleydb and networkx from tests_require and tests extra
    group as they are not always needed for tests.
  - Added separate extra groups for berkleydb and networkx
  - Added a dev extras group with dev tools.
- Update github actions
  - Use tox to run tests.
  - Run flake8 only once.
  - Change docs job to rather be more generic for potential future tox
    environments.

.github/workflows/validate.yaml

@@ -20,29 +20,33 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-latest, macos-latest, windows-latest]
         include:
-          - {
-              python-version: "3.7",
-              os: ubuntu-latest,
-              extensive-tests: true,
-            }
+          - python-version: "3.7"
+            os: ubuntu-latest
+            extensive-tests: true
+          - python-version: "3.10"
+            os: ubuntu-latest
+            TOX_EXTRA_COMMAND: "flake8 --exit-zero rdflib"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-
       - uses: actions/setup-java@v2
         if: ${{ matrix.extensive-tests }}
         with:
           distribution: "temurin"
           java-version: "17"
-
+      - name: Setup env
+        shell: bash
+        run: |
+          MATRIX_PYTHON_VERSION=${{ matrix.python-version }}
+          echo "TOX_PYENV=py${MATRIX_PYTHON_VERSION//./}" >> ${GITHUB_ENV}
       - name: Get pip cache dir
         id: pip-cache
         shell: bash
         run: |
-          python -m ensurepip --upgrade
+          python -m ensurepip
           echo "::set-output name=dir::$(pip cache dir)"
       - name: Cache pip
         uses: actions/cache@v2
@@ -59,13 +63,15 @@ jobs:
           key: ${{ matrix.os }}-xdg-v1-${{ hashFiles('**/with-fuseki.sh') }}
           restore-keys: |
             ${{ matrix.os }}-xdg-v1-
-      - name: Install dependencies
+      - name: Install python dependencies
         shell: bash
+        # Installing tox-gh-actions to get some enhancement to output rendering
+        # in github actions. Eventually we can maybe collapse the tox-envs job
+        # into this one but there are some limitations of tox-gh-actions which
+        # preclude doing that at the moment.
         run: |
-          pip install --default-timeout 60 -r requirements.txt
-          pip install --default-timeout 60 -r requirements.dev.txt
-          python setup.py install
-      - name: Install extra dev dependencies
+          python -m pip install tox tox-gh-actions
+      - name: Install system depdendencies for extensive tests
         if: ${{ matrix.extensive-tests }}
         shell: bash
         run: |
@@ -77,7 +83,6 @@ jobs:
               brew install berkeley-db@4
               export BERKELEYDB_DIR=$(brew --prefix berkeley-db@4)
           fi
-          pip install --default-timeout 60 -r requirements.dev-extra.txt
       - name: Validate
         shell: bash
         run: |
@@ -86,13 +91,21 @@ jobs:
           then
             1>&2 echo "Running with fuseki"
             test_harness+="./with-fuseki.sh"
+            TOX_PYENV_SUFFIX=-extensive
           fi
-          black --config black.toml --check ./rdflib || true
-          flake8 --exit-zero rdflib
-          mypy --show-error-context --show-error-codes
-          "${test_harness[@]}" pytest -ra --cov
-  docs:
+          TOXENV="${{ matrix.TOXENV }}"
+          TOXENV="${TOX_PYENV}${TOX_PYENV_SUFFIX}${TOXENV:+,${TOXENV}}"
+          export TOXENV
+          export TOX_EXTRA_COMMAND="${{ matrix.TOX_EXTRA_COMMAND }}"
+          "${test_harness[@]}" python -m tox
+        env:
+          GH_ACTIONS_KEY: ${{matrix.os}}-${{matrix.python-version}}
+  tox-envs:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        tox-env: ["docs"]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{env.DEFAULT_PYTHON}}
@@ -109,15 +122,15 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
-          key: docs-pip-v1-${{
+          key: tox-${{ matrix.tox-env }}-pip-v1-${{
             hashFiles('**/setup.py', '**/requirements*.txt') }}
           restore-keys: |
-            docs-pip-v1-
+            tox-${{ matrix.tox-env }}-pip-v1-
       - name: Install dependencies
         shell: bash
         run: |
-          python -m pip install tox
-      - name: Build docs
+          python -m pip install tox tox-gh-actions
+      - name: Run ${{ matrix.tox-env }}
         shell: bash
         run: |
-          python -m tox -e docs
+          python -m tox -e ${{ matrix.tox-env }}

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+---
+ci:
+  # https://pre-commit.ci/#configuration
+  autoupdate_schedule: weekly
+  autofix_prs: false
+
+repos:
+  - repo: https://github.com/hadialqattan/pycln
+    rev: v1.1.0
+    hooks:
+      - id: pycln
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        additional_dependencies: [toml]
+  - repo: https://github.com/psf/black
+    # WARNING: rev should be the same as in `black.toml`.
+    rev: 21.9b0
+    hooks:
+      - id: black

black.toml

@@ -1,4 +1,5 @@
 [tool.black]
+# WARNING: This should be the same as in `.pre-commit-config.yaml`.
 required-version = "21.9b0"
 line-length = "88"
 skip-string-normalization = true

docs/developers.rst

@@ -108,6 +108,54 @@ Check types with `mypy <http://mypy-lang.org/>`_:
 
     python -m mypy --show-error-context --show-error-codes rdflib
 
+pre-commit and pre-commit ci
+----------------------------
+
+We have `pre-commit <https://pre-commit.com/>`_ configured with `pycln
+<https://github.com/hadialqattan/pycln>`_ for removing unused imports,
+`isort <https://github.com/PyCQA/isort>`_ for sorting imports, and `black
+<https://github.com/psf/black>`_ for formatting out code.
+
+Some useful commands for using pre-commit:
+
+.. code-block:: bash
+
+    # Install pre-commit.
+    pip install --user --upgrade pre-commit
+
+    # Install pre-commit hooks, this will run pre-commit
+    # every time you make a git commit.
+    pre-commit install
+
+    # Run pre-commit on changed files.
+    pre-commit run
+
+    # Run pre-commit on all files.
+    pre-commit run --all-files
+
+There is also two tox environments for pre-commit:
+
+.. code-block:: bash
+
+    # run pre-commit on changed files.
+    tox -e precommit
+
+    # run pre-commit on all files.
+    tox -e precommitall
+
+
+There is no hard requirement for pull requests to be processed with pre-commit (or the underlying processors), however doing this makes for a less noisy codebase with cleaner history.
+
+We have enabled `https://pre-commit.ci/ <https://pre-commit.ci/>`_ and this can
+be used to automatically fix pull requests by commenting ``pre-commit.ci
+autofix`` on a pull request.
+
+There is also a pre-commit environment for tox:
+
+.. code-block:: bash
+
+    tox -e precommit
+
 Using tox
 ---------------------
 
@@ -116,21 +164,26 @@ makes it easier to run validation on all supported python versions.
 
 .. code-block:: bash
 
-    # install tox
+    # Install tox.
     pip install tox
 
-    # list tox environments that run by default
+    # List the tox environments that run by default.
     tox -e
 
-    # list all tox environments
-    tox -a
-
-    # run default environment for all python versions
+    # Run the default environments.
     tox
 
-    # run a specific environment
+    # List all tox environments, including ones that don't run by default.
+    tox -a
+
+    # Run a specific environment.
     tox -e py37 # default environment with py37
-    tox -e py39-mypy # mypy environment with py39
+    tox -e py39-extra # extra tests with py39
+
+    # Override the test command.
+    # the below command will run `pytest test/test_translate_algebra.py`
+    # instead of the default pytest command.
+    tox -e py37,py39 -- pytest test/test_translate_algebra.py
 
 Writing documentation
 ---------------------

requirements.dev.txt

@@ -8,7 +8,8 @@ mypy
 pytest
 pytest-cov
 pytest-subtests
-sphinx
+sphinx < 5
 sphinxcontrib-apidoc
 myst-parser
 types-setuptools
+pre-commit

setup.cfg

@@ -42,3 +42,6 @@ filterwarnings =
     ignore:Code. pytest_mock_example_attribute_that_shouldnt_exist is not defined in namespace .*:UserWarning
     # The below warning is a consequence of how pytest detects fixtures and how DefinedNamespace behaves when an undefined attribute is being accessed.
     ignore:Code. _pytestfixturefunction is not defined in namespace .*:UserWarning
+
+[isort]
+profile = black

setup.py

@@ -13,9 +13,7 @@
     "importlib-metadata; python_version < '3.8.0'",
 ]
 kwargs["tests_require"] = [
-    "berkeleydb",
     "html5lib",
-    "networkx",
     "pytest",
     "pytest-cov",
     "pytest-subtests",
@@ -24,6 +22,17 @@
     "html": ["html5lib"],
     "tests": kwargs["tests_require"],
     "docs": ["sphinx < 5", "sphinxcontrib-apidoc", "myst-parser"],
+    "berkeleydb": ["berkeleydb"],
+    "networkx": ["networkx"],
+    "dev": [
+        "isort",
+        "black",
+        "mypy",
+        "flake8",
+        "flake8-black",
+        "types-setuptools",
+        "pre-commit",
+    ],
 }
 
 

tox.ini

@@ -1,51 +1,50 @@
 [tox]
 envlist =
-    py37,py38,py39,py310
+    py3{7,8,9,10},docs,covreport,precommit
 
 [testenv]
 setenv =
-    BERKELEYDB_DIR = /usr
-commands =
-    {envpython} -m mypy rdflib --show-error-context --show-error-codes
-    {envpython} setup.py clean --all
-    {envpython} setup.py build
-    {envpython} -m pytest
-deps =
-	-rrequirements.txt
-	-rrequirements.dev.txt
-
-[testenv:cover]
-basepython =
-    python3.7
+    extensive: BERKELEYDB_DIR = /usr
+    COVERAGE_FILE={toxinidir}/.coverage.{envname}
+    MYPY_CACHE_DIR={envdir}/.mypy_cache
+extras =
+    tests
+    dev
+    extensive: berkeleydb
+    extensive: networkx
 commands =
-    {envpython} -m pytest \
-		--cov-report term \
-		--cov-report html \
-		--cov
-
-deps =
-	-rrequirements.txt
-	-rrequirements.dev.txt
+    {env:TOX_EXTRA_COMMAND:}
+    {env:TOX_MYPY_COMMAND:{envpython} -m mypy --show-error-context --show-error-codes}
+    {posargs:{envpython} -m pytest --cov --cov-report=}
 
-[testenv:py3{7,8,9,10}-mypy]
+[testenv:covreport]
+basepython = python3.7
+deps = coverage
+skip_install = true
+parallel_show_output = true
+depends = py3{7,8,9,10}{-extensive,}
+setenv =
+    COVERAGE_FILE=
 commands =
-    {envpython} -m mypy --show-error-context --show-error-codes
-deps =
-	-rrequirements.txt
-	-rrequirements.dev.txt
+    {envpython} -m coverage combine
+    {envpython} -m coverage report
 
 [testenv:docs]
-basepython =
-    python3.7
-deps =
-extras =
-    docs
+extras = docs
 passenv = TERM
 setenv =
     PYTHONHASHSEED = 0
 commands =
     sphinx-build -n -T -b html -d {envtmpdir}/doctrees docs docs/_build/html
 
+[testenv:precommit{,all}]
+skip_install = true
+deps = pre-commit
+passenv = HOMEPATH  # needed on Windows
+commands =
+    precommit: pre-commit run
+    precommitall: pre-commit run --all-files
+
 [pytest]
 # log_cli = true
 # log_cli_level = DEBUG