.github/workflows/publish.yaml

@@ -8,19 +8,29 @@ jobs:
   build:
     name: Build distribution package
     runs-on: ubuntu-latest
-
     steps:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
+      - name: Fetch tags
+        if: github.ref_type != 'tag'
+        run: git fetch --prune --unshallow --tags
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install pypa/build
-        run: python -m pip install --user build
+        run: python -m pip install build twine
+      - name: Add '.devN' to version for non-tag builds
+        if: github.ref_type != 'tag'
+        run:
+          sed -i
+          "/^APP_VERSION = /s/'$/.dev$(git describe --tags | cut -d- -f2)'/"
+          yamllint/__init__.py
       - name: Build a binary wheel and a source tarball
         run: python -m build
+      - name: Twine check the distribution packages
+        run: python -Im twine check --strict dist/yamllint-*
       - name: Store the distribution packages
         uses: actions/upload-artifact@v4
         with:
@@ -29,6 +39,7 @@ jobs:
 
   publish-to-testpypi:
     name: Publish distribution package to TestPyPI
+    if: github.ref_name == github.event.repository.default_branch
     needs: build
     runs-on: ubuntu-latest
     environment:
@@ -46,11 +57,10 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           repository-url: https://test.pypi.org/legacy/
-          skip-existing: true
 
   publish-to-pypi:
     name: Publish distribution package to PyPI
-    if: startsWith(github.ref, 'refs/tags/')  # only for tags
+    if: github.ref_type == 'tag'
     needs: build
     runs-on: ubuntu-latest
     environment:

.github/workflows/tests.yaml

@@ -57,6 +57,13 @@ jobs:
       - run: pip install .
       # https://github.com/AndreMiras/coveralls-python-action/issues/18
       - run: echo -e "[run]\nrelative_files = True" > .coveragerc
-      - run: coverage run -m unittest discover
+      - run: >-
+          python
+          -X warn_default_encoding
+          -W error::EncodingWarning
+          -m coverage
+          run
+          -m unittest
+          discover
       - name: Coveralls
         uses: AndreMiras/coveralls-python-action@develop

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+1.37.0 (2025-03-23)
+-------------------
+
+- Automatically detect Unicode character encoding of files
+- Publish pushes to master branch to TestPyPI
+
 1.36.2 (2025-03-17)
 -------------------
 

docs/character_encoding.rst

@@ -0,0 +1,36 @@
+Character Encoding
+==================
+
+When yamllint reads a file (whether its a configuration file or a file to
+lint), yamllint will try to automatically detect that file’s character
+encoding. In order for the automatic detection to work properly, files must
+follow these two rules (see `this section of the YAML specification for details
+<https://yaml.org/spec/1.2.2/#52-character-encodings>`_):
+
+* The file must be encoded in UTF-8, UTF-16 or UTF-32.
+
+* The file must begin with either a byte order mark or an ASCII character.
+
+Override character encoding
+---------------------------
+
+Previous versions of yamllint did not try to autodetect the character encoding
+of files. Previous versions of yamllint assumed that files used the current
+locale’s character encoding. This meant that older versions of yamllint would
+sometimes correctly decode files that didn’t follow those two rules. For the
+sake of backwards compatibility, the current version of yamllint allows you to
+disable automatic character encoding detection by setting the
+``YAMLLINT_FILE_ENCODING`` environment variable. If you set the
+``YAMLLINT_FILE_ENCODING`` environment variable to the `the name of one of
+Python’s standard character encodings
+<https://docs.python.org/3/library/codecs.html#standard-encodings>`_, then
+yamllint will use that character encoding instead of trying to autodetect the
+character encoding.
+
+The ``YAMLLINT_FILE_ENCODING`` environment variable should only be used as a
+stopgap solution. If you need to use ``YAMLLINT_FILE_ENCODING``, then you
+should really update your YAML files so that their character encoding can
+automatically be detected, or else you may run into compatibility problems.
+Future versions of yamllint may remove support for the
+``YAMLLINT_FILE_ENCODING`` environment variable, and other YAML processors may
+misinterpret your YAML files.

docs/configuration.rst

@@ -228,6 +228,10 @@ or:
 
 .. note:: However, this is mutually exclusive with the ``ignore`` key.
 
+.. note:: Files on the ``ignore-from-file`` list should use either UTF-8,
+   UTF-16 or UTF-32. See :doc:`Character Encoding <character_encoding>` for
+   details and workarounds.
+
 If you need to know the exact list of files that yamllint would process,
 without really linting them, you can use ``--list-files``:
 

docs/index.rst

@@ -27,3 +27,4 @@ Table of contents
    development
    text_editors
    integration
+   character_encoding

tests/__init__.py

@@ -1,4 +1,5 @@
 # Copyright (C) 2016 Adrien Vergé
+# Copyright (C) 2025 Jason Yundt
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,5 +15,24 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import locale
+import os
 
 locale.setlocale(locale.LC_ALL, 'C')
+env_vars_that_could_interfere_with_tests = (
+    'YAMLLINT_FILE_ENCODING',
+    # yamllint uses these environment variables to find a config file.
+    'YAMLLINT_CONFIG_FILE',
+    'XDG_CONFIG_HOME',
+    # These variables are used to determine where the user’s home
+    # directory is. See
+    # https://docs.python.org/3/library/os.path.html#os.path.expanduser
+    'HOME',
+    'USERPROFILE',
+    'HOMEPATH',
+    'HOMEDRIVE'
+)
+for name in env_vars_that_could_interfere_with_tests:
+    try:
+        del os.environ[name]
+    except KeyError:
+        pass

tests/common.py

@@ -1,4 +1,5 @@
 # Copyright (C) 2016 Adrien Vergé
+# Copyright (C) 2023–2025 Jason Yundt
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,20 +14,173 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import codecs
 import contextlib
 from io import StringIO
 import os
 import shutil
 import sys
 import tempfile
 import unittest
+import warnings
+from codecs import CodecInfo
 
 import yaml
 
 from yamllint import linter
 from yamllint.config import YamlLintConfig
 
 
+# Encoding related stuff:
+UTF_CODECS = (
+    'utf_32_be',
+    'utf_32_be_sig',
+    'utf_32_le',
+    'utf_32_le_sig',
+    'utf_16_be',
+    'utf_16_be_sig',
+    'utf_16_le',
+    'utf_16_le_sig',
+    'utf_8',
+    'utf_8_sig'
+)
+
+
+def encode_utf_32_be_sig(obj):
+    return (
+        codecs.BOM_UTF32_BE + codecs.encode(obj, 'utf_32_be', 'strict'),
+        len(obj)
+    )
+
+
+def encode_utf_32_le_sig(obj):
+    return (
+        codecs.BOM_UTF32_LE + codecs.encode(obj, 'utf_32_le', 'strict'),
+        len(obj)
+    )
+
+
+def encode_utf_16_be_sig(obj):
+    return (
+        codecs.BOM_UTF16_BE + codecs.encode(obj, 'utf_16_be', 'strict'),
+        len(obj)
+    )
+
+
+def encode_utf_16_le_sig(obj):
+    return (
+        codecs.BOM_UTF16_LE + codecs.encode(obj, 'utf_16_le', 'strict'),
+        len(obj)
+    )
+
+
+test_codec_infos = {
+    'utf_32_be_sig':
+    CodecInfo(encode_utf_32_be_sig, codecs.getdecoder('utf_32')),
+    'utf_32_le_sig':
+    CodecInfo(encode_utf_32_le_sig, codecs.getdecoder('utf_32')),
+    'utf_16_be_sig':
+    CodecInfo(encode_utf_16_be_sig, codecs.getdecoder('utf_16')),
+    'utf_16_le_sig':
+    CodecInfo(encode_utf_16_le_sig, codecs.getdecoder('utf_16')),
+}
+
+
+def register_test_codecs():
+    codecs.register(test_codec_infos.get)
+
+
+def unregister_test_codecs():
+    if sys.version_info >= (3, 10, 0):
+        codecs.unregister(test_codec_infos.get)
+    else:
+        warnings.warn(
+            "This version of Python doesn’t allow us to unregister codecs.",
+            stacklevel=1
+        )
+
+
+def is_test_codec(codec):
+    return codec in test_codec_infos.keys()
+
+
+def test_codec_built_in_equivalent(test_codec):
+    return_value = test_codec
+    for suffix in ('_sig', '_be', '_le'):
+        return_value = return_value.replace(suffix, '')
+    return return_value
+
+
+def uses_bom(codec):
+    for suffix in ('_32', '_16', '_sig'):
+        if codec.endswith(suffix):
+            return True
+    return False
+
+
+def encoding_detectable(string, codec):
+    """
+    Returns True if encoding can be detected after string is encoded
+
+    Encoding detection only works if you’re using a BOM or the first character
+    is ASCII. See yamllint.decoder.auto_decode()’s docstring.
+    """
+    return uses_bom(codec) or (len(string) > 0 and string[0].isascii())
+
+
+# Workspace related stuff:
+class Blob:
+    def __init__(self, text, encoding):
+        self.text = text
+        self.encoding = encoding
+
+
+def build_temp_workspace(files):
+    tempdir = tempfile.mkdtemp(prefix='yamllint-tests-')
+
+    for path, content in files.items():
+        path = os.fsencode(os.path.join(tempdir, path))
+        if not os.path.exists(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+
+        if isinstance(content, list):
+            os.mkdir(path)
+        elif isinstance(content, str) and content.startswith('symlink://'):
+            os.symlink(content[10:], path)
+        else:
+            if isinstance(content, Blob):
+                content = content.text.encode(content.encoding)
+            elif isinstance(content, str):
+                content = content.encode('utf_8')
+            with open(path, 'wb') as f:
+                f.write(content)
+
+    return tempdir
+
+
+@contextlib.contextmanager
+def temp_workspace(files):
+    """Provide a temporary workspace that is automatically cleaned up."""
+    backup_wd = os.getcwd()
+    wd = build_temp_workspace(files)
+
+    try:
+        os.chdir(wd)
+        yield
+    finally:
+        os.chdir(backup_wd)
+        shutil.rmtree(wd)
+
+
+def temp_workspace_with_files_in_many_codecs(path_template, text):
+    workspace = {}
+    for codec in UTF_CODECS:
+        if encoding_detectable(text, codec):
+            workspace[path_template.format(codec)] = Blob(text, codec)
+    return workspace
+
+
+# Miscellaneous stuff:
 class RuleTestCase(unittest.TestCase):
     def build_fake_config(self, conf):
         if conf is None:
@@ -81,37 +235,3 @@ def __exit__(self, *exc_info):
     @property
     def returncode(self):
         return self._raises_ctx.exception.code
-
-
-def build_temp_workspace(files):
-    tempdir = tempfile.mkdtemp(prefix='yamllint-tests-')
-
-    for path, content in files.items():
-        path = os.path.join(tempdir, path).encode('utf-8')
-        if not os.path.exists(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-
-        if isinstance(content, list):
-            os.mkdir(path)
-        elif isinstance(content, str) and content.startswith('symlink://'):
-            os.symlink(content[10:], path)
-        else:
-            mode = 'wb' if isinstance(content, bytes) else 'w'
-            with open(path, mode) as f:
-                f.write(content)
-
-    return tempdir
-
-
-@contextlib.contextmanager
-def temp_workspace(files):
-    """Provide a temporary workspace that is automatically cleaned up."""
-    backup_wd = os.getcwd()
-    wd = build_temp_workspace(files)
-
-    try:
-        os.chdir(wd)
-        yield
-    finally:
-        os.chdir(backup_wd)
-        shutil.rmtree(wd)