From 61ac761c6c13a8eadde8770c549d617266dbbb71 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 30 Oct 2022 23:29:00 +0000 Subject: [PATCH 01/23] Add test coverage to confirm that inventory file contents differ per-locale --- tests/test_util_inventory.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index e7960233001..085baf10dd0 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -5,6 +5,7 @@ from io import BytesIO from sphinx.ext.intersphinx import InventoryFile +from sphinx.testing.util import SphinxTestApp inventory_v1 = b'''\ # Sphinx inventory version 1 @@ -83,3 +84,29 @@ def test_read_inventory_v2_not_having_version(): invdata = InventoryFile.load(f, '/util', posixpath.join) assert invdata['py:module']['module1'] == \ ('foo', '', '/util/foo.html#module-module1', 'Long Module desc') + + +def _app_language(tempdir, language): + (tempdir / language).makedirs() + (tempdir / language / 'conf.py').write_text(f'language = "{language}"', encoding='utf8') + (tempdir / language / 'index.rst').write_text('index.rst', encoding='utf8') + assert (tempdir / language).listdir() == ['conf.py', 'index.rst'] + assert (tempdir / language / 'index.rst').exists() + return SphinxTestApp(srcdir=(tempdir / language)) + + +def test_inventory_localization(tempdir): + # Build an app using Estonian (EE) locale + app_et = _app_language(tempdir, "et") + app_et.build() + inventory_et = (app_et.outdir / 'objects.inv').read_bytes() + app_et.cleanup() + + # Build the same app using English (US) locale + app_en = _app_language(tempdir, "en") + app_en.build() + inventory_en = (app_en.outdir / 'objects.inv').read_bytes() + app_en.cleanup() + + # Ensure that the inventory contents differ + assert inventory_et != inventory_en From 25927666ad55af721834fd1b7b0b9c31a03858aa Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 30 Oct 2022 23:30:40 +0000 Subject: [PATCH 02/23] Use lazy-loading for all gettext-translated resource strings --- sphinx/domains/std.py | 2 +- sphinx/locale/__init__.py | 9 ++------- sphinx/util/i18n.py | 2 +- sphinx/writers/text.py | 2 +- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index 898c6f94d5f..b59a7cf80dd 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -245,7 +245,7 @@ def add_target_and_index(self, firstname: str, sig: str, signode: desc_signature else: descr = _('command line option') for option in signode.get('allnames', []): - entry = '; '.join([descr, option]) + entry = '; '.join([str(descr), option]) self.indexnode['entries'].append(('pair', entry, signode['ids'][0], '', None)) def make_old_id(self, prefix: str, optname: str) -> str: diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 05eaf16a89f..0a84db1fff6 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -220,13 +220,8 @@ def setup(app): .. versionadded:: 1.8 """ - def gettext(message: str) -> str: - if not is_translator_registered(catalog, namespace): - # not initialized yet - return _TranslationProxy(_lazy_translate, catalog, namespace, message) # type: ignore[return-value] # noqa: E501 - else: - translator = get_translator(catalog, namespace) - return translator.gettext(message) + def gettext(message: str, *args: Any) -> str: + return _TranslationProxy(_lazy_translate, catalog, namespace, message) # type: ignore[return-value] # NOQA return gettext diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py index 6915d309187..0dbbf8f93a2 100644 --- a/sphinx/util/i18n.py +++ b/sphinx/util/i18n.py @@ -208,7 +208,7 @@ def format_date( language = 'en' result = [] - tokens = date_format_re.split(format) + tokens = date_format_re.split(str(format)) for token in tokens: if token in date_format_mappings: babel_format = date_format_mappings.get(token, '') diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index 8df01012ce8..da09feab81a 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -420,7 +420,7 @@ def do_format() -> None: result.append((indent, res)) for itemindent, item in content: if itemindent == -1: - toformat.append(item) # type: ignore + toformat.append(str(item)) # type: ignore else: do_format() result.append((indent + itemindent, item)) # type: ignore From c1d2473b1b6fef747f7bf5b04c608ed8bac936f5 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 30 Oct 2022 23:32:13 +0000 Subject: [PATCH 03/23] Add test coverage to confirm that inventory contents are identical (regardless of environment locale) when reproducible builds are enabled --- tests/test_util_inventory.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index 085baf10dd0..031fca7d42c 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -110,3 +110,24 @@ def test_inventory_localization(tempdir): # Ensure that the inventory contents differ assert inventory_et != inventory_en + + +def test_inventory_reproducible(tempdir, monkeypatch): + # Configure reproducible builds + # See: https://reproducible-builds.org/docs/source-date-epoch/ + monkeypatch.setenv("SOURCE_DATE_EPOCH", "0") + + # Build an app using Estonian (EE) locale + app_et = _app_language(tempdir, "et") + app_et.build() + inventory_et = (app_et.outdir / 'objects.inv').read_bytes() + app_et.cleanup() + + # Build the same app using English (US) locale + app_en = _app_language(tempdir, "en") + app_en.build() + inventory_en = (app_en.outdir / 'objects.inv').read_bytes() + app_en.cleanup() + + # Ensure that the inventory contents are identical + assert inventory_et == inventory_en From babbb27aeb6dce45069d1f9c9f8a88663c09835e Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 30 Oct 2022 23:38:24 +0000 Subject: [PATCH 04/23] Disable localization of objects.inv (inventory files) when the build is configured for reproducible output --- sphinx/locale/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 0a84db1fff6..13364b21fc3 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -2,7 +2,7 @@ import locale from gettext import NullTranslations, translation -from os import path +from os import getenv, path from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union @@ -111,7 +111,11 @@ def init( # the None entry is the system's default locale path has_translation = True - if language and '_' in language: + if getenv('SOURCE_DATE_EPOCH') is not None: + # Disable localization during reproducible source builds + # See https://reproducible-builds.org/docs/source-date-epoch/ + languages = None + elif language and '_' in language: # for language having country code (like "de_AT") languages: Optional[List[str]] = [language, language.split('_')[0]] elif language: From f17cf4a1bdbc3fde4fe75aacc96aa50b1511bd25 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 30 Oct 2022 23:45:34 +0000 Subject: [PATCH 05/23] Fixup: mypy: relocate type hint --- sphinx/locale/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 13364b21fc3..381c05e97ae 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -114,10 +114,10 @@ def init( if getenv('SOURCE_DATE_EPOCH') is not None: # Disable localization during reproducible source builds # See https://reproducible-builds.org/docs/source-date-epoch/ - languages = None + languages: Optional[List[str]] = None elif language and '_' in language: # for language having country code (like "de_AT") - languages: Optional[List[str]] = [language, language.split('_')[0]] + languages = [language, language.split('_')[0]] elif language: languages = [language] else: From ed704c3dc69a77b79e2db9f68556a282eab862f7 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 30 Oct 2022 23:46:15 +0000 Subject: [PATCH 06/23] Fixup: mypy: remove unused type-ignore comment --- sphinx/writers/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index da09feab81a..c6d8d9e4be7 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -420,7 +420,7 @@ def do_format() -> None: result.append((indent, res)) for itemindent, item in content: if itemindent == -1: - toformat.append(str(item)) # type: ignore + toformat.append(str(item)) else: do_format() result.append((indent + itemindent, item)) # type: ignore From fae1b1ae902fb20909c85d3ab40f194b930b6252 Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 10:54:26 +0000 Subject: [PATCH 07/23] Cleanup / readability: de-duplicate test code for localized inventory creation --- tests/test_util_inventory.py | 49 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index 031fca7d42c..f8c7b73b044 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -86,30 +86,33 @@ def test_read_inventory_v2_not_having_version(): ('foo', '', '/util/foo.html#module-module1', 'Long Module desc') -def _app_language(tempdir, language): - (tempdir / language).makedirs() - (tempdir / language / 'conf.py').write_text(f'language = "{language}"', encoding='utf8') - (tempdir / language / 'index.rst').write_text('index.rst', encoding='utf8') - assert (tempdir / language).listdir() == ['conf.py', 'index.rst'] - assert (tempdir / language / 'index.rst').exists() - return SphinxTestApp(srcdir=(tempdir / language)) +def _write_appconfig(dir, language): + (dir / language).makedirs() + (dir / language / 'conf.py').write_text(f'language = "{language}"', encoding='utf8') + (dir / language / 'index.rst').write_text('index.rst', encoding='utf8') + assert (dir / language).listdir() == ['conf.py', 'index.rst'] + assert (dir / language / 'index.rst').exists() + return (dir / language) + + +def _build_inventory(srcdir): + app = SphinxTestApp(srcdir=srcdir) + app.build() + app.cleanup() + return (app.outdir / 'objects.inv') def test_inventory_localization(tempdir): # Build an app using Estonian (EE) locale - app_et = _app_language(tempdir, "et") - app_et.build() - inventory_et = (app_et.outdir / 'objects.inv').read_bytes() - app_et.cleanup() + srcdir_et = _write_appconfig(tempdir, "et") + inventory_et = _build_inventory(srcdir_et) # Build the same app using English (US) locale - app_en = _app_language(tempdir, "en") - app_en.build() - inventory_en = (app_en.outdir / 'objects.inv').read_bytes() - app_en.cleanup() + srcdir_en = _write_appconfig(tempdir, "en") + inventory_en = _build_inventory(srcdir_en) # Ensure that the inventory contents differ - assert inventory_et != inventory_en + assert inventory_et.read_bytes() != inventory_en.read_bytes() def test_inventory_reproducible(tempdir, monkeypatch): @@ -118,16 +121,12 @@ def test_inventory_reproducible(tempdir, monkeypatch): monkeypatch.setenv("SOURCE_DATE_EPOCH", "0") # Build an app using Estonian (EE) locale - app_et = _app_language(tempdir, "et") - app_et.build() - inventory_et = (app_et.outdir / 'objects.inv').read_bytes() - app_et.cleanup() + srcdir_et = _write_appconfig(tempdir, "et") + inventory_et = _build_inventory(srcdir_et) # Build the same app using English (US) locale - app_en = _app_language(tempdir, "en") - app_en.build() - inventory_en = (app_en.outdir / 'objects.inv').read_bytes() - app_en.cleanup() + srcdir_en = _write_appconfig(tempdir, "en") + inventory_en = _build_inventory(srcdir_en) # Ensure that the inventory contents are identical - assert inventory_et == inventory_en + assert inventory_et.read_bytes() == inventory_en.read_bytes() From 1cb30df8765df7078b5740324151c48217f2e12c Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 11:07:29 +0000 Subject: [PATCH 08/23] Make use of monkeypatch context management more explicit --- tests/test_util_inventory.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index f8c7b73b044..9e3b0930e9f 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -116,17 +116,18 @@ def test_inventory_localization(tempdir): def test_inventory_reproducible(tempdir, monkeypatch): - # Configure reproducible builds - # See: https://reproducible-builds.org/docs/source-date-epoch/ - monkeypatch.setenv("SOURCE_DATE_EPOCH", "0") - - # Build an app using Estonian (EE) locale - srcdir_et = _write_appconfig(tempdir, "et") - inventory_et = _build_inventory(srcdir_et) - - # Build the same app using English (US) locale - srcdir_en = _write_appconfig(tempdir, "en") - inventory_en = _build_inventory(srcdir_en) + with monkeypatch.context() as m: + # Configure reproducible builds + # See: https://reproducible-builds.org/docs/source-date-epoch/ + m.setenv("SOURCE_DATE_EPOCH", "0") + + # Build an app using Estonian (EE) locale + srcdir_et = _write_appconfig(tempdir, "et") + inventory_et = _build_inventory(srcdir_et) + + # Build the same app using English (US) locale + srcdir_en = _write_appconfig(tempdir, "en") + inventory_en = _build_inventory(srcdir_en) # Ensure that the inventory contents are identical assert inventory_et.read_bytes() == inventory_en.read_bytes() From b300d8bbd8a1034cd3d64e3883b3e2d13d61966c Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 11:09:55 +0000 Subject: [PATCH 09/23] Permit custom directory prefix for test app during reproducible build localization test cases --- tests/test_util_inventory.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index 9e3b0930e9f..f565c78aa96 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -86,13 +86,14 @@ def test_read_inventory_v2_not_having_version(): ('foo', '', '/util/foo.html#module-module1', 'Long Module desc') -def _write_appconfig(dir, language): - (dir / language).makedirs() - (dir / language / 'conf.py').write_text(f'language = "{language}"', encoding='utf8') - (dir / language / 'index.rst').write_text('index.rst', encoding='utf8') - assert (dir / language).listdir() == ['conf.py', 'index.rst'] - assert (dir / language / 'index.rst').exists() - return (dir / language) +def _write_appconfig(dir, language, prefix=None): + prefix = prefix or language + (dir / prefix).makedirs() + (dir / prefix / 'conf.py').write_text(f'language = "{language}"', encoding='utf8') + (dir / prefix / 'index.rst').write_text('index.rst', encoding='utf8') + assert (dir / prefix).listdir() == ['conf.py', 'index.rst'] + assert (dir / prefix / 'index.rst').exists() + return (dir / prefix) def _build_inventory(srcdir): @@ -123,11 +124,11 @@ def test_inventory_reproducible(tempdir, monkeypatch): # Build an app using Estonian (EE) locale srcdir_et = _write_appconfig(tempdir, "et") - inventory_et = _build_inventory(srcdir_et) + reproducible_inventory_et = _build_inventory(srcdir_et) # Build the same app using English (US) locale srcdir_en = _write_appconfig(tempdir, "en") - inventory_en = _build_inventory(srcdir_en) + reproducible_inventory_en = _build_inventory(srcdir_en) - # Ensure that the inventory contents are identical - assert inventory_et.read_bytes() == inventory_en.read_bytes() + # Ensure that the reproducible inventory contents are identical + assert reproducible_inventory_et.read_bytes() == reproducible_inventory_en.read_bytes() From 366784253734abac552ce6a09fef4644dc21473f Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 11:12:01 +0000 Subject: [PATCH 10/23] During reproducible build inventory localization test case, verify that localization has an effect on the test application's inventory output --- tests/test_util_inventory.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index f565c78aa96..4e938e4749f 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -130,5 +130,12 @@ def test_inventory_reproducible(tempdir, monkeypatch): srcdir_en = _write_appconfig(tempdir, "en") reproducible_inventory_en = _build_inventory(srcdir_en) + # Also build the app using Estonian (EE) locale without build reproducibility enabled + srcdir_et = _write_appconfig(tempdir, "et", prefix="localized") + localized_inventory_et = _build_inventory(srcdir_et) + # Ensure that the reproducible inventory contents are identical assert reproducible_inventory_et.read_bytes() == reproducible_inventory_en.read_bytes() + + # Ensure that inventory contents are different between a localized and non-localized build + assert reproducible_inventory_et.read_bytes() != localized_inventory_et.read_bytes() From 336ea1c700025eac592731b5420da1f0f6d92f25 Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 13:47:10 +0000 Subject: [PATCH 11/23] Use ISO-639-3 'undetermined' language code during reproducible builds, since this should not have an associated translation catalog --- sphinx/locale/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 381c05e97ae..5f021587e6b 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -114,7 +114,7 @@ def init( if getenv('SOURCE_DATE_EPOCH') is not None: # Disable localization during reproducible source builds # See https://reproducible-builds.org/docs/source-date-epoch/ - languages: Optional[List[str]] = None + languages: Optional[List[str]] = ['und'] elif language and '_' in language: # for language having country code (like "de_AT") languages = [language, language.split('_')[0]] From 9b0acb3f7d11e543c5161de3cf1e893626dc4c93 Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 14:03:06 +0000 Subject: [PATCH 12/23] Add explanatory comment --- sphinx/locale/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 5f021587e6b..d485f3b7062 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -114,6 +114,14 @@ def init( if getenv('SOURCE_DATE_EPOCH') is not None: # Disable localization during reproducible source builds # See https://reproducible-builds.org/docs/source-date-epoch/ + # + # Note: Providing an empty/none value to gettext.translation causes + # it to consult various language-related environment variables to find + # locale(s). We don't want that during a reproducible build; we want + # to run through the same code path, but to return NullTranslations. + # + # To achieve that, specify the ISO-639-3 'undetermined' language code, + # which should not match any translation catalogs. languages: Optional[List[str]] = ['und'] elif language and '_' in language: # for language having country code (like "de_AT") From 873a3509f74808744de44c50fee4b3e444e02c50 Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 15:04:12 +0000 Subject: [PATCH 13/23] Add test coverage for resource translation when language is empty and LANGUAGE environment variable is configured --- .../locale1/et/LC_MESSAGES/myext.mo | Bin 0 -> 80 bytes .../locale1/et/LC_MESSAGES/myext.po | 2 ++ tests/test_locale.py | 19 ++++++++++++++++++ 3 files changed, 21 insertions(+) create mode 100644 tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.mo create mode 100644 tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po diff --git a/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.mo b/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.mo new file mode 100644 index 0000000000000000000000000000000000000000..c99a36846a83f2fcd3d66fe637d8ef19060831df GIT binary patch literal 80 zcmca7#4?ou2pEA_28dOFm>Gz5fS4PIEugdukcI(}T94G6oP34y{Gyx`hLF^vRE6Bc J#LS#r1^^kF3hDp= literal 0 HcmV?d00001 diff --git a/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po b/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po new file mode 100644 index 00000000000..1ecf6e3ee90 --- /dev/null +++ b/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po @@ -0,0 +1,2 @@ +msgid "Hello world" +msgstr "Tere maailm" diff --git a/tests/test_locale.py b/tests/test_locale.py index 1dcad64eb94..441e8ef69fe 100644 --- a/tests/test_locale.py +++ b/tests/test_locale.py @@ -55,3 +55,22 @@ def test_add_message_catalog(app, rootdir): assert _('Hello world') == 'HELLO WORLD' assert _('Hello sphinx') == 'Hello sphinx' assert _('Hello reST') == 'Hello reST' + + +def _empty_language_translation(rootdir): + locale_dirs, catalog = [rootdir / 'test-locale' / 'locale1'], 'myext' + locale.translators.clear() + locale.init(locale_dirs, language=None, catalog=catalog) + return locale.get_translation(catalog) + + +def test_init_environment_language(rootdir, monkeypatch): + with monkeypatch.context() as m: + m.setenv("LANGUAGE", "en_US:en") + _ = _empty_language_translation(rootdir) + assert _('Hello world') == 'HELLO WORLD' + + with monkeypatch.context() as m: + m.setenv("LANGUAGE", "et_EE:et") + _ = _empty_language_translation(rootdir) + assert _('Hello world') == 'Tere maailm' From 121fc1d8cf14737c41f1c7e9616504815bd14fe3 Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 15:04:43 +0000 Subject: [PATCH 14/23] Add test coverage for resource translation during reproducible builds (and confirming that LANGUAGE variable does not affect the result) --- tests/test_locale.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_locale.py b/tests/test_locale.py index 441e8ef69fe..9d4db2dd256 100644 --- a/tests/test_locale.py +++ b/tests/test_locale.py @@ -74,3 +74,25 @@ def test_init_environment_language(rootdir, monkeypatch): m.setenv("LANGUAGE", "et_EE:et") _ = _empty_language_translation(rootdir) assert _('Hello world') == 'Tere maailm' + + +def test_init_reproducible_build_language(rootdir, monkeypatch): + with monkeypatch.context() as m: + m.setenv("SOURCE_DATE_EPOCH", "0") + m.setenv("LANGUAGE", "en_US:en") + _ = _empty_language_translation(rootdir) + sde_en_translation = str(_('Hello world')) # str cast to evaluate lazy method + + with monkeypatch.context() as m: + m.setenv("SOURCE_DATE_EPOCH", "0") + m.setenv("LANGUAGE", "et_EE:et") + _ = _empty_language_translation(rootdir) + sde_et_translation = str(_('Hello world')) # str cast to evaluate lazy method + + with monkeypatch.context() as m: + m.setenv("LANGUAGE", "et_EE:et") + _ = _empty_language_translation(rootdir) + loc_et_translation = str(_('Hello world')) # str cast to evaluate lazy method + + assert sde_en_translation == sde_et_translation + assert sde_et_translation != loc_et_translation From 658553813ff28771bb94e1da057b1bd57c0d1e21 Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 31 Oct 2022 15:23:57 +0000 Subject: [PATCH 15/23] Retain support for pluralized resource translation by adding support for additional plural forms to the lazy loader translation method --- sphinx/locale/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index d485f3b7062..b44eec9856e 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -199,12 +199,15 @@ def is_translator_registered(catalog: str = 'sphinx', namespace: str = 'general' return (namespace, catalog) in translators -def _lazy_translate(catalog: str, namespace: str, message: str) -> str: +def _lazy_translate(catalog: str, namespace: str, message: str, *args: Any) -> str: """Used instead of _ when creating TranslationProxy, because _ is not bound yet at that time. """ translator = get_translator(catalog, namespace) - return translator.gettext(message) + if len(args) <= 1: + return translator.gettext(message) + else: # support pluralization + return translator.ngettext(message, args[0], args[1]) def get_translation(catalog: str, namespace: str = 'general') -> Callable[[str], str]: @@ -233,7 +236,7 @@ def setup(app): .. versionadded:: 1.8 """ def gettext(message: str, *args: Any) -> str: - return _TranslationProxy(_lazy_translate, catalog, namespace, message) # type: ignore[return-value] # NOQA + return _TranslationProxy(_lazy_translate, catalog, namespace, message, *args) # type: ignore[return-value] # NOQA return gettext From 04f705811c743a5abfd3bbf8d53c64a138508f27 Mon Sep 17 00:00:00 2001 From: James Addison Date: Wed, 9 Nov 2022 11:51:11 +0000 Subject: [PATCH 16/23] Typing: be a bit stricter about providing a str-type argument to the util.i18n.format_date method (note: mypy doesn't appear to catch this?) --- sphinx/builders/html/__init__.py | 2 +- sphinx/builders/latex/__init__.py | 2 +- sphinx/transforms/__init__.py | 2 +- sphinx/util/i18n.py | 2 +- sphinx/writers/manpage.py | 2 +- sphinx/writers/texinfo.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sphinx/builders/html/__init__.py b/sphinx/builders/html/__init__.py index 7ea70fa4dd4..9cd67d96d2d 100644 --- a/sphinx/builders/html/__init__.py +++ b/sphinx/builders/html/__init__.py @@ -499,7 +499,7 @@ def prepare_writing(self, docnames: Set[str]) -> None: # typically doesn't include the time of day lufmt = self.config.html_last_updated_fmt if lufmt is not None: - self.last_updated = format_date(lufmt or _('%b %d, %Y'), + self.last_updated = format_date(lufmt or str(_('%b %d, %Y')), language=self.config.language) else: self.last_updated = None diff --git a/sphinx/builders/latex/__init__.py b/sphinx/builders/latex/__init__.py index b80ce01e586..3e66cec7087 100644 --- a/sphinx/builders/latex/__init__.py +++ b/sphinx/builders/latex/__init__.py @@ -177,7 +177,7 @@ def init_context(self) -> None: if self.config.today: self.context['date'] = self.config.today else: - self.context['date'] = format_date(self.config.today_fmt or _('%b %d, %Y'), + self.context['date'] = format_date(self.config.today_fmt or str(_('%b %d, %Y')), language=self.config.language) if self.config.latex_logo: diff --git a/sphinx/transforms/__init__.py b/sphinx/transforms/__init__.py index fdffa0f7c93..cd9ba337489 100644 --- a/sphinx/transforms/__init__.py +++ b/sphinx/transforms/__init__.py @@ -105,7 +105,7 @@ def apply(self, **kwargs: Any) -> None: text = self.config[refname] if refname == 'today' and not text: # special handling: can also specify a strftime format - text = format_date(self.config.today_fmt or _('%b %d, %Y'), + text = format_date(self.config.today_fmt or str(_('%b %d, %Y')), language=self.config.language) ref.replace_self(nodes.Text(text)) diff --git a/sphinx/util/i18n.py b/sphinx/util/i18n.py index 0dbbf8f93a2..6915d309187 100644 --- a/sphinx/util/i18n.py +++ b/sphinx/util/i18n.py @@ -208,7 +208,7 @@ def format_date( language = 'en' result = [] - tokens = date_format_re.split(str(format)) + tokens = date_format_re.split(format) for token in tokens: if token in date_format_mappings: babel_format = date_format_mappings.get(token, '') diff --git a/sphinx/writers/manpage.py b/sphinx/writers/manpage.py index a550156473a..3214103dfad 100644 --- a/sphinx/writers/manpage.py +++ b/sphinx/writers/manpage.py @@ -92,7 +92,7 @@ def __init__(self, document: nodes.document, builder: Builder) -> None: if self.config.today: self._docinfo['date'] = self.config.today else: - self._docinfo['date'] = format_date(self.config.today_fmt or _('%b %d, %Y'), + self._docinfo['date'] = format_date(self.config.today_fmt or str(_('%b %d, %Y')), language=self.config.language) self._docinfo['copyright'] = self.config.copyright self._docinfo['version'] = self.config.version diff --git a/sphinx/writers/texinfo.py b/sphinx/writers/texinfo.py index cbac28a2fc0..007aa086827 100644 --- a/sphinx/writers/texinfo.py +++ b/sphinx/writers/texinfo.py @@ -219,7 +219,7 @@ def init_settings(self) -> None: 'project': self.escape(self.config.project), 'copyright': self.escape(self.config.copyright), 'date': self.escape(self.config.today or - format_date(self.config.today_fmt or _('%b %d, %Y'), + format_date(self.config.today_fmt or str(_('%b %d, %Y')), language=self.config.language)) }) # title From b264cb2bbf13a1e5a275b4ea9cbe4655295bca15 Mon Sep 17 00:00:00 2001 From: James Addison Date: Wed, 9 Nov 2022 11:56:56 +0000 Subject: [PATCH 17/23] Revert "Fixup: mypy: remove unused type-ignore comment" This reverts commit c8d9de33a68de113dfd8f4b24aa7987d0f656b8d. --- sphinx/writers/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index c6d8d9e4be7..da09feab81a 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -420,7 +420,7 @@ def do_format() -> None: result.append((indent, res)) for itemindent, item in content: if itemindent == -1: - toformat.append(str(item)) + toformat.append(str(item)) # type: ignore else: do_format() result.append((indent + itemindent, item)) # type: ignore From 837753486a9353947c2bcc228107c7e8d955be59 Mon Sep 17 00:00:00 2001 From: James Addison Date: Wed, 9 Nov 2022 11:57:32 +0000 Subject: [PATCH 18/23] Typing: only invoke str typecasting for localized items within writers.text module --- sphinx/writers/text.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index da09feab81a..ba7aca37bad 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -420,7 +420,7 @@ def do_format() -> None: result.append((indent, res)) for itemindent, item in content: if itemindent == -1: - toformat.append(str(item)) # type: ignore + toformat.append(item) # type: ignore else: do_format() result.append((indent + itemindent, item)) # type: ignore @@ -792,8 +792,8 @@ def visit_acks(self, node: Element) -> None: def visit_image(self, node: Element) -> None: if 'alt' in node.attributes: - self.add_text(_('[image: %s]') % node['alt']) - self.add_text(_('[image]')) + self.add_text(str(_('[image: %s]') % node['alt'])) + self.add_text(str(_('[image]'))) raise nodes.SkipNode def visit_transition(self, node: Element) -> None: From aa7e1652327fd7a432925cc693969e8ef6cbdbbb Mon Sep 17 00:00:00 2001 From: James Addison Date: Wed, 9 Nov 2022 12:04:40 +0000 Subject: [PATCH 19/23] Typing: be a bit stricter about joining str-type values within domains.std module --- sphinx/domains/std.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index b59a7cf80dd..0b2b77134b7 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -241,11 +241,11 @@ def add_target_and_index(self, firstname: str, sig: str, signode: desc_signature # create an index entry if currprogram: - descr = _('%s command line option') % currprogram + descr = str(_('%s command line option') % currprogram) else: - descr = _('command line option') + descr = str(_('command line option')) for option in signode.get('allnames', []): - entry = '; '.join([str(descr), option]) + entry = '; '.join([descr, option]) self.indexnode['entries'].append(('pair', entry, signode['ids'][0], '', None)) def make_old_id(self, prefix: str, optname: str) -> str: From bb02c9204671000d3c16836426d41180d6c71d2d Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 30 Dec 2022 00:21:52 +0000 Subject: [PATCH 20/23] Remove support for ngettext (pluralized forms) See bb37309a6da824664e385d43ca60a3161e620195 --- sphinx/locale/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index b44eec9856e..6cf10c32757 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -204,10 +204,7 @@ def _lazy_translate(catalog: str, namespace: str, message: str, *args: Any) -> s not bound yet at that time. """ translator = get_translator(catalog, namespace) - if len(args) <= 1: - return translator.gettext(message) - else: # support pluralization - return translator.ngettext(message, args[0], args[1]) + return translator.gettext(message) def get_translation(catalog: str, namespace: str = 'general') -> Callable[[str], str]: From 6a880addadc9f116a85a63bc334003ddd379ed5d Mon Sep 17 00:00:00 2001 From: James Addison Date: Tue, 21 Mar 2023 20:59:34 +0000 Subject: [PATCH 21/23] Fixup: assert on sorted directory contents in test_util_inventory.test_inventory_localization --- tests/test_util_inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index 4b594fbd38b..675bba06b77 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -91,7 +91,7 @@ def _write_appconfig(dir, language, prefix=None): (dir / prefix).makedirs() (dir / prefix / 'conf.py').write_text(f'language = "{language}"', encoding='utf8') (dir / prefix / 'index.rst').write_text('index.rst', encoding='utf8') - assert (dir / prefix).listdir() == ['conf.py', 'index.rst'] + assert sorted((dir / prefix).listdir()) == ['conf.py', 'index.rst'] assert (dir / prefix / 'index.rst').exists() return (dir / prefix) From dd4142060375d4ca3cb0d5c5edc78a2f01913e3c Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 7 Apr 2023 10:11:12 +0100 Subject: [PATCH 22/23] CHANGES: Add bugfix entry --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index a85952dd6b9..ccc2167633d 100644 --- a/CHANGES +++ b/CHANGES @@ -83,6 +83,7 @@ Bugs fixed * #11192: Restore correct parallel search index building. Patch by Jeremy Maitin-Shepard * Use the new Transifex ``tx`` client +* #9778: Disable localisation of the ``objects.inv`` file during Reproducible Builds Testing -------- From a9c910006f585e08accf4d8e0355f54a17b3c43c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Fri, 7 Apr 2023 15:27:22 +0100 Subject: [PATCH 23/23] Reword CHANGES entry to be more general --- CHANGES | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index ccc2167633d..a66c2c46039 100644 --- a/CHANGES +++ b/CHANGES @@ -83,7 +83,8 @@ Bugs fixed * #11192: Restore correct parallel search index building. Patch by Jeremy Maitin-Shepard * Use the new Transifex ``tx`` client -* #9778: Disable localisation of the ``objects.inv`` file during Reproducible Builds +* #9778: Disable localisation when the ``SOURCE_DATE_EPOCH`` environment + variable is set, to assist with 'reproducible builds'. Patch by James Addison Testing --------