From c1ab2ed578421dda2729af43be723e8cfd404ad1 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 9 Apr 2023 17:29:09 +0100 Subject: [PATCH 1/3] Revert "Refactor ``_TranslationProxy``" This reverts commit 2c83af0aab7080e0b78d4a16981eed878b2cac4c. --- sphinx/locale/__init__.py | 74 +++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 8bc7a267320..2291fc00bf3 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -10,22 +10,29 @@ class _TranslationProxy: """ + Class for proxy strings from gettext translations. This is a helper for the + lazy_* functions from this module. + The proxy implementation attempts to be as complete as possible, so that the lazy objects should mostly work as expected, for example for sorting. """ - __slots__ = '_catalogue', '_namespace', '_message' + __slots__ = ('_func', '_args') + + def __new__(cls, func: Callable[..., str], *args: str) -> _TranslationProxy: + if not args: + # not called with "function" and "arguments", but a plain string + return str(func) # type: ignore[return-value] + return object.__new__(cls) + + def __getnewargs__(self) -> tuple[str]: + return (self._func,) + self._args # type: ignore - def __init__(self, catalogue: str, namespace: str, message: str) -> None: - self._catalogue = catalogue - self._namespace = namespace - self._message = message + def __init__(self, func: Callable[..., str], *args: str) -> None: + self._func = func + self._args = args def __str__(self) -> str: - try: - return translators[self._namespace, self._catalogue].gettext(self._message) - except KeyError: - # NullTranslations().gettext(self._message) == self._message - return self._message + return str(self._func(*self._args)) def __dir__(self) -> list[str]: return dir(str) @@ -33,21 +40,20 @@ def __dir__(self) -> list[str]: def __getattr__(self, name: str) -> Any: return getattr(self.__str__(), name) - def __getstate__(self) -> tuple[str, str, str]: - return self._catalogue, self._namespace, self._message + def __getstate__(self) -> tuple[Callable[..., str], tuple[str, ...]]: + return self._func, self._args - def __setstate__(self, tup: tuple[str, str, str]) -> None: - self._catalogue, self._namespace, self._message = tup + def __setstate__(self, tup: tuple[Callable[..., str], tuple[str]]) -> None: + self._func, self._args = tup def __copy__(self) -> _TranslationProxy: - return _TranslationProxy(self._catalogue, self._namespace, self._message) + return _TranslationProxy(self._func, *self._args) def __repr__(self) -> str: try: - return f'i{self.__str__()!r}' + return 'i' + repr(str(self.__str__())) except Exception: - return (self.__class__.__name__ - + f'({self._catalogue}, {self._namespace}, {self._message})') + return f'<{self.__class__.__name__} broken>' def __add__(self, other: str) -> str: return self.__str__() + other @@ -104,6 +110,8 @@ def init( # ignore previously failed attempts to find message catalogs if translator.__class__ is NullTranslations: translator = None + # the None entry is the system's default locale path + has_translation = True if getenv('SOURCE_DATE_EPOCH') is not None: # Disable localization during reproducible source builds @@ -117,17 +125,15 @@ def init( # To achieve that, specify the ISO-639-3 'undetermined' language code, # which should not match any translation catalogs. languages: list[str] | None = ['und'] + elif language and '_' in language: + # for language having country code (like "de_AT") + languages = [language, language.split('_')[0]] elif language: - if '_' in language: - # for language having country code (like "de_AT") - languages = [language, language.split('_')[0]] - else: - languages = [language] + languages = [language] else: languages = None # loading - # the None entry is the system's default locale path for dir_ in locale_dirs: try: trans = translation(catalog, localedir=dir_, languages=languages) @@ -138,13 +144,11 @@ def init( except Exception: # Language couldn't be found in the specified path pass - if translator is not None: - has_translation = True - else: + # guarantee translators[(namespace, catalog)] exists + if translator is None: translator = NullTranslations() has_translation = False - # guarantee translators[(namespace, catalog)] exists - translators[namespace, catalog] = translator + translators[(namespace, catalog)] = translator return translator, has_translation @@ -177,6 +181,14 @@ def is_translator_registered(catalog: str = 'sphinx', namespace: str = 'general' return (namespace, catalog) in translators +def _lazy_translate(catalog: str, namespace: str, message: str, *args: Any) -> str: + """Used instead of _ when creating TranslationProxy, because _ is + not bound yet at that time. + """ + translator = get_translator(catalog, namespace) + return translator.gettext(message) + + def get_translation(catalog: str, namespace: str = 'general') -> Callable[[str], str]: """Get a translation function based on the *catalog* and *namespace*. @@ -202,8 +214,8 @@ def setup(app): .. versionadded:: 1.8 """ - def gettext(message: str) -> str: - return _TranslationProxy(catalog, namespace, message) # type: ignore[return-value] + def gettext(message: str, *args: Any) -> str: + return _TranslationProxy(_lazy_translate, catalog, namespace, message, *args) # type: ignore[return-value] # NOQA return gettext From a8c1ab498dcdc6cec6aab6ca9eb44a7376ba2679 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 9 Apr 2023 17:30:01 +0100 Subject: [PATCH 2/3] Revert "Disable localisation when ``SOURCE_DATE_EPOCH`` is set (#10949)" This reverts commit f82c3c99126e644125d243ba0d0788197401e416. --- CHANGES | 2 - sphinx/builders/html/__init__.py | 2 +- sphinx/builders/latex/__init__.py | 2 +- sphinx/domains/std.py | 4 +- sphinx/locale/__init__.py | 29 ++++----- sphinx/transforms/__init__.py | 2 +- sphinx/writers/manpage.py | 2 +- sphinx/writers/texinfo.py | 2 +- sphinx/writers/text.py | 4 +- .../locale1/et/LC_MESSAGES/myext.mo | Bin 80 -> 0 bytes .../locale1/et/LC_MESSAGES/myext.po | 2 - tests/test_locale.py | 41 ------------- tests/test_util_inventory.py | 56 ------------------ 13 files changed, 20 insertions(+), 128 deletions(-) delete mode 100644 tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.mo delete mode 100644 tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po diff --git a/CHANGES b/CHANGES index 8a77524fe4d..bb24941b7c4 100644 --- a/CHANGES +++ b/CHANGES @@ -92,8 +92,6 @@ Bugs fixed * #11192: Restore correct parallel search index building. Patch by Jeremy Maitin-Shepard * Use the new Transifex ``tx`` client -* #9778: Disable localisation when the ``SOURCE_DATE_EPOCH`` environment - variable is set, to assist with 'reproducible builds'. Patch by James Addison Testing -------- diff --git a/sphinx/builders/html/__init__.py b/sphinx/builders/html/__init__.py index 5b7478e775d..2bf14672d46 100644 --- a/sphinx/builders/html/__init__.py +++ b/sphinx/builders/html/__init__.py @@ -502,7 +502,7 @@ def prepare_writing(self, docnames: set[str]) -> None: # typically doesn't include the time of day lufmt = self.config.html_last_updated_fmt if lufmt is not None: - self.last_updated = format_date(lufmt or str(_('%b %d, %Y')), + self.last_updated = format_date(lufmt or _('%b %d, %Y'), language=self.config.language) else: self.last_updated = None diff --git a/sphinx/builders/latex/__init__.py b/sphinx/builders/latex/__init__.py index b8472a3c0e9..8efa3ad1ca1 100644 --- a/sphinx/builders/latex/__init__.py +++ b/sphinx/builders/latex/__init__.py @@ -179,7 +179,7 @@ def init_context(self) -> None: if self.config.today: self.context['date'] = self.config.today else: - self.context['date'] = format_date(self.config.today_fmt or str(_('%b %d, %Y')), + self.context['date'] = format_date(self.config.today_fmt or _('%b %d, %Y'), language=self.config.language) if self.config.latex_logo: diff --git a/sphinx/domains/std.py b/sphinx/domains/std.py index ad28c9f2c9f..d1a6b048fa9 100644 --- a/sphinx/domains/std.py +++ b/sphinx/domains/std.py @@ -242,9 +242,9 @@ def add_target_and_index(self, firstname: str, sig: str, signode: desc_signature # create an index entry if currprogram: - descr = str(_('%s command line option') % currprogram) + descr = _('%s command line option') % currprogram else: - descr = str(_('command line option')) + descr = _('command line option') for option in signode.get('allnames', []): entry = '; '.join([descr, option]) self.indexnode['entries'].append(('pair', entry, signode['ids'][0], '', None)) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 2291fc00bf3..4b21218afc7 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -4,7 +4,7 @@ import locale from gettext import NullTranslations, translation -from os import getenv, path +from os import path from typing import Any, Callable @@ -113,21 +113,9 @@ def init( # the None entry is the system's default locale path has_translation = True - if getenv('SOURCE_DATE_EPOCH') is not None: - # Disable localization during reproducible source builds - # See https://reproducible-builds.org/docs/source-date-epoch/ - # - # Note: Providing an empty/none value to gettext.translation causes - # it to consult various language-related environment variables to find - # locale(s). We don't want that during a reproducible build; we want - # to run through the same code path, but to return NullTranslations. - # - # To achieve that, specify the ISO-639-3 'undetermined' language code, - # which should not match any translation catalogs. - languages: list[str] | None = ['und'] - elif language and '_' in language: + if language and '_' in language: # for language having country code (like "de_AT") - languages = [language, language.split('_')[0]] + languages: Optional[List[str]] = [language, language.split('_')[0]] elif language: languages = [language] else: @@ -181,7 +169,7 @@ def is_translator_registered(catalog: str = 'sphinx', namespace: str = 'general' return (namespace, catalog) in translators -def _lazy_translate(catalog: str, namespace: str, message: str, *args: Any) -> str: +def _lazy_translate(catalog: str, namespace: str, message: str) -> str: """Used instead of _ when creating TranslationProxy, because _ is not bound yet at that time. """ @@ -214,8 +202,13 @@ def setup(app): .. versionadded:: 1.8 """ - def gettext(message: str, *args: Any) -> str: - return _TranslationProxy(_lazy_translate, catalog, namespace, message, *args) # type: ignore[return-value] # NOQA + def gettext(message: str) -> str: + if not is_translator_registered(catalog, namespace): + # not initialized yet + return _TranslationProxy(_lazy_translate, catalog, namespace, message) # type: ignore[return-value] # noqa: E501 + else: + translator = get_translator(catalog, namespace) + return translator.gettext(message) return gettext diff --git a/sphinx/transforms/__init__.py b/sphinx/transforms/__init__.py index 2045a7426ee..f3647308484 100644 --- a/sphinx/transforms/__init__.py +++ b/sphinx/transforms/__init__.py @@ -106,7 +106,7 @@ def apply(self, **kwargs: Any) -> None: text = self.config[refname] if refname == 'today' and not text: # special handling: can also specify a strftime format - text = format_date(self.config.today_fmt or str(_('%b %d, %Y')), + text = format_date(self.config.today_fmt or _('%b %d, %Y'), language=self.config.language) ref.replace_self(nodes.Text(text)) diff --git a/sphinx/writers/manpage.py b/sphinx/writers/manpage.py index 0731e972280..1e57f48addc 100644 --- a/sphinx/writers/manpage.py +++ b/sphinx/writers/manpage.py @@ -93,7 +93,7 @@ def __init__(self, document: nodes.document, builder: Builder) -> None: if self.config.today: self._docinfo['date'] = self.config.today else: - self._docinfo['date'] = format_date(self.config.today_fmt or str(_('%b %d, %Y')), + self._docinfo['date'] = format_date(self.config.today_fmt or _('%b %d, %Y'), language=self.config.language) self._docinfo['copyright'] = self.config.copyright self._docinfo['version'] = self.config.version diff --git a/sphinx/writers/texinfo.py b/sphinx/writers/texinfo.py index 6979437686c..927e74f3487 100644 --- a/sphinx/writers/texinfo.py +++ b/sphinx/writers/texinfo.py @@ -220,7 +220,7 @@ def init_settings(self) -> None: 'project': self.escape(self.config.project), 'copyright': self.escape(self.config.copyright), 'date': self.escape(self.config.today or - format_date(self.config.today_fmt or str(_('%b %d, %Y')), + format_date(self.config.today_fmt or _('%b %d, %Y'), language=self.config.language)), }) # title diff --git a/sphinx/writers/text.py b/sphinx/writers/text.py index d78d600f80d..3bce03ac6cb 100644 --- a/sphinx/writers/text.py +++ b/sphinx/writers/text.py @@ -791,8 +791,8 @@ def visit_acks(self, node: Element) -> None: def visit_image(self, node: Element) -> None: if 'alt' in node.attributes: - self.add_text(str(_('[image: %s]') % node['alt'])) - self.add_text(str(_('[image]'))) + self.add_text(_('[image: %s]') % node['alt']) + self.add_text(_('[image]')) raise nodes.SkipNode def visit_transition(self, node: Element) -> None: diff --git a/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.mo b/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.mo deleted file mode 100644 index c99a36846a83f2fcd3d66fe637d8ef19060831df..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 zcmca7#4?ou2pEA_28dOFm>Gz5fS4PIEugdukcI(}T94G6oP34y{Gyx`hLF^vRE6Bc J#LS#r1^^kF3hDp= diff --git a/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po b/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po deleted file mode 100644 index 1ecf6e3ee90..00000000000 --- a/tests/roots/test-locale/locale1/et/LC_MESSAGES/myext.po +++ /dev/null @@ -1,2 +0,0 @@ -msgid "Hello world" -msgstr "Tere maailm" diff --git a/tests/test_locale.py b/tests/test_locale.py index 1d90473ca8d..4861079ec33 100644 --- a/tests/test_locale.py +++ b/tests/test_locale.py @@ -55,44 +55,3 @@ def test_add_message_catalog(app, rootdir): assert _('Hello world') == 'HELLO WORLD' assert _('Hello sphinx') == 'Hello sphinx' assert _('Hello reST') == 'Hello reST' - - -def _empty_language_translation(rootdir): - locale_dirs, catalog = [rootdir / 'test-locale' / 'locale1'], 'myext' - locale.translators.clear() - locale.init(locale_dirs, language=None, catalog=catalog) - return locale.get_translation(catalog) - - -def test_init_environment_language(rootdir, monkeypatch): - with monkeypatch.context() as m: - m.setenv("LANGUAGE", "en_US:en") - _ = _empty_language_translation(rootdir) - assert _('Hello world') == 'HELLO WORLD' - - with monkeypatch.context() as m: - m.setenv("LANGUAGE", "et_EE:et") - _ = _empty_language_translation(rootdir) - assert _('Hello world') == 'Tere maailm' - - -def test_init_reproducible_build_language(rootdir, monkeypatch): - with monkeypatch.context() as m: - m.setenv("SOURCE_DATE_EPOCH", "0") - m.setenv("LANGUAGE", "en_US:en") - _ = _empty_language_translation(rootdir) - sde_en_translation = str(_('Hello world')) # str cast to evaluate lazy method - - with monkeypatch.context() as m: - m.setenv("SOURCE_DATE_EPOCH", "0") - m.setenv("LANGUAGE", "et_EE:et") - _ = _empty_language_translation(rootdir) - sde_et_translation = str(_('Hello world')) # str cast to evaluate lazy method - - with monkeypatch.context() as m: - m.setenv("LANGUAGE", "et_EE:et") - _ = _empty_language_translation(rootdir) - loc_et_translation = str(_('Hello world')) # str cast to evaluate lazy method - - assert sde_en_translation == sde_et_translation - assert sde_et_translation != loc_et_translation diff --git a/tests/test_util_inventory.py b/tests/test_util_inventory.py index 675bba06b77..583edf6691f 100644 --- a/tests/test_util_inventory.py +++ b/tests/test_util_inventory.py @@ -4,7 +4,6 @@ import zlib from io import BytesIO -from sphinx.testing.util import SphinxTestApp from sphinx.util.inventory import InventoryFile inventory_v1 = b'''\ @@ -84,58 +83,3 @@ def test_read_inventory_v2_not_having_version(): invdata = InventoryFile.load(f, '/util', posixpath.join) assert invdata['py:module']['module1'] == \ ('foo', '', '/util/foo.html#module-module1', 'Long Module desc') - - -def _write_appconfig(dir, language, prefix=None): - prefix = prefix or language - (dir / prefix).makedirs() - (dir / prefix / 'conf.py').write_text(f'language = "{language}"', encoding='utf8') - (dir / prefix / 'index.rst').write_text('index.rst', encoding='utf8') - assert sorted((dir / prefix).listdir()) == ['conf.py', 'index.rst'] - assert (dir / prefix / 'index.rst').exists() - return (dir / prefix) - - -def _build_inventory(srcdir): - app = SphinxTestApp(srcdir=srcdir) - app.build() - app.cleanup() - return (app.outdir / 'objects.inv') - - -def test_inventory_localization(tempdir): - # Build an app using Estonian (EE) locale - srcdir_et = _write_appconfig(tempdir, "et") - inventory_et = _build_inventory(srcdir_et) - - # Build the same app using English (US) locale - srcdir_en = _write_appconfig(tempdir, "en") - inventory_en = _build_inventory(srcdir_en) - - # Ensure that the inventory contents differ - assert inventory_et.read_bytes() != inventory_en.read_bytes() - - -def test_inventory_reproducible(tempdir, monkeypatch): - with monkeypatch.context() as m: - # Configure reproducible builds - # See: https://reproducible-builds.org/docs/source-date-epoch/ - m.setenv("SOURCE_DATE_EPOCH", "0") - - # Build an app using Estonian (EE) locale - srcdir_et = _write_appconfig(tempdir, "et") - reproducible_inventory_et = _build_inventory(srcdir_et) - - # Build the same app using English (US) locale - srcdir_en = _write_appconfig(tempdir, "en") - reproducible_inventory_en = _build_inventory(srcdir_en) - - # Also build the app using Estonian (EE) locale without build reproducibility enabled - srcdir_et = _write_appconfig(tempdir, "et", prefix="localized") - localized_inventory_et = _build_inventory(srcdir_et) - - # Ensure that the reproducible inventory contents are identical - assert reproducible_inventory_et.read_bytes() == reproducible_inventory_en.read_bytes() - - # Ensure that inventory contents are different between a localized and non-localized build - assert reproducible_inventory_et.read_bytes() != localized_inventory_et.read_bytes() From 78a7c617644236bd541855c154b4b538391831f7 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 9 Apr 2023 17:31:23 +0100 Subject: [PATCH 3/3] Partial re-application of commit 31ca9627136b7660f96fddce25e0f6f8e839fcae. ('Use PEP 585 and 604 type annotations in sphinx.locale') --- sphinx/locale/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sphinx/locale/__init__.py b/sphinx/locale/__init__.py index 4b21218afc7..877fbbb715a 100644 --- a/sphinx/locale/__init__.py +++ b/sphinx/locale/__init__.py @@ -115,7 +115,7 @@ def init( if language and '_' in language: # for language having country code (like "de_AT") - languages: Optional[List[str]] = [language, language.split('_')[0]] + languages: list[str] | None = [language, language.split('_')[0]] elif language: languages = [language] else: