test_catalog.py

#
# Copyright (C) 2007-2011 Edgewall Software, 2013-2023 the Babel team
# All rights reserved.
#
# This software is licensed as described in the file LICENSE, which
# you should have received as part of this distribution. The terms
# are also available at http://babel.edgewall.org/wiki/License.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://babel.edgewall.org/log/.

import copy
import datetime
import unittest
from io import StringIO

from babel.dates import UTC, format_datetime
from babel.messages import catalog, pofile
from babel.util import FixedOffsetTimezone


class MessageTestCase(unittest.TestCase):

    def test_python_format(self):
        assert catalog.PYTHON_FORMAT.search('foo %d bar')
        assert catalog.PYTHON_FORMAT.search('foo %s bar')
        assert catalog.PYTHON_FORMAT.search('foo %r bar')
        assert catalog.PYTHON_FORMAT.search('foo %(name).1f')
        assert catalog.PYTHON_FORMAT.search('foo %(name)3.3f')
        assert catalog.PYTHON_FORMAT.search('foo %(name)3f')
        assert catalog.PYTHON_FORMAT.search('foo %(name)06d')
        assert catalog.PYTHON_FORMAT.search('foo %(name)Li')
        assert catalog.PYTHON_FORMAT.search('foo %(name)#d')
        assert catalog.PYTHON_FORMAT.search('foo %(name)-4.4hs')
        assert catalog.PYTHON_FORMAT.search('foo %(name)*.3f')
        assert catalog.PYTHON_FORMAT.search('foo %(name).*f')
        assert catalog.PYTHON_FORMAT.search('foo %(name)3.*f')
        assert catalog.PYTHON_FORMAT.search('foo %(name)*.*f')
        assert catalog.PYTHON_FORMAT.search('foo %()s')

    def test_translator_comments(self):
        mess = catalog.Message('foo', user_comments=['Comment About `foo`'])
        assert mess.user_comments == ['Comment About `foo`']
        mess = catalog.Message('foo',
                               auto_comments=['Comment 1 About `foo`',
                                              'Comment 2 About `foo`'])
        assert mess.auto_comments == ['Comment 1 About `foo`', 'Comment 2 About `foo`']

    def test_clone_message_object(self):
        msg = catalog.Message('foo', locations=[('foo.py', 42)])
        clone = msg.clone()
        clone.locations.append(('bar.py', 42))
        assert msg.locations == [('foo.py', 42)]
        msg.flags.add('fuzzy')
        assert not clone.fuzzy and msg.fuzzy


class CatalogTestCase(unittest.TestCase):

    def test_add_returns_message_instance(self):
        cat = catalog.Catalog()
        message = cat.add('foo')
        assert message.id == 'foo'

    def test_two_messages_with_same_singular(self):
        cat = catalog.Catalog()
        cat.add('foo')
        cat.add(('foo', 'foos'))
        assert len(cat) == 1

    def test_duplicate_auto_comment(self):
        cat = catalog.Catalog()
        cat.add('foo', auto_comments=['A comment'])
        cat.add('foo', auto_comments=['A comment', 'Another comment'])
        assert cat['foo'].auto_comments == ['A comment', 'Another comment']

    def test_duplicate_user_comment(self):
        cat = catalog.Catalog()
        cat.add('foo', user_comments=['A comment'])
        cat.add('foo', user_comments=['A comment', 'Another comment'])
        assert cat['foo'].user_comments == ['A comment', 'Another comment']

    def test_duplicate_location(self):
        cat = catalog.Catalog()
        cat.add('foo', locations=[('foo.py', 1)])
        cat.add('foo', locations=[('foo.py', 1)])
        assert cat['foo'].locations == [('foo.py', 1)]

    def test_update_message_changed_to_plural(self):
        cat = catalog.Catalog()
        cat.add('foo', 'Voh')
        tmpl = catalog.Catalog()
        tmpl.add(('foo', 'foos'))
        cat.update(tmpl)
        assert cat['foo'].string == ('Voh', '')
        assert cat['foo'].fuzzy

    def test_update_message_changed_to_simple(self):
        cat = catalog.Catalog()
        cat.add('foo' 'foos', ('Voh', 'Vöhs'))
        tmpl = catalog.Catalog()
        tmpl.add('foo')
        cat.update(tmpl)
        assert cat['foo'].string == 'Voh'
        assert cat['foo'].fuzzy

    def test_update_message_updates_comments(self):
        cat = catalog.Catalog()
        cat['foo'] = catalog.Message('foo', locations=[('main.py', 5)])
        assert cat['foo'].auto_comments == []
        assert cat['foo'].user_comments == []
        # Update cat[u'foo'] with a new location and a comment
        cat['foo'] = catalog.Message('foo', locations=[('main.py', 7)],
                                     user_comments=['Foo Bar comment 1'])
        assert cat['foo'].user_comments == ['Foo Bar comment 1']
        # now add yet another location with another comment
        cat['foo'] = catalog.Message('foo', locations=[('main.py', 9)],
                                     auto_comments=['Foo Bar comment 2'])
        assert cat['foo'].auto_comments == ['Foo Bar comment 2']

    def test_update_fuzzy_matching_with_case_change(self):
        cat = catalog.Catalog()
        cat.add('FOO', 'Voh')
        cat.add('bar', 'Bahr')
        tmpl = catalog.Catalog()
        tmpl.add('foo')
        cat.update(tmpl)
        assert len(cat.obsolete) == 1
        assert 'FOO' not in cat

        assert cat['foo'].string == 'Voh'
        assert cat['foo'].fuzzy is True

    def test_update_fuzzy_matching_with_char_change(self):
        cat = catalog.Catalog()
        cat.add('fo', 'Voh')
        cat.add('bar', 'Bahr')
        tmpl = catalog.Catalog()
        tmpl.add('foo')
        cat.update(tmpl)
        assert len(cat.obsolete) == 1
        assert 'fo' not in cat

        assert cat['foo'].string == 'Voh'
        assert cat['foo'].fuzzy is True

    def test_update_fuzzy_matching_no_msgstr(self):
        cat = catalog.Catalog()
        cat.add('fo', '')
        tmpl = catalog.Catalog()
        tmpl.add('fo')
        tmpl.add('foo')
        cat.update(tmpl)
        assert 'fo' in cat
        assert 'foo' in cat

        assert cat['fo'].string == ''
        assert cat['fo'].fuzzy is False
        assert cat['foo'].string is None
        assert cat['foo'].fuzzy is False

    def test_update_fuzzy_matching_with_new_context(self):
        cat = catalog.Catalog()
        cat.add('foo', 'Voh')
        cat.add('bar', 'Bahr')
        tmpl = catalog.Catalog()
        tmpl.add('Foo', context='Menu')
        cat.update(tmpl)
        assert len(cat.obsolete) == 1
        assert 'foo' not in cat

        message = cat.get('Foo', 'Menu')
        assert message.string == 'Voh'
        assert message.fuzzy is True
        assert message.context == 'Menu'

    def test_update_fuzzy_matching_with_changed_context(self):
        cat = catalog.Catalog()
        cat.add('foo', 'Voh', context='Menu|File')
        cat.add('bar', 'Bahr', context='Menu|File')
        tmpl = catalog.Catalog()
        tmpl.add('Foo', context='Menu|Edit')
        cat.update(tmpl)
        assert len(cat.obsolete) == 1
        assert cat.get('Foo', 'Menu|File') is None

        message = cat.get('Foo', 'Menu|Edit')
        assert message.string == 'Voh'
        assert message.fuzzy is True
        assert message.context == 'Menu|Edit'

    def test_update_fuzzy_matching_no_cascading(self):
        cat = catalog.Catalog()
        cat.add('fo', 'Voh')
        cat.add('foo', 'Vohe')
        tmpl = catalog.Catalog()
        tmpl.add('fo')
        tmpl.add('foo')
        tmpl.add('fooo')
        cat.update(tmpl)
        assert 'fo' in cat
        assert 'foo' in cat

        assert cat['fo'].string == 'Voh'
        assert cat['fo'].fuzzy is False
        assert cat['foo'].string == 'Vohe'
        assert cat['foo'].fuzzy is False
        assert cat['fooo'].string == 'Vohe'
        assert cat['fooo'].fuzzy is True

    def test_update_fuzzy_matching_long_string(self):
        lipsum = "\
Lorem Ipsum is simply dummy text of the printing and typesetting \
industry. Lorem Ipsum has been the industry's standard dummy text ever \
since the 1500s, when an unknown printer took a galley of type and \
scrambled it to make a type specimen book. It has survived not only \
five centuries, but also the leap into electronic typesetting, \
remaining essentially unchanged. It was popularised in the 1960s with \
the release of Letraset sheets containing Lorem Ipsum passages, and \
more recently with desktop publishing software like Aldus PageMaker \
including versions of Lorem Ipsum."
        cat = catalog.Catalog()
        cat.add("ZZZZZZ " + lipsum, "foo")
        tmpl = catalog.Catalog()
        tmpl.add(lipsum + " ZZZZZZ")
        cat.update(tmpl)
        assert cat[lipsum + " ZZZZZZ"].fuzzy is True
        assert len(cat.obsolete) == 0

    def test_update_without_fuzzy_matching(self):
        cat = catalog.Catalog()
        cat.add('fo', 'Voh')
        cat.add('bar', 'Bahr')
        tmpl = catalog.Catalog()
        tmpl.add('foo')
        cat.update(tmpl, no_fuzzy_matching=True)
        assert len(cat.obsolete) == 2

    def test_fuzzy_matching_regarding_plurals(self):
        cat = catalog.Catalog()
        cat.add(('foo', 'foh'), ('foo', 'foh'))
        ru = copy.copy(cat)
        ru.locale = 'ru_RU'
        ru.update(cat)
        assert ru['foo'].fuzzy is True
        ru = copy.copy(cat)
        ru.locale = 'ru_RU'
        ru['foo'].string = ('foh', 'fohh', 'fohhh')
        ru.update(cat)
        assert ru['foo'].fuzzy is False

    def test_update_no_template_mutation(self):
        tmpl = catalog.Catalog()
        tmpl.add('foo')
        cat1 = catalog.Catalog()
        cat1.add('foo', 'Voh')
        cat1.update(tmpl)
        cat2 = catalog.Catalog()
        cat2.update(tmpl)

        assert cat2['foo'].string is None
        assert cat2['foo'].fuzzy is False

    def test_update_po_updates_pot_creation_date(self):
        template = catalog.Catalog()
        localized_catalog = copy.deepcopy(template)
        localized_catalog.locale = 'de_DE'
        assert template.mime_headers != localized_catalog.mime_headers
        assert template.creation_date == localized_catalog.creation_date
        template.creation_date = datetime.datetime.now() - \
            datetime.timedelta(minutes=5)
        localized_catalog.update(template)
        assert template.creation_date == localized_catalog.creation_date

    def test_update_po_keeps_po_revision_date(self):
        template = catalog.Catalog()
        localized_catalog = copy.deepcopy(template)
        localized_catalog.locale = 'de_DE'
        fake_rev_date = datetime.datetime.now() - datetime.timedelta(days=5)
        localized_catalog.revision_date = fake_rev_date
        assert template.mime_headers != localized_catalog.mime_headers
        assert template.creation_date == localized_catalog.creation_date
        template.creation_date = datetime.datetime.now() - \
            datetime.timedelta(minutes=5)
        localized_catalog.update(template)
        assert localized_catalog.revision_date == fake_rev_date

    def test_stores_datetime_correctly(self):
        localized = catalog.Catalog()
        localized.locale = 'de_DE'
        localized[''] = catalog.Message('',
                                        "POT-Creation-Date: 2009-03-09 15:47-0700\n" +
                                        "PO-Revision-Date: 2009-03-09 15:47-0700\n")
        for key, value in localized.mime_headers:
            if key in ('POT-Creation-Date', 'PO-Revision-Date'):
                assert value == '2009-03-09 15:47-0700'

    def test_mime_headers_contain_same_information_as_attributes(self):
        cat = catalog.Catalog()
        cat[''] = catalog.Message('',
                                  "Last-Translator: Foo Bar <foo.bar@example.com>\n" +
                                  "Language-Team: de <de@example.com>\n" +
                                  "POT-Creation-Date: 2009-03-01 11:20+0200\n" +
                                  "PO-Revision-Date: 2009-03-09 15:47-0700\n")
        assert cat.locale is None
        mime_headers = dict(cat.mime_headers)

        assert cat.last_translator == 'Foo Bar <foo.bar@example.com>'
        assert mime_headers['Last-Translator'] == 'Foo Bar <foo.bar@example.com>'

        assert cat.language_team == 'de <de@example.com>'
        assert mime_headers['Language-Team'] == 'de <de@example.com>'

        dt = datetime.datetime(2009, 3, 9, 15, 47, tzinfo=FixedOffsetTimezone(-7 * 60))
        assert cat.revision_date == dt
        formatted_dt = format_datetime(dt, 'yyyy-MM-dd HH:mmZ', locale='en')
        assert mime_headers['PO-Revision-Date'] == formatted_dt


def test_message_fuzzy():
    assert not catalog.Message('foo').fuzzy
    msg = catalog.Message('foo', 'foo', flags=['fuzzy'])
    assert msg.fuzzy
    assert msg.id == 'foo'


def test_message_pluralizable():
    assert not catalog.Message('foo').pluralizable
    assert catalog.Message(('foo', 'bar')).pluralizable


def test_message_python_format():
    assert catalog.Message('foo %(name)s bar').python_format
    assert catalog.Message(('foo %(name)s', 'foo %(name)s')).python_format


def test_catalog():
    cat = catalog.Catalog(project='Foobar', version='1.0',
                          copyright_holder='Foo Company')
    assert cat.header_comment == (
        '# Translations template for Foobar.\n'
        '# Copyright (C) %(year)d Foo Company\n'
        '# This file is distributed under the same '
        'license as the Foobar project.\n'
        '# FIRST AUTHOR <EMAIL@ADDRESS>, %(year)d.\n'
        '#') % {'year': datetime.date.today().year}

    cat = catalog.Catalog(project='Foobar', version='1.0',
                          copyright_holder='Foo Company')
    cat.header_comment = (
        '# The POT for my really cool PROJECT project.\n'
        '# Copyright (C) 1990-2003 ORGANIZATION\n'
        '# This file is distributed under the same license as the PROJECT\n'
        '# project.\n'
        '#\n')
    assert cat.header_comment == (
        '# The POT for my really cool Foobar project.\n'
        '# Copyright (C) 1990-2003 Foo Company\n'
        '# This file is distributed under the same license as the Foobar\n'
        '# project.\n'
        '#\n')


def test_catalog_mime_headers():
    created = datetime.datetime(1990, 4, 1, 15, 30, tzinfo=UTC)
    cat = catalog.Catalog(project='Foobar', version='1.0',
                          creation_date=created)
    assert cat.mime_headers == [
        ('Project-Id-Version', 'Foobar 1.0'),
        ('Report-Msgid-Bugs-To', 'EMAIL@ADDRESS'),
        ('POT-Creation-Date', '1990-04-01 15:30+0000'),
        ('PO-Revision-Date', 'YEAR-MO-DA HO:MI+ZONE'),
        ('Last-Translator', 'FULL NAME <EMAIL@ADDRESS>'),
        ('Language-Team', 'LANGUAGE <LL@li.org>'),
        ('MIME-Version', '1.0'),
        ('Content-Type', 'text/plain; charset=utf-8'),
        ('Content-Transfer-Encoding', '8bit'),
        ('Generated-By', f'Babel {catalog.VERSION}\n'),
    ]


def test_catalog_mime_headers_set_locale():
    created = datetime.datetime(1990, 4, 1, 15, 30, tzinfo=UTC)
    revised = datetime.datetime(1990, 8, 3, 12, 0, tzinfo=UTC)
    cat = catalog.Catalog(locale='de_DE', project='Foobar', version='1.0',
                          creation_date=created, revision_date=revised,
                          last_translator='John Doe <jd@example.com>',
                          language_team='de_DE <de@example.com>')
    assert cat.mime_headers == [
        ('Project-Id-Version', 'Foobar 1.0'),
        ('Report-Msgid-Bugs-To', 'EMAIL@ADDRESS'),
        ('POT-Creation-Date', '1990-04-01 15:30+0000'),
        ('PO-Revision-Date', '1990-08-03 12:00+0000'),
        ('Last-Translator', 'John Doe <jd@example.com>'),
        ('Language', 'de_DE'),
        ('Language-Team', 'de_DE <de@example.com>'),
        ('Plural-Forms', 'nplurals=2; plural=(n != 1);'),
        ('MIME-Version', '1.0'),
        ('Content-Type', 'text/plain; charset=utf-8'),
        ('Content-Transfer-Encoding', '8bit'),
        ('Generated-By', f'Babel {catalog.VERSION}\n'),
    ]


def test_catalog_num_plurals():
    assert catalog.Catalog(locale='en').num_plurals == 2
    assert catalog.Catalog(locale='ga').num_plurals == 5


def test_catalog_plural_expr():
    assert catalog.Catalog(locale='en').plural_expr == '(n != 1)'
    assert (catalog.Catalog(locale='ga').plural_expr
            == '(n==1 ? 0 : n==2 ? 1 : n>=3 && n<=6 ? 2 : n>=7 && n<=10 ? 3 : 4)')


def test_catalog_plural_forms():
    assert (catalog.Catalog(locale='en').plural_forms
            == 'nplurals=2; plural=(n != 1);')
    assert (catalog.Catalog(locale='pt_BR').plural_forms
            == 'nplurals=2; plural=(n > 1);')


def test_catalog_setitem():
    cat = catalog.Catalog()
    cat['foo'] = catalog.Message('foo')
    assert cat['foo'].id == 'foo'

    cat = catalog.Catalog()
    cat['foo'] = catalog.Message('foo', locations=[('main.py', 1)])
    assert cat['foo'].locations == [('main.py', 1)]
    cat['foo'] = catalog.Message('foo', locations=[('utils.py', 5)])
    assert cat['foo'].locations == [('main.py', 1), ('utils.py', 5)]


def test_catalog_add():
    cat = catalog.Catalog()
    foo = cat.add('foo')
    assert foo.id == 'foo'
    assert cat['foo'] is foo


def test_catalog_update():
    template = catalog.Catalog(header_comment="# A Custom Header")
    template.add('green', locations=[('main.py', 99)])
    template.add('blue', locations=[('main.py', 100)])
    template.add(('salad', 'salads'), locations=[('util.py', 42)])
    cat = catalog.Catalog(locale='de_DE')
    cat.add('blue', 'blau', locations=[('main.py', 98)])
    cat.add('head', 'Kopf', locations=[('util.py', 33)])
    cat.add(('salad', 'salads'), ('Salat', 'Salate'),
            locations=[('util.py', 38)])

    cat.update(template)
    assert len(cat) == 3

    msg1 = cat['green']
    assert msg1.string
    assert msg1.locations == [('main.py', 99)]

    msg2 = cat['blue']
    assert msg2.string == 'blau'
    assert msg2.locations == [('main.py', 100)]

    msg3 = cat['salad']
    assert msg3.string == ('Salat', 'Salate')
    assert msg3.locations == [('util.py', 42)]

    assert 'head' not in cat
    assert list(cat.obsolete.values())[0].id == 'head'

    cat.update(template, update_header_comment=True)
    assert cat.header_comment == template.header_comment  # Header comment also gets updated


def test_datetime_parsing():
    val1 = catalog._parse_datetime_header('2006-06-28 23:24+0200')
    assert val1.year == 2006
    assert val1.month == 6
    assert val1.day == 28
    assert val1.tzinfo.zone == 'Etc/GMT+120'

    val2 = catalog._parse_datetime_header('2006-06-28 23:24')
    assert val2.year == 2006
    assert val2.month == 6
    assert val2.day == 28
    assert val2.tzinfo is None


def test_update_catalog_comments():
    # Based on https://web.archive.org/web/20100710131029/http://babel.edgewall.org/attachment/ticket/163/cat-update-comments.py

    catalog = pofile.read_po(StringIO('''
    # A user comment
    #. An auto comment
    #: main.py:1
    #, fuzzy, python-format
    msgid "foo %(name)s"
    msgstr "foo %(name)s"
    '''))

    assert all(message.user_comments and message.auto_comments for message in catalog if message.id)

    # NOTE: in the POT file, there are no comments
    template = pofile.read_po(StringIO('''
    #: main.py:1
    #, fuzzy, python-format
    msgid "bar %(name)s"
    msgstr ""
    '''))

    catalog.update(template)

    # Auto comments will be obliterated here
    assert all(message.user_comments for message in catalog if message.id)