File Explorer

1"""Internationalization and localization support.2 3This module provides internationalization (I18N) and localization (L10N)4support for your Python programs by providing an interface to the GNU gettext5message catalog library.6 7I18N refers to the operation by which a program is made aware of multiple8languages.  L10N refers to the adaptation of your program, once9internationalized, to the local language and cultural habits.10 11"""12 13# This module represents the integration of work, contributions, feedback, and14# suggestions from the following people:15#16# Martin von Loewis, who wrote the initial implementation of the underlying17# C-based libintlmodule (later renamed _gettext), along with a skeletal18# gettext.py implementation.19#20# Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,21# which also included a pure-Python implementation to read .mo files if22# intlmodule wasn't available.23#24# James Henstridge, who also wrote a gettext.py module, which has some25# interesting, but currently unsupported experimental features: the notion of26# a Catalog class and instances, and the ability to add to a catalog file via27# a Python API.28#29# Barry Warsaw integrated these modules, wrote the .install() API and code,30# and conformed all C and Python code to Python's coding standards.31#32# Francois Pinard and Marc-Andre Lemburg also contributed valuably to this33# module.34#35# J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.36#37# TODO:38# - Lazy loading of .mo files.  Currently the entire catalog is loaded into39#   memory, but that's probably bad for large translated programs.  Instead,40#   the lexical sort of original strings in GNU .mo files should be exploited41#   to do binary searches and lazy initializations.  Or you might want to use42#   the undocumented double-hash algorithm for .mo files with hash tables, but43#   you'll need to study the GNU gettext code to do this.44#45# - Support Solaris .mo file formats.  Unfortunately, we've been unable to46#   find this format documented anywhere.47 48 49import os50import re51import sys52 53 54__all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',55           'find', 'translation', 'install', 'textdomain', 'bindtextdomain',56           'bind_textdomain_codeset',57           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',58           'ldngettext', 'lngettext', 'ngettext',59           'pgettext', 'dpgettext', 'npgettext', 'dnpgettext',60           ]61 62_default_localedir = os.path.join(sys.base_prefix, 'share', 'locale')63 64# Expression parsing for plural form selection.65#66# The gettext library supports a small subset of C syntax.  The only67# incompatible difference is that integer literals starting with zero are68# decimal.69#70# https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms71# http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y72 73_token_pattern = re.compile(r"""74        (?P<WHITESPACES>[ \t]+)                    | # spaces and horizontal tabs75        (?P<NUMBER>[0-9]+\b)                       | # decimal integer76        (?P<NAME>n\b)                              | # only n is allowed77        (?P<PARENTHESIS>[()])                      |78        (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >,79                                                     # <=, >=, ==, !=, &&, ||,80                                                     # ? :81                                                     # unary and bitwise ops82                                                     # not allowed83        (?P<INVALID>\w+|.)                           # invalid token84    """, re.VERBOSE|re.DOTALL)85 86def _tokenize(plural):87    for mo in re.finditer(_token_pattern, plural):88        kind = mo.lastgroup89        if kind == 'WHITESPACES':90            continue91        value = mo.group(kind)92        if kind == 'INVALID':93            raise ValueError('invalid token in plural form: %s' % value)94        yield value95    yield ''96 97def _error(value):98    if value:99        return ValueError('unexpected token in plural form: %s' % value)100    else:101        return ValueError('unexpected end of plural form')102 103_binary_ops = (104    ('||',),105    ('&&',),106    ('==', '!='),107    ('<', '>', '<=', '>='),108    ('+', '-'),109    ('*', '/', '%'),110)111_binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops}112_c2py_ops = {'||': 'or', '&&': 'and', '/': '//'}113 114def _parse(tokens, priority=-1):115    result = ''116    nexttok = next(tokens)117    while nexttok == '!':118        result += 'not '119        nexttok = next(tokens)120 121    if nexttok == '(':122        sub, nexttok = _parse(tokens)123        result = '%s(%s)' % (result, sub)124        if nexttok != ')':125            raise ValueError('unbalanced parenthesis in plural form')126    elif nexttok == 'n':127        result = '%s%s' % (result, nexttok)128    else:129        try:130            value = int(nexttok, 10)131        except ValueError:132            raise _error(nexttok) from None133        result = '%s%d' % (result, value)134    nexttok = next(tokens)135 136    j = 100137    while nexttok in _binary_ops:138        i = _binary_ops[nexttok]139        if i < priority:140            break141        # Break chained comparisons142        if i in (3, 4) and j in (3, 4):  # '==', '!=', '<', '>', '<=', '>='143            result = '(%s)' % result144        # Replace some C operators by their Python equivalents145        op = _c2py_ops.get(nexttok, nexttok)146        right, nexttok = _parse(tokens, i + 1)147        result = '%s %s %s' % (result, op, right)148        j = i149    if j == priority == 4:  # '<', '>', '<=', '>='150        result = '(%s)' % result151 152    if nexttok == '?' and priority <= 0:153        if_true, nexttok = _parse(tokens, 0)154        if nexttok != ':':155            raise _error(nexttok)156        if_false, nexttok = _parse(tokens)157        result = '%s if %s else %s' % (if_true, result, if_false)158        if priority == 0:159            result = '(%s)' % result160 161    return result, nexttok162 163def _as_int(n):164    try:165        i = round(n)166    except TypeError:167        raise TypeError('Plural value must be an integer, got %s' %168                        (n.__class__.__name__,)) from None169    import warnings170    warnings.warn('Plural value must be an integer, got %s' %171                  (n.__class__.__name__,),172                  DeprecationWarning, 4)173    return n174 175def c2py(plural):176    """Gets a C expression as used in PO files for plural forms and returns a177    Python function that implements an equivalent expression.178    """179 180    if len(plural) > 1000:181        raise ValueError('plural form expression is too long')182    try:183        result, nexttok = _parse(_tokenize(plural))184        if nexttok:185            raise _error(nexttok)186 187        depth = 0188        for c in result:189            if c == '(':190                depth += 1191                if depth > 20:192                    # Python compiler limit is about 90.193                    # The most complex example has 2.194                    raise ValueError('plural form expression is too complex')195            elif c == ')':196                depth -= 1197 198        ns = {'_as_int': _as_int}199        exec('''if True:200            def func(n):201                if not isinstance(n, int):202                    n = _as_int(n)203                return int(%s)204            ''' % result, ns)205        return ns['func']206    except RecursionError:207        # Recursion error can be raised in _parse() or exec().208        raise ValueError('plural form expression is too complex')209 210 211def _expand_lang(loc):212    import locale213    loc = locale.normalize(loc)214    COMPONENT_CODESET   = 1 << 0215    COMPONENT_TERRITORY = 1 << 1216    COMPONENT_MODIFIER  = 1 << 2217    # split up the locale into its base components218    mask = 0219    pos = loc.find('@')220    if pos >= 0:221        modifier = loc[pos:]222        loc = loc[:pos]223        mask |= COMPONENT_MODIFIER224    else:225        modifier = ''226    pos = loc.find('.')227    if pos >= 0:228        codeset = loc[pos:]229        loc = loc[:pos]230        mask |= COMPONENT_CODESET231    else:232        codeset = ''233    pos = loc.find('_')234    if pos >= 0:235        territory = loc[pos:]236        loc = loc[:pos]237        mask |= COMPONENT_TERRITORY238    else:239        territory = ''240    language = loc241    ret = []242    for i in range(mask+1):243        if not (i & ~mask):  # if all components for this combo exist ...244            val = language245            if i & COMPONENT_TERRITORY: val += territory246            if i & COMPONENT_CODESET:   val += codeset247            if i & COMPONENT_MODIFIER:  val += modifier248            ret.append(val)249    ret.reverse()250    return ret251 252 253 254class NullTranslations:255    def __init__(self, fp=None):256        self._info = {}257        self._charset = None258        self._output_charset = None259        self._fallback = None260        if fp is not None:261            self._parse(fp)262 263    def _parse(self, fp):264        pass265 266    def add_fallback(self, fallback):267        if self._fallback:268            self._fallback.add_fallback(fallback)269        else:270            self._fallback = fallback271 272    def gettext(self, message):273        if self._fallback:274            return self._fallback.gettext(message)275        return message276 277    def lgettext(self, message):278        import warnings279        warnings.warn('lgettext() is deprecated, use gettext() instead',280                      DeprecationWarning, 2)281        import locale282        if self._fallback:283            with warnings.catch_warnings():284                warnings.filterwarnings('ignore', r'.*\blgettext\b.*',285                                        DeprecationWarning)286                return self._fallback.lgettext(message)287        if self._output_charset:288            return message.encode(self._output_charset)289        return message.encode(locale.getpreferredencoding())290 291    def ngettext(self, msgid1, msgid2, n):292        if self._fallback:293            return self._fallback.ngettext(msgid1, msgid2, n)294        if n == 1:295            return msgid1296        else:297            return msgid2298 299    def lngettext(self, msgid1, msgid2, n):300        import warnings301        warnings.warn('lngettext() is deprecated, use ngettext() instead',302                      DeprecationWarning, 2)303        import locale304        if self._fallback:305            with warnings.catch_warnings():306                warnings.filterwarnings('ignore', r'.*\blngettext\b.*',307                                        DeprecationWarning)308                return self._fallback.lngettext(msgid1, msgid2, n)309        if n == 1:310            tmsg = msgid1311        else:312            tmsg = msgid2313        if self._output_charset:314            return tmsg.encode(self._output_charset)315        return tmsg.encode(locale.getpreferredencoding())316 317    def pgettext(self, context, message):318        if self._fallback:319            return self._fallback.pgettext(context, message)320        return message321 322    def npgettext(self, context, msgid1, msgid2, n):323        if self._fallback:324            return self._fallback.npgettext(context, msgid1, msgid2, n)325        if n == 1:326            return msgid1327        else:328            return msgid2329 330    def info(self):331        return self._info332 333    def charset(self):334        return self._charset335 336    def output_charset(self):337        import warnings338        warnings.warn('output_charset() is deprecated',339                      DeprecationWarning, 2)340        return self._output_charset341 342    def set_output_charset(self, charset):343        import warnings344        warnings.warn('set_output_charset() is deprecated',345                      DeprecationWarning, 2)346        self._output_charset = charset347 348    def install(self, names=None):349        import builtins350        builtins.__dict__['_'] = self.gettext351        if names is not None:352            allowed = {'gettext', 'lgettext', 'lngettext',353                       'ngettext', 'npgettext', 'pgettext'}354            for name in allowed & set(names):355                builtins.__dict__[name] = getattr(self, name)356 357 358class GNUTranslations(NullTranslations):359    # Magic number of .mo files360    LE_MAGIC = 0x950412de361    BE_MAGIC = 0xde120495362 363    # The encoding of a msgctxt and a msgid in a .mo file is364    # msgctxt + "\x04" + msgid (gettext version >= 0.15)365    CONTEXT = "%s\x04%s"366 367    # Acceptable .mo versions368    VERSIONS = (0, 1)369 370    def _get_versions(self, version):371        """Returns a tuple of major version, minor version"""372        return (version >> 16, version & 0xffff)373 374    def _parse(self, fp):375        """Override this method to support alternative .mo formats."""376        # Delay struct import for speeding up gettext import when .mo files377        # are not used.378        from struct import unpack379        filename = getattr(fp, 'name', '')380        # Parse the .mo file header, which consists of 5 little endian 32381        # bit words.382        self._catalog = catalog = {}383        self.plural = lambda n: int(n != 1) # germanic plural by default384        buf = fp.read()385        buflen = len(buf)386        # Are we big endian or little endian?387        magic = unpack('<I', buf[:4])[0]388        if magic == self.LE_MAGIC:389            version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])390            ii = '<II'391        elif magic == self.BE_MAGIC:392            version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])393            ii = '>II'394        else:395            raise OSError(0, 'Bad magic number', filename)396 397        major_version, minor_version = self._get_versions(version)398 399        if major_version not in self.VERSIONS:400            raise OSError(0, 'Bad version number ' + str(major_version), filename)401 402        # Now put all messages from the .mo file buffer into the catalog403        # dictionary.404        for i in range(0, msgcount):405            mlen, moff = unpack(ii, buf[masteridx:masteridx+8])406            mend = moff + mlen407            tlen, toff = unpack(ii, buf[transidx:transidx+8])408            tend = toff + tlen409            if mend < buflen and tend < buflen:410                msg = buf[moff:mend]411                tmsg = buf[toff:tend]412            else:413                raise OSError(0, 'File is corrupt', filename)414            # See if we're looking at GNU .mo conventions for metadata415            if mlen == 0:416                # Catalog description417                lastk = None418                for b_item in tmsg.split(b'\n'):419                    item = b_item.decode().strip()420                    if not item:421                        continue422                    # Skip over comment lines:423                    if item.startswith('#-#-#-#-#') and item.endswith('#-#-#-#-#'):424                        continue425                    k = v = None426                    if ':' in item:427                        k, v = item.split(':', 1)428                        k = k.strip().lower()429                        v = v.strip()430                        self._info[k] = v431                        lastk = k432                    elif lastk:433                        self._info[lastk] += '\n' + item434                    if k == 'content-type':435                        self._charset = v.split('charset=')[1]436                    elif k == 'plural-forms':437                        v = v.split(';')438                        plural = v[1].split('plural=')[1]439                        self.plural = c2py(plural)440            # Note: we unconditionally convert both msgids and msgstrs to441            # Unicode using the character encoding specified in the charset442            # parameter of the Content-Type header.  The gettext documentation443            # strongly encourages msgids to be us-ascii, but some applications444            # require alternative encodings (e.g. Zope's ZCML and ZPT).  For445            # traditional gettext applications, the msgid conversion will446            # cause no problems since us-ascii should always be a subset of447            # the charset encoding.  We may want to fall back to 8-bit msgids448            # if the Unicode conversion fails.449            charset = self._charset or 'ascii'450            if b'\x00' in msg:451                # Plural forms452                msgid1, msgid2 = msg.split(b'\x00')453                tmsg = tmsg.split(b'\x00')454                msgid1 = str(msgid1, charset)455                for i, x in enumerate(tmsg):456                    catalog[(msgid1, i)] = str(x, charset)457            else:458                catalog[str(msg, charset)] = str(tmsg, charset)459            # advance to next entry in the seek tables460            masteridx += 8461            transidx += 8462 463    def lgettext(self, message):464        import warnings465        warnings.warn('lgettext() is deprecated, use gettext() instead',466                      DeprecationWarning, 2)467        import locale468        missing = object()469        tmsg = self._catalog.get(message, missing)470        if tmsg is missing:471            if self._fallback:472                return self._fallback.lgettext(message)473            tmsg = message474        if self._output_charset:475            return tmsg.encode(self._output_charset)476        return tmsg.encode(locale.getpreferredencoding())477 478    def lngettext(self, msgid1, msgid2, n):479        import warnings480        warnings.warn('lngettext() is deprecated, use ngettext() instead',481                      DeprecationWarning, 2)482        import locale483        try:484            tmsg = self._catalog[(msgid1, self.plural(n))]485        except KeyError:486            if self._fallback:487                return self._fallback.lngettext(msgid1, msgid2, n)488            if n == 1:489                tmsg = msgid1490            else:491                tmsg = msgid2492        if self._output_charset:493            return tmsg.encode(self._output_charset)494        return tmsg.encode(locale.getpreferredencoding())495 496    def gettext(self, message):497        missing = object()498        tmsg = self._catalog.get(message, missing)499        if tmsg is missing:500            if self._fallback:501                return self._fallback.gettext(message)502            return message503        return tmsg504 505    def ngettext(self, msgid1, msgid2, n):506        try:507            tmsg = self._catalog[(msgid1, self.plural(n))]508        except KeyError:509            if self._fallback:510                return self._fallback.ngettext(msgid1, msgid2, n)511            if n == 1:512                tmsg = msgid1513            else:514                tmsg = msgid2515        return tmsg516 517    def pgettext(self, context, message):518        ctxt_msg_id = self.CONTEXT % (context, message)519        missing = object()520        tmsg = self._catalog.get(ctxt_msg_id, missing)521        if tmsg is missing:522            if self._fallback:523                return self._fallback.pgettext(context, message)524            return message525        return tmsg526 527    def npgettext(self, context, msgid1, msgid2, n):528        ctxt_msg_id = self.CONTEXT % (context, msgid1)529        try:530            tmsg = self._catalog[ctxt_msg_id, self.plural(n)]531        except KeyError:532            if self._fallback:533                return self._fallback.npgettext(context, msgid1, msgid2, n)534            if n == 1:535                tmsg = msgid1536            else:537                tmsg = msgid2538        return tmsg539 540 541# Locate a .mo file using the gettext strategy542def find(domain, localedir=None, languages=None, all=False):543    # Get some reasonable defaults for arguments that were not supplied544    if localedir is None:545        localedir = _default_localedir546    if languages is None:547        languages = []548        for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):549            val = os.environ.get(envar)550            if val:551                languages = val.split(':')552                break553        if 'C' not in languages:554            languages.append('C')555    # now normalize and expand the languages556    nelangs = []557    for lang in languages:558        for nelang in _expand_lang(lang):559            if nelang not in nelangs:560                nelangs.append(nelang)561    # select a language562    if all:563        result = []564    else:565        result = None566    for lang in nelangs:567        if lang == 'C':568            break569        mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)570        if os.path.exists(mofile):571            if all:572                result.append(mofile)573            else:574                return mofile575    return result576 577 578 579# a mapping between absolute .mo file path and Translation object580_translations = {}581_unspecified = ['unspecified']582 583def translation(domain, localedir=None, languages=None,584                class_=None, fallback=False, codeset=_unspecified):585    if class_ is None:586        class_ = GNUTranslations587    mofiles = find(domain, localedir, languages, all=True)588    if not mofiles:589        if fallback:590            return NullTranslations()591        from errno import ENOENT592        raise FileNotFoundError(ENOENT,593                                'No translation file found for domain', domain)594    # Avoid opening, reading, and parsing the .mo file after it's been done595    # once.596    result = None597    for mofile in mofiles:598        key = (class_, os.path.abspath(mofile))599        t = _translations.get(key)600        if t is None:601            with open(mofile, 'rb') as fp:602                t = _translations.setdefault(key, class_(fp))603        # Copy the translation object to allow setting fallbacks and604        # output charset. All other instance data is shared with the605        # cached object.606        # Delay copy import for speeding up gettext import when .mo files607        # are not used.608        import copy609        t = copy.copy(t)610        if codeset is not _unspecified:611            import warnings612            warnings.warn('parameter codeset is deprecated',613                          DeprecationWarning, 2)614            if codeset:615                with warnings.catch_warnings():616                    warnings.filterwarnings('ignore', r'.*\bset_output_charset\b.*',617                                            DeprecationWarning)618                    t.set_output_charset(codeset)619        if result is None:620            result = t621        else:622            result.add_fallback(t)623    return result624 625 626def install(domain, localedir=None, codeset=_unspecified, names=None):627    t = translation(domain, localedir, fallback=True, codeset=codeset)628    t.install(names)629 630 631 632# a mapping b/w domains and locale directories633_localedirs = {}634# a mapping b/w domains and codesets635_localecodesets = {}636# current global domain, `messages' used for compatibility w/ GNU gettext637_current_domain = 'messages'638 639 640def textdomain(domain=None):641    global _current_domain642    if domain is not None:643        _current_domain = domain644    return _current_domain645 646 647def bindtextdomain(domain, localedir=None):648    global _localedirs649    if localedir is not None:650        _localedirs[domain] = localedir651    return _localedirs.get(domain, _default_localedir)652 653 654def bind_textdomain_codeset(domain, codeset=None):655    import warnings656    warnings.warn('bind_textdomain_codeset() is deprecated',657                  DeprecationWarning, 2)658    global _localecodesets659    if codeset is not None:660        _localecodesets[domain] = codeset661    return _localecodesets.get(domain)662 663 664def dgettext(domain, message):665    try:666        t = translation(domain, _localedirs.get(domain, None))667    except OSError:668        return message669    return t.gettext(message)670 671def ldgettext(domain, message):672    import warnings673    warnings.warn('ldgettext() is deprecated, use dgettext() instead',674                  DeprecationWarning, 2)675    import locale676    codeset = _localecodesets.get(domain)677    try:678        with warnings.catch_warnings():679            warnings.filterwarnings('ignore', r'.*\bparameter codeset\b.*',680                                    DeprecationWarning)681            t = translation(domain, _localedirs.get(domain, None), codeset=codeset)682    except OSError:683        return message.encode(codeset or locale.getpreferredencoding())684    with warnings.catch_warnings():685        warnings.filterwarnings('ignore', r'.*\blgettext\b.*',686                                DeprecationWarning)687        return t.lgettext(message)688 689def dngettext(domain, msgid1, msgid2, n):690    try:691        t = translation(domain, _localedirs.get(domain, None))692    except OSError:693        if n == 1:694            return msgid1695        else:696            return msgid2697    return t.ngettext(msgid1, msgid2, n)698 699def ldngettext(domain, msgid1, msgid2, n):700    import warnings701    warnings.warn('ldngettext() is deprecated, use dngettext() instead',702                  DeprecationWarning, 2)703    import locale704    codeset = _localecodesets.get(domain)705    try:706        with warnings.catch_warnings():707            warnings.filterwarnings('ignore', r'.*\bparameter codeset\b.*',708                                    DeprecationWarning)709            t = translation(domain, _localedirs.get(domain, None), codeset=codeset)710    except OSError:711        if n == 1:712            tmsg = msgid1713        else:714            tmsg = msgid2715        return tmsg.encode(codeset or locale.getpreferredencoding())716    with warnings.catch_warnings():717        warnings.filterwarnings('ignore', r'.*\blngettext\b.*',718                                DeprecationWarning)719        return t.lngettext(msgid1, msgid2, n)720 721 722def dpgettext(domain, context, message):723    try:724        t = translation(domain, _localedirs.get(domain, None))725    except OSError:726        return message727    return t.pgettext(context, message)728 729 730def dnpgettext(domain, context, msgid1, msgid2, n):731    try:732        t = translation(domain, _localedirs.get(domain, None))733    except OSError:734        if n == 1:735            return msgid1736        else:737            return msgid2738    return t.npgettext(context, msgid1, msgid2, n)739 740 741def gettext(message):742    return dgettext(_current_domain, message)743 744def lgettext(message):745    import warnings746    warnings.warn('lgettext() is deprecated, use gettext() instead',747                  DeprecationWarning, 2)748    with warnings.catch_warnings():749        warnings.filterwarnings('ignore', r'.*\bldgettext\b.*',750                                DeprecationWarning)751        return ldgettext(_current_domain, message)752 753def ngettext(msgid1, msgid2, n):754    return dngettext(_current_domain, msgid1, msgid2, n)755 756def lngettext(msgid1, msgid2, n):757    import warnings758    warnings.warn('lngettext() is deprecated, use ngettext() instead',759                  DeprecationWarning, 2)760    with warnings.catch_warnings():761        warnings.filterwarnings('ignore', r'.*\bldngettext\b.*',762                                DeprecationWarning)763        return ldngettext(_current_domain, msgid1, msgid2, n)764 765 766def pgettext(context, message):767    return dpgettext(_current_domain, context, message)768 769 770def npgettext(context, msgid1, msgid2, n):771    return dnpgettext(_current_domain, context, msgid1, msgid2, n)772 773 774# dcgettext() has been deemed unnecessary and is not implemented.775 776# James Henstridge's Catalog constructor from GNOME gettext.  Documented usage777# was:778#779#    import gettext780#    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)781#    _ = cat.gettext782#    print _('Hello World')783 784# The resulting catalog object currently don't support access through a785# dictionary API, which was supported (but apparently unused) in GNOME786# gettext.787 788Catalog = translation789