#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2010 Zuza Software Foundation
#
# This file is part of the Translate Toolkit.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

"""Module to guess the language ISO code based on the 'Language-Team' entry in
the header of a Gettext PO file.
"""

import re

from translate.misc.typecheck import accepts, returns, IsOneOf
from translate.misc.typecheck.typeclasses import String

__all__ = ['LANG_TEAM_CONTACT_SNIPPETS', 'guess_language']

LANG_TEAM_REGEX = (
   ("@li.org", "([a-z_A-Z]{2,})@li.org", ["LL", "XX", "TEAM"]),
   ("translation-team",
    "translation-team-([a-z_A-Z]+)<EMAIL>", None),
   ("fedora-trans", "fedora-trans-([a-z_A-Z]+)@redhat.com", ["list"]),
   ("ubuntu-l10n", "ubuntu-l10n-([a-z_A-Z]+)<EMAIL>", None),
   ("translate-discuss",
    "translate-discuss-([a-z_A-Z]+)@lists.sourceforge.net", None),
   ("kde-i18n", "kde-i18n-([a-z_A-Z]+)@(?:lists\.|mail\.|)kde.org", ["doc"]),
   ("kde-l10n", "kde-l10n-([a-z_A-Z]+)<EMAIL>", None),
   ("fedoraproject", "<EMAIL>-([a-z_A-Z]+)@<EMAIL>", None),
   ("gnome.org", "gnome-([a-z_A-Z]+)-<EMAIL>", ["latin"]),
)
"""Data for regular expression based extraction.  The fieds are: prefilter
information, regex with single group that contains the language code,
postfilter."""

LANG_TEAM_CONTACT_SNIPPETS = {
    "af": ("<EMAIL>", "<NAME>",),
    "am": ("<EMAIL>", ),
    "ar": ("arabeyes.org", "Arabeyes", ),
    "as": ("<EMAIL>", ),
    "ast": ("<EMAIL>", "launchpad.net/~ubuntu-l10n-ast",
            "<EMAIL>", "Softastur",),
    "az": ("<EMAIL>", "<EMAIL>", u"<EMAIL>",),
    "az_IR": ("<EMAIL>",),
    "be": ("<EMAIL>", "<EMAIL>", "<EMAIL>",),
    "be@latin": ("<EMAIL>", "be-latin.open-tran.eu",),
    "bg": ("<EMAIL>", "<EMAIL>", ),
    "bn": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", ),
    "bn_IN": ("<EMAIL>", ),
    "br": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "bs": ("<EMAIL>", "<EMAIL>", ),
    "ca": ("@<EMAIL>ala.<EMAIL>",),
    "crh": ("<EMAIL>", ),
    "cs": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", "<EMAIL>"),
    "cy": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "da": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "de": ("<EMAIL>", "<EMAIL>", ),
    "dz": ("<EMAIL>", "<EMAIL>", ),
    "el": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "en_AU": ("<EMAIL>", ),
    "en_CA": ("<EMAIL>", "<EMAIL>", ),
    "en_GB": ("<EMAIL>", ),
    "en@shaw": ("<EMAIL>",
                "ubuntu-l10n-<EMAIL>", ),
    "eo": ("<EMAIL>",
           "<EMAIL>",
           "<EMAIL>",
           "eo-tradukado.tuxfamily.org", ),
    "es": ("<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "et": ("<EMAIL>", "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "eu": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "fa": ("<EMAIL>", "Farsiweb.info", ),
    "fi": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "fr": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "ga": ("<EMAIL>",
           "<EMAIL>", ),
    "gl": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "gu": ("<EMAIL>", ),
    "he": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "hi": ("<EMAIL>",
           "<EMAIL>", ),
    "hr": ("translator-shop.org", "<EMAIL>", ),
    "hu": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "id": ("@id.gnome.org", "@gnome.linux.or.id", "<EMAIL>",
           "linux.or.id", "<EMAIL>"),
    "io": ("<EMAIL>", ),
    "is": ("<EMAIL>", "<EMAIL>", "<EMAIL>", ),
    "it": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "ja": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "ka": ("<EMAIL>",
           "Ubuntu-<EMAIL>", ),
    "kk": ("<EMAIL>", ),
    "km": ("@khmeros.info", ),
    "kn": ("<EMAIL>", ),
    "ko": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", ),
    "ks": ("<EMAIL>", ),
    "ku": ("<EMAIL>", ),
    "ky": ("<EMAIL>", "<EMAIL>", ),
    "la": ("<EMAIL>", ),
    "li": ("<EMAIL>", ),
    "lt": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "lv": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "mai": ("maithili.sf.net", ),
    "mg": ("<EMAIL>", ),
    "mi": ("<EMAIL>", ),
    "mk": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", ),
    "ml": ("<EMAIL>", ),
    "mn": ("openmn-", "openmn.org", ),
    "ms": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "nb": ("<EMAIL>", ),
    "nds": ("<EMAIL>", ),
    "ne": ("<EMAIL>", ),
    "nl": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "nn": ("<EMAIL>", ),
    "nso": ("<EMAIL>", ),
    "or": ("<EMAIL>", "<EMAIL>", ),
    "pa": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "pl": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "ps": ("<EMAIL>", ),
    "pt": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "pt_BR": ("<EMAIL>",
              "<EMAIL>", "<EMAIL>",
              "<EMAIL>", "<EMAIL>",
              "<EMAIL>",
              "<EMAIL>",
              "<EMAIL>", ),
    "ro": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "ru": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "sk": ("<EMAIL>", "<EMAIL>", ),
    "sl": ("<EMAIL>", ),
    "sq": ("<EMAIL>",
           "<EMAIL>", ),
    "sr": ("<EMAIL>", "<EMAIL>", ),
    "sv": ("<EMAIL>", "<EMAIL>", ),
    "ta": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "te": ("<EMAIL>",
           "<EMAIL>", ),
    "th": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "l10n.opentle.org", ),
    "tk": ("<EMAIL>", ),
    "tl": ("<EMAIL>", ),
    "tr": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "tt": ("<EMAIL>", ),
    "ug": ("<EMAIL>", ),
    "uk": ("<EMAIL>", ),
    "ur": ("<EMAIL>", "<EMAIL>", ),
    "ve": ("<EMAIL>", ),
    "vi": ("<EMAIL>", "<EMAIL>", ),
    "wa": ("linux-wa@", ),
    "xh": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "zh_CN": ("<EMAIL>",
              "<EMAIL>",
              "<EMAIL>",
              "<EMAIL>", ),
    "zh_TW": ("<EMAIL>", "<EMAIL>",
              "<EMAIL>", "<EMAIL>", ),
    "zu": ("<EMAIL>", ),
}
"""Language codes with snippets of contact information that can be used to
uniquely identify the language"""

LANG_TEAM_LANGUAGE_SNIPPETS = {
    "af": ("Afrikaans",),
    "am": ("Amharic",),
    "ang": ("Old English",),
    "ar": ("Arabic", ),
    "as": ("Assamese", ),
    "ast": ("Asturian", ),
    "az": ("Azerbaijani", u"Azərbaycan", ),
    "bg": ("Bulgarian", ),
    "be@latin": ("Belarusian Latin", ),
    "be": ("Belarusian", "Belorussian", ),
    "bn_IN": ("Bengali (India)", "Bengali INDIA", "Bengali India", ),
    "bn": ("Bangladeshi", "Bengali", ),
    "br": ("Breton", "Britton", ),
    "bs": ("Bosanski", "Bosnian", ),
    "byn": ("Blin", ),
    "ca": ("Catalan", ),
    "ckb": ("Kurdish (Sorani)", ),
    "crh": ("Crimean Tatar", "Crimean Turkish", ),
    "cs": ("Czech", ),
    "cy": ("Cymru", "Welsh", ),
    "da": ("Danish", "Dansk", ),
    "de": ("Deutsch", "German", ),
    "dz": ("Dzongkha", ),
    "el": ("Greek", ),
    "en_GB": ("British English", "en_GB", "English (Great Britain)", ),
    "eo": ("Esperanto", ),
    "es": ("Spanish", "es_ES", u"Español", ),
    "et": ("Eesti", "Estonian", ),
    "eu": ("Basque", "Euskara", ),
    "fa": ("Persian", ),
    "fi": ("Finnish", "Suomi", ),
    "fo": ("Faroese", ),
    "fr": ("French", u"Français", ),
    "fur": ("Friulian", ),
    "ga": ("Irish", ),
    "gez": ("Geez", ),
    "gl": ("Galego", "Galician", "Gallegan", "gl_ES", ),
    "gu": ("Gujarati", ),
    "haw": ("Hawaiian", ),
    "he": ("Hebrew", ),
    "hi": ("Hindi", ),
    "hr": ("Croatian", ),
    "hu": ("Hungarian", ),
    "hy": ("Armenian", ),
    "ia": ("Interlingua", ),
    "id": ("Bahasa Indonesia", "Indonesia", "Indonesian", ),
    "ig": ("Igbo", ),
    "is": ("Icelandic", ),
    "it": ("Italian", ),
    "ja": ("Japanese", ),
    "ka": ("Georgian", ),
    "kk": ("Kazakh", ),
    "km": ("Khmer", ),
    "kn": ("Kannada", ),
    "ko": ("Korean", "Hangul", ),
    "kok": ("Konkani", ),
    "ks": ("Kashmiri", ),
    "ku": ("Kurdish", ),
    "ky": ("Kitghiz", "Kirghiz", ),
    "lg": ("Luganda", ),
    "li": ("Limburgish", ),
    "lt": ("Lithuanian", ),
    "lv": ("Latvian", "lv_LV", "Valoda", u"Latviešu", ),
    "mal": ("Malayalam", ),
    "mg": ("Malagasy", ),
    "mi": ("Maori", ),
    "mk": ("Macedonian", ),
    "ml": ("Malayalam", ),
    "mn": ("Mongolian", ),
    "mt": ("Marathi", ),
    "ms": ("Malay", "Bahasa Melayu", ),
    "my": ("Burmese", ),
    "nb": ("Norwegian Bokmaal", u"Norsk bokmål", u"Norwegian Bokmål",
           u"Norwegian bokmål", ),
    "nds": ("Low Saxon", ),
    "nl": ("Dutch", "Nederlands", ),
    "nn": ("Norwegian nynorsk", "Nynorsk", ),
    "oc": ("Occitan", ),
    "or": ("Oriya", ),
    "pa": ("Punjabi", "Panjabi", ),
    "pl": ("Polish", ),
    "ps": ("Pashto", "Pushto", ),
    "pt_BR": ("Brazilian Portuguese", u"Português/Brasil",
              u"Português do Brasil", ),
    "pt": ("Portuguese", ),
    "rm": ("Rhaeto-Romance", ),
    "ro": ("Romania", "Romanian", u"Română", ),
    "ru": ("Russian", ),
    "si": ("Sinhala", "Sinhalese", ),
    "sk": ("Slovak", ),
    "sl": ("Slovene", "Slovenian", ),
    "so": ("Somali", ),
    "sq": ("Albanian", ),
    "sr": ("Serbian", ),
    "sv": ("Swedish", ),
    "sw": ("Swahili", ),
    "ta": ("Tamil", ),
    "te": ("Telugu", ),
    "tet": ("Tetum", ),
    "tg": ("Tajik", ),
    "th": ("Thai", ),
    "ti": ("Tigrinya", ),
    "tig": ("Tigre", ),
    "tl": ("Tagalog", ),
    "tr": ("Turkish", u"Türkçe", u"Türkiye", ),
    "tt": ("Tatarish", ),
    "ug": ("Uighur", ),
    "uk": ("Ukrainian", ),
    "ur": ("Urdu", ),
    "uz": ("Uzbek", ),
    "ve": ("Venda", u"Tshivenḓa", "Tshivenda", ),
    "vi": ("Vietnamese", ),
    "wa": ("Walloon", ),
    "wal": ("Walamo", ),
    "wo": ("Wolof", ),
    "xh": ("Xhosa", "IsiXhosa", "isiXhosa", ),
    "yi": ("Yiddish", ),
    "yo": ("Yoruba", ),
    "zh_CN": ("Chinese Simplified", "Chinese/Simplified",
              "Chinese (simplified)", "Simplified Chinese", ),
    "zh_HK": ("Chinese (Hong Kong)", ),
    "zh_TW": ("Chinese (traditional)", "Chinese/Traditional",
              "Traditional Chinese", ),
}
"""Language codes with snippets of language names, including English, native
spelling and varients, that can be used to uniquely identify the language"""


def _regex_guesser(prefilter, regex, string, postfilter=None):
    """Use regular expressions to extract the language team

    :param prefilter: simple filter to apply before attempting the regex
    :param regex: regular expression with one group that will contain
    the language code
    :param string: the language team string that should be examined
    :param postfilter: filter to apply to reject any potential matches
    after they have been retreived by the regex
    :return: ISO language code for the found language
    """
    # TODO instead of a posfilter, have a dictionary of transform rules
    # e.g. for debian-l10n-albanian a dict of {'russian': 'ru' would allow
    # transformation.  {'default': None} would ensure that anything we
    # don't understand gets ignored.  Or {'default': 'nothing'} means to
    # nothing.
    if prefilter in string:
        found = re.search(regex, string)
        if found:
            regex_lang = found.groups()[0]
        else:
            return None
        if postfilter is not None and regex_lang in postfilter:
            return None
        if regex_lang and regex_lang != 'en':
            return regex_lang
    return None


def _nofilter(text):
    """Return the supplied text unchanged"""
    return text


def _lower(text):
    """Convert the supplied text to lowercase"""
    return text.lower()


def _snippet_guesser(snippets_dict, string, filter_=_nofilter):
    """Guess the language based on a snippet of text in the language team
    string.

    :param snippets_dict: A dict of snippets that can be used to identify a
    language in the format {'lang': ('snippet1', 'snippet2'), 'lang2'...}
    :param string: The language string to be analysed
    :param filter_: a function to be applied to the string and snippets
    before examination
    """
    string = filter_(string)
    for possible_lang, snippets in snippets_dict.iteritems():
        for snippet in snippets:
            if filter_(snippet) in string:
                return possible_lang
    return None


@accepts(unicode)
@returns(IsOneOf(String, type(None)))
def guess_language(team_string):
    """Gueses the language of a PO file based on the Language-Team entry"""

    for prefilter, regex, postfilter in LANG_TEAM_REGEX:
        lang = _regex_guesser(prefilter, regex, team_string, postfilter)
        if lang:
            break

    if not lang:
        lang = _snippet_guesser(LANG_TEAM_CONTACT_SNIPPETS, team_string,
                                _lower)

    if not lang:
        lang = _snippet_guesser(LANG_TEAM_LANGUAGE_SNIPPETS, team_string)

    # TODO Maybe clean everything and see of we have a language code only

    if not lang:
        #print (u"MISSED: '%s'" % team_string).encode('utf-8')
        return None
    return lang

if __name__ == "__main__":
    from sys import argv
    from translate.storage import factory
    for fname in argv[1:]:
        store = factory.getobject(fname)
        print fname, guess_language(store.parseheader().get('Language-Team', u""))

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2010 Zuza Software Foundation
#
# This file is part of the Translate Toolkit.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

"""Module to guess the language ISO code based on the 'Language-Team' entry in
the header of a Gettext PO file.
"""

import re

from translate.misc.typecheck import accepts, returns, IsOneOf
from translate.misc.typecheck.typeclasses import String

__all__ = ['LANG_TEAM_CONTACT_SNIPPETS', 'guess_language']

LANG_TEAM_REGEX = (
   ("@li.org", "([a-z_A-Z]{2,})@li.org", ["LL", "XX", "TEAM"]),
   ("translation-team",
    "translation-team-([a-z_A-Z]+)<EMAIL>", None),
   ("fedora-trans", "fedora-trans-([a-z_A-Z]+)@redhat.com", ["list"]),
   ("ubuntu-l10n", "ubuntu-l10n-([a-z_A-Z]+)<EMAIL>", None),
   ("translate-discuss",
    "translate-discuss-([a-z_A-Z]+)@lists.sourceforge.net", None),
   ("kde-i18n", "kde-i18n-([a-z_A-Z]+)@(?:lists\.|mail\.|)kde.org", ["doc"]),
   ("kde-l10n", "kde-l10n-([a-z_A-Z]+)<EMAIL>", None),
   ("fedoraproject", "<EMAIL>-([a-z_A-Z]+)@<EMAIL>", None),
   ("gnome.org", "gnome-([a-z_A-Z]+)-<EMAIL>", ["latin"]),
)
"""Data for regular expression based extraction.  The fieds are: prefilter
information, regex with single group that contains the language code,
postfilter."""

LANG_TEAM_CONTACT_SNIPPETS = {
    "af": ("<EMAIL>", "<NAME>",),
    "am": ("<EMAIL>", ),
    "ar": ("arabeyes.org", "Arabeyes", ),
    "as": ("<EMAIL>", ),
    "ast": ("<EMAIL>", "launchpad.net/~ubuntu-l10n-ast",
            "<EMAIL>", "Softastur",),
    "az": ("<EMAIL>", "<EMAIL>", u"<EMAIL>",),
    "az_IR": ("<EMAIL>",),
    "be": ("<EMAIL>", "<EMAIL>", "<EMAIL>",),
    "be@latin": ("<EMAIL>", "be-latin.open-tran.eu",),
    "bg": ("<EMAIL>", "<EMAIL>", ),
    "bn": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", ),
    "bn_IN": ("<EMAIL>", ),
    "br": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "bs": ("<EMAIL>", "<EMAIL>", ),
    "ca": ("@<EMAIL>ala.<EMAIL>",),
    "crh": ("<EMAIL>", ),
    "cs": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", "<EMAIL>"),
    "cy": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "da": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "de": ("<EMAIL>", "<EMAIL>", ),
    "dz": ("<EMAIL>", "<EMAIL>", ),
    "el": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "en_AU": ("<EMAIL>", ),
    "en_CA": ("<EMAIL>", "<EMAIL>", ),
    "en_GB": ("<EMAIL>", ),
    "en@shaw": ("<EMAIL>",
                "ubuntu-l10n-<EMAIL>", ),
    "eo": ("<EMAIL>",
           "<EMAIL>",
           "<EMAIL>",
           "eo-tradukado.tuxfamily.org", ),
    "es": ("<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "et": ("<EMAIL>", "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "eu": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "fa": ("<EMAIL>", "Farsiweb.info", ),
    "fi": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "fr": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "ga": ("<EMAIL>",
           "<EMAIL>", ),
    "gl": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "gu": ("<EMAIL>", ),
    "he": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "hi": ("<EMAIL>",
           "<EMAIL>", ),
    "hr": ("translator-shop.org", "<EMAIL>", ),
    "hu": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "id": ("@id.gnome.org", "@gnome.linux.or.id", "<EMAIL>",
           "linux.or.id", "<EMAIL>"),
    "io": ("<EMAIL>", ),
    "is": ("<EMAIL>", "<EMAIL>", "<EMAIL>", ),
    "it": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "ja": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "ka": ("<EMAIL>",
           "Ubuntu-<EMAIL>", ),
    "kk": ("<EMAIL>", ),
    "km": ("@khmeros.info", ),
    "kn": ("<EMAIL>", ),
    "ko": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", ),
    "ks": ("<EMAIL>", ),
    "ku": ("<EMAIL>", ),
    "ky": ("<EMAIL>", "<EMAIL>", ),
    "la": ("<EMAIL>", ),
    "li": ("<EMAIL>", ),
    "lt": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "lv": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "mai": ("maithili.sf.net", ),
    "mg": ("<EMAIL>", ),
    "mi": ("<EMAIL>", ),
    "mk": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>",
           "<EMAIL>", ),
    "ml": ("<EMAIL>", ),
    "mn": ("openmn-", "openmn.org", ),
    "ms": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "nb": ("<EMAIL>", ),
    "nds": ("<EMAIL>", ),
    "ne": ("<EMAIL>", ),
    "nl": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "nn": ("<EMAIL>", ),
    "nso": ("<EMAIL>", ),
    "or": ("<EMAIL>", "<EMAIL>", ),
    "pa": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "pl": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "ps": ("<EMAIL>", ),
    "pt": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "pt_BR": ("<EMAIL>",
              "<EMAIL>", "<EMAIL>",
              "<EMAIL>", "<EMAIL>",
              "<EMAIL>",
              "<EMAIL>",
              "<EMAIL>", ),
    "ro": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "ru": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "sk": ("<EMAIL>", "<EMAIL>", ),
    "sl": ("<EMAIL>", ),
    "sq": ("<EMAIL>",
           "<EMAIL>", ),
    "sr": ("<EMAIL>", "<EMAIL>", ),
    "sv": ("<EMAIL>", "<EMAIL>", ),
    "ta": ("<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "te": ("<EMAIL>",
           "<EMAIL>", ),
    "th": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>",
           "l10n.opentle.org", ),
    "tk": ("<EMAIL>", ),
    "tl": ("<EMAIL>", ),
    "tr": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", "<EMAIL>", ),
    "tt": ("<EMAIL>", ),
    "ug": ("<EMAIL>", ),
    "uk": ("<EMAIL>", ),
    "ur": ("<EMAIL>", "<EMAIL>", ),
    "ve": ("<EMAIL>", ),
    "vi": ("<EMAIL>", "<EMAIL>", ),
    "wa": ("linux-wa@", ),
    "xh": ("<EMAIL>", "<EMAIL>",
           "<EMAIL>", ),
    "zh_CN": ("<EMAIL>",
              "<EMAIL>",
              "<EMAIL>",
              "<EMAIL>", ),
    "zh_TW": ("<EMAIL>", "<EMAIL>",
              "<EMAIL>", "<EMAIL>", ),
    "zu": ("<EMAIL>", ),
}
"""Language codes with snippets of contact information that can be used to
uniquely identify the language"""

LANG_TEAM_LANGUAGE_SNIPPETS = {
    "af": ("Afrikaans",),
    "am": ("Amharic",),
    "ang": ("Old English",),
    "ar": ("Arabic", ),
    "as": ("Assamese", ),
    "ast": ("Asturian", ),
    "az": ("Azerbaijani", u"Azərbaycan", ),
    "bg": ("Bulgarian", ),
    "be@latin": ("Belarusian Latin", ),
    "be": ("Belarusian", "Belorussian", ),
    "bn_IN": ("Bengali (India)", "Bengali INDIA", "Bengali India", ),
    "bn": ("Bangladeshi", "Bengali", ),
    "br": ("Breton", "Britton", ),
    "bs": ("Bosanski", "Bosnian", ),
    "byn": ("Blin", ),
    "ca": ("Catalan", ),
    "ckb": ("Kurdish (Sorani)", ),
    "crh": ("Crimean Tatar", "Crimean Turkish", ),
    "cs": ("Czech", ),
    "cy": ("Cymru", "Welsh", ),
    "da": ("Danish", "Dansk", ),
    "de": ("Deutsch", "German", ),
    "dz": ("Dzongkha", ),
    "el": ("Greek", ),
    "en_GB": ("British English", "en_GB", "English (Great Britain)", ),
    "eo": ("Esperanto", ),
    "es": ("Spanish", "es_ES", u"Español", ),
    "et": ("Eesti", "Estonian", ),
    "eu": ("Basque", "Euskara", ),
    "fa": ("Persian", ),
    "fi": ("Finnish", "Suomi", ),
    "fo": ("Faroese", ),
    "fr": ("French", u"Français", ),
    "fur": ("Friulian", ),
    "ga": ("Irish", ),
    "gez": ("Geez", ),
    "gl": ("Galego", "Galician", "Gallegan", "gl_ES", ),
    "gu": ("Gujarati", ),
    "haw": ("Hawaiian", ),
    "he": ("Hebrew", ),
    "hi": ("Hindi", ),
    "hr": ("Croatian", ),
    "hu": ("Hungarian", ),
    "hy": ("Armenian", ),
    "ia": ("Interlingua", ),
    "id": ("Bahasa Indonesia", "Indonesia", "Indonesian", ),
    "ig": ("Igbo", ),
    "is": ("Icelandic", ),
    "it": ("Italian", ),
    "ja": ("Japanese", ),
    "ka": ("Georgian", ),
    "kk": ("Kazakh", ),
    "km": ("Khmer", ),
    "kn": ("Kannada", ),
    "ko": ("Korean", "Hangul", ),
    "kok": ("Konkani", ),
    "ks": ("Kashmiri", ),
    "ku": ("Kurdish", ),
    "ky": ("Kitghiz", "Kirghiz", ),
    "lg": ("Luganda", ),
    "li": ("Limburgish", ),
    "lt": ("Lithuanian", ),
    "lv": ("Latvian", "lv_LV", "Valoda", u"Latviešu", ),
    "mal": ("Malayalam", ),
    "mg": ("Malagasy", ),
    "mi": ("Maori", ),
    "mk": ("Macedonian", ),
    "ml": ("Malayalam", ),
    "mn": ("Mongolian", ),
    "mt": ("Marathi", ),
    "ms": ("Malay", "Bahasa Melayu", ),
    "my": ("Burmese", ),
    "nb": ("Norwegian Bokmaal", u"Norsk bokmål", u"Norwegian Bokmål",
           u"Norwegian bokmål", ),
    "nds": ("Low Saxon", ),
    "nl": ("Dutch", "Nederlands", ),
    "nn": ("Norwegian nynorsk", "Nynorsk", ),
    "oc": ("Occitan", ),
    "or": ("Oriya", ),
    "pa": ("Punjabi", "Panjabi", ),
    "pl": ("Polish", ),
    "ps": ("Pashto", "Pushto", ),
    "pt_BR": ("Brazilian Portuguese", u"Português/Brasil",
              u"Português do Brasil", ),
    "pt": ("Portuguese", ),
    "rm": ("Rhaeto-Romance", ),
    "ro": ("Romania", "Romanian", u"Română", ),
    "ru": ("Russian", ),
    "si": ("Sinhala", "Sinhalese", ),
    "sk": ("Slovak", ),
    "sl": ("Slovene", "Slovenian", ),
    "so": ("Somali", ),
    "sq": ("Albanian", ),
    "sr": ("Serbian", ),
    "sv": ("Swedish", ),
    "sw": ("Swahili", ),
    "ta": ("Tamil", ),
    "te": ("Telugu", ),
    "tet": ("Tetum", ),
    "tg": ("Tajik", ),
    "th": ("Thai", ),
    "ti": ("Tigrinya", ),
    "tig": ("Tigre", ),
    "tl": ("Tagalog", ),
    "tr": ("Turkish", u"Türkçe", u"Türkiye", ),
    "tt": ("Tatarish", ),
    "ug": ("Uighur", ),
    "uk": ("Ukrainian", ),
    "ur": ("Urdu", ),
    "uz": ("Uzbek", ),
    "ve": ("Venda", u"Tshivenḓa", "Tshivenda", ),
    "vi": ("Vietnamese", ),
    "wa": ("Walloon", ),
    "wal": ("Walamo", ),
    "wo": ("Wolof", ),
    "xh": ("Xhosa", "IsiXhosa", "isiXhosa", ),
    "yi": ("Yiddish", ),
    "yo": ("Yoruba", ),
    "zh_CN": ("Chinese Simplified", "Chinese/Simplified",
              "Chinese (simplified)", "Simplified Chinese", ),
    "zh_HK": ("Chinese (Hong Kong)", ),
    "zh_TW": ("Chinese (traditional)", "Chinese/Traditional",
              "Traditional Chinese", ),
}
"""Language codes with snippets of language names, including English, native
spelling and varients, that can be used to uniquely identify the language"""


def _regex_guesser(prefilter, regex, string, postfilter=None):
    """Use regular expressions to extract the language team

    :param prefilter: simple filter to apply before attempting the regex
    :param regex: regular expression with one group that will contain
    the language code
    :param string: the language team string that should be examined
    :param postfilter: filter to apply to reject any potential matches
    after they have been retreived by the regex
    :return: ISO language code for the found language
    """
    # TODO instead of a posfilter, have a dictionary of transform rules
    # e.g. for debian-l10n-albanian a dict of {'russian': 'ru' would allow
    # transformation.  {'default': None} would ensure that anything we
    # don't understand gets ignored.  Or {'default': 'nothing'} means to
    # nothing.
    if prefilter in string:
        found = re.search(regex, string)
        if found:
            regex_lang = found.groups()[0]
        else:
            return None
        if postfilter is not None and regex_lang in postfilter:
            return None
        if regex_lang and regex_lang != 'en':
            return regex_lang
    return None


def _nofilter(text):
    """Return the supplied text unchanged"""
    return text


def _lower(text):
    """Convert the supplied text to lowercase"""
    return text.lower()


def _snippet_guesser(snippets_dict, string, filter_=_nofilter):
    """Guess the language based on a snippet of text in the language team
    string.

    :param snippets_dict: A dict of snippets that can be used to identify a
    language in the format {'lang': ('snippet1', 'snippet2'), 'lang2'...}
    :param string: The language string to be analysed
    :param filter_: a function to be applied to the string and snippets
    before examination
    """
    string = filter_(string)
    for possible_lang, snippets in snippets_dict.iteritems():
        for snippet in snippets:
            if filter_(snippet) in string:
                return possible_lang
    return None


@accepts(unicode)
@returns(IsOneOf(String, type(None)))
def guess_language(team_string):
    """Gueses the language of a PO file based on the Language-Team entry"""

    for prefilter, regex, postfilter in LANG_TEAM_REGEX:
        lang = _regex_guesser(prefilter, regex, team_string, postfilter)
        if lang:
            break

    if not lang:
        lang = _snippet_guesser(LANG_TEAM_CONTACT_SNIPPETS, team_string,
                                _lower)

    if not lang:
        lang = _snippet_guesser(LANG_TEAM_LANGUAGE_SNIPPETS, team_string)

    # TODO Maybe clean everything and see of we have a language code only

    if not lang:
        #print (u"MISSED: '%s'" % team_string).encode('utf-8')
        return None
    return lang

if __name__ == "__main__":
    from sys import argv
    from translate.storage import factory
    for fname in argv[1:]:
        store = factory.getobject(fname)
        print fname, guess_language(store.parseheader().get('Language-Team', u""))

#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright 2010 Zuza Software Foundation # # This file is part of the Translate Toolkit. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, see <http://www.gnu.org/licenses/>. Module to guess the language ISO code based on the 'Language-Team' entry in the header of a Gettext PO file. Data for regular expression based extraction.  The fieds are: prefilter information, regex with single group that contains the language code, postfilter. Language codes with snippets of contact information that can be used to uniquely identify the language Language codes with snippets of language names, including English, native spelling and varients, that can be used to uniquely identify the language Use regular expressions to extract the language team      :param prefilter: simple filter to apply before attempting the regex     :param regex: regular expression with one group that will contain     the language code     :param string: the language team string that should be examined     :param postfilter: filter to apply to reject any potential matches     after they have been retreived by the regex     :return: ISO language code for the found language # TODO instead of a posfilter, have a dictionary of transform rules # e.g. for debian-l10n-albanian a dict of {'russian': 'ru' would allow # transformation.  {'default': None} would ensure that anything we # don't understand gets ignored.  Or {'default': 'nothing'} means to # nothing. Return the supplied text unchanged Convert the supplied text to lowercase Guess the language based on a snippet of text in the language team     string.      :param snippets_dict: A dict of snippets that can be used to identify a     language in the format {'lang': ('snippet1', 'snippet2'), 'lang2'...}     :param string: The language string to be analysed     :param filter_: a function to be applied to the string and snippets     before examination Gueses the language of a PO file based on the Language-Team entry # TODO Maybe clean everything and see of we have a language code only #print (u"MISSED: '%s'" % team_string).encode('utf-8')