#!/usr/bin/python

# Converts .xlf files into .json files for use at http://translatewiki.net.
#
# Copyright 2013 Google Inc.
# http://blockly.googlecode.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import codecs
from datetime import datetime
import os
import re
import sys
from xml.dom import minidom
from common import InputError
from common import parse_trans_unit

# Global variables
args = None      # parsed command-line arguments


def _create_qqq_file():
    """Creates a qqq.json file with message documentation for translatewiki.net.

    The file consists of key-value pairs, where the keys are message ids and
    the values are descriptions for the translators of the messages.
    What documentation exists for the format can be found at:
    http://translatewiki.net/wiki/Translating:Localisation_for_developers#Message_documentation

    The file should be closed by _close_qqq_file().

    Returns:
        A pointer to a file to which a left brace and newline have been written.

    Raises:
        IOError: An error occurred while opening or writing the file.
    """
    qqq_file_name = os.path.join(os.curdir, args.output_dir, 'qqq.json')
    qqq_file = codecs.open(qqq_file_name, 'w', 'utf-8')
    print 'Created file: ' + qqq_file_name
    qqq_file.write('{\n')
    return qqq_file


def _close_qqq_file(qqq_file):
    """Closes a qqq.json file created and opened by _create_qqq_file().

    This writes the final newlines and right brace.

    Args:
        qqq_file: A file created by _create_qqq_file().

    Raises:
        IOError: An error occurred while writing to or closing the file.
    """
    qqq_file.write('\n}\n')
    qqq_file.close()


def _create_key_file():
    """Creates a keys.json file mapping Closure keys to Blockly keys.

    Raises:
        IOError: An error occurred while creating the file.
    """
    key_file_name = os.path.join(os.curdir, args.output_dir, 'keys.json')
    key_file = open(key_file_name, 'w')
    key_file.write('{\n')
    print 'Created file: ' + key_file_name
    return key_file


def _close_key_file(key_file):
    """Closes a key file created and opened with _create_key_file().

    Args:
        key_file: A file created by _create_key_file().

    Raises:
        IOError: An error occurred while writing to or closing the file.
    """
    key_file.write('\n}\n')
    key_file.close()


def _create_lang_file():
    """Creates a <lang>.json file for translatewiki.net.

    The file consists of metadata, followed by key-value pairs, where the keys
    are message ids and the values are the messages in the language specified
    by the corresponding command-line argument.  The file should be closed by
    _close_lang_file().

    Returns:
        A pointer to a file to which the metadata has been written.

    Raises:
        IOError: An error occurred while opening or writing the file.
    """
    lang_file_name = os.path.join(
        os.curdir, args.output_dir, args.lang + '.json')
    lang_file = codecs.open(lang_file_name, 'w', 'utf-8')
    print 'Created file: ' + lang_file_name
    # string.format doesn't like printing braces, so break up our writes.
    lang_file.write('{\n    "@metadata": {')
    lang_file.write("""
        "author": "{0}",
        "lastupdated": "{1}",
        "locale": "{2}",
        "messagedocumentation" : "qqq"
""".format(args.author, str(datetime.now()), args.lang))
    lang_file.write('    },\n')
    return lang_file


def _close_lang_file(lang_file):
    """Closes a <lang>.json file created with _create_lang_file().

    This also writes the terminating left brace and newline.

    Args:
        lang_file: A file opened with _create_lang_file().

    Raises:
        IOError: An error occurred while writing to or closing the file.
    """
    lang_file.write('\n}\n')
    lang_file.close()


def _process_file(filename):
    """Builds list of translation units from input file.

    Each translation unit in the input file includes:
    - an id (opaquely generated by Soy)
    - the Blockly name for the message
    - the text in the source language (generally English)
    - a description for the translator

    The Soy and Blockly ids are joined with a hyphen and serve as the
    keys in both output files.  The value is the corresponding text (in the
    <lang>.json file) or the description (in the qqq.json file).

    Args:
        filename: The name of an .xlf file produced by Closure.

    Raises:
        IOError: An I/O error occurred with an input or output file.
        InputError: The input file could not be parsed or lacked required
            fields.

    Returns:
        A list of dictionaries produced by parse_trans_unit().
    """
    try:
        results = []  # list of dictionaries (return value)
        names = []    # list of names of encountered keys (local variable)
        try:
            parsed_xml = minidom.parse(filename)
        except IOError:
            # Don't get caught by below handler
            raise
        except Exception, e:
            print
            raise InputError(filename, str(e))

        # Make sure needed fields are present and non-empty.
        for trans_unit in parsed_xml.getElementsByTagName('trans-unit'):
            unit = parse_trans_unit(trans_unit)
            for key in ['description', 'meaning', 'source']:
                if not key in unit or not unit[key]:
                    raise InputError(filename + ':' + unit['key'],
                                     key + ' not found')
            if unit['description'].lower() == 'ibid':
              if unit['meaning'] not in names:
                # If the term has not already been described, the use of 'ibid'
                # is an error.
                raise InputError(filename,
                                 'First definition of: ' + unit['meaning']
                                 + ' has definition: ' + unit['description'])
              else:
                # If term has already been described, 'ibid' was used correctly,
                # and we output nothing.
                pass
            else:
              if unit['meaning'] in names:
                raise InputError(filename,
                                 'Second definition of: ' + unit['meaning'])
              names.append(unit['meaning'])
              results.append(unit)

        return results
    except IOError, e:
        print 'Error with file {0}: {1}'.format(filename, e.strerror)
        sys.exit(1)


def sort_units(units, templates):
    """Sorts the translation units by their definition order in the template.

    Args:
        units: A list of dictionaries produced by parse_trans_unit()
            that have a non-empty value for the key 'meaning'.
        templates: A string containing the Soy templates in which each of
            the units' meanings is defined.

    Returns:
        A new list of translation units, sorted by the order in which
        their meaning is defined in the templates.

    Raises:
        InputError: If a meaning definition cannot be found in the
            templates.
    """
    def key_function(unit):
        match = re.search(
            '\\smeaning\\s*=\\s*"{0}"\\s'.format(unit['meaning']),
            templates)
        if match:
            return match.start()
        else:
            raise InputError(args.templates,
                             'msg definition for meaning not found: ' +
                             unit['meaning'])
    return sorted(units, key=key_function)


def _write_files(units):
    """Writes the output files for the given units.

    There are three output files:
    * lang_file: JSON file mapping meanings (e.g., Maze.turnLeft) to the
      English text.  The base name of the language file is specified by the
      "lang" command-line argument.
    * key_file: JSON file mapping meanings to Closure-generated keys (long hash
      codes).
    * qqq_file: JSON file mapping meanings to descriptions.

    Args:
        units: A list of dictionaries produced by parse_trans_unit(),
            in the order desired in the output files.

    Raises:
        IOError: An error occurs opening, writing to, or closing a file.
    """
    lang_file = _create_lang_file()
    qqq_file = _create_qqq_file()
    key_file = _create_key_file()
    first_entry = True
    for unit in units:
        if not first_entry:
            lang_file.write(',\n')
            key_file.write(',\n')
            qqq_file.write(',\n')
        lang_file.write(u'    "{0}": "{1}"'.format(
            unit['meaning'],
            unit['source'].replace('"', "'")))
        key_file.write('"{0}": "{1}"'.format(unit['meaning'], unit['key']))
        qqq_file.write(u'    "{0}": "{1}"'.format(
            unit['meaning'],
            unit['description'].replace('"', "'").replace(
                '{lb}', '{').replace('{rb}', '}')));
        first_entry = False
    _close_lang_file(lang_file)
    _close_key_file(key_file)
    _close_qqq_file(qqq_file)


def main():
    """Parses arguments and processes the specified file.

    Raises:
        IOError: An I/O error occurred with an input or output file.
        InputError: Input files lacked required fields.
    """
    # Set up argument parser.
    parser = argparse.ArgumentParser(description='Create translation files.')
    parser.add_argument(
        '--author',
        default='<NAME> <<EMAIL>>',
        help='name and email address of contact for translators')
    parser.add_argument('--lang', default='en',
                        help='ISO 639-1 source language code')
    parser.add_argument('--output_dir', default='.',
                        help='relative directory for output files')
    parser.add_argument('--xlf', help='file containing xlf definitions')
    parser.add_argument('--templates', default=['template.soy'], nargs='+',
                        help='relative path to Soy templates '
                        '(used for ordering messages)')
    global args
    args = parser.parse_args()

    # Process the input file, and sort the entries.
    units = _process_file(args.xlf)
    files = []
    for filename in args.templates:
      with open(filename) as myfile:
        files.append(' '.join(line.strip() for line in myfile))
    sorted_units = sort_units(units, ' '.join(files))

    # Write the output files.
    _write_files(sorted_units)


if __name__ == '__main__':
    main()

#!/usr/bin/python

# Converts .xlf files into .json files for use at http://translatewiki.net.
#
# Copyright 2013 Google Inc.
# http://blockly.googlecode.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import codecs
from datetime import datetime
import os
import re
import sys
from xml.dom import minidom
from common import InputError
from common import parse_trans_unit

# Global variables
args = None      # parsed command-line arguments


def _create_qqq_file():
    """Creates a qqq.json file with message documentation for translatewiki.net.

    The file consists of key-value pairs, where the keys are message ids and
    the values are descriptions for the translators of the messages.
    What documentation exists for the format can be found at:
    http://translatewiki.net/wiki/Translating:Localisation_for_developers#Message_documentation

    The file should be closed by _close_qqq_file().

    Returns:
        A pointer to a file to which a left brace and newline have been written.

    Raises:
        IOError: An error occurred while opening or writing the file.
    """
    qqq_file_name = os.path.join(os.curdir, args.output_dir, 'qqq.json')
    qqq_file = codecs.open(qqq_file_name, 'w', 'utf-8')
    print 'Created file: ' + qqq_file_name
    qqq_file.write('{\n')
    return qqq_file


def _close_qqq_file(qqq_file):
    """Closes a qqq.json file created and opened by _create_qqq_file().

    This writes the final newlines and right brace.

    Args:
        qqq_file: A file created by _create_qqq_file().

    Raises:
        IOError: An error occurred while writing to or closing the file.
    """
    qqq_file.write('\n}\n')
    qqq_file.close()


def _create_key_file():
    """Creates a keys.json file mapping Closure keys to Blockly keys.

    Raises:
        IOError: An error occurred while creating the file.
    """
    key_file_name = os.path.join(os.curdir, args.output_dir, 'keys.json')
    key_file = open(key_file_name, 'w')
    key_file.write('{\n')
    print 'Created file: ' + key_file_name
    return key_file


def _close_key_file(key_file):
    """Closes a key file created and opened with _create_key_file().

    Args:
        key_file: A file created by _create_key_file().

    Raises:
        IOError: An error occurred while writing to or closing the file.
    """
    key_file.write('\n}\n')
    key_file.close()


def _create_lang_file():
    """Creates a <lang>.json file for translatewiki.net.

    The file consists of metadata, followed by key-value pairs, where the keys
    are message ids and the values are the messages in the language specified
    by the corresponding command-line argument.  The file should be closed by
    _close_lang_file().

    Returns:
        A pointer to a file to which the metadata has been written.

    Raises:
        IOError: An error occurred while opening or writing the file.
    """
    lang_file_name = os.path.join(
        os.curdir, args.output_dir, args.lang + '.json')
    lang_file = codecs.open(lang_file_name, 'w', 'utf-8')
    print 'Created file: ' + lang_file_name
    # string.format doesn't like printing braces, so break up our writes.
    lang_file.write('{\n    "@metadata": {')
    lang_file.write("""
        "author": "{0}",
        "lastupdated": "{1}",
        "locale": "{2}",
        "messagedocumentation" : "qqq"
""".format(args.author, str(datetime.now()), args.lang))
    lang_file.write('    },\n')
    return lang_file


def _close_lang_file(lang_file):
    """Closes a <lang>.json file created with _create_lang_file().

    This also writes the terminating left brace and newline.

    Args:
        lang_file: A file opened with _create_lang_file().

    Raises:
        IOError: An error occurred while writing to or closing the file.
    """
    lang_file.write('\n}\n')
    lang_file.close()


def _process_file(filename):
    """Builds list of translation units from input file.

    Each translation unit in the input file includes:
    - an id (opaquely generated by Soy)
    - the Blockly name for the message
    - the text in the source language (generally English)
    - a description for the translator

    The Soy and Blockly ids are joined with a hyphen and serve as the
    keys in both output files.  The value is the corresponding text (in the
    <lang>.json file) or the description (in the qqq.json file).

    Args:
        filename: The name of an .xlf file produced by Closure.

    Raises:
        IOError: An I/O error occurred with an input or output file.
        InputError: The input file could not be parsed or lacked required
            fields.

    Returns:
        A list of dictionaries produced by parse_trans_unit().
    """
    try:
        results = []  # list of dictionaries (return value)
        names = []    # list of names of encountered keys (local variable)
        try:
            parsed_xml = minidom.parse(filename)
        except IOError:
            # Don't get caught by below handler
            raise
        except Exception, e:
            print
            raise InputError(filename, str(e))

        # Make sure needed fields are present and non-empty.
        for trans_unit in parsed_xml.getElementsByTagName('trans-unit'):
            unit = parse_trans_unit(trans_unit)
            for key in ['description', 'meaning', 'source']:
                if not key in unit or not unit[key]:
                    raise InputError(filename + ':' + unit['key'],
                                     key + ' not found')
            if unit['description'].lower() == 'ibid':
              if unit['meaning'] not in names:
                # If the term has not already been described, the use of 'ibid'
                # is an error.
                raise InputError(filename,
                                 'First definition of: ' + unit['meaning']
                                 + ' has definition: ' + unit['description'])
              else:
                # If term has already been described, 'ibid' was used correctly,
                # and we output nothing.
                pass
            else:
              if unit['meaning'] in names:
                raise InputError(filename,
                                 'Second definition of: ' + unit['meaning'])
              names.append(unit['meaning'])
              results.append(unit)

        return results
    except IOError, e:
        print 'Error with file {0}: {1}'.format(filename, e.strerror)
        sys.exit(1)


def sort_units(units, templates):
    """Sorts the translation units by their definition order in the template.

    Args:
        units: A list of dictionaries produced by parse_trans_unit()
            that have a non-empty value for the key 'meaning'.
        templates: A string containing the Soy templates in which each of
            the units' meanings is defined.

    Returns:
        A new list of translation units, sorted by the order in which
        their meaning is defined in the templates.

    Raises:
        InputError: If a meaning definition cannot be found in the
            templates.
    """
    def key_function(unit):
        match = re.search(
            '\\smeaning\\s*=\\s*"{0}"\\s'.format(unit['meaning']),
            templates)
        if match:
            return match.start()
        else:
            raise InputError(args.templates,
                             'msg definition for meaning not found: ' +
                             unit['meaning'])
    return sorted(units, key=key_function)


def _write_files(units):
    """Writes the output files for the given units.

    There are three output files:
    * lang_file: JSON file mapping meanings (e.g., Maze.turnLeft) to the
      English text.  The base name of the language file is specified by the
      "lang" command-line argument.
    * key_file: JSON file mapping meanings to Closure-generated keys (long hash
      codes).
    * qqq_file: JSON file mapping meanings to descriptions.

    Args:
        units: A list of dictionaries produced by parse_trans_unit(),
            in the order desired in the output files.

    Raises:
        IOError: An error occurs opening, writing to, or closing a file.
    """
    lang_file = _create_lang_file()
    qqq_file = _create_qqq_file()
    key_file = _create_key_file()
    first_entry = True
    for unit in units:
        if not first_entry:
            lang_file.write(',\n')
            key_file.write(',\n')
            qqq_file.write(',\n')
        lang_file.write(u'    "{0}": "{1}"'.format(
            unit['meaning'],
            unit['source'].replace('"', "'")))
        key_file.write('"{0}": "{1}"'.format(unit['meaning'], unit['key']))
        qqq_file.write(u'    "{0}": "{1}"'.format(
            unit['meaning'],
            unit['description'].replace('"', "'").replace(
                '{lb}', '{').replace('{rb}', '}')));
        first_entry = False
    _close_lang_file(lang_file)
    _close_key_file(key_file)
    _close_qqq_file(qqq_file)


def main():
    """Parses arguments and processes the specified file.

    Raises:
        IOError: An I/O error occurred with an input or output file.
        InputError: Input files lacked required fields.
    """
    # Set up argument parser.
    parser = argparse.ArgumentParser(description='Create translation files.')
    parser.add_argument(
        '--author',
        default='<NAME> <<EMAIL>>',
        help='name and email address of contact for translators')
    parser.add_argument('--lang', default='en',
                        help='ISO 639-1 source language code')
    parser.add_argument('--output_dir', default='.',
                        help='relative directory for output files')
    parser.add_argument('--xlf', help='file containing xlf definitions')
    parser.add_argument('--templates', default=['template.soy'], nargs='+',
                        help='relative path to Soy templates '
                        '(used for ordering messages)')
    global args
    args = parser.parse_args()

    # Process the input file, and sort the entries.
    units = _process_file(args.xlf)
    files = []
    for filename in args.templates:
      with open(filename) as myfile:
        files.append(' '.join(line.strip() for line in myfile))
    sorted_units = sort_units(units, ' '.join(files))

    # Write the output files.
    _write_files(sorted_units)


if __name__ == '__main__':
    main()

#!/usr/bin/python # Converts .xlf files into .json files for use at http://translatewiki.net. # # Copyright 2013 Google Inc. # http://blockly.googlecode.com/ # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #   http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Global variables # parsed command-line arguments Creates a qqq.json file with message documentation for translatewiki.net.      The file consists of key-value pairs, where the keys are message ids and     the values are descriptions for the translators of the messages.     What documentation exists for the format can be found at:     http://translatewiki.net/wiki/Translating:Localisation_for_developers#Message_documentation      The file should be closed by _close_qqq_file().      Returns:         A pointer to a file to which a left brace and newline have been written.      Raises:         IOError: An error occurred while opening or writing the file. Closes a qqq.json file created and opened by _create_qqq_file().      This writes the final newlines and right brace.      Args:         qqq_file: A file created by _create_qqq_file().      Raises:         IOError: An error occurred while writing to or closing the file. Creates a keys.json file mapping Closure keys to Blockly keys.      Raises:         IOError: An error occurred while creating the file. Closes a key file created and opened with _create_key_file().      Args:         key_file: A file created by _create_key_file().      Raises:         IOError: An error occurred while writing to or closing the file. Creates a <lang>.json file for translatewiki.net.      The file consists of metadata, followed by key-value pairs, where the keys     are message ids and the values are the messages in the language specified     by the corresponding command-line argument.  The file should be closed by     _close_lang_file().      Returns:         A pointer to a file to which the metadata has been written.      Raises:         IOError: An error occurred while opening or writing the file. # string.format doesn't like printing braces, so break up our writes. "author": "{0}",         "lastupdated": "{1}",         "locale": "{2}",         "messagedocumentation" : "qqq" Closes a <lang>.json file created with _create_lang_file().      This also writes the terminating left brace and newline.      Args:         lang_file: A file opened with _create_lang_file().      Raises:         IOError: An error occurred while writing to or closing the file. Builds list of translation units from input file.      Each translation unit in the input file includes:     - an id (opaquely generated by Soy)     - the Blockly name for the message     - the text in the source language (generally English)     - a description for the translator      The Soy and Blockly ids are joined with a hyphen and serve as the     keys in both output files.  The value is the corresponding text (in the     <lang>.json file) or the description (in the qqq.json file).      Args:         filename: The name of an .xlf file produced by Closure.      Raises:         IOError: An I/O error occurred with an input or output file.         InputError: The input file could not be parsed or lacked required             fields.      Returns:         A list of dictionaries produced by parse_trans_unit(). # list of dictionaries (return value) # list of names of encountered keys (local variable) # Don't get caught by below handler # Make sure needed fields are present and non-empty. # If the term has not already been described, the use of 'ibid' # is an error. # If term has already been described, 'ibid' was used correctly, # and we output nothing. Sorts the translation units by their definition order in the template.      Args:         units: A list of dictionaries produced by parse_trans_unit()             that have a non-empty value for the key 'meaning'.         templates: A string containing the Soy templates in which each of             the units' meanings is defined.      Returns:         A new list of translation units, sorted by the order in which         their meaning is defined in the templates.      Raises:         InputError: If a meaning definition cannot be found in the             templates. Writes the output files for the given units.      There are three output files:     * lang_file: JSON file mapping meanings (e.g., Maze.turnLeft) to the       English text.  The base name of the language file is specified by the       "lang" command-line argument.     * key_file: JSON file mapping meanings to Closure-generated keys (long hash       codes).     * qqq_file: JSON file mapping meanings to descriptions.      Args:         units: A list of dictionaries produced by parse_trans_unit(),             in the order desired in the output files.      Raises:         IOError: An error occurs opening, writing to, or closing a file. Parses arguments and processes the specified file.      Raises:         IOError: An I/O error occurred with an input or output file.         InputError: Input files lacked required fields. # Set up argument parser. # Process the input file, and sort the entries. # Write the output files.