def find_dependencies(input_file, templatedirs, env, exceptions=[]):
    """
    Find jinja2 dependency list.  Files listed in 'exceptions' do not
    generate exceptions if not found.
    """

    total_set = set()

    def find_dependencies_recurse(file_path):
        new_deps = []

        # Parse the file and extract the list of references

        with open(file_path, "r") as f:
            ast = env.parse(f.read().decode('utf-8'))

            # For each reference, find the absolute path.  If no file
            # is found and the reference was not listed in exceptions,
            # throw an error.

            for reference in meta.find_referenced_templates(ast):
                reference_path = find_file_in_dirs(reference, templatedirs)
                if reference_path is None:
                    if reference in exceptions:
                        continue
                    raise Exception("cannot find file '%s' referenced in "
                                    "'%s'" % (reference, file_path))
                new_deps.append(reference_path)

        for dep in new_deps:
            # Make sure we don't have a circular reference
            if dep not in total_set:
                total_set.add(dep)
                find_dependencies_recurse(dep)
        return

    top_file = find_file_in_dirs(input_file, templatedirs)
    if top_file is None:
        raise Exception("cannot find file '%s'" % input_file)
    total_set.add(top_file)
    find_dependencies_recurse(top_file)

    sorted_total = []
    for x in total_set:
        sorted_total.append(x)

    sorted_total.sort()
    return sorted_total class SubProc(object):
    """Encapsulation for running subprocesses, capturing the output and processing to return in a response."""
    def __init__(self, command, cwd=None):
        self.command = command
        self.cwd = cwd
        self.retcode = 0
        self.time_delta = datetime.timedelta()
        self.stdout_report, self.stderr_report = ('','')

    def update_command(self, command, cwd=None, env=None):
        """Set the command to execute and optionally the path to execute form."""
        self.command = command
        self.cwd = cwd

    def time_popen(self):
        """Time a subprocess command and return process retcode. This method will block until the process completes."""
        time_start = datetime.datetime.now()

        proc = subprocess.Popen(self.command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cwd)
        stdout_report, stderr_report = proc.communicate()
        self.retcode = proc.wait()

        time_end = datetime.datetime.now()
        time_delta = time_end - time_start

        self.time_delta += time_delta
        self.stdout_report += stdout_report
        self.stderr_report += stderr_report

        return self.retcode

    def command_str(self):
        """Generate the command string."""
        return ' '.join(self.command) class Profiler(object):

    _profiler_impl = ProfilerDummyImpl()

    @classmethod
    def enable(cls):
        if not isinstance(cls._profiler_impl, ProfilerDummyImpl):
            raise Exception("Profiler.enable_profiler() called twice")
        cls._profiler_impl = ProfilerImpl()

    @classmethod
    def start(cls, section_name):
        cls._profiler_impl.start(section_name)

    @classmethod
    def stop(cls, section_name):
        cls._profiler_impl.stop(section_name)

    @classmethod
    def get_root_nodes(cls):
        return cls._profiler_impl.get_root_nodes()

    @classmethod
    def dump_data(cls):
        cls._profiler_impl.dump_data() def read_file_utf8(filename):
    with open(filename, 'rb') as f:
        text = f.read().decode('utf-8-sig')

        # This is unpleasant, but it is thorough.  This is the
        # single-pass equivalent of:
        # return (in_data.replace('\r\n', '\n').replace('\n\r', '\n'))\
        #     .replace('\r', '\n');
        return LF_RE.sub(_sanitize_crlf, text) def env_create(options, default_template=None):
    """
    Setup a jinja env based on the tool options
    """

    LOG.info("Template dirs:")
    for t in options.templatedirs:
        LOG.info(" - '%s'", t)

    loaders = [ UTF8FileSystemLoader(t) for t in options.templatedirs ]
    if default_template is not None:
        loaders.append(DefaultTemplateLoader('default', default_template))

    _loader = ChoiceLoader(loaders)
    env = Environment(loader = _loader,
                      block_start_string = '/*{%',
                      block_end_string = '%}*/',
                      variable_start_string = '/*{{',
                      variable_end_string = '}}*/',
                      comment_start_string = '/*{#',
                      comment_end_string = '#}*/')
    return env def env_load_templates(env, inputs):
    """
    Load an array of templates into the environment.  Returns the
    loaded templates as a list
    """

    return [env_load_template(env, i) for i in inputs] def render_js(context, options, templates_js, inject_js):
    """
    Renders the templates in templates_js, as if the first template
    began with include declarations for each of the files in
    inject_js.  Returns the result of rendering, and the list of
    includes that were not inlined.  (rendered_js, inc_js)

    For dev modes, the list of includes is returned in inc_js as
    relative paths from the output file.  For release modes, includes
    are all inlined (inc_js == []).
    """

    regex_use_strict = re_compile('"use strict";')

    out = []
    inc_js = []
    outfile_dir = os.path.abspath(os.path.dirname(options.output)) + os.sep

    includes_seen = []

    # Any headers

    if options.use_strict:
        out.append('"use strict";')

    if options.mode in [ 'plugin', 'canvas' ]:
        out.append('(function () {')

    # Functions for handling includes

    def _find_include_or_error(name):
        try:
            f = find_file_in_dirs(name, options.templatedirs)
        except Exception, ex:
            raise ToolsException(str(ex))
        if f is None:
            raise ToolsException("No file '%s' in any template dir" % name)
        LOG.info(" resolved '%s' to path '%s'", name, f)
        return f

    def handle_javascript_dev(name):
        file_path = _find_include_or_error(name)
        if file_path in includes_seen:
            LOG.info(" include '%s' (%s) already listed", name, file_path)
            return ""
        includes_seen.append(file_path)

        # Calculate relative path name
        # rel_path = file_path.replace(outfile_dir, '').replace('\\', '/')
        # if rel_path == file_path:
        #     raise ToolsException("Included file '%s' found at '%s', which is "
        #                          "not in a child directory of the output file "
        #                          "'%s' in directory %s" % (name, file_path,
        #                                                    options.output,
        #                                                    outfile_dir))
        rel_path = os.path.relpath(file_path, outfile_dir).replace('\\', '/')

        inc_js.append(rel_path)
        return ""

    def handle_javascript_webworker_dev(name):
        file_path = _find_include_or_error(name)
        if file_path in includes_seen:
            LOG.info(" include '%s' (%s) already listed", name, file_path)
            return ""
        includes_seen.append(file_path)

        rel_path = os.path.relpath(file_path, outfile_dir).replace('\\', '/')

        return ('importScripts("%s");' % rel_path).encode('utf-8')

    def handle_javascript_release(name):
        if options.stripdebug and os.path.basename(name) == "debug.js":
            LOG.warning("App attempting to include debug.js.  Removing.")
            return ""
        file_path = _find_include_or_error(name)
        if file_path in includes_seen:
            LOG.info(" include '%s' (%s) already listed", name, file_path)
            return ""
        includes_seen.append(file_path)

        d = read_file_utf8(file_path)

        if options.include_use_strict:
            return d
        else:
            # strip out any "use strict"; lines
            return regex_use_strict.sub('', d)

    if options.mode in [ 'plugin', 'canvas', 'webworker' ]:
        handle_javascript = handle_javascript_release
    elif options.mode == 'webworker-debug':
        handle_javascript = handle_javascript_webworker_dev
    else:
        handle_javascript = handle_javascript_dev
    context['javascript'] = handle_javascript

    # Inject any includes at the start, either embedding them or
    # adding to the inc_js list.

    for inj in inject_js:
        js_line = handle_javascript(inj)
        if js_line:
            out.append(js_line)

    # Render templates

    out += [t.render(context) for t in templates_js]
    del context['javascript']

    # Any footer code

    if options.mode == 'plugin':
        out.append("""
    if (!TurbulenzEngine.onload)
    {
        window.alert("Entry point 'TurbulenzEngine.onload' must be defined.");
        return;
    }
    TurbulenzEngine.onload.call(this);
}());""")

    if options.mode == 'canvas':
        out.append('window.TurbulenzEngine = TurbulenzEngine;}());')

    # Combine all parts into a single string

    return ("\n".join(out), inc_js) def render_js_extract_includes(context, options, templates_js, injects):
    """
    Renders the templates in templates_js against the given context
    and just collects the set of 'javascript('...')' includes.  Will
    optionally handle a list of files to be injected.

    Returns an array of absolute paths, removing duplicates.
    """

    includes = []

    def _find_in_dirs_or_error(name):
        file_path = find_file_in_dirs(name, options.templatedirs)
        if file_path is None:
            raise ToolsException("No file '%s' in any template dir" % name)
        if file_path in includes:
            LOG.info(" include '%s' (%s) already listed", name, file_path)
            return
        LOG.info(" resolved '%s' to path '%s'", name, file_path)
        includes.append(file_path)

    # In release mode, filter out debug.js

    if options.mode in [ 'plugin', 'canvas' ] and options.stripdebug:
        _do_find_in_dirs_or_error = _find_in_dirs_or_error
        # pylint: disable=E0102
        def _find_in_dirs_or_error(name):
            if os.path.basename(name) == "debug.js":
                LOG.warning("App attempting to include debug.js.  Removing.")
                return
            _do_find_in_dirs_or_error(name)
        # pylint: enable=E0102

    # Deal with any injects

    for i in injects:
        _find_in_dirs_or_error(i)

    # Use the templating engine to deal with remaining includes

    def handle_javascipt_extract_includes(name):
        _find_in_dirs_or_error(name)
        return ""

    context['javascript'] = handle_javascipt_extract_includes

    for t in templates_js:
        t.render(context)

    del context['javascript']

    return includes def inject_js_from_options(options):
    """
    Given the build options, find (if necessary), all includes that
    must be injected for canvas mode to work.  This is done by
    searching for webgl_engine_file in any of the
    template directories, and collecting the list of all .js files
    that reside there.
    """

    inject_list = []

    if options.noinject:
        return inject_list

    mode = options.mode

    # Put debug.js at the top (if in debug mode), and ALWAYS include
    # vmath.js

    if mode in [ 'plugin-debug', 'canvas-debug', 'webworker-debug' ] or not options.stripdebug:
        inject_list.append('jslib/debug.js')
    inject_list.append('jslib/vmath.js')

    # Include webgl includes in canvas mode

    if mode in [ 'canvas', 'canvas-debug' ]:
        LOG.info("Looking for jslib/webgl ...")

        webgl_engine_file = 'jslib/webgl/turbulenzengine.js'
        webgl_engine_dir = os.path.dirname(webgl_engine_file)

        # Find absolute path of webgl_engine_file

        webgl_abs_path = None

        for t in options.templatedirs:
            p = os.path.join(t, webgl_engine_file)
            if os.path.exists(p):
                webgl_abs_path = os.path.dirname(p)
                LOG.info("Found at: %s", webgl_abs_path)
                break

        if webgl_abs_path is None:
            raise ToolsException("No '%s' in any template dir" \
                                     % webgl_engine_file)

        webgl_abs_files = glob.glob(webgl_abs_path + "/*.js")
        inject_list += [ 'jslib/utilities.js',
                         'jslib/aabbtree.js',
                         'jslib/observer.js' ]
        inject_list += \
            [ webgl_engine_dir + "/" + os.path.basename(f) for f in webgl_abs_files ]

    return inject_list def context_from_options(options, title):

    # Sanity check

    if options.hybrid:
        if options.mode not in [ 'canvas', 'canvas-debug' ]:
            raise ToolsException("--hybrid option available only in canvas and "
                                 "canvas_dev modes")

    # Set up the context

    context = {}
    context['tz_app_title_name_var'] = title
    context['tz_app_title_var'] = title

    context['tz_development'] = options.mode in [ 'plugin-debug', 'canvas-debug', 'webworker-debug' ]
    context['tz_canvas'] = options.mode in [ 'canvas', 'canvas-debug' ]
    context['tz_webworker'] = options.mode in [ 'webworker', 'webworker-debug' ]
    context['tz_hybrid'] = options.hybrid

    return context def default_parser_options(parser):
    """
    Command line options shared by make*.py tools
    """

    parser.add_option("--version", action="store_true", dest="output_version",
                      default=False, help="output version number")
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
                      default=False, help="verbose output")
    parser.add_option("-s", "--silent", action="store_true", dest="silent",
                      default=False, help="silent running")

    # Input / Output (input .html and .js files don't need a prefix)
    parser.add_option("-o", "--output", action="store", dest="output",
                      help="output file to process")
    parser.add_option("-t", "--templatedir", action="append", dest="templatedirs",
                      default=[], help="template directory (multiple allowed)")

    # Dependency generation
    parser.add_option("-M", "--dependency", action="store_true",
                      dest="dependency", default=False,
                      help="output dependencies")
    parser.add_option("--MF", action="store", dest="dependency_file",
                      help="dependencies output to file")

    # "use strict" options
    parser.add_option("--use-strict", action="store_true", dest="use_strict",
                      default=False, help='enforce "use strict"; statement. '
                      'This adds a single "use strict"; line at the top of the '
                      'JavaScript code.')
    parser.add_option("--include-use-strict", action="store_true",
                      dest="include_use_strict", default=False,
                      help='don\'t strip out "use strict"; statements. '
                      'By default all "use strict"; statements are removed '
                      'from the output file.')

    # Hybrid
    parser.add_option("--hybrid", action="store_true", dest="hybrid",
                      default=False, help="canvas, canvas_dev modes only. "
                      "Start up a plugin as well as a canvas-based "
                      "TurbulenzEngine. The plugin will be available as "
                      "TurbulenzEnginePlugin.")

    # Profiling
    def _enable_profiler(_options, _opt_str, _value, _parser):
        Profiler.enable()
    parser.add_option("--profile", action="callback", callback=_enable_profiler,
                      help="enable the collection and output of profiling "
                      "information")

    # Injecting files
    parser.add_option("--no-inject", action="store_true", dest="noinject",
                      default=False, help="Don't inject default library files") def output_dependency_info(dependency_file, output_file, dependencies):
    """
    This dependency write outputs dependency information in a format
    consistent with the GCC -M flags.
    """

    try:
        with open(dependency_file, "wb") as f:
            f.write(output_file)
            f.write(" ")
            f.write(dependency_file)
            f.write(" : \\\n")
            for d in dependencies:
                f.write("    ")
                f.write(d)
                f.write(" \\\n")
            f.write("\n\n")
            for d in dependencies:
                f.write(d)
                f.write(" :\n\n")
    except IOError:
        raise ToolsException("failed to write file: %s" % dependency_file) class ToolsException(Exception):
    pass def simple_options(parser_fn, version, dependencies, input_required=True):
    parser = parser_fn()
    (options, args) = parser.parse_args()

    if options.output_version:
        standard_output_version(version, dependencies, getattr(options, 'output', None))
        exit(0)

    if input_required:
        # Not all tools have an input file, so we print help for no args as well.
        try:
            if options.input is None:
                print "ERROR: no input files specified"
                parser.print_help()
                exit(1)
        except AttributeError:
            if len(args) == 0:
                print "ERROR: no input files specified"
                parser.print_help()
                exit(1)

    # Not all tools have a metrics option.
    try:
        metrics = options.metrics
    except AttributeError:
        metrics = False

    if options.silent:
        logging.basicConfig(level=logging.CRITICAL)
    elif options.verbose or metrics:
        logging.basicConfig(level=logging.INFO)
    else:
        logging.basicConfig(level=logging.WARNING)

    return (options, args, parser)
from turbulenz_tools.utils.dependencies import find_dependencies
from turbulenz_tools.utils.subproc import SubProc
from turbulenz_tools.utils.profiler import Profiler
from turbulenz_tools.tools.templates import read_file_utf8
from turbulenz_tools.tools.templates import env_create
from turbulenz_tools.tools.templates import env_load_templates
from turbulenz_tools.tools.appcodegen import render_js
from turbulenz_tools.tools.appcodegen import render_js_extract_includes
from turbulenz_tools.tools.appcodegen import inject_js_from_options
from turbulenz_tools.tools.appcodegen import context_from_options
from turbulenz_tools.tools.appcodegen import default_parser_options
from turbulenz_tools.tools.appcodegen import output_dependency_info
from turbulenz_tools.tools.toolsexception import ToolsException
from turbulenz_tools.tools.stdtool import simple_options
from logging import getLogger
from os import remove
from os.path import relpath, abspath, normpath
from tempfile import NamedTemporaryFile
from optparse import OptionParser, TitledHelpFormatter
import subprocess
    Profiler.stop('find_inject_code')

    if 0 < len(inject_js):
        LOG.info("Files to inject:")
        for i in inject_js:
            LOG.info(" - '%s'", i)

    # Create a context and render the template

    Profiler.start('load_templates')
    context = context_from_options(options, input_js[0])
    templates_js = env_load_templates(env, input_js)
    Profiler.stop('load_templates')

    Profiler.start('render_js')
    (rendered_js, inc_js) = render_js(context, options, templates_js,
                                      inject_js)
    rendered_js = rendered_js.encode('utf-8')
    Profiler.stop('render_js')

    if 0 != len(inc_js):
        raise ToolsException("internal error")

    # If required, remove all calls to 'debug.*' methods BEFORE
    # compacting

    # TODO: We write and read the files too many times.  Better to
    # write once to a temporary, keep track of the name and invoke
    # each external command on files, creating subsequent temporaries
    # as required.

    if options.stripdebug:

        strip_path = "strip-debug"
        if options.stripdebugpath:
            strip_path = normpath(abspath(options.stripdebugpath))

        LOG.info("Stripping debug method calls ...")

        # Check we can actually run strip debug, with the given path
        p = subprocess.Popen('%s -h' % strip_path,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT,
                             shell=True)
        p.communicate()
        if p.returncode != 0:
            raise ToolsException( \
                "\n\tstrip-debug tool could not be found, check it's on your path\n"
                "\tor supply the path with --strip-debug <path>. To run maketzjs\n"
                "\twithout stripping debug code run with --no-strip-debug." )

        Profiler.start('strip_debug')

        strip_debug_flags = "-Ddebug=false"

        # Add the default flags first, in case the custom flags
        # override them.

        if options.verbose:
            strip_debug_flags += " -v"
        for s in options.stripnamespaces:
            strip_debug_flags += " --namespace %s" % s
        for v in options.stripvars:
            strip_debug_flags += " -D %s" % v
        if options.ignoreerrors:
            strip_debug_flags += " --ignore-errors"

        # Launch the strip command and pass in the full script via
        # streams.

        with NamedTemporaryFile(delete = False) as t:
            LOG.info("Writing temp JS to '%s'", t.name)
            t.write(rendered_js)

        with NamedTemporaryFile(delete = False) as tmp_out:
            pass

        strip_cmd = "%s %s -o %s %s" % (strip_path, strip_debug_flags,
                                        tmp_out.name, t.name)
        LOG.info("Strip cmd: %s", strip_cmd)
        strip_retval = subprocess.call(strip_cmd, shell=True)

        if 0 != strip_retval:
            raise ToolsException( \
                "strip-debug tool exited with code %d\n"
                "The (merged) input probably contains a syntax error:\n"
                "  %s" % (strip_retval, t.name))

        rendered_js = read_file_utf8(tmp_out.name).encode('utf-8')
        remove(tmp_out.name)
        remove(t.name)

        Profiler.stop('strip_debug')

    # If required, compact the JS via a temporary file, otherwise just
    # write out directly to the output file.

    if options.mode != 'webworker-debug' and (options.yui or options.closure or options.uglifyjs):

        Profiler.start('compact')

        with NamedTemporaryFile(delete = False) as t:
            LOG.info("Writing temp JS to '%s'", t.name)
            t.write(rendered_js)

        LOG.info("Compacting temp JS to '%s'", options.output)
        tzjs_compact(options, t.name, options.output)
        remove(t.name)
        Profiler.stop('compact')

    else:

        LOG.info("Writing JS to '%s'", options.output)
        Profiler.start('write_out')
        try:
            with open(options.output, 'wb') as f:
                f.write(rendered_js)
                LOG.info("Succeeded")
        except IOError:
            raise ToolsException("failed to write file: %s" % options.output)

        Profiler.stop('write_out')