package org.codehaus.mojo.exec;

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;

import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteException;
import org.apache.commons.exec.ExecuteResultHandler;
import org.apache.commons.exec.Executor;
import org.apache.commons.exec.OS;
import org.apache.commons.exec.ProcessDestroyer;
import org.apache.commons.exec.PumpStreamHandler;
import org.apache.commons.exec.ShutdownHookProcessDestroyer;
import org.apache.maven.artifact.Artifact;
import org.apache.maven.artifact.resolver.filter.AndArtifactFilter;
import org.apache.maven.artifact.resolver.filter.IncludesArtifactFilter;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
import org.apache.maven.project.MavenProject;
import org.apache.maven.toolchain.Toolchain;
import org.apache.maven.toolchain.ToolchainManager;
import org.codehaus.plexus.component.repository.exception.ComponentLookupException;
import org.codehaus.plexus.util.IOUtil;
import org.codehaus.plexus.util.StringUtils;
import org.codehaus.plexus.util.cli.CommandLineException;
import org.codehaus.plexus.util.cli.CommandLineUtils;
import org.codehaus.plexus.util.cli.Commandline;
import org.codehaus.plexus.util.cli.DefaultConsumer;
import org.codehaus.plexus.util.cli.StreamConsumer;

/**
 * A Plugin for executing external programs.
 *
 * @author Jerome Lacoste (jerome@coffeebreaks.org)
 * @version $Id$
 * @since 1.0
 */
@Mojo( name = "exec", threadSafe = true, requiresDependencyResolution = ResolutionScope.TEST )
public class ExecMojo
    extends AbstractExecMojo
{
    /**
     *

     * The executable. Can be a full path or the name of the executable. In the latter case, the executable must be in
     * the PATH for the execution to work. Omit when using executableDependency.
     *

     *

     * The plugin will search for the executable in the following order:
     *

     *
relative to the root of the project

     *
as toolchain executable

     *
relative to the working directory (Windows only)

     *
relative to the directories specified in the system property PATH (Windows Only)

     *

     * Otherwise use the executable as is.
     *

     *
     * @since 1.0
     */
    @Parameter( property = "exec.executable" )
    private String executable;

    /**
     *

     * The toolchain. If omitted, "jdk" is assumed.
     *

     */
    @Parameter( property = "exec.toolchain", defaultValue = "jdk" )
    private String toolchain;

    /**
     * The current working directory. Optional. If not specified, basedir will be used.
     *
     * @since 1.0
     */
    @Parameter( property = "exec.workingdir" )
    private File workingDirectory;

    /**
     * Program standard and error output will be redirected to the file specified by this optional field. If not
     * specified the standard Maven logging is used.

     * Note: Be aware that System.out and System.err use buffering, so don't
     * rely on the order!
     *
     * @since 1.1-beta-2
     * @see java.lang.System#err
     * @see java.lang.System#in
     */
    @Parameter( property = "exec.outputFile" )
    private File outputFile;

    /**
     *

     * A list of arguments passed to the {@code executable}, which should be of type <argument> or
     * <classpath>. Can be overridden by using the exec.args environment variable.
     *

     *
     * @since 1.0
     */
    @Parameter
    private List<?> arguments; // TODO: Change ? into something more meaningful

    /**
     * @since 1.0
     */
    @Parameter( readonly = true, required = true, defaultValue = "${basedir}" )
    private File basedir;

    /**
     * @since 3.0.0
     */
    @Parameter( readonly = true, required = true, defaultValue = "${project.build.directory}" )
    private File buildDirectory;

    /**
     *
Environment variables to pass to the executed program. For example if you want to set the LANG var:
     * <environmentVariables>
     *     <LANG>en_US</LANG>
     * </environmentVariables>
     *
     *

     *
     * @since 1.1-beta-2
     */
    @Parameter
    private Map<String, String> environmentVariables = new HashMap<String, String>();

    /**
     * Environment script to be merged with environmentVariables This script is platform specifics, on Unix its
     * must be Bourne shell format. Use this feature if you have a need to create environment variable dynamically such
     * as invoking Visual Studio environment script file
     *
     * @since 1.4.0
     */
    @Parameter
    private File environmentScript = null;

    /**
     * The current build session instance. This is used for toolchain manager API calls.
     */
    @Parameter( defaultValue = "${session}", readonly = true )
    private MavenSession session;

    /**
     * Exit codes to be resolved as successful execution for non-compliant applications (applications not returning 0
     * for success).
     *
     * @since 1.1.1
     */
    @Parameter
    private int[] successCodes;

    /**
     * If set to true the classpath and the main class will be written to a MANIFEST.MF file and wrapped into a jar.
     * Instead of '-classpath/-cp CLASSPATH mainClass' the exec plugin executes '-jar maven-exec.jar'.
     *
     * @since 1.1.2
     */
    @Parameter( property = "exec.longClasspath", defaultValue = "false" )
    private boolean longClasspath;

    /**
     * If set to true the modulepath and the main class will be written as an @arg file
     * Instead of '--module-path/-p MODULEPATH ' the exec plugin executes '@modulepath'.
     *
     * @since 1.1.2
     */
    @Parameter( property = "exec.longModulepath", defaultValue = "true" )
    private boolean longModulepath;

    /**
     * If set to true the child process executes asynchronously and build execution continues in parallel.
     */
    @Parameter( property = "exec.async", defaultValue = "false" )
    private boolean async;

    /**
     * If set to true, the asynchronous child process is destroyed upon JVM shutdown. If set to false, asynchronous
     * child process continues execution after JVM shutdown. Applies only to asynchronous processes; ignored for
     * synchronous processes.
     */
    @Parameter( property = "exec.asyncDestroyOnShutdown", defaultValue = "true" )
    private boolean asyncDestroyOnShutdown = true;

    public static final String CLASSPATH_TOKEN = "%classpath";

    public static final String MODULEPATH_TOKEN = "%modulepath";

    /**
     * priority in the execute method will be to use System properties arguments over the pom specification.
     *
     * @throws MojoExecutionException if a failure happens
     */
    public void execute()
        throws MojoExecutionException
    {
        if ( executable == null )
        {
            if (executableDependency == null)
            {
                throw new MojoExecutionException( "The parameter 'executable' is missing or invalid" );
            }

            executable = findExecutableArtifact().getFile().getAbsolutePath();
            getLog().debug( "using executable dependency " + executable);
        }

        if ( isSkip() )
        {
            getLog().info( "skipping execute as per configuration" );
            return;
        }

        if ( basedir == null )
        {
            throw new IllegalStateException( "basedir is null. Should not be possible." );
        }

        try
        {

            handleWorkingDirectory();

            String argsProp = getSystemProperty( "exec.args" );

            List<String> commandArguments = new ArrayList<String>();

            if ( hasCommandlineArgs() )
            {
                handleCommandLineArgs( commandArguments );
            }
            else if ( !StringUtils.isEmpty( argsProp ) )
            {
                handleSystemPropertyArguments( argsProp, commandArguments );
            }
            else
            {
                if ( arguments != null )
                {
                    handleArguments( commandArguments );
                }
            }

            Map<String, String> enviro = handleSystemEnvVariables();

            CommandLine commandLine = getExecutablePath( enviro, workingDirectory );

            String[] args = commandArguments.toArray( new String[commandArguments.size()] );

            commandLine.addArguments( args, false );

            Executor exec = new DefaultExecutor();
            exec.setWorkingDirectory( workingDirectory );
            fillSuccessCodes( exec );

            getLog().debug( "Executing command line: " + commandLine );

            try
            {
                int resultCode;
                if ( outputFile != null )
                {
                    if ( !outputFile.getParentFile().exists() && !outputFile.getParentFile().mkdirs() )
                    {
                        getLog().warn( "Could not create non existing parent directories for log file: " + outputFile );
                    }

                    FileOutputStream outputStream = null;
                    try
                    {
                        outputStream = new FileOutputStream( outputFile );

                        resultCode = executeCommandLine( exec, commandLine, enviro, outputStream );
                    }
                    finally
                    {
                        IOUtil.close( outputStream );
                    }
                }
                else
                {
                    resultCode = executeCommandLine( exec, commandLine, enviro, System.out, System.err );
                }

                if ( isResultCodeAFailure( resultCode ) )
                {
                    String message = "Result of " + commandLine.toString() + " execution is: '" + resultCode + "'.";
                    getLog().error( message );
                    throw new MojoExecutionException( message );
                }
            }
            catch ( ExecuteException e )
            {
                getLog().error( "Command execution failed.", e );
                throw new MojoExecutionException( "Command execution failed.", e );
            }
            catch ( IOException e )
            {
                getLog().error( "Command execution failed.", e );
                throw new MojoExecutionException( "Command execution failed.", e );
            }

            registerSourceRoots();
        }
        catch ( IOException e )
        {
            throw new MojoExecutionException( "I/O Error", e );
        }
    }

    private Map<String, String> handleSystemEnvVariables()
        throws MojoExecutionException
    {

        Map<String, String> enviro = new HashMap<String, String>();
        try
        {
            Properties systemEnvVars = CommandLineUtils.getSystemEnvVars();
            for ( Map.Entry<?, ?> entry : systemEnvVars.entrySet() )
            {
                enviro.put( (String) entry.getKey(), (String) entry.getValue() );
            }
        }
        catch ( IOException x )
        {
            getLog().error( "Could not assign default system enviroment variables.", x );
        }

        if ( environmentVariables != null )
        {
            enviro.putAll( environmentVariables );
        }

        if ( this.environmentScript != null )
        {
            getLog().info( "Pick up external environment script: " + this.environmentScript );
            Map<String, String> envVarsFromScript = this.createEnvs( this.environmentScript );
            if ( envVarsFromScript != null )
            {
                enviro.putAll( envVarsFromScript );
            }
        }

        if ( this.getLog().isDebugEnabled() )
        {
            Set<String> keys = new TreeSet<String>();
            keys.addAll( enviro.keySet() );
            for ( String key : keys )
            {
                this.getLog().debug( "env: " + key + "=" + enviro.get( key ) );
            }
        }

        return enviro;
    }

    /**
     * This is a convenient method to make the execute method a little bit more readable. It will define the
     * workingDirectory to be the baseDir in case of workingDirectory is null. If the workingDirectory does not exist it
     * will created.
     *
     * @throws MojoExecutionException
     */
    private void handleWorkingDirectory()
        throws MojoExecutionException
    {
        if ( workingDirectory == null )
        {
            workingDirectory = basedir;
        }

        if ( !workingDirectory.exists() )
        {
            getLog().debug( "Making working directory '" + workingDirectory.getAbsolutePath() + "'." );
            if ( !workingDirectory.mkdirs() )
            {
                throw new MojoExecutionException( "Could not make working directory: '"
                    + workingDirectory.getAbsolutePath() + "'" );
            }
        }
    }

    private void handleSystemPropertyArguments( String argsProp, List<String> commandArguments )
        throws MojoExecutionException
    {
        getLog().debug( "got arguments from system properties: " + argsProp );

        try
        {
            String[] args = CommandLineUtils.translateCommandline( argsProp );
            commandArguments.addAll( Arrays.asList( args ) );
        }
        catch ( Exception e )
        {
            throw new MojoExecutionException( "Couldn't parse systemproperty 'exec.args'" );
        }
    }

    private void handleCommandLineArgs( List<String> commandArguments )
        throws MojoExecutionException, IOException
    {
        String[] args = parseCommandlineArgs();
        for ( int i = 0; i < args.length; i++ )
        {
            if ( isLongClassPathArgument( args[i] ) )
            {
                // it is assumed that starting from -cp or -classpath the arguments
                // are: -classpath/-cp %classpath mainClass
                // the arguments are replaced with: -jar $TMP/maven-exec.jar
                // NOTE: the jar will contain the classpath and the main class
                commandArguments.add( "-jar" );
                File tmpFile = createJar( computePath( null ), args[i + 2] );
                commandArguments.add( tmpFile.getAbsolutePath() );
                i += 2;
            }
            else if ( args[i].contains( CLASSPATH_TOKEN ) )
            {
                commandArguments.add( args[i].replace( CLASSPATH_TOKEN, computeClasspathString( null ) ) );
            }
            else
            {
                commandArguments.add( args[i] );
            }
        }
    }

    private void handleArguments( List<String> commandArguments )
        throws MojoExecutionException, IOException
    {
        String specialArg = null;

        for ( int i = 0; i < arguments.size(); i++ )
        {
            Object argument = arguments.get( i );

            if ( specialArg != null )
            {
                if ( isLongClassPathArgument( specialArg ) && argument instanceof Classpath )
                {
                    // it is assumed that starting from -cp or -classpath the arguments
                    // are: -classpath/-cp %classpath mainClass
                    // the arguments are replaced with: -jar $TMP/maven-exec.jar
                    // NOTE: the jar will contain the classpath and the main class
                    commandArguments.add( "-jar" );

                    File tmpFile = createJar( computePath( (Classpath) argument ),
                                              (String) arguments.get( ++i ) );
                    commandArguments.add( tmpFile.getAbsolutePath() );
                }
                else if ( isLongModulePathArgument( specialArg ) && argument instanceof Modulepath )
                {
                    String filePath = new File( buildDirectory, "modulepath" ).getAbsolutePath();

                    StringBuilder modulePath = new StringBuilder();
                    modulePath.append( '"' );

                    for ( Iterator<String> it = computePath( (Modulepath) argument ).iterator(); it.hasNext(); )
                    {
                        modulePath.append( it.next().replace( "\\", "\\\\" ) );
                        if ( it.hasNext() )
                        {
                            modulePath.append( File.pathSeparatorChar );
                        }
                    }

                    modulePath.append( '"' );

                    createArgFile( filePath, Arrays.asList( "-p", modulePath.toString() ) );
                    commandArguments.add( '@' + filePath );
                }
                else
                {
                    commandArguments.add( specialArg );
                }

                specialArg = null;

                continue;
            }

            if ( argument instanceof Classpath )
            {
                Classpath specifiedClasspath = (Classpath) argument;
                commandArguments.add( computeClasspathString( specifiedClasspath ) );
            }
            else if ( argument instanceof Modulepath )
            {
                Modulepath specifiedModulepath = (Modulepath) argument;
                commandArguments.add( computeClasspathString( specifiedModulepath ) );
            }
            else if ( isLongModulePathArgument( specialArg ) || isLongClassPathArgument( specialArg ) )
            {
                specialArg = (String) argument;
            }
            else
            {
                commandArguments.add( (String) argument );
            }
        }
    }

    private void fillSuccessCodes( Executor exec )
    {
        if ( successCodes != null && successCodes.length > 0 )
        {
            exec.setExitValues( successCodes );
        }
    }

    boolean isResultCodeAFailure( int result )
    {
        if ( successCodes == null || successCodes.length == 0 )
        {
            return result != 0;
        }
        for ( int successCode : successCodes )
        {
            if ( successCode == result )
            {
                return false;
            }
        }
        return true;
    }

    private boolean isLongClassPathArgument( String arg )
    {
        return longClasspath && ( "-classpath".equals( arg ) || "-cp".equals( arg ) );
    }

    private boolean isLongModulePathArgument( String arg )
    {
        return longModulepath && ( "--module-path".equals( arg ) || "-p".equals( arg ) );
    }

    /**
     * Compute the classpath from the specified Classpath. The computed classpath is based on the classpathScope. The
     * plugin cannot know from maven the phase it is executed in. So we have to depend on the user to tell us he wants
     * the scope in which the plugin is expected to be executed.
     *
     * @param specifiedClasspath Non null when the user restricted the dependencies, null otherwise (the
     *            default classpath will be used)
     * @return a platform specific String representation of the classpath
     */
    private String computeClasspathString( AbstractPath specifiedClasspath )
    {
        List<String> resultList = computePath( specifiedClasspath );
        StringBuffer theClasspath = new StringBuffer();

        for ( String str : resultList )
        {
            addToClasspath( theClasspath, str );
        }

        return theClasspath.toString();
    }

    /**
     * Compute the classpath from the specified Classpath. The computed classpath is based on the classpathScope. The
     * plugin cannot know from maven the phase it is executed in. So we have to depend on the user to tell us he wants
     * the scope in which the plugin is expected to be executed.
     *
     * @param specifiedClasspath Non null when the user restricted the dependencies, null otherwise (the
     *            default classpath will be used)
     * @return a list of class path elements
     */
    private List<String> computePath( AbstractPath specifiedClasspath )
    {
        List<Artifact> artifacts = new ArrayList<>();
        List<Path> theClasspathFiles = new ArrayList<>();
        List<String> resultList = new ArrayList<>();

        collectProjectArtifactsAndClasspath( artifacts, theClasspathFiles );

        if ( ( specifiedClasspath != null ) && ( specifiedClasspath.getDependencies() != null ) )
        {
            artifacts = filterArtifacts( artifacts, specifiedClasspath.getDependencies() );
        }

        for ( Path f : theClasspathFiles )
        {
            resultList.add( f.toAbsolutePath().toString() );
        }

        for ( Artifact artifact : artifacts )
        {
            getLog().debug( "dealing with " + artifact );
            resultList.add( artifact.getFile().getAbsolutePath() );
        }

        return resultList;
    }

    private static void addToClasspath( StringBuffer theClasspath, String toAdd )
    {
        if ( theClasspath.length() > 0 )
        {
            theClasspath.append( File.pathSeparator );
        }
        theClasspath.append( toAdd );
    }

    private List<Artifact> filterArtifacts( List<Artifact> artifacts, Collection<String> dependencies )
    {
        AndArtifactFilter filter = new AndArtifactFilter();

        filter.add( new IncludesArtifactFilter( new ArrayList<String>( dependencies ) ) ); // gosh

        List<Artifact> filteredArtifacts = new ArrayList<Artifact>();
        for ( Artifact artifact : artifacts )
        {
            if ( filter.include( artifact ) )
            {
                getLog().debug( "filtering in " + artifact );
                filteredArtifacts.add( artifact );
            }
        }
        return filteredArtifacts;
    }

    private ProcessDestroyer processDestroyer;

    CommandLine getExecutablePath( Map<String, String> enviro, File dir )
    {
        File execFile = new File( executable );
        String exec = null;
        if ( execFile.isFile() )
        {
            getLog().debug( "Toolchains are ignored, 'executable' parameter is set to " + executable );
            exec = execFile.getAbsolutePath();
        }

        if ( exec == null )
        {
            Toolchain tc = getToolchain();

            // if the file doesn't exist & toolchain is null, the exec is probably in the PATH...
            // we should probably also test for isFile and canExecute, but the second one is only
            // available in SDK 6.
            if ( tc != null )
            {
                getLog().info( "Toolchain in exec-maven-plugin: " + tc );
                exec = tc.findTool( executable );
            }
            else
            {
                if ( OS.isFamilyWindows() )
                {
                    List<String> paths = this.getExecutablePaths( enviro );
                    paths.add( 0, dir.getAbsolutePath() );

                    exec = findExecutable( executable, paths );
                }
            }
        }

        if ( exec == null )
        {
            exec = executable;
        }

        CommandLine toRet;
        if ( OS.isFamilyWindows() && !hasNativeExtension( exec ) && hasExecutableExtension( exec ) )
        {
            // run the windows batch script in isolation and exit at the end
            final String comSpec = System.getenv( "ComSpec" );
            toRet = new CommandLine( comSpec == null ? "cmd" : comSpec );
            toRet.addArgument( "/c" );
            toRet.addArgument( exec );
        }
        else
        {
            toRet = new CommandLine( exec );
        }

        return toRet;
    }

    static String findExecutable( final String executable, final List<String> paths )
    {
        File f = null;
        search: for ( final String path : paths )
        {
            f = new File( path, executable );
            if ( !OS.isFamilyWindows() && f.isFile() )
                break;
            else
                for ( final String extension : getExecutableExtensions() )
                {
                    f = new File( path, executable + extension );
                    if ( f.isFile() )
                        break search;
                }
        }

        if ( f == null || !f.exists() )
            return null;

        return f.getAbsolutePath();
    }

    private static boolean hasNativeExtension( final String exec )
    {
        final String lowerCase = exec.toLowerCase();
        return lowerCase.endsWith( ".exe" ) || lowerCase.endsWith( ".com" );
    }

    private static boolean hasExecutableExtension( final String exec )
    {
        final String lowerCase = exec.toLowerCase();
        for ( final String ext : getExecutableExtensions() )
            if ( lowerCase.endsWith( ext ) )
                return true;

        return false;
    }

    private static List<String> getExecutableExtensions()
    {
        final String pathExt = System.getenv( "PATHEXT" );
        return pathExt == null ? Arrays.asList( ".bat", ".cmd" )
                        : Arrays.asList( StringUtils.split( pathExt.toLowerCase(), File.pathSeparator ) );
    }

    private List<String> getExecutablePaths( Map<String, String> enviro )
    {
        List<String> paths = new ArrayList<String>();
        paths.add( "" );

        String path = enviro.get( "PATH" );
        if ( path != null )
        {
            paths.addAll( Arrays.asList( StringUtils.split( path, File.pathSeparator ) ) );
        }

        return paths;
    }

    protected int executeCommandLine( Executor exec, CommandLine commandLine, Map<String, String> enviro,
                                      OutputStream out, OutputStream err )
                                          throws ExecuteException, IOException
    {
        // note: don't use BufferedOutputStream here since it delays the outputs MEXEC-138
        PumpStreamHandler psh = new PumpStreamHandler( out, err, System.in );
        return executeCommandLine( exec, commandLine, enviro, psh );
    }

    protected int executeCommandLine( Executor exec, CommandLine commandLine, Map<String, String> enviro,
                                      FileOutputStream outputFile )
                                          throws ExecuteException, IOException
    {
        BufferedOutputStream bos = new BufferedOutputStream( outputFile );
        PumpStreamHandler psh = new PumpStreamHandler( bos );
        return executeCommandLine( exec, commandLine, enviro, psh );
    }

    protected int executeCommandLine( Executor exec, final CommandLine commandLine, Map<String, String> enviro,
                                      final PumpStreamHandler psh )
                                          throws ExecuteException, IOException
    {
        exec.setStreamHandler( psh );

        int result;
        try
        {
            psh.start();
            if ( async )
            {
                if ( asyncDestroyOnShutdown )
                {
                    exec.setProcessDestroyer( getProcessDestroyer() );
                }

                exec.execute( commandLine, enviro, new ExecuteResultHandler()
                {
                    public void onProcessFailed( ExecuteException e )
                    {
                        getLog().error( "Async process failed for: " + commandLine, e );
                    }

                    public void onProcessComplete( int exitValue )
                    {
                        getLog().info( "Async process complete, exit value = " + exitValue + " for: " + commandLine );
                        try
                        {
                            psh.stop();
                        }
                        catch ( IOException e )
                        {
                            getLog().error( "Error stopping async process stream handler for: " + commandLine, e );
                        }
                    }
                } );
                result = 0;
            }
            else
            {
                result = exec.execute( commandLine, enviro );
            }
        }
        finally
        {
            if ( !async )
            {
                psh.stop();
            }
        }
        return result;
    }

    //
    // methods used for tests purposes - allow mocking and simulate automatic setters
    //

    void setExecutable( String executable )
    {
        this.executable = executable;
    }

    String getExecutable()
    {
        return executable;
    }

    void setWorkingDirectory( String workingDir )
    {
        setWorkingDirectory( new File( workingDir ) );
    }

    void setWorkingDirectory( File workingDir )
    {
        this.workingDirectory = workingDir;
    }

    void setArguments( List<?> arguments )
    {
        this.arguments = arguments;
    }

    void setBasedir( File basedir )
    {
        this.basedir = basedir;
    }

    void setProject( MavenProject project )
    {
        this.project = project;
    }

    protected String getSystemProperty( String key )
    {
        return System.getProperty( key );
    }

    public void setSuccessCodes( Integer... list )
    {
        this.successCodes = new int[list.length];
        for ( int index = 0; index < list.length; index++ )
        {
            successCodes[index] = list[index];
        }
    }

    public int[] getSuccessCodes()
    {
        return successCodes;
    }

    private Toolchain getToolchain()
    {
        Toolchain tc = null;

        try
        {
            if ( session != null ) // session is null in tests..
            {
                ToolchainManager toolchainManager =
                    (ToolchainManager) session.getContainer().lookup( ToolchainManager.ROLE );

                if ( toolchainManager != null )
                {
                    tc = toolchainManager.getToolchainFromBuildContext( toolchain, session );
                }
            }
        }
        catch ( ComponentLookupException componentLookupException )
        {
            // just ignore, could happen in pre-2.0.9 builds..
        }
        return tc;
    }

    /**
     * Create a jar with just a manifest containing a Main-Class entry for SurefireBooter and a Class-Path entry for all
     * classpath elements. Copied from surefire (ForkConfiguration#createJar())
     *
     * @param classPath List<String> of all classpath elements.
     * @return
     * @throws IOException
     */
    private File createJar( List<String> classPath, String mainClass )
        throws IOException
    {
        File file = File.createTempFile( "maven-exec", ".jar" );
        file.deleteOnExit();
        FileOutputStream fos = new FileOutputStream( file );
        JarOutputStream jos = new JarOutputStream( fos );
        jos.setLevel( JarOutputStream.STORED );
        JarEntry je = new JarEntry( "META-INF/MANIFEST.MF" );
        jos.putNextEntry( je );

        Manifest man = new Manifest();

        // we can't use StringUtils.join here since we need to add a '/' to
        // the end of directory entries - otherwise the jvm will ignore them.
        StringBuilder cp = new StringBuilder();
        for ( String el : classPath )
        {
            // NOTE: if File points to a directory, this entry MUST end in '/'.
            cp.append( new URL( new File( el ).toURI().toASCIIString() ).toExternalForm() + " " );
        }

        man.getMainAttributes().putValue( "Manifest-Version", "1.0" );
        man.getMainAttributes().putValue( "Class-Path", cp.toString().trim() );
        man.getMainAttributes().putValue( "Main-Class", mainClass );

        man.write( jos );
        jos.close();

        return file;
    }

    private void createArgFile( String filePath, List<String> lines )
        throws IOException
    {
        final String EOL = System.getProperty( "line.separator", "\\n" );

        FileWriter writer = null;
        try
        {
            writer = new FileWriter( filePath );
            for ( String line : lines )
            {
                writer.append( line ).append( EOL );
            }

        }
        finally
        {
            IOUtil.close( writer );
        }
    }

    protected Map<String, String> createEnvs( File envScriptFile )
        throws MojoExecutionException
    {
        Map<String, String> results = null;

        File tmpEnvExecFile = null;
        try
        {
            tmpEnvExecFile = this.createEnvWrapperFile( envScriptFile );

            Commandline cl = new Commandline();// commons-exec instead?
            cl.setExecutable( tmpEnvExecFile.getAbsolutePath() );
            if ( !OS.isFamilyWindows() )
            {
                cl.setExecutable( "sh" );
                cl.createArg().setFile( tmpEnvExecFile );
            }

            // pickup the initial env vars so that the env script can used if necessary
            if ( environmentVariables != null )
            {
                for ( Map.Entry<String, String> item : environmentVariables.entrySet() )
                {
                    cl.addEnvironment( item.getKey(), item.getValue() );
                }
            }

            EnvStreamConsumer stdout = new EnvStreamConsumer();
            StreamConsumer stderr = new DefaultConsumer();

            CommandLineUtils.executeCommandLine( cl, stdout, stderr );

            results = stdout.getParsedEnv();
        }
        catch ( CommandLineException e )
        {
            throw new MojoExecutionException( e.getMessage() );
        }
        catch ( IOException e )
        {
            throw new MojoExecutionException( e.getMessage() );
        }
        finally
        {
            if ( tmpEnvExecFile != null )
            {
                tmpEnvExecFile.delete();
            }
        }

        return results;

    }

    protected File createEnvWrapperFile( File envScript )
        throws IOException
    {
        PrintWriter writer = null;
        File tmpFile = null;
        try
        {

            if ( OS.isFamilyWindows() )
            {
                tmpFile = File.createTempFile( "env", ".bat" );
                writer = new PrintWriter( tmpFile );
                writer.append( "@echo off" ).println();
                writer.append( "call \"" ).append( envScript.getCanonicalPath() ).append( "\"" ).println();
                writer.append( "echo " + EnvStreamConsumer.START_PARSING_INDICATOR ).println();
                writer.append( "set" ).println();
                writer.flush();
            }
            else
            {
                tmpFile = File.createTempFile( "env", ".sh" );
                // tmpFile.setExecutable( true );//java 6 only
                writer = new PrintWriter( tmpFile );
                writer.append( "#! /bin/sh" ).println();
                writer.append( ". " ).append( envScript.getCanonicalPath() ).println(); // works on all unix??
                writer.append( "echo " + EnvStreamConsumer.START_PARSING_INDICATOR ).println();
                writer.append( "env" ).println();
                writer.flush();
            }
        }
        finally
        {
            IOUtil.close( writer );
        }

        return tmpFile;

    }

    protected ProcessDestroyer getProcessDestroyer()
    {
        if ( processDestroyer == null )
        {
            processDestroyer = new ShutdownHookProcessDestroyer();
        }
        return processDestroyer;
    }
}


package com.conveyal.gtfs.stats;

import com.conveyal.gtfs.GTFSFeed;
import com.conveyal.gtfs.model.Agency;
import com.conveyal.gtfs.model.Service;
import com.conveyal.gtfs.model.Stop;
import com.conveyal.gtfs.model.Trip;
import com.conveyal.gtfs.stats.model.AgencyStatistic;
import com.vividsolutions.jts.geom.Geometry;

import java.awt.geom.Point2D;
import java.awt.geom.Rectangle2D;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

/**
 * Retrieves a base set of statistics from the GTFS.
 *
 */
public class FeedStats {

    private GTFSFeed feed = null;
    public String feed_id = null;
    public PatternStats pattern = null;
    public StopStats stop = null;
    public RouteStats route = null;

    public FeedStats(GTFSFeed f) {
        this.feed = f;
        this.feed_id = f.feedId;
        this.pattern = new PatternStats(feed, this);
        this.stop = new StopStats(feed, this);
        this.route = new RouteStats(feed, this);
    }

    public Integer getAgencyCount() {
        return feed.agency.size();
    }

    public Integer getRouteCount() {
        return feed.routes.size();
    }

    public Integer getTripCount() {
        return feed.trips.size();
    }

    public Integer getTripCount(LocalDate date) {
        return getTripsForDate(date).size();
    }

    public Integer getStopCount() {
        return feed.stops.size();
    }

    public Integer getStopTimesCount() {
        return feed.stop_times.size();
    }

    // calendar date range start/end assume a service calendar based schedule
    // returns null for schedules without calendar service schedules

    public LocalDate getCalendarServiceRangeStart() {

        int startDate = 0;
        for (Service service : feed.services.values()) {
            if (service.calendar == null)
                continue;
//            if (startDate == 0 || service.calendar.start_date < startDate) {
//                startDate = service.calendar.start_date;
//            }
        }
        if (startDate == 0)
            return null;

        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
        return LocalDate.parse(String.valueOf(startDate), formatter);
    }

    public LocalDate getCalendarServiceRangeEnd() {

        int endDate = 0;

        for (Service service : feed.services.values()) {
            if (service.calendar == null)
                continue;

//            if (endDate == 0 || service.calendar.end_date > endDate) {
//                endDate = service.calendar.end_date;
//            }
        }
        if (endDate == 0)
            return null;

        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
        return LocalDate.parse(String.valueOf(endDate), formatter);
    }

    public LocalDate getCalendarDateStart() {
        LocalDate startDate = null;
        for (Service service : feed.services.values()) {
            for (LocalDate date : service.calendar_dates.keySet()) {
                if (startDate == null
                        || date.isBefore(startDate))
                startDate = date;
            }
        }
        return startDate;
    }

    public LocalDate getCalendarDateEnd() {
        LocalDate endDate = null;
        for (Service service : feed.services.values()) {
            for (LocalDate date : service.calendar_dates.keySet()) {
                if (endDate == null
                        || date.isAfter(endDate))
                    endDate = date;
            }
        }
        return endDate;
    }

    public Map<LocalDate, Integer> getTripCountPerDateOfService() {

        Map<LocalDate, List<Trip>> tripsPerDate = getTripsPerDateOfService();
        Map<LocalDate, Integer> tripCountPerDate = new HashMap<>();
        for (Map.Entry<LocalDate, List<Trip>> entry : tripsPerDate.entrySet()) {
            LocalDate date = entry.getKey();
            Integer count = entry.getValue().size();
            tripCountPerDate.put(date, count);
        }

        return tripCountPerDate;
    }
    public LocalDate getStartDate() {
        LocalDate startDate = null;

        if (feed.hasFeedInfo()) startDate = feed.getFeedInfo().feed_start_date;
        if (startDate == null) startDate = getCalendarServiceRangeStart();
        if (startDate == null) startDate = getCalendarDateStart();

        return startDate;
    }

    public LocalDate getEndDate() {
        LocalDate endDate = null;

        if (feed.hasFeedInfo()) endDate = feed.getFeedInfo().feed_end_date;
        if (endDate == null) endDate = getCalendarServiceRangeEnd();
        if (endDate == null) endDate = getCalendarDateEnd();

        return endDate;
    }

    public LocalTime getStartTime (LocalDate date) {
        return this.pattern.getStartTimeForTrips(getTripsForDate(date));
    }

    public LocalTime getEndTime (LocalDate date) {
        return this.pattern.getEndTimeForTrips(getTripsForDate(date));
    }

    /** Get total revenue time (in seconds) for all trips on a given date. */
    public long getTotalRevenueTimeForDate (LocalDate date) {
        return this.pattern.getTotalRevenueTimeForTrips(getTripsForDate(date));
    }

    public long getAverageDailyRevenueTime (int dow) {
        // int value of dow from 1 (Mon) to 7 (Sun)
        DayOfWeek dayOfWeek = DayOfWeek.of(dow);
        List<LocalDate> dates = feed.getDatesOfService().stream()
                .filter(date -> date.getDayOfWeek().equals(dayOfWeek))
                .collect(Collectors.toList());

        return getRevenueTimeForDates(dates) / dates.size();
    }

    public long getAverageWeekdayRevenueTime () {
        List<LocalDate> dates = feed.getDatesOfService().stream()
                .filter(date -> {
                    int dow = date.getDayOfWeek().getValue();
                    boolean isWeekday = ((dow >= DayOfWeek.MONDAY.getValue()) && (dow <= DayOfWeek.FRIDAY.getValue()));
                    return isWeekday;
                })
                .collect(Collectors.toList());
        if (dates.size() == 0) {
            return 0;
        }
        return getRevenueTimeForDates(dates) / dates.size();
    }
    public long getRevenueTimeForDates (List<LocalDate> dates) {
        Map<String, Long> timePerService = new HashMap<>();

        // First, get revenue time for each service calendar used in feed dates
        // NOTE: we don't simply get revenue time for each individual date of service
        // because that ends up duplicating a lot of operations if service calendars
        // are reused often.
        dates.stream()
                .map(date -> feed.getServicesForDate(date))
                .flatMap(List::stream)
                .collect(Collectors.toSet())
                .stream()
                .forEach(s -> {
                    List<Trip> trips = feed.getTripsForService(s.service_id);
                    long time = this.pattern.getTotalRevenueTimeForTrips(trips);
                    timePerService.put(s.service_id, time);
                });

        // Next, sum up service calendars by dates of service
        long total = dates.stream()
                .map(date ->  // get sum of services per date
                    feed.getServicesForDate(date).stream()
                            .map(s -> timePerService.get(s.service_id))
                            .mapToLong(time -> time)
                            .sum()
                )
                .mapToLong(time -> time)
                .sum();
        return total;
    }

    public long getTotalRevenueTime () {
        return getRevenueTimeForDates(feed.getDatesOfService());
    }

    /** Get total revenue distance (in meters) for all trips on a given date. */
    public double getTotalDistanceForTrips (LocalDate date) {
        return this.pattern.getTotalDistanceForTrips(getTripsForDate(date));
    }

    /** in seconds */
    public int getDailyAverageHeadway (LocalDate date, LocalTime from, LocalTime to) {

        OptionalDouble avg =  feed.stops.values().stream()
                .map(s -> this.stop.getAverageHeadwayForStop(s.stop_id, date, from, to))
                .mapToDouble(headway -> headway)
                .average();

        return (int) avg.getAsDouble();
    }

    public double getAverageTripSpeed (LocalDate date, LocalTime from, LocalTime to) {
        List<Trip> trips = getTripsForDate(date);
        return this.pattern.getAverageSpeedForTrips(trips, from, to);
    }
    public Map<LocalDate, List<Trip>> getTripsPerDateOfService () {
        Map<LocalDate, List<Trip>> tripsPerDate = new HashMap<>();
        Map<String, List<Trip>> tripsPerService = new HashMap<>();

        LocalDate startDate = getStartDate();
        LocalDate endDate = getEndDate();
        if (startDate == null || endDate == null) {
            return tripsPerDate;
        }
        int allDates = (int) ChronoUnit.DAYS.between(startDate, endDate.plus(1, ChronoUnit.DAYS));
        List<LocalDate> dates = IntStream.range(0, allDates)
                .mapToObj(offset -> startDate.plusDays(offset))
                .collect(Collectors.toList());
        dates.stream()
                .map(date -> feed.getServicesForDate(date))
                .flatMap(List::stream)
                .collect(Collectors.toSet())
                .stream()
                .forEach(s -> {
                    List<Trip> trips = feed.getTripsForService(s.service_id);
                    tripsPerService.put(s.service_id, trips);
                });
        dates.stream()
                .forEach(date -> {
                    List<Trip> trips = feed.getServicesForDate(date).stream()
                            .map(s -> tripsPerService.get(s.service_id))
                            .flatMap(List::stream)
                            .collect(Collectors.toList());
                    tripsPerDate.put(date, trips);
                });
        return tripsPerDate;
    }

    public List<Trip> getTripsForDate (LocalDate date) {
        return feed.getServicesForDate(date).stream()
                .map(s -> feed.getTripsForService(s.service_id))
                .flatMap(List::stream)
                .collect(Collectors.toList());
    }

    public Collection<Agency> getAllAgencies() {
        return feed.agency.values();
    }

    /**
     * Get the bounding box of this GTFS feed.
     * We use a Rectangle2D rather than a Geotools envelope because GTFS is always in WGS 84.
     * Note that stops do not have agencies in GTFS.
     */
    public Rectangle2D getBounds () {
        Rectangle2D ret = null;

        for (Stop stop : feed.stops.values()) {

            // skip over stops that don't have any stop times
            if (!feed.stopCountByStopTime.containsKey(stop.stop_id)) {
                continue;
            }
            if (ret == null) {
                ret = new Rectangle2D.Double(stop.stop_lon, stop.stop_lat, 0, 0);
            }
            else {
                ret.add(new Point2D.Double(stop.stop_lon, stop.stop_lat));
            }
        }

        return ret;
    }

    public AgencyStatistic getStatistic(String agencyId) {
        AgencyStatistic gs = new AgencyStatistic();
        gs.setAgencyId(agencyId);
        gs.setRouteCount(AgencyStats.getRouteCount(feed, agencyId));
        gs.setTripCount(AgencyStats.getTripCount(feed, agencyId));
        gs.setStopCount(AgencyStats.getStopCount(feed, agencyId));
        gs.setStopTimeCount(AgencyStats.getStopTimesCount(feed, agencyId));
        gs.setCalendarStartDate(AgencyStats.getCalendarDateStart(feed, agencyId));
        gs.setCalendarEndDate(AgencyStats.getCalendarDateEnd(feed, agencyId));
        gs.setCalendarServiceStart(AgencyStats.getCalendarServiceRangeStart(feed, agencyId));
        gs.setCalendarServiceEnd(AgencyStats.getCalendarServiceRangeEnd(feed, agencyId));
        gs.setBounds(getBounds());
        return gs;
    }

    public String getStatisticAsCSV(String agencyId) {
        AgencyStatistic s = getStatistic(agencyId);
        return formatStatisticAsCSV(s);
    }

    public static String formatStatisticAsCSV(AgencyStatistic s) {
        StringBuffer buff = new StringBuffer();
        buff.append(s.getAgencyId());
        buff.append(",");
        buff.append(s.getRouteCount());
        buff.append(",");
        buff.append(s.getTripCount());
        buff.append(",");
        buff.append(s.getStopCount());
        buff.append(",");
        buff.append(s.getStopTimeCount());
        buff.append(",");
        buff.append(s.getCalendarServiceStart());
        buff.append(",");
        buff.append(s.getCalendarServiceEnd());
        buff.append(",");
        buff.append(s.getCalendarStartDate());
        buff.append(",");
        buff.append(s.getCalendarEndDate());
        return buff.toString();
    }

    public long getFrequencyCount() {
        return feed.frequencies.size();
    }

    public long getShapePointCount() {
        return feed.shape_points.size();
    }

    public long getFareAttributeCount() {
        return feed.fares.size();
    }

    public long getFareRulesCount() {
        return feed.fares.values().stream()
                .mapToInt(fare -> fare.fare_rules.size())
                .sum();
    }

    public long getServiceCount() {
        return feed.services.size();
    }

    public List<LocalDate> getDatesOfService() {
        return feed.getDatesOfService();
    }

    public Geometry getMergedBuffers() {
        return feed.getMergedBuffers();
    }
}


package com.conveyal.gtfs.validator;

import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.SQLErrorStorage;
import com.conveyal.gtfs.loader.Feed;
import com.conveyal.gtfs.model.Entity;
import com.conveyal.gtfs.model.Route;
import com.conveyal.gtfs.model.Stop;
import com.conveyal.gtfs.model.StopTime;
import com.conveyal.gtfs.model.Trip;

import java.util.HashSet;
import java.util.List;
import java.util.Set;

import static com.conveyal.gtfs.error.NewGTFSErrorType.*;
import static com.conveyal.gtfs.util.Util.fastDistance;
import static com.conveyal.gtfs.validator.NewTripTimesValidator.*;

/**
 * Created by abyrd on 2017-04-18
 */
public class SpeedTripValidator extends TripValidator {

    public static final double MIN_SPEED_KPH = 0.5;
    private boolean allTravelTimesAreRounded = true;
    private Set<NewGTFSError> travelTimeZeroErrors = new HashSet<>();

    public SpeedTripValidator(Feed feed, SQLErrorStorage errorStorage) {
        super(feed, errorStorage);
    }

    @Override
    public void validateTrip(Trip trip, Route route, List<StopTime> stopTimes, List<Stop> stops) {
        // The specific maximum speed for this trip's route's mode of travel.
        double maxSpeedKph = getMaxSpeedKph(route);
        // Skip over any initial stop times that won't allow calculating speeds.
        int beginIndex = 0;
        while (missingBothTimes(stopTimes.get(beginIndex))) {
            beginIndex++;
            if (beginIndex == stopTimes.size()) return;
        }
        // Unfortunately we can't work on each stop pair in isolation,
        // because we want to accumulate distance when stop times are missing.
        StopTime prevStopTime = stopTimes.get(beginIndex);
        Stop prevStop = stops.get(beginIndex);
        double distanceMeters = 0;
        for (int i = beginIndex + 1; i < stopTimes.size(); i++) {
            StopTime currStopTime = stopTimes.get(i);
            Stop currStop = stops.get(i);
            // Distance is accumulated in case times are not provided for some StopTimes.
            distanceMeters += fastDistance(currStop.stop_lat, currStop.stop_lon, prevStop.stop_lat, prevStop.stop_lon);
            // Check that shape_dist_traveled is increasing.
            checkShapeDistTraveled(prevStopTime, currStopTime);
            if (missingBothTimes(currStopTime)) {
                // FixMissingTimes has already been called, so both arrival and departure time are missing.
                // The spec allows this. Other than accumulating distance, skip this StopTime.
                continue;
            }
            if (currStopTime.departure_time < currStopTime.arrival_time) {
                registerError(currStopTime, DEPARTURE_BEFORE_ARRIVAL);
            }
            // Detect if travel times are rounded off to minutes.
            boolean bothTravelTimesRounded = areTravelTimesRounded(prevStopTime) && areTravelTimesRounded(currStopTime);
            double travelTimeSeconds = currStopTime.arrival_time - prevStopTime.departure_time;
            // If travel times are rounded and travel time is zero, determine the maximum and minimum possible speed
            // by adding/removing one minute of slack.
            if (bothTravelTimesRounded && travelTimeSeconds == 0) {
                travelTimeSeconds += 60;
            }
            if (checkDistanceAndTime(distanceMeters, travelTimeSeconds, currStopTime)) {
                // If distance and time are OK, we've got valid numbers to calculate a travel speed.
                double kph = (distanceMeters / 1000D) / (travelTimeSeconds / 60D / 60D);
                if (kph < MIN_SPEED_KPH) {
                    registerError(currStopTime, TRAVEL_TOO_SLOW, String.format("%2.1f km/h", kph));
                } else if (kph > maxSpeedKph) {
                    registerError(currStopTime, TRAVEL_TOO_FAST, String.format("%2.1f km/h", kph));
                }
            }
            // Reset accumulated distance, we've processed a stop time with arrival or departure time specified.
            distanceMeters = 0;
            // Record current stop and stopTime for the next iteration.
            prevStopTime = currStopTime;
            prevStop = currStop;
        }
    }

    /**
     * Register shape dist traveled error if current stop time has a value AND either and the previous value is
     * missing (if at least one stop time has a value, all stop times for the trip should) OR if current value
     * is less than or equal to the previous value. Note: if the previous shape_dist_traveled value is present and the
     * current value is missing, the previous value will be greater than the current stop time's value because
     * {@link Entity#DOUBLE_MISSING} is the lowest possible double value. This in turn will register an error.
     */
    private void checkShapeDistTraveled(StopTime previous, StopTime current) {
        if (
            current.shape_dist_traveled != Entity.DOUBLE_MISSING &&
            (
                previous.shape_dist_traveled == Entity.DOUBLE_MISSING ||
                current.shape_dist_traveled <= previous.shape_dist_traveled
            )
        ) {
            registerError(current, SHAPE_DIST_TRAVELED_NOT_INCREASING, current.shape_dist_traveled);
        }
    }

    /**
     * Completing this feed validator means checking if there were any unrounded travel times in the feed and (if so)
     * registering any zero travel time errors that were passed over before the first unrounded travel time was
     * encountered. If in fact all travel times are rounded to the minute, store a special feed-wide error in this case.
     */
    public void complete (ValidationResult validationResult) {
        if (!allTravelTimesAreRounded) storeErrors(travelTimeZeroErrors);
        else registerError(NewGTFSError.forFeed(FEED_TRAVEL_TIMES_ROUNDED, null));
    }

    /**
     * Check that arrival and departure time for a stop time are rounded to the minute and update
     * {@link #allTravelTimesAreRounded} accordingly.
     */
    private boolean areTravelTimesRounded(StopTime stopTime) {
        boolean bothTravelTimesAreRounded = stopTime.departure_time % 60 == 0 && stopTime.arrival_time % 60 == 0;
        if (!bothTravelTimesAreRounded) this.allTravelTimesAreRounded = false;
        return bothTravelTimesAreRounded;
    }

    /**
     * This just pulls some of the range checking logic out of the main trip checking loop so it's more readable.
     * @return true if all values are OK
     */
    private boolean checkDistanceAndTime (double distanceMeters, double travelTimeSeconds, StopTime stopTime) {
        boolean good = true;
        // TODO Use Epsilon for very tiny travel e.g. < 5 meters
        if (distanceMeters == 0) {
            registerError(stopTime, TRAVEL_DISTANCE_ZERO);
            good = false;
        }
        if (travelTimeSeconds < 0) {
            registerError(stopTime, TRAVEL_TIME_NEGATIVE, travelTimeSeconds);
            good = false;
        } else if (travelTimeSeconds == 0) {
            // Only register the travel time zero error if not all travel times are rounded. Otherwise, hold onto the
            // error in the travelTimeZeroErrors collection until the completion of this validator.
            if (!allTravelTimesAreRounded) registerError(stopTime, TRAVEL_TIME_ZERO);
            else travelTimeZeroErrors.add(createUnregisteredError(stopTime, TRAVEL_TIME_ZERO));
            good = false;
        }
        return good;
    }

    /**
     * @return max speed in km/hour.
     */
    private static double getMaxSpeedKph (Route route) {
        int type = -1;
        if (route != null) type = route.route_type;
        switch (type) {
            case Route.SUBWAY:
                return 140; // Speed of HK airport line.
            case Route.RAIL:
                return 310; // European HSR max speed is around 300kph, Chinese HSR runs at about 310kph.
            case Route.FERRY:
                return 107; // World's fastest ferry is 107kph.
            default:
                return 130; // 130 kph is max highway speed.
        }
    }

}


package com.conveyal.gtfs.model;

import com.google.common.base.Joiner;
import com.vividsolutions.jts.geom.LineString;

import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Represents a collection of trips that all visit the same stops in the same sequence.
 */
public class Pattern extends Entity {
    public static final long serialVersionUID = 1L;
    private static final Logger LOG = LoggerFactory.getLogger(Pattern.class);

    @Override
    public String getId () {
        return pattern_id;
    }

    // A unique ID for this journey pattern / stop pattern
    public String pattern_id;

    // The segment of the pattern's geometry (which is always a LineString) on which each stop in the sequence falls.
    public int[] segmentIndex;

    // The percentage in [0..1] along the line segment at which each stop in the sequence falls.
    public double[] segmentFraction;

    public List<String> orderedStops;
    // TODO: change list of trips to set
    public List<String> associatedTrips;
    // TODO: add set of shapes
    public Set<String> associatedShapes;
    public LineString geometry;
    public String name;
    public String route_id;
    public int direction_id = INT_MISSING;
    public static Joiner joiner = Joiner.on("-").skipNulls();
    public String feed_id;

    // TODO: Should a Pattern be generated for a single trip or a set of trips that share the same ordered stop list?

    /**
     *
     * @param orderedStops
     * @param trips the first trip will serve as an exemplar for all the others.
     * @param patternGeometry
     */
    public Pattern (List<String> orderedStops, Collection<Trip> trips, LineString patternGeometry){

        // Temporarily make a random ID for the pattern, which might be overwritten in a later step ?
        this.pattern_id = UUID.randomUUID().toString();

        // Assign ordered list of stop IDs to be the key of this pattern.
        // FIXME what about pickup / dropoff type?
        this.orderedStops = orderedStops;

        // Save the string IDs of the trips on this pattern.
        this.associatedTrips = trips.stream().map(t -> t.trip_id).collect(Collectors.toList());

        // In theory all trips could take different paths and be on different routes.
        // Here we're using only the first one as an exemplar.
        String trip_id = associatedTrips.get(0);

        Trip exemplarTrip = trips.iterator().next();
        this.geometry = patternGeometry;

        // feed.getTripGeometry(exemplarTrip.trip_id);

        // Patterns have one and only one route.
        // FIXME are we certain we're only passing in trips on one route? or are we losing information here?
        this.route_id = exemplarTrip.route_id;
        this.direction_id = exemplarTrip.direction_id;

        // A name is assigned to this pattern based on the headsign, short name, direction ID or stop IDs.
        // This is not at all guaranteed to be unique, it's just to help identify the pattern.
        if (exemplarTrip.trip_headsign != null){
            name = exemplarTrip.trip_headsign;
        }
        else if (exemplarTrip.trip_short_name != null) {
            name = exemplarTrip.trip_short_name;
        }
        else if (exemplarTrip.direction_id >= 0){
            name = String.valueOf(exemplarTrip.direction_id);
        }
        else{
            name = joiner.join(orderedStops);
        }

        // TODO: Implement segmentIndex using JTS to segment out LineString by stops.

        // TODO: Implement segmentFraction using JTS to segment out LineString by stops.

    }

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#PATTERNS}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        statement.setString(oneBasedIndex++, pattern_id);
        statement.setString(oneBasedIndex++, route_id);
        statement.setString(oneBasedIndex++, name);
        // Editor-specific fields
        setIntParameter(statement, oneBasedIndex++, direction_id);
        // Note: pattern#use_frequency is set in JdbcGtfsSnapshotter here:
        // https://github.com/conveyal/gtfs-lib/blob/0c6aca98a83d534853b74011e6cc7bf376592581/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsSnapshotter.java#L196-L211
        setIntParameter(statement, oneBasedIndex++, INT_MISSING); // use_frequency
        // FIXME: Shape set might be null?
        statement.setString(oneBasedIndex++, associatedShapes.iterator().next());
    }
}


package com.conveyal.gtfs.loader;

import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.NewGTFSErrorType;
import com.conveyal.gtfs.error.SQLErrorStorage;
import com.conveyal.gtfs.storage.StorageException;
import com.csvreader.CsvReader;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import com.google.common.io.Files;
import org.apache.commons.dbutils.DbUtils;
import org.postgresql.copy.CopyManager;
import org.postgresql.core.BaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.sql.DataSource;
import java.io.*;
import java.sql.*;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import static com.conveyal.gtfs.error.NewGTFSErrorType.*;
import static com.conveyal.gtfs.model.Entity.human;
import static com.conveyal.gtfs.util.Util.randomIdString;

/**
 * This class loads CSV tables from zipped GTFS into an SQL relational database management system with a JDBC driver.
 * By comparing the GTFS specification for a table with the headers present in the GTFS CSV, it dynamically builds up
 * table definitions and SQL statements to interact with those tables. It retains all columns present in the GTFS,
 * including optional columns, known extensions, and unrecognized proprietary extensions.
 *
 * It supports several ways of putting the data into the tables: batched prepared inserts or loading from an
 * intermediate tab separated text file.
 *
 * Our previous approach involved loading GTFS CSV tables into Java objects and then using an object-relational mapping
 * to put those objects into a database. In that case a fixed number of fields are represented. If the GTFS feed
 * contains extra proprietary fields, they are lost immediately on import. The Java model objects must contain fields
 * for all GTFS columns that will ever be retrieved, and memory and database space are required to represent all those
 * fields even when they are not present in a particular feed. The same would be true of a direct-to-database approach
 * if multiple feeds were loaded into the same tables: a "lowest common denominator" set of fields would need to be
 * selected. However, we create one set of tables per GTFS feed loaded into the same database and namespace them with
 * what the SQL spec calls "schemas".
 *
 * The structure of the CSV file and the data it contains are validated line by line during the load process. This
 * allows us to handle error recovery ourselves, recording detailed messages about the largest possible number of errors
 * rather than failing at the first error.
 *
 * This is important because of the generally long turnaround for GTFS publication, repair, and validation. If a newly
 * submitted feed fails to import because of a missing file, we don't want to report that single error to the feed
 * producer, only to discover and report additional errors when the repaired feed is re-submitted weeks later.
 *
 * The fact that existing libraries would abort a GTFS import upon encountering very common, recoverable errors was the
 * original motivation for creating gtfs-lib. Error recovery is all the more important when bulk-loading data into
 * database systems - the Postgres 'copy' import is particularly brittle and does not provide error messages that would
 * help the feed producer repair their feed.
 *
 * The validation performed during CSV loading includes:
 * - columns are present for all required fields
 * - all rows have the same number of fields as there are headers
 * - fields do not contain problematic characters
 * - field contents can be converted to the target data types and are in range
 * - referential integrity
 */
public class JdbcGtfsLoader {

    public static final long INSERT_BATCH_SIZE = 500;
    // Represents null in Postgres text format
    private static final String POSTGRES_NULL_TEXT = "\\N";
    private static final Logger LOG = LoggerFactory.getLogger(JdbcGtfsLoader.class);

    private String gtfsFilePath;
    protected ZipFile zip;

    private File tempTextFile;
    private PrintStream tempTextFileStream;
    private PreparedStatement insertStatement = null;

    private final DataSource dataSource;

    // These fields will be filled in once feed loading begins.
    private Connection connection;
    private String tablePrefix;
    private SQLErrorStorage errorStorage;

    // Contains references to unique entity IDs during load stage used for referential integrity check.
    private ReferenceTracker referenceTracker = new ReferenceTracker();

    public JdbcGtfsLoader(String gtfsFilePath, DataSource dataSource) {
        this.gtfsFilePath = gtfsFilePath;
        this.dataSource = dataSource;
    }

    // Hash to uniquely identify files.
    // We can't use CRC32, the probability of collision on 10k items is about 1%.
    // https://stackoverflow.com/a/1867252
    // http://preshing.com/20110504/hash-collision-probabilities/
    // On the full NL feed:
    // MD5 took 820 msec,    cabb18e43798f92c52d5d0e49f52c988
    // Murmur took 317 msec, 5e5968f9bf5e1cdf711f6f48fcd94355
    // SHA1 took 1072 msec,  9fb356af4be2750f20955203787ec6f95d32ef22

    // There appears to be no advantage to loading tables in parallel, as the whole loading process is I/O bound.
    public FeedLoadResult loadTables () {

        // This result object will be returned to the caller to summarize the feed and report any critical errors.
        FeedLoadResult result = new FeedLoadResult();

        try {
            // Begin tracking time. FIXME: should this follow the connect/register and begin with the table loads?
            long startTime = System.currentTimeMillis();
            // We get a single connection object and share it across several different methods.
            // This ensures that actions taken in one method are visible to all subsequent SQL statements.
            // If we create a schema or table on one connection, then access it in a separate connection, we have no
            // guarantee that it exists when the accessing statement is executed.
            connection = dataSource.getConnection();
            File gtfsFile = new File(gtfsFilePath);
            this.zip = new ZipFile(gtfsFilePath);
            // Generate a unique prefix that will identify this feed.
            // Prefixes ("schema" names) based on feed_id and feed_version get very messy, so we use random unique IDs.
            // We don't want to use an auto-increment numeric primary key because these need to be alphabetical.
            // Although ID collisions are theoretically possible, they are improbable in the extreme because our IDs
            // are long enough to have as much entropy as a UUID. So we don't really need to check for uniqueness and
            // retry in a loop.
            // TODO handle the case where we don't want any prefix.
            this.tablePrefix = randomIdString();
            result.filename = gtfsFilePath;
            result.uniqueIdentifier = tablePrefix;

            // The order of the following four lines should not be changed because the schema needs to be in place
            // before the error storage can be constructed, which in turn needs to exist in case any errors are
            // encountered during the loading process.
            {
                createSchema(connection, tablePrefix);
                //the SQLErrorStorage constructor expects the tablePrefix to contain the dot separator.
                this.errorStorage = new SQLErrorStorage(connection, tablePrefix + ".", true);
                //registerFeed accesses this.tablePrefix which shouldn't contain the dot separator.
                registerFeed(gtfsFile);
                // Include the dot separator in the table prefix from this point onwards.
                // This allows everything to work even when there's no prefix.
                this.tablePrefix += ".";
            }
            // Load each table in turn, saving some summary information about what happened during each table load
            result.agency = load(Table.AGENCY);
            result.calendar = load(Table.CALENDAR);
            result.calendarDates = load(Table.CALENDAR_DATES);
            result.routes = load(Table.ROUTES);
            result.fareAttributes = load(Table.FARE_ATTRIBUTES);
            result.fareRules = load(Table.FARE_RULES);
            result.feedInfo = load(Table.FEED_INFO);
            result.shapes = load(Table.SHAPES);
            result.stops = load(Table.STOPS);
            result.transfers = load(Table.TRANSFERS);
            result.trips = load(Table.TRIPS); // refs routes
            result.frequencies = load(Table.FREQUENCIES); // refs trips
            result.stopTimes = load(Table.STOP_TIMES);
            result.errorCount = errorStorage.getErrorCount();
            // This will commit and close the single connection that has been shared between all preceding load steps.
            errorStorage.commitAndClose();
            zip.close();
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            LOG.info("Loading tables took {} sec", result.loadTimeMillis / 1000);
        } catch (Exception ex) {
            // TODO catch exceptions separately while loading each table so load can continue, store in TableLoadResult
            LOG.error("Exception while loading GTFS file: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.toString();
        }
        return result;
    }

    /**
     * Creates a schema/namespace in the database WITHOUT committing the changes.
     * This does *not* setup any other tables or enter the schema name in a registry (@see #registerFeed).
     *
     * @param connection Connection to the database to create the schema on.
     * @param schemaName Name of the schema (i.e. table prefix). Should not include the dot suffix.
     */
    static void createSchema (Connection connection, String schemaName) {
        try {
            Statement statement = connection.createStatement();
            // FIXME do the following only on databases that support schemas.
            // SQLite does not support them. Is there any advantage of schemas over flat tables?
            statement.execute("create schema " + schemaName);
            LOG.info("Created new feed schema: {}", statement);
        } catch (Exception ex) {
            LOG.error("Exception while registering new feed namespace in feeds table: {}", ex.getMessage());
            DbUtils.closeQuietly(connection);
        }
    }

    /**
     * Add a line to the list of loaded feeds showing that this feed has been loaded.
     * We used to inspect feed_info here so we could make our table prefix based on feed ID and version.
     * Now we just load feed_info like any other table.
     *         // Create a row in the table of loaded feeds for this feed
     * Really this is not just making the table prefix - it's loading the feed_info and should also calculate hashes.
     *
     * Originally we were flattening all feed_info files into one root-level table, but that forces us to drop any
     * custom fields in feed_info.
     */
    private void registerFeed (File gtfsFile) {

        // FIXME is this extra CSV reader used anymore? Check comment below.
        // First, inspect feed_info.txt to extract the ID and version.
        // We could get this with SQL after loading, but feed_info, feed_id and feed_version are all optional.
        CsvReader csvReader = Table.FEED_INFO.getCsvReader(zip, errorStorage);
        String feedId = "", feedVersion = "";
        if (csvReader != null) {
            // feed_info.txt has been found and opened.
            try {
                csvReader.readRecord();
                // csvReader.get() returns the empty string for missing columns
                feedId = csvReader.get("feed_id");
                feedVersion = csvReader.get("feed_version");
            } catch (IOException e) {
                LOG.error("Exception while inspecting feed_info: {}", e);
            }
            csvReader.close();
        }

        try {
            HashCode md5 = Files.hash(gtfsFile, Hashing.md5());
            String md5Hex = md5.toString();
            HashCode sha1 = Files.hash(gtfsFile, Hashing.sha1());
            String shaHex = sha1.toString();
            createFeedRegistryIfNotExists(connection);
            // TODO try to get the feed_id and feed_version out of the feed_info table
            // statement.execute("select * from feed_info");

            // current_timestamp seems to be the only standard way to get the current time across all common databases.
            // Record total load processing time?
            PreparedStatement insertStatement = connection.prepareStatement(
                    "insert into feeds values (?, ?, ?, ?, ?, ?, current_timestamp, null)");
            insertStatement.setString(1, tablePrefix);
            insertStatement.setString(2, md5Hex);
            insertStatement.setString(3, shaHex);
            insertStatement.setString(4, feedId.isEmpty() ? null : feedId);
            insertStatement.setString(5, feedVersion.isEmpty() ? null : feedVersion);
            insertStatement.setString(6, zip.getName());
            insertStatement.execute();
            connection.commit();
            LOG.info("Created new feed namespace: {}", insertStatement);
        } catch (Exception ex) {
            LOG.error("Exception while registering new feed namespace in feeds table", ex);
            DbUtils.closeQuietly(connection);
        }
    }

    /**
     * Creates the feed registry table if it does not already exist. This must occur before the first attempt to load a
     * GTFS feed or create an empty snapshot. Note: the connection MUST be committed after this method call.
     */
    static void createFeedRegistryIfNotExists(Connection connection) throws SQLException {
        Statement statement = connection.createStatement();
        statement.execute("create table if not exists feeds (namespace varchar primary key, md5 varchar, " +
                              "sha1 varchar, feed_id varchar, feed_version varchar, filename varchar, loaded_date timestamp, " +
                              "snapshot_of varchar)");
    }

    /**
     * This wraps the main internal table loader method to catch exceptions and figure out how many errors happened.
     */
    private TableLoadResult load (Table table) {
        // This object will be returned to the caller to summarize the contents of the table and any errors.
        TableLoadResult tableLoadResult = new TableLoadResult();
        int initialErrorCount = errorStorage.getErrorCount();
        try {
            tableLoadResult.rowCount = loadInternal(table);
            tableLoadResult.fileSize = getTableSize(table);
            LOG.info(String.format("loaded in %d %s records", tableLoadResult.rowCount, table.name));
        } catch (Exception ex) {
            LOG.error("Fatal error loading table", ex);
            tableLoadResult.fatalException = ex.toString();
            // Rollback connection so that fatal exception does not impact loading of other tables.
            try {
                connection.rollback();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        } finally {
            // Explicitly delete the tmp file now that load is finished (either success or failure).
            // Otherwise these multi-GB files clutter the drive.
            if (tempTextFile != null) {
                tempTextFile.delete();
            }
        }
        int finalErrorCount = errorStorage.getErrorCount();
        tableLoadResult.errorCount = finalErrorCount - initialErrorCount;
        return tableLoadResult;
    }

    /**
     * Get the uncompressed file size in bytes for the specified GTFS table.
     */
    private int getTableSize(Table table) {
        ZipEntry zipEntry = zip.getEntry(table.name + ".txt");
        if (zipEntry == null) return 0;
        return (int) zipEntry.getSize();
    }

    /**
     * This function will throw any exception that occurs. Those exceptions will be handled by the outer load method.
     * @return number of rows that were loaded.
     */
    private int loadInternal (Table table) throws Exception {
        CsvReader csvReader = table.getCsvReader(zip, errorStorage);
        if (csvReader == null) {
            LOG.info(String.format("file %s.txt not found in gtfs zipfile", table.name));
            // This GTFS table could not be opened in the zip, even in a subdirectory.
            if (table.isRequired()) errorStorage.storeError(NewGTFSError.forTable(table, MISSING_TABLE));
            return 0;
        }
        LOG.info("Loading GTFS table {}", table.name);
        // Use the Postgres text load format if we're connected to that DBMS.
        boolean postgresText = (connection.getMetaData().getDatabaseProductName().equals("PostgreSQL"));

        // TODO Strip out line returns, tabs in field contents.
        // By default the CSV reader trims leading and trailing whitespace in fields.
        // Build up a list of fields in the same order they appear in this GTFS CSV file.
        Field[] fields = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), errorStorage);
        int keyFieldIndex = table.getKeyFieldIndex(fields);
        // Create separate fields array with filtered list that does not include null values (for duplicate headers or
        // ID field). This is solely used to construct the table and array of values to load.
        Field[] cleanFields = Arrays.stream(fields).filter(field -> field != null).toArray(Field[]::new);
        if (cleanFields.length == 0) {
            // Do not create the table if there are no valid fields.
            errorStorage.storeError(NewGTFSError.forTable(table, TABLE_MISSING_COLUMN_HEADERS));
            return 0;
        }
        // Replace the GTFS spec Table with one representing the SQL table we will populate, with reordered columns.
        // FIXME this is confusing, we only create a new table object so we can call a couple of methods on it, all of which just need a list of fields.
        Table targetTable = new Table(tablePrefix + table.name, table.entityClass, table.required, cleanFields);

        // NOTE H2 doesn't seem to work with schemas (or create schema doesn't work).
        // With bulk loads it takes 140 seconds to load the data and additional 120 seconds just to index the stop times.
        // SQLite also doesn't support schemas, but you can attach additional database files with schema-like naming.
        // We'll just literally prepend feed identifiers to table names when supplied.
        // Some databases require the table to exist before a statement can be prepared.
        targetTable.createSqlTable(connection);

        // TODO are we loading with or without a header row in our Postgres text file?
        if (postgresText) {
            // No need to output headers to temp text file, our SQL table column order exactly matches our text file.
            tempTextFile = File.createTempFile(targetTable.name, "text");
            tempTextFileStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(tempTextFile)));
            LOG.info("Loading via temporary text file at " + tempTextFile.getAbsolutePath());
        } else {
            insertStatement = connection.prepareStatement(targetTable.generateInsertSql());
            LOG.info(insertStatement.toString()); // Logs the SQL for the prepared statement
        }

        // When outputting text, accumulate transformed strings to allow skipping rows when errors are encountered.
        // One extra position in the array for the CSV line number.
        String[] transformedStrings = new String[cleanFields.length + 1];

        // Iterate over each record and prepare the record for storage in the table either through batch insert
        // statements or postgres text copy operation.
        while (csvReader.readRecord()) {
            // The CSV reader's current record is zero-based and does not include the header line.
            // Convert to a CSV file line number that will make more sense to people reading error messages.
            if (csvReader.getCurrentRecord() + 2 > Integer.MAX_VALUE) {
                errorStorage.storeError(NewGTFSError.forTable(table, TABLE_TOO_LONG));
                break;
            }
            int lineNumber = ((int) csvReader.getCurrentRecord()) + 2;
            if (lineNumber % 500_000 == 0) LOG.info("Processed {}", human(lineNumber));
            if (csvReader.getColumnCount() != fields.length) {
                String badValues = String.format("expected=%d; found=%d", fields.length, csvReader.getColumnCount());
                errorStorage.storeError(NewGTFSError.forLine(table, lineNumber, WRONG_NUMBER_OF_FIELDS, badValues));
                continue;
            }
            // Store value of key field for use in checking duplicate IDs
            // FIXME: If the key field is missing (keyFieldIndex is still -1) from a loaded table, this will crash.
            String keyValue = csvReader.get(keyFieldIndex);
            // The first field holds the line number of the CSV file. Prepared statement parameters are one-based.
            if (postgresText) transformedStrings[0] = Integer.toString(lineNumber);
            else insertStatement.setInt(1, lineNumber);
            // Maintain a separate columnIndex from for loop because some fields may be null and not included in the set
            // of fields for this table.
            int columnIndex = 0;
            for (int f = 0; f < fields.length; f++) {
                Field field = fields[f];
                // If the field is null, it represents a duplicate header or ID field and must be skipped to maintain
                // table integrity.
                if (field == null) continue;
                // CSV reader get on an empty field will be an empty string literal.
                String string = csvReader.get(f);
                // Use spec table to check that references are valid and IDs are unique.
                Set<NewGTFSError> errors = referenceTracker
                    .checkReferencesAndUniqueness(keyValue, lineNumber, field, string, table);
                // Check for special case with calendar_dates where added service should not trigger ref. integrity
                // error.
                if (
                    table.name.equals("calendar_dates") &&
                    "service_id".equals(field.name) &&
                    "1".equals(csvReader.get(Field.getFieldIndex(fields, "exception_type")))

                ){
                    for (NewGTFSError error : errors) {
                        if (NewGTFSErrorType.REFERENTIAL_INTEGRITY.equals(error.errorType)) {
                            // Do not record bad service_id reference errors for calendar date entries that add service
                            // (exception type=1) because a corresponding service_id in calendars.txt is not required in
                            // this case.
                            LOG.info(
                                "A calendar_dates.txt entry added service (exception_type=1) for service_id={}, which does not have (or necessarily need) a corresponding entry in calendars.txt.",
                                keyValue
                            );
                        } else {
                            errorStorage.storeError(error);
                        }
                    }
                }
                // In all other cases (i.e., outside of the calendar_dates special case), store the reference errors found.
                else {
                    errorStorage.storeErrors(errors);
                }
                // Add value for entry into table
                setValueForField(table, columnIndex, lineNumber, field, string, postgresText, transformedStrings);
                // Increment column index.
                columnIndex += 1;
            }
            if (postgresText) {
                // Print a new line in the standard postgres text format:
                // https://www.postgresql.org/docs/9.1/static/sql-copy.html#AEN64380
                tempTextFileStream.println(String.join("\t", transformedStrings));
            } else {
                insertStatement.addBatch();
                if (lineNumber % INSERT_BATCH_SIZE == 0) insertStatement.executeBatch();
            }
        }
        // Record number is zero based but includes the header record, which we don't want to count.
        // But if we are working with Postgres text file (without a header row) we have to add 1
        // Iteration over all rows has finished, so We are now one record past the end of the file.
        int numberOfRecordsLoaded = (int) csvReader.getCurrentRecord();
        if (postgresText) {
            numberOfRecordsLoaded = numberOfRecordsLoaded + 1;
        }
        if (table.isRequired() && numberOfRecordsLoaded == 0) {
            errorStorage.storeError(NewGTFSError.forTable(table, REQUIRED_TABLE_EMPTY));
        }
        csvReader.close();

        // Finalize loading the table, either by copying the pre-validated text file into the database (for Postgres)
        // or inserting any remaining rows (for all others).
        if (postgresText) {
            LOG.info("Loading into database table {} from temporary text file...", targetTable.name);
            tempTextFileStream.close();
            copyFromFile(connection, tempTextFile, targetTable.name);
        } else {
            insertStatement.executeBatch();
        }
        // Create indexes using spec table. Target table must not be used because fields could be in the wrong order
        // (and the order is currently important to determining the index fields).
        table.createIndexes(connection, tablePrefix);

        LOG.info("Committing transaction...");
        connection.commit();
        LOG.info("Done.");
        return numberOfRecordsLoaded;
    }

    /**
     * Method that uses the PostgreSQL-specific copy from file command to load csv data into a table on the provided
     * connection. NOTE: This method does not commit the transaction or close the connection.
     */
    public static void copyFromFile(Connection connection, File file, String targetTableName) throws IOException, SQLException {
        // Allows sending over network. This is only slightly slower than a local file copy.
        final String copySql = String.format("copy %s from stdin", targetTableName);
        // FIXME we should be reading the COPY text from a stream in parallel, not from a temporary text file.
        InputStream stream = new BufferedInputStream(new FileInputStream(file.getAbsolutePath()));
        // Our connection pool wraps the Connection objects, so we need to unwrap the Postgres connection interface.
        CopyManager copyManager = new CopyManager(connection.unwrap(BaseConnection.class));
        copyManager.copyIn(copySql, stream, 1024*1024);
        stream.close();
        // It is also possible to load from local file if this code is running on the database server.
        // statement.execute(String.format("copy %s from '%s'", table.name, tempTextFile.getAbsolutePath()));
    }

    /**
     * Set value for a field either as a prepared statement parameter or (if using postgres text-loading) in the
     * transformed strings array provided. This also handles the case where the string is empty (i.e., field is null)
     * and when an exception is encountered while setting the field value (usually due to a bad data type), in which case
     * the field is set to null.
     */
    public void setValueForField(Table table, int fieldIndex, int lineNumber, Field field, String string, boolean postgresText, String[] transformedStrings) {
        if (string.isEmpty()) {
            // CSV reader always returns empty strings, not nulls
            if (field.isRequired() && !field.isEmptyValuePermitted() && errorStorage != null) {
                errorStorage.storeError(NewGTFSError.forLine(table, lineNumber, MISSING_FIELD, field.name));
            }
            setFieldToNull(postgresText, transformedStrings, fieldIndex, field);
        } else {
            // Micro-benchmarks show it's only 4-5% faster to call typed parameter setter methods
            // rather than setObject with a type code. I think some databases don't have setObject though.
            // The Field objects throw exceptions to avoid passing the line number, table name etc. into them.
            try {
                // Here, we set the transformed string element even when an error occurs.
                // Ideally, no errors should be signaled with exceptions, but this happens in a try/catch in case
                // something goes wrong (we don't necessarily want to abort loading the feed altogether).
                // FIXME Also, we should probably not be converting any GTFS field values, but some of them are coerced
                //  to null if they are unparseable (e.g., DateField).
                //  We should be saving it as-is in the database and converting upon load into our model objects.
                Set<NewGTFSError> errors;
                if (postgresText) {
                    ValidateFieldResult<String> result = field.validateAndConvert(string);
                    // If the result is null, use the null-setting method.
                    if (result.clean == null) setFieldToNull(postgresText, transformedStrings, fieldIndex, field);
                    // Otherwise, set the cleaned field according to its index.
                    else transformedStrings[fieldIndex + 1] = result.clean;
                    errors = result.errors;
                } else {
                    errors = field.setParameter(insertStatement, fieldIndex + 2, string);
                }
                // Store any errors encountered after field value has been set.
                for (NewGTFSError error : errors) {
                    error.entityType = table.getEntityClass();
                    error.lineNumber = lineNumber;
                    if (errorStorage != null) errorStorage.storeError(error);
                }
            } catch (StorageException ex) {
                // FIXME many exceptions don't have an error type
                if (errorStorage != null) {
                    errorStorage.storeError(NewGTFSError.forLine(table, lineNumber, ex.errorType, ex.badValue));
                }
                // Set transformedStrings or prepared statement param to null
                setFieldToNull(postgresText, transformedStrings, fieldIndex, field);
            }
        }
    }

    /**
     * Sets field to null in statement or string array depending on whether postgres is being used.
     */
    private void setFieldToNull(boolean postgresText, String[] transformedStrings, int fieldIndex, Field field) {
        if (postgresText) transformedStrings[fieldIndex + 1] = POSTGRES_NULL_TEXT;
        // Adjust parameter index by two: indexes are one-based and the first one is the CSV line number.
        else try {
            // LOG.info("setting {} index to null", fieldIndex + 2);
            field.setNull(insertStatement, fieldIndex + 2);
        } catch (SQLException e) {
            e.printStackTrace();
            // FIXME: store error here? It appears that an exception should only be thrown if the type value is invalid,
            // the connection is closed, or the index is out of bounds. So storing an error may be unnecessary.
        }
    }

    /**
     * Protect against SQL injection.
     * The only place we include arbitrary input in SQL is the column names of tables.
     * Implicitly (looking at all existing table names) these should consist entirely of
     * lowercase letters and underscores.
     *
     * TODO add a test including SQL injection text (quote and semicolon)
     */
    public static String sanitize (String string, SQLErrorStorage errorStorage) {
        String clean = string.replaceAll("[^\\p{Alnum}_]", "");
        if (!clean.equals(string)) {
            LOG.warn("SQL identifier '{}' was sanitized to '{}'", string, clean);
            if (errorStorage != null) errorStorage.storeError(NewGTFSError.forFeed(COLUMN_NAME_UNSAFE, string));
        }
        return clean;
    }
}


package com.conveyal.gtfs.model;

import com.conveyal.gtfs.GTFSFeed;
import com.conveyal.gtfs.error.ReferentialIntegrityError;

import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.Map;

public class FareRule extends Entity {

    private static final long serialVersionUID = 3209660216692732272L;
    public String fare_id;
    public String route_id;
    public String origin_id;
    public String destination_id;
    public String contains_id;

    @Override
    public String getId () {
        return fare_id;
    }

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#FARE_RULES}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        statement.setString(oneBasedIndex++, fare_id);
        statement.setString(oneBasedIndex++, route_id);
        statement.setString(oneBasedIndex++, origin_id);
        statement.setString(oneBasedIndex++, destination_id);
        statement.setString(oneBasedIndex++, contains_id);
    }

    public static class Loader extends Entity.Loader<FareRule> {

        private final Map<String, Fare> fares;

        public Loader(GTFSFeed feed, Map<String, Fare> fares) {
            super(feed, "fare_rules");
            this.fares = fares;
        }

        @Override
        protected boolean isRequired() {
            return false;
        }

        @Override
        public void loadOneRow() throws IOException {

            /* Calendars and Fares are special: they are stored as joined tables rather than simple maps. */
            String fareId = getStringField("fare_id", true);

            /* Referential integrity check for fare id */
            if (!fares.containsKey(fareId)) {
                this.feed.errors.add(new ReferentialIntegrityError(tableName, row, "fare_id", fareId));
            }

            Fare fare = fares.computeIfAbsent(fareId, Fare::new);
            FareRule fr = new FareRule();
            fr.id = row + 1; // offset line number by 1 to account for 0-based row index
            fr.fare_id = fare.fare_id;
            fr.route_id = getStringField("route_id", false);
            fr.origin_id = getStringField("origin_id", false);
            fr.destination_id = getStringField("destination_id", false);
            fr.contains_id = getStringField("contains_id", false);
            fr.feed = feed;
            fare.fare_rules.add(fr);

        }

    }

    public static class Writer extends Entity.Writer<FareRule> {
        public Writer(GTFSFeed feed) {
            super(feed, "fare_rules");
        }

        @Override
        public void writeHeaders() throws IOException {
            writer.writeRecord(new String[] {"fare_id", "route_id", "origin_id", "destination_id",
                    "contains_id"});
        }

        @Override
        public void writeOneRow(FareRule fr) throws IOException {
            writeStringField(fr.fare_id);
            writeStringField(fr.route_id);
            writeStringField(fr.origin_id);
            writeStringField(fr.destination_id);
            writeStringField(fr.contains_id);
            endRecord();
        }

        @Override
        public Iterator<FareRule> iterator() {
            return feed.fares.values().stream()
                    .map(f -> f.fare_rules)
                    .flatMap(fr -> fr.stream())
                    .iterator();
        }
    }

}