package com.conveyal.gtfs.loader;

import com.conveyal.gtfs.model.Calendar;
import com.conveyal.gtfs.model.CalendarDate;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.apache.commons.dbutils.DbUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import static com.conveyal.gtfs.loader.JdbcGtfsLoader.createFeedRegistryIfNotExists;
import static com.conveyal.gtfs.loader.JdbcGtfsLoader.createSchema;
import static com.conveyal.gtfs.util.Util.randomIdString;

/**
 * This class takes a feedId that represents a feed already in the database and creates a copy of the entire feed.
 * All tables except for the derived error and service tables are copied over (the derived pattern and pattern stop
 * tables ARE copied over).
 *
 * This copy functionality is intended to make the feed editable and so the resulting feed
 * tables are somewhat modified from their original read-only source. For instance, the ID column has been modified
 * so that it is an auto-incrementing serial integer, changing the meaning of the column from csv_line (for feeds
 * loaded from GTFS) to a unique identifier used to reference entities in an API.
 */
public class JdbcGtfsSnapshotter {

    private static final Logger LOG = LoggerFactory.getLogger(JdbcGtfsSnapshotter.class);

    private final DataSource dataSource;

    // These fields will be filled in once feed snapshot begins.
    private Connection connection;
    private String tablePrefix;
    // The reference feed ID (namespace) to copy.
    private final String feedIdToSnapshot;

    /**
     * @param feedId namespace (schema) to snapshot. If null, a blank snapshot will be created.
     * @param dataSource the JDBC data source with database connection details
     */
    public JdbcGtfsSnapshotter(String feedId, DataSource dataSource) {
        this.feedIdToSnapshot = feedId;
        this.dataSource = dataSource;
    }

    /**
     * Copy primary entity tables as well as Pattern and PatternStops tables.
     */
    public SnapshotResult copyTables() {
        // This result object will be returned to the caller to summarize the feed and report any critical errors.
        SnapshotResult result = new SnapshotResult();

        try {
            long startTime = System.currentTimeMillis();
            // We get a single connection object and share it across several different methods.
            // This ensures that actions taken in one method are visible to all subsequent SQL statements.
            // If we create a schema or table on one connection, then access it in a separate connection, we have no
            // guarantee that it exists when the accessing statement is executed.
            connection = dataSource.getConnection();
            // Generate a unique prefix that will identify this feed.
            this.tablePrefix = randomIdString();
            result.uniqueIdentifier = tablePrefix;
            // Create entry in snapshots table.
            registerSnapshot();
            // Include the dot separator in the table prefix.
            // This allows everything to work even when there's no prefix.
            this.tablePrefix += ".";
            // Copy each table in turn
            // FIXME: NO non-fatal exception errors are being captured during copy operations.
            result.agency = copy(Table.AGENCY, true);
            result.calendar = copy(Table.CALENDAR, true);
            result.calendarDates = copy(Table.CALENDAR_DATES, true);
            result.fareAttributes = copy(Table.FARE_ATTRIBUTES, true);
            result.fareRules = copy(Table.FARE_RULES, true);
            result.feedInfo = copy(Table.FEED_INFO, true);
            result.frequencies = copy(Table.FREQUENCIES, true);
            result.routes = copy(Table.ROUTES, true);
            // FIXME: Find some place to store errors encountered on copy for patterns and pattern stops.
            copy(Table.PATTERNS, true);
            copy(Table.PATTERN_STOP, true);
            // see method comments fo why different logic is needed for this table
            result.scheduleExceptions = createScheduleExceptionsTable();
            result.shapes = copy(Table.SHAPES, true);
            result.stops = copy(Table.STOPS, true);
            // TODO: Should we defer index creation on stop times?
            // Copying all tables for STIF w/ stop times idx = 156 sec; w/o = 28 sec
            // Other feeds w/ stop times AC Transit = 3 sec; Brooklyn bus =
            result.stopTimes = copy(Table.STOP_TIMES, true);
            result.transfers = copy(Table.TRANSFERS, true);
            result.trips = copy(Table.TRIPS, true);
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            LOG.info("Copying tables took {} sec", (result.loadTimeMillis) / 1000);
        } catch (Exception ex) {
            // Note: Exceptions that occur during individual table loads are separately caught and stored in
            // TableLoadResult.
            LOG.error("Exception while creating snapshot: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.toString();
        }
        return result;
    }

    /**
     * This is the main table copy method that wraps a call to Table#createSqlTableFrom and creates indexes for
     * the table.
     */
    private TableLoadResult copy (Table table, boolean createIndexes) {
        // This object will be returned to the caller to summarize the contents of the table and any errors.
        // FIXME: Should there be a separate TableSnapshotResult? Load result is empty except for fatal exception.
        TableLoadResult tableLoadResult = new TableLoadResult();
        try {
            // FIXME this is confusing, we only create a new table object so we can call a couple of methods on it,
            // all of which just need a list of fields.
            Table targetTable = new Table(tablePrefix + table.name, table.entityClass, table.required, table.fields);
            boolean success;
            if (feedIdToSnapshot == null) {
                // If there is no feedId to snapshot (i.e., we're making an empty snapshot), simply create the table.
                success = targetTable.createSqlTable(connection, true);
            } else {
                // Otherwise, use the createTableFrom method to copy the data from the original.
                String fromTableName = String.format("%s.%s", feedIdToSnapshot, table.name);
                LOG.info("Copying table {} to {}", fromTableName, targetTable.name);
                success = targetTable.createSqlTableFrom(connection, fromTableName);
            }
            // Only create indexes if table creation was successful.
            if (success && createIndexes) {
                addEditorSpecificFields(connection, tablePrefix, table);
                // Use spec table to create indexes. See createIndexes method for more info on why.
                table.createIndexes(connection, tablePrefix);
                // Populate default values for editor fields, including normalization of stop time stop sequences.
                populateDefaultEditorValues(connection, tablePrefix, table);
            }
            LOG.info("Committing transaction...");
            connection.commit();
            LOG.info("Done.");
        } catch (Exception ex) {
            tableLoadResult.fatalException = ex.toString();
            LOG.error("Error: ", ex);
            try {
                connection.rollback();
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
        return tableLoadResult;
    }

    /**
     * Special logic is needed for creating the schedule_exceptions table.
     *
     * gtfs-lib has some more advanced data types in addition to those available in the calendar_dates.txt file of the
     * GTFS specification.  The schedule_exceptions table is the source of truth for exceptions to regular schedules.
     * When exporting to a GTFS, the calendar_dates table is completely ignored if the schedule_exceptions table exists.
     *
     * When creating a snapshot, if the schedule_exceptions table doesn't currently exist, it is assumed that the feed
     * being copied has just been imported and additional data to explain schedule_exceptions has not been generated yet.
     * If the schedule_exceptions does already exist, that table is simply copied over.
     *
     * If the calendar table does not exist in the feed being copied from, it might have been the case that the
     * imported feed did not have a calendar.txt file. If that was the case, then the schedule exceptions need to be
     * generated from the calendar_dates table and also the calendar table needs to be populated with dummy entries so
     * that it is possible to export the data to a GTFS because of current logic in JdbcGtfsExporter.  The way
     * JdbcGtfsExporter will only export calendar_dates that have a corresponding entry in the calendar table and have a
     * service span that applies to a schedule_exception.  Furthermore, the dummy calendar entries are currently needed
     * for the downstream library datatools-server/datatools-ui to work properly.
     */
    private TableLoadResult createScheduleExceptionsTable() {
        // check to see if the schedule_exceptions table exists
        boolean scheduleExceptionsTableExists = tableExists(feedIdToSnapshot, "schedule_exceptions");
        String scheduleExceptionsTableName = tablePrefix + "schedule_exceptions";

        if (scheduleExceptionsTableExists) {
            // schedule_exceptions table already exists in namespace being copied from.  Therefore, we simply copy it.
            return copy(Table.SCHEDULE_EXCEPTIONS, true);
        } else {
            // schedule_exceptions does not exist.  Therefore, we generate schedule_exceptions from the calendar_dates.
            TableLoadResult tableLoadResult = new TableLoadResult();
            try {
                Table.SCHEDULE_EXCEPTIONS.createSqlTable(
                    connection,
                    tablePrefix.replace(".", ""),
                    true
                );
                String sql = String.format(
                    "insert into %s (name, dates, exemplar, added_service, removed_service) values (?, ?, ?, ?, ?)",
                    scheduleExceptionsTableName
                );
                PreparedStatement scheduleExceptionsStatement = connection.prepareStatement(sql);
                final BatchTracker scheduleExceptionsTracker = new BatchTracker(
                    "schedule_exceptions",
                    scheduleExceptionsStatement
                );

                JDBCTableReader<CalendarDate> calendarDatesReader = new JDBCTableReader(
                    Table.CALENDAR_DATES,
                    dataSource,
                    feedIdToSnapshot + ".",
                    EntityPopulator.CALENDAR_DATE
                );
                Iterable<CalendarDate> calendarDates = calendarDatesReader.getAll();

                // Keep track of calendars by service id in case we need to add dummy calendar entries.
                Map<String, Calendar> calendarsByServiceId = new HashMap<>();

                // Iterate through calendar dates to build up to get maps from exceptions to their dates.
                Multimap<String, String> removedServiceForDate = HashMultimap.create();
                Multimap<String, String> addedServiceForDate = HashMultimap.create();
                for (CalendarDate calendarDate : calendarDates) {
                    // Skip any null dates
                    if (calendarDate.date == null) {
                        LOG.warn("Encountered calendar date record with null value for date field. Skipping.");
                        continue;
                    }
                    String date = calendarDate.date.format(DateTimeFormatter.BASIC_ISO_DATE);
                    if (calendarDate.exception_type == 1) {
                        addedServiceForDate.put(date, calendarDate.service_id);
                        // create (if needed) and extend range of dummy calendar that would need to be created if we are
                        // copying from a feed that doesn't have the calendar.txt file
                        Calendar calendar = calendarsByServiceId.getOrDefault(calendarDate.service_id, new Calendar());
                        calendar.service_id = calendarDate.service_id;
                        if (calendar.start_date == null || calendar.start_date.isAfter(calendarDate.date)) {
                            calendar.start_date = calendarDate.date;
                        }
                        if (calendar.end_date == null || calendar.end_date.isBefore(calendarDate.date)) {
                            calendar.end_date = calendarDate.date;
                        }
                        calendarsByServiceId.put(calendarDate.service_id, calendar);
                    } else {
                        removedServiceForDate.put(date, calendarDate.service_id);
                    }
                }
                // Iterate through dates with added or removed service and add to database.
                // For usability and simplicity of code, don't attempt to find all dates with similar
                // added and removed services, but simply create an entry for each found date.
                for (String date : Sets.union(removedServiceForDate.keySet(), addedServiceForDate.keySet())) {
                    scheduleExceptionsStatement.setString(1, date);
                    String[] dates = {date};
                    scheduleExceptionsStatement.setArray(2, connection.createArrayOf("text", dates));
                    scheduleExceptionsStatement.setInt(3, 9); // FIXME use better static type
                    scheduleExceptionsStatement.setArray(
                        4,
                        connection.createArrayOf("text", addedServiceForDate.get(date).toArray())
                    );
                    scheduleExceptionsStatement.setArray(
                        5,
                        connection.createArrayOf("text", removedServiceForDate.get(date).toArray())
                    );
                    scheduleExceptionsTracker.addBatch();
                }
                scheduleExceptionsTracker.executeRemaining();

                // determine if we appear to be working with a calendar_dates-only feed.
                // If so, we must also add dummy entries to the calendar table
                if (
                    feedIdToSnapshot != null &&
                    !tableExists(feedIdToSnapshot, "calendar") &&
                    calendarDatesReader.getRowCount() > 0
                ) {
                    sql = String.format(
                        "insert into %s (service_id, description, start_date, end_date, " +
                            "monday, tuesday, wednesday, thursday, friday, saturday, sunday)" +
                            "values (?, ?, ?, ?, 0, 0, 0, 0, 0, 0, 0)",
                        tablePrefix + "calendar"
                    );
                    PreparedStatement calendarStatement = connection.prepareStatement(sql);
                    final BatchTracker calendarsTracker = new BatchTracker(
                        "calendar",
                        calendarStatement
                    );
                    for (Calendar calendar : calendarsByServiceId.values()) {
                        calendarStatement.setString(1, calendar.service_id);
                        calendarStatement.setString(
                            2,
                            String.format("%s (auto-generated)", calendar.service_id)
                        );
                        calendarStatement.setString(
                            3,
                            calendar.start_date.format(DateTimeFormatter.BASIC_ISO_DATE)
                        );
                        calendarStatement.setString(
                            4,
                            calendar.end_date.format(DateTimeFormatter.BASIC_ISO_DATE)
                        );
                        calendarsTracker.addBatch();
                    }
                    calendarsTracker.executeRemaining();
                }

                connection.commit();
            } catch (Exception e) {
                tableLoadResult.fatalException = e.toString();
                LOG.error("Error creating schedule Exceptions: ", e);
                e.printStackTrace();
                try {
                    connection.rollback();
                } catch (SQLException ex) {
                    ex.printStackTrace();
                }
            }
            LOG.info("done creating schedule exceptions");
            return tableLoadResult;
        }
    }

    /**
     * Helper method to determine if a table exists within a namespace.
     */
    private boolean tableExists(String namespace, String tableName) {
        // Preempt SQL check with null check of either namespace or table name.
        if (namespace == null || tableName == null) return false;
        try {
            // This statement is postgres-specific.
            PreparedStatement tableExistsStatement = connection.prepareStatement(
                "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = ? AND table_name = ?)"
            );
            tableExistsStatement.setString(1, namespace);
            tableExistsStatement.setString(2, tableName);
            ResultSet resultSet = tableExistsStatement.executeQuery();
            resultSet.next();
            return resultSet.getBoolean(1);
        } catch (SQLException e) {
            e.printStackTrace();
            return false;
        }
    }

    /**
     * Add columns for any required, optional, or editor fields that don't already exist as columns on the table.
     * This method contains a SQL statement that requires PostgreSQL 9.6+.
     */
    private void addEditorSpecificFields(Connection connection, String tablePrefix, Table table) throws SQLException {
        LOG.info("Adding any missing columns for {}", tablePrefix + table.name);
        Statement statement = connection.createStatement();
        for (Field field : table.editorFields()) {
            // The following statement requires PostgreSQL 9.6+.
            String addColumnSql = String.format("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s",
                    tablePrefix + table.name,
                    field.name,
                    field.getSqlTypeName());
            LOG.info(addColumnSql);
            statement.execute(addColumnSql);
        }
    }

    /**
     * Populates editor-specific fields added during GTFS-to-snapshot operation with default values. This method also
     * "normalizes" the stop sequences for stop times to zero-based, incremented integers. NOTE: stop time normalization
     * can take a significant amount of time (for large feeds up to 5 minutes) if the update is large.
     */
    private void populateDefaultEditorValues(Connection connection, String tablePrefix, Table table) throws SQLException {
        Statement statement = connection.createStatement();
        if (Table.ROUTES.name.equals(table.name)) {
            // Set default values for route status and publicly visible to "Approved" and "Public", respectively.
            // This prevents unexpected results when users attempt to export a GTFS feed from the editor and no
            // routes are exported due to undefined values for status and publicly visible.
            String updateStatusSql = String.format(
                    "update %sroutes set status = 2, publicly_visible = 1 where status is NULL AND publicly_visible is NULL",
                    tablePrefix);
            int updatedRoutes = statement.executeUpdate(updateStatusSql);
            LOG.info("Updated status for {} routes", updatedRoutes);
        }
        if (Table.CALENDAR.name.equals(table.name)) {
            // Set default values for description field. Basically constructs a description from the days of the week
            // for which the calendar is active.
            LOG.info("Updating calendar descriptions");
            String[] daysOfWeek = new String[]{"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"};
            String concatenatedDaysOfWeek =  String.join(", ",
                    Arrays.stream(daysOfWeek)
                        .map(d -> String.format(
                                    "case %s when 1 then '%s' else '' end",
                                    d,
                                // Capitalize first letter. Converts days of week from "monday" -> "Mo".
                                d.substring(0, 1).toUpperCase() + d.substring(1, 2))).toArray(String[]::new));
            String updateOtherSql = String.format(
                    "update %scalendar set description = concat(%s) where description is NULL",
                    tablePrefix,
                    concatenatedDaysOfWeek);
            LOG.info(updateOtherSql);
            int calendarsUpdated = statement.executeUpdate(updateOtherSql);
            LOG.info("Updated description for {} calendars", calendarsUpdated);
        }
        if (Table.TRIPS.name.equals(table.name)) {
            // Update use_frequency field for patterns. This sets all patterns that have a frequency trip to use
            // frequencies. NOTE: This is performed after copying the TRIPS table rather than after PATTERNS because
            // both tables (plus, frequencies) need to exist for the successful operation.
            // TODO: How should this handle patterns that have both timetable- and frequency-based trips?
            // NOTE: The below substitution uses argument indexing. All values "%1$s" reference the first argument
            // supplied (i.e., tablePrefix).
            String updatePatternsSql = String.format(
                    "update %1$spatterns set use_frequency = 1 " +
                    "from (select distinct %1$strips.pattern_id from %1$strips, %1$sfrequencies where %1$sfrequencies.trip_id = %1$strips.trip_id) freq " +
                    "where freq.pattern_id = %1$spatterns.pattern_id",
                    tablePrefix);
            LOG.info(updatePatternsSql);
            int patternsUpdated = statement.executeUpdate(updatePatternsSql);
            LOG.info("Updated use_frequency for {} patterns", patternsUpdated);
        }
        // TODO: Add simple conversion from calendar_dates to schedule_exceptions if no exceptions exist? See
        // https://github.com/catalogueglobal/datatools-server/issues/80
    }

    /**
     * Add a line to the list of loaded feeds to record the snapshot and which feed the snapshot replicates.
     */
    private void registerSnapshot () {
        try {
            // We cannot simply insert into the feeds table because if we are creating an empty snapshot (to create/edit
            // a GTFS feed from scratch), the feed registry table will not exist.
            // TODO copy over feed_id and feed_version from source namespace?
            // TODO: Record total snapshot processing time?
            createFeedRegistryIfNotExists(connection);
            createSchema(connection, tablePrefix);
            PreparedStatement insertStatement = connection.prepareStatement(
                    "insert into feeds values (?, null, null, null, null, null, current_timestamp, ?)");
            insertStatement.setString(1, tablePrefix);
            insertStatement.setString(2, feedIdToSnapshot);
            insertStatement.execute();
            connection.commit();
            LOG.info("Created new snapshot namespace: {}", insertStatement);
        } catch (Exception ex) {
            LOG.error("Exception while registering snapshot namespace in feeds table", ex);
            DbUtils.closeQuietly(connection);
        }
    }
}


package com.conveyal.gtfs.model;

import com.conveyal.gtfs.GTFSFeed;
import com.conveyal.gtfs.error.DuplicateKeyError;
import com.conveyal.gtfs.loader.DateField;
import com.conveyal.gtfs.loader.Table;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;

import java.io.IOException;
import java.io.Serializable;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.Iterator;
import java.util.Map;

public class Calendar extends Entity implements Serializable {

    private static final long serialVersionUID = 6634236680822635875L;

    public String service_id;

    public LocalDate start_date;
    public LocalDate end_date;

    // Should these be booleans? Oh... the implications.
    public int monday;
    public int tuesday;
    public int wednesday;
    public int thursday;
    public int friday;
    public int saturday;
    public int sunday;

    public String feed_id;

    @Override
    public String getId () {
        return service_id;
    }

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#CALENDAR}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        DateField startDateField = (DateField) Table.CALENDAR.getFieldForName("start_date");
        DateField endDateField = ((DateField) Table.CALENDAR.getFieldForName("end_date"));
        statement.setString(oneBasedIndex++, service_id);
        setIntParameter(statement, oneBasedIndex++, monday);
        setIntParameter(statement, oneBasedIndex++, tuesday);
        setIntParameter(statement, oneBasedIndex++, wednesday);
        setIntParameter(statement, oneBasedIndex++, thursday);
        setIntParameter(statement, oneBasedIndex++, friday);
        setIntParameter(statement, oneBasedIndex++, saturday);
        setIntParameter(statement, oneBasedIndex++, sunday);
        startDateField.setParameter(statement, oneBasedIndex++, start_date);
        endDateField.setParameter(statement, oneBasedIndex++, end_date);
        statement.setString(oneBasedIndex++, null); // description
    }

    public static class Loader extends Entity.Loader<Calendar> {

        private final Map<String, Service> services;

        /**
         * Create a loader. The map parameter should be an in-memory map that will be modified. We can't write directly
         * to MapDB because we modify services as we load calendar dates, and this creates
         * ConcurrentModificationExceptions.
         */
        public Loader(GTFSFeed feed, Map<String, Service> services) {
            super(feed, "calendar");
            this.services = services;
        }

        @Override
        protected boolean isRequired() {
            return true;
        }

        @Override
        public void loadOneRow() throws IOException {

            /* Calendars and Fares are special: they are stored as joined tables rather than simple maps. */
            String service_id = getStringField("service_id", true); // TODO service_id can reference either calendar or calendar_dates.
            Service service = services.computeIfAbsent(service_id, Service::new);
            if (service.calendar != null) {
                feed.errors.add(new DuplicateKeyError(tableName, row, "service_id"));
            } else {
                Calendar c = new Calendar();
                c.id = row + 1; // offset line number by 1 to account for 0-based row index
                c.service_id = service.service_id;
                c.monday = getIntField("monday", true, 0, 1);
                c.tuesday = getIntField("tuesday", true, 0, 1);
                c.wednesday = getIntField("wednesday", true, 0, 1);
                c.thursday = getIntField("thursday", true, 0, 1);
                c.friday = getIntField("friday", true, 0, 1);
                c.saturday = getIntField("saturday", true, 0, 1);
                c.sunday = getIntField("sunday", true, 0, 1);
                // TODO check valid dates
                c.start_date = getDateField("start_date", true);
                c.end_date = getDateField("end_date", true);
                c.feed = feed;
                c.feed_id = feed.feedId;
                service.calendar = c;
            }

        }
    }

    public static class Writer extends Entity.Writer<Calendar> {
        public Writer(GTFSFeed feed) {
            super(feed, "calendar");
        }

        @Override
        protected void writeHeaders() throws IOException {
            writer.writeRecord(new String[] {"service_id", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", "start_date", "end_date"});
        }

        @Override
        protected void writeOneRow(Calendar c) throws IOException {
            writeStringField(c.service_id);
            writeIntField(c.monday);
            writeIntField(c.tuesday);
            writeIntField(c.wednesday);
            writeIntField(c.thursday);
            writeIntField(c.friday);
            writeIntField(c.saturday);
            writeIntField(c.sunday);
            writeDateField(c.start_date);
            writeDateField(c.end_date);
            endRecord();
        }

        @Override
        protected Iterator<Calendar> iterator() {
            // wrap an iterator over services
            Iterator<Calendar> calIt = Iterators.transform(feed.services.values().iterator(), new Function<Service, Calendar> () {
                @Override
                public Calendar apply (Service s) {
                    return s.calendar;
                }
            });

            // not every service has a calendar (e.g. TriMet has no calendars, just calendar dates).
            // This is legal GTFS, so skip services with no calendar
            return Iterators.filter(calIt, new Predicate<Calendar> () {
                @Override
                public boolean apply(Calendar c) {
                    return c != null;
                }
            });

        }
    }
}


package com.conveyal.gtfs.model;

import com.conveyal.gtfs.GTFSFeed;
import org.mapdb.Fun;

import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Iterator;

import static com.conveyal.gtfs.model.Entity.Writer.convertToGtfsTime;

public class Frequency extends Entity implements Comparable<Frequency> {
    /**
     * Frequency entries have no ID in GTFS so we define one based on the fields in the frequency entry.
     *
     * It is possible to have two identical frequency entries in the GTFS, which under our understanding of the situation
     * would mean that two sets of vehicles were randomly running the same trip at the same headway, but uncorrelated
     * with each other, which is almost certain to be an error.
     */
     public String getId() {
        StringBuilder sb = new StringBuilder();
         sb.append(trip_id);
         sb.append('_');
         sb.append(convertToGtfsTime(start_time));
         sb.append("_to_");
         sb.append(convertToGtfsTime(end_time));
         sb.append("_every_");
         sb.append(String.format("%dm%02ds", headway_secs / 60, headway_secs % 60));
         if (exact_times == 1) sb.append("_exact");
         return sb.toString();
     }

    private static final long serialVersionUID = -7182161664471704133L;
    public String trip_id;
    public int start_time;
    public int end_time;
    public int headway_secs;
    public int exact_times;

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#FREQUENCIES}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        statement.setString(oneBasedIndex++, trip_id);
        setIntParameter(statement, oneBasedIndex++, start_time);
        setIntParameter(statement, oneBasedIndex++, end_time);
        setIntParameter(statement, oneBasedIndex++, headway_secs);
        setIntParameter(statement, oneBasedIndex++, exact_times);
    }

    /** must have a comparator since they go in a navigable set that is serialized */
    @Override
    public int compareTo(Frequency o) {
        return this.start_time - o.start_time;
    }

    public static class Loader extends Entity.Loader<Frequency> {

        public Loader(GTFSFeed feed) {
            super(feed, "frequencies");
        }

        @Override
        protected boolean isRequired() {
            return false;
        }

        @Override
        public void loadOneRow() throws IOException {
            Frequency f = new Frequency();
            Trip trip = getRefField("trip_id", true, feed.trips);
            f.id = row + 1; // offset line number by 1 to account for 0-based row index
            f.trip_id = trip.trip_id;
            f.start_time = getTimeField("start_time", true);
            f.end_time = getTimeField("end_time", true);
            f.headway_secs = getIntField("headway_secs", true, 1, 24 * 60 * 60);
            f.exact_times = getIntField("exact_times", false, 0, 1);
            f.feed = feed;
            feed.frequencies.add(Fun.t2(f.trip_id, f));
        }
    }

    public static class Writer extends Entity.Writer<Frequency> {
        public Writer (GTFSFeed feed) {
            super(feed, "frequencies");
        }

        @Override
        public void writeHeaders() throws IOException {
            writer.writeRecord(new String[] {"trip_id", "start_time", "end_time", "headway_secs", "exact_times"});
        }

        @Override
        public void writeOneRow(Frequency f) throws IOException {
            writeStringField(f.trip_id);
            writeTimeField(f.start_time);
            writeTimeField(f.end_time);
            writeIntField(f.headway_secs);
            writeIntField(f.exact_times);
            endRecord();
        }

        @Override
        public Iterator<Frequency> iterator() {
            return feed.frequencies.stream()
                    .map(t2 -> t2.b)
                    .iterator();
        }

    }

}


package com.conveyal.gtfs.loader;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.PreparedStatement;
import java.sql.SQLException;

/**
 * Avoid Java's "effectively final" nonsense when using prepared statements in foreach loops.
 * Automatically push execute batches of prepared statements before the batch gets too big.
 * TODO there's probably something like this in an Apache Commons util library
 */
public class BatchTracker {
    private static final Logger LOG = LoggerFactory.getLogger(BatchTracker.class);

    private final String recordType;
    private PreparedStatement preparedStatement;
    private int currentBatchSize = 0;
    private int totalRecordsProcessed = 0;

    public BatchTracker(String recordType, PreparedStatement preparedStatement) {
        this.preparedStatement = preparedStatement;
        this.recordType = recordType;
    }

    public void addBatch() throws SQLException {
        preparedStatement.addBatch();
        currentBatchSize += 1;
        if (currentBatchSize > JdbcGtfsLoader.INSERT_BATCH_SIZE) {
            preparedStatement.executeBatch();
            totalRecordsProcessed += currentBatchSize;
            currentBatchSize = 0;
        }
    }

    /**
     * Execute any remaining statements and return the total records processed.
     */
    public int executeRemaining() throws SQLException {
        if (currentBatchSize > 0) {
            totalRecordsProcessed += currentBatchSize;
            preparedStatement.executeBatch();
            currentBatchSize = 0;
        }
        // Avoid reuse, signal that this was cleanly closed.
        preparedStatement = null;
        LOG.info(String.format("Processed %d %s records", totalRecordsProcessed, recordType));
        return totalRecordsProcessed;
    }

    public void finalize () {
        if (preparedStatement != null || currentBatchSize > 0) {
            throw new RuntimeException("BUG: It looks like someone did not call executeRemaining on a BatchTracker.");
        }
    }
}


package com.conveyal.gtfs.storage;

import java.io.*;
import java.util.BitSet;
import java.util.zip.GZIPInputStream;

/**
 * Loads an ESRI ASCII grid containing integers and allows looking up values as booleans (where > 0).
 * This is used for
 */
public class BooleanAsciiGrid {

    int ncols;
    int nrows;
    double xllcorner;
    double yllcorner;
    double cellsize;
    double nodata;

    BitSet grid;

    public BooleanAsciiGrid (InputStream inputStream, boolean gzipped) {

        try {
            if (gzipped) inputStream = new GZIPInputStream(inputStream);
            BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
            boolean inHeader = true;
            int nDataItemsRead = 0;
            for(String line = reader.readLine(); line != null; line = reader.readLine()) {
                String[] fields = line.trim().split("\\s+"); // split on one or more whitespace characters
                if (inHeader) {
                    inHeader = handleHeaderRow(fields);
                    if (inHeader) continue;
                    grid = new BitSet(ncols * nrows);
                }
                if (fields.length != ncols) {
                    throw new RuntimeException("Wrong number of data columns: " +  fields.length);
                }
                for (String field : fields) {
                    int value = Integer.parseInt(field);
                    grid.set(nDataItemsRead++, value > 0);
                }
            }
            if (nDataItemsRead != ncols * nrows) throw new RuntimeException("Too few data items: " + nDataItemsRead);
        } catch (IOException exception) {
            throw new RuntimeException(exception);
        }
    }

    /**
     * Get a grid for places where population density is over 5 people per square kilometer.
     * We use the Gridded Population of the World v3 data set for 2015 UN-adjusted population density.
     * This data set was downloaded at 1/4 degree resolution in ESRI ASCII grid format. The grid file was edited
     * manually to eliminate the no-data value header, since I could not find a way to operate on no-value cells in the
     * QGIS raster calculator. Then the raster calculator in QGIS was used with the formula ("glds00ag15@1" > 5),
     * which makes all cells with population density above the threshold have a value of one,
     * and all others a value of zero (since the no data value in the grid is -9999). This was then exported as another
     * ASCII grid file, which zips well. The license for this data set is Creative Commons Attribution.
     * See http://sedac.ciesin.columbia.edu/data/collection/gpw-v3
     */
    public static BooleanAsciiGrid forEarthPopulation() {
        try {
            InputStream gridStream = BooleanAsciiGrid.class.getResourceAsStream("gpwv3-quarter-boolean.asc");
            return new BooleanAsciiGrid(gridStream, false);
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }

    /**
     * @param fields
     * @return whether we appear to still be in the header lines of the file
     */
    private boolean handleHeaderRow (String[] fields) {
        if (fields.length != 2) return false;
        String key = fields[0];
        if ("ncols".equals(key)) {
            ncols = Integer.parseInt(fields[1]);
        } else if ("nrows".equalsIgnoreCase(key)) {
            nrows = Integer.parseInt(fields[1]);
        } else if ("xllcorner".equalsIgnoreCase(key)) {
            xllcorner = Double.parseDouble(fields[1]);
        } else if ("yllcorner".equalsIgnoreCase(key)) {
            yllcorner = Double.parseDouble(fields[1]);
        } else if ("cellsize".equalsIgnoreCase(key)) {
            cellsize = Double.parseDouble(fields[1]);
        } else if ("NODATA_value".equalsIgnoreCase(key)) {
            nodata = Double.parseDouble(fields[1]);
        } else {
            return false;
        }
        return true;
    }

    public boolean getValueForCoords (double x, double y) {
        int xCell = (int)((x - xllcorner) / cellsize);
        int yCell = (int)((y - yllcorner) / cellsize);
        int index = (nrows - yCell - 1) * ncols + xCell; // Vertical flip
        if (index < 0 || index > grid.length()) return false;
        return grid.get(index);
    }

}


package com.conveyal.gtfs.validator;

import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.NewGTFSErrorType;
import com.conveyal.gtfs.error.SQLErrorStorage;
import com.conveyal.gtfs.loader.BatchTracker;
import com.conveyal.gtfs.loader.DateField;
import com.conveyal.gtfs.loader.Feed;
import com.conveyal.gtfs.loader.Table;
import com.conveyal.gtfs.model.Calendar;
import com.conveyal.gtfs.model.CalendarDate;
import com.conveyal.gtfs.model.Entity;
import com.conveyal.gtfs.model.Route;
import com.conveyal.gtfs.model.Stop;
import com.conveyal.gtfs.model.StopTime;
import com.conveyal.gtfs.model.Trip;
import com.conveyal.gtfs.storage.StorageException;
import gnu.trove.map.hash.TIntIntHashMap;
import org.apache.commons.dbutils.DbUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.DayOfWeek;
import java.time.LocalDate;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static com.conveyal.gtfs.error.NewGTFSErrorType.TRIP_OVERLAP_IN_BLOCK;

/**
 * This will validate that service date information is coherent, and attempt to deduce or validate the range of dates
 * covered by a GTFS feed.
 *
 * It turns the GTFS system of repeating weekly calendars and exceptions (calendar dates) into a single large table
 * listing which services run on which days. This in turn allows us to build a histogram of service duration on each
 * day.
 *
 * As an intermediate result it builds a table of service duration by service ID and mode of transport.
 *
 * Makes one object representing each service ID.
 * That object will contain a calendar (for repeating service on specific days of the week)
 * and potentially multiple CalendarDates defining exceptions to the base calendar.
 * TODO build histogram of stop times, check against calendar and declared feed validity dates
 */
public class ServiceValidator extends TripValidator {

    private static final Logger LOG = LoggerFactory.getLogger(ServiceValidator.class);
    private HashMap<String, List<BlockInterval>> blockIntervals = new HashMap<>();
    private Map<String, ServiceInfo> serviceInfoForServiceId = new HashMap<>();

    private Map<LocalDate, DateInfo> dateInfoForDate = new HashMap<>();

    public ServiceValidator(Feed feed, SQLErrorStorage errorStorage) {
        super(feed, errorStorage);
    }

    @Override
    public void validateTrip(Trip trip, Route route, List<StopTime> stopTimes, List<Stop> stops) {
        if (trip.block_id != null) {
            // If the trip has a block_id, add a new block interval to the map.
            BlockInterval blockInterval = new BlockInterval();
            blockInterval.trip = trip;
            StopTime firstStopTime = stopTimes.get(0);
            blockInterval.startTime = firstStopTime.departure_time;
            blockInterval.firstStop = firstStopTime;
            blockInterval.lastStop = stopTimes.get(stopTimes.size() - 1);
            // Construct new list of intervals if none exists for encountered block_id.
            blockIntervals
                .computeIfAbsent(trip.block_id, k -> new ArrayList<>())
                .add(blockInterval);
        }
        int firstStopDeparture = stopTimes.get(0).departure_time;
        int lastStopArrival = stopTimes.get(stopTimes.size() - 1).arrival_time;
        if (firstStopDeparture == Entity.INT_MISSING || lastStopArrival == Entity.INT_MISSING) {
            // ERR
            return;
        }
        int tripDurationSeconds = lastStopArrival - firstStopDeparture;
        if (tripDurationSeconds <= 0) {
            // ERR
            return;
        }
        // Get the map from modes to service durations in seconds for this trip's service ID.
        // Create a new empty map if it doesn't yet exist.
        ServiceInfo serviceInfo = serviceInfoForServiceId.computeIfAbsent(trip.service_id, ServiceInfo::new);
        if (route != null) {
            // Increment the service duration for this trip's transport mode and service ID.
            serviceInfo.durationByRouteType.adjustOrPutValue(route.route_type, tripDurationSeconds, tripDurationSeconds);
        }
        // Record which trips occur on each service_id.
        serviceInfo.tripIds.add(trip.trip_id);
        // TODO validate mode codes
    }

    /**
     * You'd think we'd want to do this during the loading phase. But during the loading phase we don't have a reading
     * connection to the entity tables in the database. Rather than make the Feed object read-write, we want to leave
     * it completely read-only.
     *
     * @param validationResult can be written into
     */
    @Override
    public void complete(ValidationResult validationResult) {
        validateServiceInfo(validationResult);
        validateBlocks();
    }

    private void validateServiceInfo(ValidationResult validationResult) {
        LOG.info("Merging calendars and calendar_dates...");

        // First handle the calendar entries, which define repeating weekly schedules.
        for (Calendar calendar : feed.calendars) {
            try {
                LocalDate endDate = calendar.end_date;
                // Loop over all days in this calendar entry, recording on which ones it is active.
                for (LocalDate date = calendar.start_date; date.isBefore(endDate) || date.isEqual(endDate); date = date.plusDays(1)) {
                    DayOfWeek dayOfWeek = date.getDayOfWeek();
                    if (    (dayOfWeek == DayOfWeek.MONDAY && calendar.monday > 0) ||
                        (dayOfWeek == DayOfWeek.TUESDAY && calendar.tuesday > 0) ||
                        (dayOfWeek == DayOfWeek.WEDNESDAY && calendar.wednesday > 0) ||
                        (dayOfWeek == DayOfWeek.THURSDAY && calendar.thursday > 0) ||
                        (dayOfWeek == DayOfWeek.FRIDAY && calendar.friday > 0) ||
                        (dayOfWeek == DayOfWeek.SATURDAY && calendar.saturday > 0) ||
                        (dayOfWeek == DayOfWeek.SUNDAY && calendar.sunday > 0)) {
                        // Service is active on this date.
                        serviceInfoForServiceId.computeIfAbsent(calendar.service_id, ServiceInfo::new).datesActive.add(date);
                    }
                }
            } catch (Exception ex) {
                LOG.error("Error validating service entries (merging calendars and calendar_dates)", ex);
                // Continue on to next calendar entry.
            }
        }

        // Next handle the calendar_dates, which specify exceptions to the repeating weekly schedules.
        for (CalendarDate calendarDate : feed.calendarDates) {
            ServiceInfo serviceInfo = serviceInfoForServiceId.computeIfAbsent(calendarDate.service_id, ServiceInfo::new);
            if (calendarDate.exception_type == 1) {
                // Service added, add to set for this date.
                serviceInfo.datesActive.add(calendarDate.date);
            } else if (calendarDate.exception_type == 2) {
                // Service removed, remove from Set for this date.
                serviceInfo.datesActive.remove(calendarDate.date);
            }
            // Otherwise exception_type is out of range. This should already have been caught during the loading phase.
        }

        /*
            A view that is similar to ServiceInfo class, but doesn't deal well with missing IDs in either subquery:
            select durations.service_id, duration_seconds, days_active from (
              (select service_id, sum(duration_seconds) as duration_seconds
                   from elwp_qhqsgzufnpvwnxtdbwcthn.service_durations group by service_id) as durations
              join
              (select service_id, count(service_date) as days_active
                   from elwp_qhqsgzufnpvwnxtdbwcthn.service_dates group by service_id) as days
              on durations.service_id = days.service_id
            );
         */

        // Check for incoherent or erroneous services.
        for (ServiceInfo serviceInfo : serviceInfoForServiceId.values()) {
            if (serviceInfo.datesActive.isEmpty()) {
                // This service must have been referenced by trips but is never active on any day.
                registerError(NewGTFSError.forFeed(NewGTFSErrorType.SERVICE_NEVER_ACTIVE, serviceInfo.serviceId));
                for (String tripId : serviceInfo.tripIds) {
                    registerError(
                        NewGTFSError.forTable(Table.TRIPS, NewGTFSErrorType.TRIP_NEVER_ACTIVE)
                                    .setEntityId(tripId)
                                    .setBadValue(tripId));
                }
            }
            if (serviceInfo.tripIds.isEmpty()) {
                registerError(NewGTFSError.forFeed(NewGTFSErrorType.SERVICE_UNUSED, serviceInfo.serviceId));
            }
        }

        // Accumulate info about services into each date that they are active.
        for (ServiceInfo serviceInfo : serviceInfoForServiceId.values()) {
            for (LocalDate date : serviceInfo.datesActive) {
                dateInfoForDate.computeIfAbsent(date, DateInfo::new).add(serviceInfo);
            }
        }

        // Check for dates that have no service within full range of dates with defined service.
        // Sum up service duration by mode for each day within that range.
        if (dateInfoForDate.isEmpty()) {
            registerError(NewGTFSError.forFeed(NewGTFSErrorType.NO_SERVICE, null));
        } else {
            LocalDate firstDate = LocalDate.MAX;
            LocalDate lastDate = LocalDate.MIN;
            for (LocalDate date : dateInfoForDate.keySet()) {
                // If the date is invalid, skip.
                if (date == null) {
                    LOG.error("Encountered null date. Did something go wrong with computeIfAbsent?");
                    continue;
                }
                if (date.isBefore(firstDate)) firstDate = date;
                if (date.isAfter(lastDate)) lastDate = date;
            }
            // Copy some useful information into the ValidationResult object to return to the caller.
            // These variables are actually not directly tied to data in the calendar_dates.txt file.  Instead, they
            // represent the first and last date respectively of any entry in the calendar.txt and calendar_dates.txt
            // files.
            validationResult.firstCalendarDate = firstDate;
            validationResult.lastCalendarDate = lastDate;
            // Is this any different? firstDate.until(lastDate, ChronoUnit.DAYS);
            // If no days were found in the dateInfoForDate, nDays is a very large negative number, so we default to 0.
            int nDays = Math.max(0, (int) ChronoUnit.DAYS.between(firstDate, lastDate) + 1);
            validationResult.dailyBusSeconds = new int[nDays];
            validationResult.dailyTramSeconds = new int[nDays];
            validationResult.dailyMetroSeconds = new int[nDays];
            validationResult.dailyRailSeconds = new int[nDays];
            validationResult.dailyTotalSeconds = new int[nDays];
            validationResult.dailyTripCounts = new int[nDays];
            for (int d = 0; d < nDays; d++) {
                LocalDate date = firstDate.plusDays(d); // current date being processed
                // Add one value per day. Trove map returns zero for missing keys.
                DateInfo dateInfo = dateInfoForDate.get(date);
                if (dateInfo == null) {
                    dateInfo = new DateInfo(date); // new empty object to get empty durations map.
                }
                validationResult.dailyBusSeconds[d] = dateInfo.durationByRouteType.get(3);
                validationResult.dailyTramSeconds[d] = dateInfo.durationByRouteType.get(0);
                validationResult.dailyMetroSeconds[d] = dateInfo.durationByRouteType.get(1);
                validationResult.dailyRailSeconds[d] = dateInfo.durationByRouteType.get(2);
                validationResult.dailyTotalSeconds[d] = dateInfo.getTotalServiceDurationSeconds();
                validationResult.dailyTripCounts[d] = dateInfo.tripCount;
                if (dateInfo.getTotalServiceDurationSeconds() <= 0) {
                    // Check for low or zero service, which seems to happen even when services are defined.
                    // This will also catch cases where dateInfo was null and the new instance contains no service.
                    registerError(NewGTFSError.forFeed(NewGTFSErrorType.DATE_NO_SERVICE,
                                                       DateField.GTFS_DATE_FORMATTER.format(date)));
                }
            }
        }

        // Now write all these calendar-date relations out to the database.
        Connection connection = null;
        try {
            connection = feed.getConnection();
            Statement statement = connection.createStatement();

            // Create a table summarizing all known service IDs.
            // This is almost just a view joining two sub-selects:
            // select * from
            //     (select service_id, count(service_date) from x.service_dates group by service_id) as days
            //   join
            //     (select service_id, sum(duration_seconds) from x.service_durations group by service_id) as durations
            //   on days.service_id = durations.service_id;
            // Except that some service IDs may have no trips on them, or may not be referenced in any calendar or
            // calendar exception, which would keep them from appearing in either of those tables. So we just create
            // this somewhat redundant materialized view to serve as a master list of all services.
            String servicesTableName = feed.tablePrefix + "services";
            String sql = String.format("create table %s (service_id varchar, n_days_active integer, duration_seconds integer, n_trips integer)", servicesTableName);
            LOG.info(sql);
            statement.execute(sql);
            sql = String.format("insert into %s values (?, ?, ?, ?)", servicesTableName);
            PreparedStatement serviceStatement = connection.prepareStatement(sql);
            final BatchTracker serviceTracker = new BatchTracker("services", serviceStatement);
            for (ServiceInfo serviceInfo : serviceInfoForServiceId.values()) {
                serviceStatement.setString(1, serviceInfo.serviceId);
                serviceStatement.setInt(2, serviceInfo.datesActive.size());
                serviceStatement.setInt(3, serviceInfo.getTotalServiceDurationSeconds());
                serviceStatement.setInt(4, serviceInfo.tripIds.size());
                serviceTracker.addBatch();
            }
            serviceTracker.executeRemaining();

            // Create a table that shows on which dates each service is active.
            String serviceDatesTableName = feed.tablePrefix + "service_dates";
            sql = String.format("create table %s (service_date varchar, service_id varchar)", serviceDatesTableName);
            LOG.info(sql);
            statement.execute(sql);
            sql = String.format("insert into %s values (?, ?)", serviceDatesTableName);
            PreparedStatement serviceDateStatement = connection.prepareStatement(sql);
            final BatchTracker serviceDateTracker = new BatchTracker("service_dates", serviceDateStatement);
            for (ServiceInfo serviceInfo : serviceInfoForServiceId.values()) {
                for (LocalDate date : serviceInfo.datesActive) {
                    if (date == null) continue; // TODO ERR? Can happen with bad data (unparseable dates).
                    try {
                        serviceDateStatement.setString(1, date.format(DateField.GTFS_DATE_FORMATTER));
                        serviceDateStatement.setString(2, serviceInfo.serviceId);
                        serviceDateTracker.addBatch();
                    } catch (SQLException ex) {
                        throw new StorageException(ex);
                    }
                }

            }
            serviceDateTracker.executeRemaining();

            LOG.info("Indexing...");
            statement.execute(String.format("create index service_dates_service_date on %s (service_date)", serviceDatesTableName));
            statement.execute(String.format("create index service_dates_service_id on %s (service_id)", serviceDatesTableName));

            // Create a table containing the total trip durations per service_id and per transit mode.
            // Using this table you can get total service duration by mode (route_type) per day, joining tables:
            // select service_date, route_type, sum(duration_seconds)
            // from x.service_dates as dates, x.service_durations as durations
            // where dates.service_id = durations.service_id
            // group by service_date, route_type order by service_date, route_type;

            String serviceDurationsTableName = feed.tablePrefix + "service_durations";
            sql = String.format("create table %s (service_id varchar, route_type integer, " +
                                    "duration_seconds integer, primary key (service_id, route_type))", serviceDurationsTableName);
            LOG.info(sql);
            statement.execute(sql);
            sql = String.format("insert into %s values (?, ?, ?)", serviceDurationsTableName);
            PreparedStatement serviceDurationStatement = connection.prepareStatement(sql);
            final BatchTracker serviceDurationTracker = new BatchTracker(
                "service_durations",
                serviceDurationStatement
            );
            for (ServiceInfo serviceInfo : serviceInfoForServiceId.values()) {
                serviceInfo.durationByRouteType.forEachEntry((routeType, serviceDurationSeconds) -> {
                    try {
                        serviceDurationStatement.setString(1, serviceInfo.serviceId);
                        serviceDurationStatement.setInt(2, routeType);
                        serviceDurationStatement.setInt(3, serviceDurationSeconds);
                        serviceDurationTracker.addBatch();
                    } catch (SQLException ex) {
                        throw new StorageException(ex);
                    }
                    return true; // Iteration continues
                });
            }
            serviceDurationTracker.executeRemaining();
            // No need to build indexes because (service_id, route_type) is already the primary key of this table.

            connection.commit();
        } catch (SQLException e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        } finally {
            DbUtils.closeQuietly(connection);
        }
        LOG.info("Done.");
    }

    static class ServiceInfo {

        final String serviceId;
        TIntIntHashMap durationByRouteType = new TIntIntHashMap();
        Set<LocalDate> datesActive = new HashSet<>();
        Set<String> tripIds = new HashSet<>();

        public ServiceInfo(String serviceId) {
            this.serviceId = serviceId;
        }

        public int getTotalServiceDurationSeconds() {
            return Arrays.stream(durationByRouteType.values()).sum();
        }

    }

    static class DateInfo {

        final LocalDate date;
        TIntIntHashMap durationByRouteType = new TIntIntHashMap();
        int tripCount = 0; // Trip count could also in theory be broken down by route type.
        Set<String> servicesActive = new HashSet<>();

        public DateInfo(LocalDate date) {
            this.date = date;
        }

        public int getTotalServiceDurationSeconds() {
            return Arrays.stream(durationByRouteType.values()).sum();
        }

        public void add (ServiceInfo serviceInfo) {
            servicesActive.add(serviceInfo.serviceId);
            serviceInfo.durationByRouteType.forEachEntry((routeType, serviceDurationSeconds) -> {
                durationByRouteType.adjustOrPutValue(routeType, serviceDurationSeconds, serviceDurationSeconds);
                return true; // Continue iteration.
            });
            tripCount += serviceInfo.tripIds.size();
        }
    }

    /**
     * Checks that trips which run on the same block (i.e., share a block_id) do not overlap. The block_id
     * represents a vehicle in service, so there must not be any trips on the same block interval that start while another
     * block trip is running.
     *
     * NOTE: This validation check happens in the {@link ServiceValidator} because it depends on information derived
     * about which service calendars operate on which feed dates ({@link #serviceInfoForServiceId}).
     */
    private void validateBlocks () {
        // Iterate over each block and determine if there are any trips that overlap one another.
        for (String blockId : blockIntervals.keySet()) {
            List<BlockInterval> intervals = blockIntervals.get(blockId);
            intervals.sort(Comparator.comparingInt(i -> i.startTime));
            // Iterate over each interval (except for the last) comparing it to every other interval (so the last interval
            // is handled through the course of iteration).
            // FIXME this has complexity of n^2, there has to be a better way.
            for (int n = 0; n < intervals.size() - 1; n++) {
                BlockInterval interval1 = intervals.get(n);
                // Compare the interval at position N with all other intervals at position N+1 to the end of the list.
                for (BlockInterval interval2 : intervals.subList(n + 1, intervals.size())) {
                    if (interval1.lastStop.departure_time <= interval2.firstStop.arrival_time || interval2.lastStop.departure_time <= interval1.firstStop.arrival_time) {
                        continue;
                    }
                    // If either trip's last departure occurs after the other's first arrival, they overlap. We still
                    // need to determine if they operate on the same day though.
                    if (interval1.trip.service_id.equals(interval2.trip.service_id)) {
                        // If the overlapping trips share a service_id, record an error.
                        registerError(interval1.trip, TRIP_OVERLAP_IN_BLOCK, interval2.trip.trip_id);
                    } else {
                        // Trips overlap but don't have the same service_id.
                        // Check to see if service days fall on the same days of the week.
                        ServiceValidator.ServiceInfo info1 = serviceInfoForServiceId.get(interval1.trip.service_id);
                        ServiceValidator.ServiceInfo info2 = serviceInfoForServiceId.get(interval2.trip.service_id);
                        Set<LocalDate> overlappingDates = new HashSet<>(info1.datesActive); // use the copy constructor
                        overlappingDates.retainAll(info2.datesActive);
                        if (overlappingDates.size() > 0) {
                            registerError(interval1.trip, TRIP_OVERLAP_IN_BLOCK, interval2.trip.trip_id);
                        }
                    }
                }
            }
        }
    }

    /**
     * A simple class used during validation to store details the run interval for a block trip.
     */
    private class BlockInterval {
        Trip trip;
        Integer startTime;
        StopTime firstStop;
        StopTime lastStop;
    }
}


package com.conveyal.gtfs.loader;

import com.conveyal.gtfs.GTFSFeed;
import com.conveyal.gtfs.model.Calendar;
import com.conveyal.gtfs.model.CalendarDate;
import com.conveyal.gtfs.model.ScheduleException;
import com.conveyal.gtfs.model.Service;
import org.postgresql.copy.CopyManager;
import org.postgresql.core.BaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.sql.DataSource;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.SQLException;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * Handles exporting a feed contained in the database to a GTFS zip file.
 */
public class JdbcGtfsExporter {
    private static final Logger LOG = LoggerFactory.getLogger(JdbcGtfsExporter.class);

    private final String outFile;
    private final DataSource dataSource;
    private final boolean fromEditor;

    // These fields will be filled in once feed snapshot begins.
    private Connection connection;
    private ZipOutputStream zipOutputStream;
    // The reference feed ID (namespace) to copy.
    private final String feedIdToExport;
    private List<String> emptyTableList = new ArrayList<>();

    public JdbcGtfsExporter(String feedId, String outFile, DataSource dataSource, boolean fromEditor) {
        this.feedIdToExport = feedId;
        this.outFile = outFile;
        this.dataSource = dataSource;
        this.fromEditor = fromEditor;
    }

    /**
     * Export primary entity tables as well as Pattern and PatternStops tables.
     *
     * FIXME: This should probably only export main GTFS tables... unless there is a benefit to keeping editor fields or
     * tables intact for future editing. Maybe this could be an option provided to the function (e.g., specTablesOnly).
     */
    public FeedLoadResult exportTables() {
        // This result object will be returned to the caller to summarize the feed and report any critical errors.
        // FIXME: use feed export result object?
        FeedLoadResult result = new FeedLoadResult();

        try {
            zipOutputStream = new ZipOutputStream(new FileOutputStream(outFile));
            long startTime = System.currentTimeMillis();
            // We get a single connection object and share it across several different methods.
            // This ensures that actions taken in one method are visible to all subsequent SQL statements.
            // If we create a schema or table on one connection, then access it in a separate connection, we have no
            // guarantee that it exists when the accessing statement is executed.
            connection = dataSource.getConnection();
            if (!connection.getMetaData().getDatabaseProductName().equals("PostgreSQL")) {
                // This code path currently requires the Postgres text "copy to" format.
                throw new RuntimeException("Export from SQL database not implemented for non-PostgreSQL databases.");
            }
            // Construct where clause for routes table to filter only "approved" routes (and entities related to routes)
            // if exporting a feed/schema that represents an editor snapshot.
            String whereRouteIsApproved = String.format("where %s.%s.status = 2", feedIdToExport, Table.ROUTES.name);
            // Export each table in turn (by placing entry in zip output stream).
            result.agency = export(Table.AGENCY, connection);
            if (fromEditor) {
                // only export calendar entries that have at least one day of service set
                // this could happen in cases where a feed was imported that only had calendar_dates.txt
                result.calendar = export(
                    Table.CALENDAR,
                    String.join(
                        " ",
                        Table.CALENDAR.generateSelectSql(feedIdToExport, Requirement.OPTIONAL),
                        "WHERE monday=1 OR tuesday=1 OR wednesday=1 OR thursday=1 OR friday=1 OR saturday=1 OR sunday=1"
                    )
                );
            } else {
                result.calendar = export(Table.CALENDAR, connection);
            }
            if (fromEditor) {
                // Export schedule exceptions in place of calendar dates if exporting a feed/schema that represents an editor snapshot.
                GTFSFeed feed = new GTFSFeed();
                // FIXME: The below table readers should probably just share a connection with the exporter.
                JDBCTableReader<ScheduleException> exceptionsReader =
                        new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".",
                                EntityPopulator.SCHEDULE_EXCEPTION);
                JDBCTableReader<Calendar> calendarsReader =
                        new JDBCTableReader(Table.CALENDAR, dataSource, feedIdToExport + ".",
                                EntityPopulator.CALENDAR);
                Iterable<Calendar> calendars = calendarsReader.getAll();
                Iterable<ScheduleException> exceptionsIterator = exceptionsReader.getAll();
                List<ScheduleException> exceptions = new ArrayList<>();
                // FIXME: Doing this causes the connection to stay open, but it is closed in the finalizer so it should
                // not be a big problem.
                for (ScheduleException exception : exceptionsIterator) {
                    exceptions.add(exception);
                }
                // check whether the feed is organized in a format with the calendars.txt file
                if (calendarsReader.getRowCount() > 0) {
                    // feed does have calendars.txt file, continue export with strategy of matching exceptions
                    // to calendar to output calendar_dates.txt
                    int calendarDateCount = 0;
                    for (Calendar cal : calendars) {
                        Service service = new Service(cal.service_id);
                        service.calendar = cal;
                        for (ScheduleException ex : exceptions) {
                            if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.SWAP) &&
                                (!ex.addedService.contains(cal.service_id) && !ex.removedService.contains(cal.service_id))) {
                                // Skip swap exception if cal is not referenced by added or removed service.
                                // This is not technically necessary, but the output is cleaner/more intelligible.
                                continue;
                            }

                            for (LocalDate date : ex.dates) {
                                if (date.isBefore(cal.start_date) || date.isAfter(cal.end_date)) {
                                    // No need to write dates that do not apply
                                    continue;
                                }

                                CalendarDate calendarDate = new CalendarDate();
                                calendarDate.date = date;
                                calendarDate.service_id = cal.service_id;
                                calendarDate.exception_type = ex.serviceRunsOn(cal) ? 1 : 2;
                                LOG.info("Adding exception {} (type={}) for calendar {} on date {}", ex.name, calendarDate.exception_type, cal.service_id, date.toString());

                                if (service.calendar_dates.containsKey(date))
                                    throw new IllegalArgumentException("Duplicate schedule exceptions on " + date.toString());

                                service.calendar_dates.put(date, calendarDate);
                                calendarDateCount += 1;
                            }
                        }
                        feed.services.put(cal.service_id, service);
                    }
                    if (calendarDateCount == 0) {
                        LOG.info("No calendar dates found. Skipping table.");
                    } else {
                        LOG.info("Writing {} calendar dates from schedule exceptions", calendarDateCount);
                        new CalendarDate.Writer(feed).writeTable(zipOutputStream);
                    }
                } else {
                    // No calendar records exist, export calendar_dates as is and hope for the best.
                    // This situation will occur in at least 2 scenarios:
                    // 1.  A GTFS has been loaded into the editor that had only the calendar_dates.txt file
                    //     and no further edits were made before exporting to a snapshot
                    // 2.  A new GTFS has been created from scratch and calendar information has yet to be added.
                    //     This will result in an invalid GTFS, but it was what the user wanted so ¯\_(ツ)_/¯
                    result.calendarDates = export(Table.CALENDAR_DATES, connection);
                }
            } else {
                // Otherwise, simply export the calendar dates as they were loaded in.
                result.calendarDates = export(Table.CALENDAR_DATES, connection);
            }
            result.fareAttributes = export(Table.FARE_ATTRIBUTES, connection);
            result.fareRules = export(Table.FARE_RULES, connection);
            result.feedInfo = export(Table.FEED_INFO, connection);
            // Only write frequencies for "approved" routes using COPY TO with results of select query
            if (fromEditor) {
                // Generate filter SQL for trips if exporting a feed/schema that represents an editor snapshot.
                // The filter clause for frequencies requires two joins to reach the routes table and a where filter on
                // route status.
                // FIXME Replace with string literal query instead of clause generators
                result.frequencies = export(
                    Table.FREQUENCIES,
                    String.join(
                        " ",
                        // Convert start_time and end_time values from seconds to time format (HH:MM:SS).
                        Table.FREQUENCIES.generateSelectSql(feedIdToExport, Requirement.OPTIONAL),
                        Table.FREQUENCIES.generateJoinSql(Table.TRIPS, feedIdToExport),
                        Table.TRIPS.generateJoinSql(
                            Table.ROUTES,
                            feedIdToExport,
                            "route_id",
                            false
                        ),
                        whereRouteIsApproved
                    )
                );
            } else {
                result.frequencies = export(Table.FREQUENCIES, connection);
            }

            // Only write "approved" routes using COPY TO with results of select query
            if (fromEditor) {
                // The filter clause for routes is simple. We're just checking that the route is APPROVED.
                result.routes = export(
                    Table.ROUTES,
                    String.join(
                        " ",
                        Table.ROUTES.generateSelectSql(feedIdToExport, Requirement.OPTIONAL),
                        whereRouteIsApproved
                    )
                );
            } else {
                result.routes = export(Table.ROUTES, connection);
            }

            // FIXME: Find some place to store errors encountered on export for patterns and pattern stops.
            // FIXME: Is there a need to export patterns or pattern stops? Should these be iterated over to ensure that
            // frequency-based pattern travel times match stop time arrivals/departures?
//            export(Table.PATTERNS);
//            export(Table.PATTERN_STOP);
            // Only write shapes for "approved" routes using COPY TO with results of select query
            if (fromEditor) {
                // Generate filter SQL for shapes if exporting a feed/schema that represents an editor snapshot.
                // The filter clause for shapes requires joining to trips and then to routes table and a where filter on
                // route status.
                // FIXME: I'm not sure that shape_id is indexed for the trips table. This could cause slow downs.
                // FIXME: this is exporting point_type, which is not a GTFS field, but its presence shouldn't hurt.
                String shapeFieldsToExport = Table.commaSeparatedNames(
                    Table.SHAPES.specFields(),
                    String.join(".", feedIdToExport, Table.SHAPES.name + "."),
                    true
                );
                // NOTE: The below substitution uses relative indexing. All values "%<s" reference the same arg as the
                // previous format specifier (i.e., feedIdToExport).
                result.shapes = export(
                    Table.SHAPES,
                    String.format(
                        "select %s " +
                            "from (select distinct %s.trips.shape_id " +
                            "from %<s.trips, %<s.routes " +
                            "where %<s.trips.route_id = %<s.routes.route_id and " +
                            "%<s.routes.status = 2) as unique_approved_shape_ids, " +
                            "%<s.shapes where unique_approved_shape_ids.shape_id = %<s.shapes.shape_id",
                        shapeFieldsToExport,
                        feedIdToExport
                    )
                );
            } else {
                result.shapes = export(Table.SHAPES, connection);
            }
            result.stops = export(Table.STOPS, connection);
            // Only write stop times for "approved" routes using COPY TO with results of select query
            if (fromEditor) {
                // Generate filter SQL for trips if exporting a feed/schema that represents an editor snapshot.
                // The filter clause for stop times requires two joins to reach the routes table and a where filter on
                // route status.
                // FIXME Replace with string literal query instead of clause generators
                result.stopTimes = export(
                    Table.STOP_TIMES,
                    String.join(" ",
                        Table.STOP_TIMES.generateSelectSql(feedIdToExport, Requirement.OPTIONAL),
                        Table.STOP_TIMES.generateJoinSql(Table.TRIPS, feedIdToExport),
                        Table.TRIPS.generateJoinSql(
                            Table.ROUTES,
                            feedIdToExport,
                            "route_id",
                            false
                        ),
                        whereRouteIsApproved
                    )
                );
            } else {
                result.stopTimes = export(Table.STOP_TIMES, connection);
            }
            result.transfers = export(Table.TRANSFERS, connection);
            if (fromEditor) {
                // Generate filter SQL for trips if exporting a feed/schema that represents an editor snapshot.
                // The filter clause for trips requires an inner join on the routes table and the same where check on
                // route status.
                // FIXME Replace with string literal query instead of clause generators
                result.trips = export(
                    Table.TRIPS,
                    String.join(" ",
                        Table.TRIPS.generateSelectSql(feedIdToExport, Requirement.OPTIONAL),
                        Table.TRIPS.generateJoinSql(
                            Table.ROUTES,
                            feedIdToExport,
                            "route_id",
                            false
                        ),
                        whereRouteIsApproved
                    )
                );
            } else {
                result.trips = export(Table.TRIPS, connection);
            }

            zipOutputStream.close();
            // Run clean up on the resulting zip file.
            cleanUpZipFile();
            result.completionTime = System.currentTimeMillis();
            result.loadTimeMillis = result.completionTime - startTime;
            // Exporting primary GTFS tables for GRTA Xpress = 12 sec
            LOG.info("Exporting tables took {} sec", (result.loadTimeMillis) / 1000);
            LOG.info("Exported feed {} to zip file: {}", feedIdToExport, outFile);
        } catch (Exception ex) {
            // Note: Exceptions that occur during individual table loads are separately caught and stored in
            // TableLoadResult.
            LOG.error("Exception while creating snapshot: {}", ex.toString());
            ex.printStackTrace();
            result.fatalException = ex.toString();
        }
        return result;
    }

    /**
     * Removes any empty zip files from the final zip file.
     */
    private void cleanUpZipFile() {
        long startTime = System.currentTimeMillis();
        // Define ZIP File System Properties in HashMap
        Map<String, String> zip_properties = new HashMap<>();
        // We want to read an existing ZIP File, so we set this to False
        zip_properties.put("create", "false");

        // Specify the path to the ZIP File that you want to read as a File System
        // (File#toURI allows this to work across different operating systems, including Windows)
        URI zip_disk = URI.create("jar:" + new File(outFile).toURI());

        // Create ZIP file System
        try (FileSystem fileSystem = FileSystems.newFileSystem(zip_disk, zip_properties)) {
            // Get the Path inside ZIP File to delete the ZIP Entry
            for (String fileName : emptyTableList) {
                Path filePath = fileSystem.getPath(fileName);
                // Execute Delete
                Files.delete(filePath);
                LOG.info("Empty file {} successfully deleted", fileName);
            }
        } catch (IOException e) {
            LOG.error("Could not remove empty zip files");
            e.printStackTrace();
        }
        LOG.info("Deleted {} empty files in {} ms", emptyTableList.size(), System.currentTimeMillis() - startTime);
    }

    private TableLoadResult export (Table table, Connection connection) {
        if (fromEditor) {
            // Default behavior for exporting editor snapshot tables is to select only the spec fields.
            return export(table, table.generateSelectSql(feedIdToExport, Requirement.OPTIONAL));
        } else {
            String existingFieldsSelect = null;
            try {
                existingFieldsSelect = table.generateSelectAllExistingFieldsSql(connection, feedIdToExport);
            } catch (SQLException e) {
                LOG.error("failed to generate select statement for existing fields");
                TableLoadResult tableLoadResult = new TableLoadResult();
                tableLoadResult.fatalException = e.toString();
                e.printStackTrace();
                return tableLoadResult;
            }
            return export(table, existingFieldsSelect);
        }
    }

    /**
     * Export a table to the zipOutputStream to be written to the GTFS.
     */
    private TableLoadResult export (Table table, String filterSql) {
        long startTime = System.currentTimeMillis();
        TableLoadResult tableLoadResult = new TableLoadResult();
        try {
            if (filterSql == null) {
                throw new IllegalArgumentException("filterSql argument cannot be null");
            } else {
                // Surround filter SQL in parentheses.
                filterSql = String.format("(%s)", filterSql);
            }

            // Create entry for table
            String textFileName = table.name + ".txt";
            ZipEntry zipEntry = new ZipEntry(textFileName);
            zipOutputStream.putNextEntry(zipEntry);

            // don't let CSVWriter close the stream when it is garbage-collected
            OutputStream protectedOut = new FilterOutputStream(zipOutputStream);
            String copySql = String.format("copy %s to STDOUT DELIMITER ',' CSV HEADER", filterSql);
            LOG.info(copySql);
            // Our connection pool wraps the Connection objects, so we need to unwrap the Postgres connection interface.
            CopyManager copyManager = new CopyManager(connection.unwrap(BaseConnection.class));
            tableLoadResult.rowCount = (int) copyManager.copyOut(copySql, protectedOut);
            if (tableLoadResult.rowCount == 0) {
                // If no rows were exported, keep track of table name for later removal.
                emptyTableList.add(textFileName);
            }
            zipOutputStream.closeEntry();
            LOG.info("Copied {} {} in {} ms.", tableLoadResult.rowCount, table.name, System.currentTimeMillis() - startTime);
            connection.commit();
        } catch (SQLException | IOException | IllegalArgumentException e) {
            // Rollback connection so that fatal exception does not impact loading of other tables.
            try {
                connection.rollback();
            } catch (SQLException ex) {
                ex.printStackTrace();
            }
            tableLoadResult.fatalException = e.toString();
            LOG.error("Exception while exporting tables", e);
        }
        return tableLoadResult;
    }
}


package com.conveyal.gtfs.model;

import com.conveyal.gtfs.GTFSFeed;

import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Iterator;

public class Trip extends Entity {

    private static final long serialVersionUID = -4869384750974542712L;
    public String route_id;
    public String service_id;
    public String trip_id;
    public String trip_headsign;
    public String trip_short_name;
    public int    direction_id;
    public String block_id;
    public String shape_id;
    public int    bikes_allowed;
    public int    wheelchair_accessible;
    public String feed_id;

    @Override
    public String getId() {
        return trip_id;
    }

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#TRIPS}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        statement.setString(oneBasedIndex++, trip_id);
        statement.setString(oneBasedIndex++, route_id);
        statement.setString(oneBasedIndex++, service_id);
        statement.setString(oneBasedIndex++, trip_headsign);
        statement.setString(oneBasedIndex++, trip_short_name);
        setIntParameter(statement, oneBasedIndex++, direction_id);
        statement.setString(oneBasedIndex++, block_id);
        statement.setString(oneBasedIndex++, shape_id);
        setIntParameter(statement, oneBasedIndex++, wheelchair_accessible);
        setIntParameter(statement, oneBasedIndex++, bikes_allowed);
        // Editor-specific field? pattern_id
        statement.setString(oneBasedIndex++, null);
    }

    public static class Loader extends Entity.Loader<Trip> {

        public Loader(GTFSFeed feed) {
            super(feed, "trips");
        }

        @Override
        protected boolean isRequired() {
            return true;
        }

        @Override
        public void loadOneRow() throws IOException {
            Trip t = new Trip();

            t.id = row + 1; // offset line number by 1 to account for 0-based row index
            t.route_id        = getStringField("route_id", true);
            t.service_id      = getStringField("service_id", true);
            t.trip_id         = getStringField("trip_id", true);
            t.trip_headsign   = getStringField("trip_headsign", false);
            t.trip_short_name = getStringField("trip_short_name", false);
            t.direction_id    = getIntField("direction_id", false, 0, 1);
            t.block_id        = getStringField("block_id", false); // make a blocks multimap
            t.shape_id        = getStringField("shape_id", false);
            t.bikes_allowed   = getIntField("bikes_allowed", false, 0, 2);
            t.wheelchair_accessible = getIntField("wheelchair_accessible", false, 0, 2);
            t.feed = feed;
            t.feed_id = feed.feedId;
            feed.trips.put(t.trip_id, t);

            /*
              Check referential integrity without storing references. Trip cannot directly reference Services or
              Routes because they would be serialized into the MapDB.
             */
            // TODO confirm existence of shape ID
            getRefField("service_id", true, feed.services);
            getRefField("route_id", true, feed.routes);
        }

    }

    public static class Writer extends Entity.Writer<Trip> {
        public Writer (GTFSFeed feed) {
            super(feed, "trips");
        }

        @Override
        protected void writeHeaders() throws IOException {
            // TODO: export shapes
            writer.writeRecord(new String[] {"route_id", "trip_id", "trip_headsign", "trip_short_name", "direction_id", "block_id",
                    "shape_id", "bikes_allowed", "wheelchair_accessible", "service_id"});
        }

        @Override
        protected void writeOneRow(Trip t) throws IOException {
            writeStringField(t.route_id);
            writeStringField(t.trip_id);
            writeStringField(t.trip_headsign);
            writeStringField(t.trip_short_name);
            writeIntField(t.direction_id);
            writeStringField(t.block_id);
            writeStringField(t.shape_id);
            writeIntField(t.bikes_allowed);
            writeIntField(t.wheelchair_accessible);
            writeStringField(t.service_id);
            endRecord();
        }

        @Override
        protected Iterator<Trip> iterator() {
            return feed.trips.values().iterator();
        }

    }

}


package com.conveyal.gtfs.model;

import com.conveyal.gtfs.GTFSFeed;

import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Iterator;

import org.mapdb.Fun.Tuple2;

public class ShapePoint extends Entity {

    private static final long serialVersionUID = 6751814959971086070L;
    public String shape_id;
    public double shape_pt_lat;
    public double shape_pt_lon;
    public int    shape_pt_sequence;
    public double shape_dist_traveled;

    @Override
    public String getId () {
        return shape_id;
    }

    @Override
    public Integer getSequenceNumber() {
        return shape_pt_sequence;
    }

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#SHAPES}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        statement.setString(oneBasedIndex++, shape_id);
        setIntParameter(statement, oneBasedIndex++, shape_pt_sequence);
        statement.setDouble(oneBasedIndex++, shape_pt_lat);
        statement.setDouble(oneBasedIndex++, shape_pt_lon);
        statement.setDouble(oneBasedIndex++, shape_dist_traveled);
        // Editor-specific field below (point_type 0 indicates no control point)
        statement.setInt(oneBasedIndex++, 0);
    }

    public ShapePoint () { }

    // Similar to stoptime, we have to have a constructor, because fields are final so as to be immutable for storage in MapDB.
    public ShapePoint(String shape_id, double shape_pt_lat, double shape_pt_lon, int shape_pt_sequence, double shape_dist_traveled) {
        this.shape_id = shape_id;
        this.shape_pt_lat = shape_pt_lat;
        this.shape_pt_lon = shape_pt_lon;
        this.shape_pt_sequence = shape_pt_sequence;
        this.shape_dist_traveled = shape_dist_traveled;
    }

    public static class Loader extends Entity.Loader<ShapePoint> {

        public Loader(GTFSFeed feed) {
            super(feed, "shapes");
        }

        @Override
        protected boolean isRequired() {
            return false;
        }

        @Override
        public void loadOneRow() throws IOException {
            String shape_id = getStringField("shape_id", true);
            double shape_pt_lat = getDoubleField("shape_pt_lat", true, -90D, 90D);
            double shape_pt_lon = getDoubleField("shape_pt_lon", true, -180D, 180D);
            int shape_pt_sequence = getIntField("shape_pt_sequence", true, 0, Integer.MAX_VALUE);
            double shape_dist_traveled = getDoubleField("shape_dist_traveled", false, 0D, Double.MAX_VALUE);

            ShapePoint s = new ShapePoint(shape_id, shape_pt_lat, shape_pt_lon, shape_pt_sequence, shape_dist_traveled);
            s.id = row + 1; // offset line number by 1 to account for 0-based row index
            s.feed = null; // since we're putting this into MapDB, we don't want circular serialization
            feed.shape_points.put(new Tuple2<String, Integer>(s.shape_id, s.shape_pt_sequence), s);
        }
    }

    public static class Writer extends Entity.Writer<ShapePoint> {
        public Writer (GTFSFeed feed) {
            super(feed, "shapes");
        }

        @Override
        protected void writeHeaders() throws IOException {
            writer.writeRecord(new String[] {"shape_id", "shape_pt_lat", "shape_pt_lon", "shape_pt_sequence", "shape_dist_traveled"});
        }

        @Override
        protected void writeOneRow(ShapePoint s) throws IOException {
            writeStringField(s.shape_id);
            writeDoubleField(s.shape_pt_lat);
            writeDoubleField(s.shape_pt_lon);
            writeIntField(s.shape_pt_sequence);
            writeDoubleField(s.shape_dist_traveled);
            endRecord();
        }

        @Override
        protected Iterator<ShapePoint> iterator() {
            return feed.shape_points.values().iterator();
        }
    }
}


package com.conveyal.gtfs.validator;

import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.NewGTFSErrorType;
import com.conveyal.gtfs.error.SQLErrorStorage;
import com.conveyal.gtfs.loader.Feed;
import com.conveyal.gtfs.model.Entity;

import java.util.Set;

/**
 * A Validator examines a whole GTFS feed or a single trip within a GTFS feed. It accumulates error messages for
 * problems it finds in that feed, optionally repairing the problems it encounters.
 */
public abstract class Validator {

    Feed feed;

    SQLErrorStorage errorStorage;

    public Validator(Feed feed, SQLErrorStorage errorStorage) {
        this.feed = feed;
        this.errorStorage = errorStorage;
    }

    /**
     * Store an error that affects the entire feed or an entire file. Wraps the underlying error constructor.
     */
//    public void registerError (Class<? extends Entity> entityType, NewGTFSErrorType errorType) {
//        errorStorage.storeError(new NewGTFSError(entityType, errorType));
//    }

    /**
     * Store an error that affects a single line of a single table. Wraps the underlying error factory method.
     */
    public void registerError(Entity entity, NewGTFSErrorType errorType) {
        errorStorage.storeError(NewGTFSError.forEntity(entity, errorType));
    }

    /**
     * Stores a set of errors.
     */
    public void storeErrors(Set<NewGTFSError> errors) {
        errorStorage.storeErrors(errors);
    }

    /**
     * WARNING: this method creates but DOES NOT STORE a new GTFS error. It should only be used in cases where a
     * collection of errors need to be temporarily held before storing in batch (e.g., waiting to store travel time zero
     * errors before it is determined that the entire feed uses travel times rounded to the minute).
     */
    NewGTFSError createUnregisteredError (Entity entity, NewGTFSErrorType errorType) {
        return NewGTFSError.forEntity(entity, errorType);
    }

//    /**
//     * Store an error that affects a single line of a single table. Add a single key-value pair to it. Wraps the
//     * underlying error constructor.
//     */
//    public void registerError(Entity entity, NewGTFSErrorType errorType, String key, String value) {
//        errorStorage.storeError(new NewGTFSError(entity, errorType).addInfo(key, value));
//    }

    /**
     * Store an error that affects a single line of a single table.
     * Add a bad value to it.
     */
    public void registerError(Entity entity, NewGTFSErrorType errorType, Object badValue) {
        errorStorage.storeError(NewGTFSError.forEntity(entity, errorType).setBadValue(badValue.toString()));
    }

    /**
     * Basic storage of user-constructed error.
     */
    public void registerError (NewGTFSError error) {
        errorStorage.storeError(error);
    }

    /**
     * This method will be called after the validation process is complete.
     * This allows the implementation to perform any analysis or checking that uses accumulated information, and
     * provides a path to output that summary information (by saving it in the provided ValidationResult object.
     */
    public void complete (ValidationResult validationResult) {}

}