package com.conveyal.gtfs;

import com.conveyal.gtfs.error.GTFSError;
import com.conveyal.gtfs.loader.FeedLoadResult;
import com.conveyal.gtfs.loader.JdbcGTFSFeedConverter;
import com.conveyal.gtfs.model.*;
import com.conveyal.gtfs.model.Calendar;
import com.conveyal.gtfs.validator.Validator;
import com.conveyal.gtfs.stats.FeedStats;
import com.conveyal.gtfs.validator.service.GeoUtils;
import com.google.common.collect.*;
import com.google.common.eventbus.EventBus;
import com.google.common.util.concurrent.ExecutionError;
import com.vividsolutions.jts.algorithm.ConvexHull;
import com.vividsolutions.jts.geom.*;
import com.vividsolutions.jts.index.strtree.STRtree;
import com.vividsolutions.jts.simplify.DouglasPeuckerSimplifier;
import org.geotools.referencing.GeodeticCalculator;
import org.mapdb.BTreeMap;
import org.mapdb.Bind;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.Fun;
import org.mapdb.Fun.Tuple2;
import org.mapdb.Serializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.sql.DataSource;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOError;
import java.io.IOException;
import java.io.OutputStream;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.StreamSupport;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

/**
 * All entities must be from a single feed namespace.
 * Composed of several GTFSTables.
 */
public class GTFSFeed implements Cloneable, Closeable {

    private static final Logger LOG = LoggerFactory.getLogger(GTFSFeed.class);
    private static final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyyMMdd");

    private DB db;

    public String feedId = null;

    // TODO make all of these Maps MapDBs so the entire GTFSFeed is persistent and uses constant memory

    /* Some of these should be multimaps since they don't have an obvious unique key. */
    public final Map<String, Agency> agency;
    public final Map<String, FeedInfo> feedInfo;
    // This is how you do a multimap in mapdb: https://github.com/jankotek/MapDB/blob/release-1.0/src/test/java/examples/MultiMap.java
    public final NavigableSet<Tuple2<String, Frequency>> frequencies;
    public final Map<String, Route> routes;
    public final Map<String, Stop> stops;
    public final Map<String, Transfer> transfers;
    public final BTreeMap<String, Trip> trips;

    public final Set<String> transitIds = new HashSet<>();
    /** CRC32 of the GTFS file this was loaded from */
    public long checksum;

    /* Map from 2-tuples of (shape_id, shape_pt_sequence) to shape points */
    public final ConcurrentNavigableMap<Tuple2<String, Integer>, ShapePoint> shape_points;

    /* Map from 2-tuples of (trip_id, stop_sequence) to stoptimes. */
    public final BTreeMap<Tuple2, StopTime> stop_times;

//    public final ConcurrentMap<String, Long> stopCountByStopTime;

    /* Map from stop (stop_id) to stopTimes tuples (trip_id, stop_sequence) */
    public final NavigableSet<Tuple2<String, Tuple2>> stopStopTimeSet;
    public final ConcurrentMap<String, Long> stopCountByStopTime;

    public final NavigableSet<Tuple2<String, String>> tripsPerService;

    public final NavigableSet<Tuple2<String, String>> servicesPerDate;

    /* A fare is a fare_attribute and all fare_rules that reference that fare_attribute. */
    public final Map<String, Fare> fares;

    /* A service is a calendar entry and all calendar_dates that modify that calendar entry. */
    public final BTreeMap<String, Service> services;

    /* A place to accumulate errors while the feed is loaded. Tolerate as many errors as possible and keep on loading. */
    public final NavigableSet<GTFSError> errors;

    /* Stops spatial index which gets built lazily by getSpatialIndex() */
    private transient STRtree spatialIndex;

    /* Convex hull of feed (based on stops) built lazily by getConvexHull() */
    private transient Polygon convexHull;

    /* Merged stop buffers polygon built lazily by getMergedBuffers() */
    private transient Geometry mergedBuffers;

    /* Create geometry factory to produce LineString geometries. */
    GeometryFactory gf = new GeometryFactory();

    /* Map routes to associated trip patterns. */
    // TODO: Hash Multimapping in guava (might need dependency).
    public final Map<String, Pattern> patterns;

    // TODO bind this to map above so that it is kept up to date automatically
    public final Map<String, String> tripPatternMap;
    private boolean loaded = false;

    /* A place to store an event bus that is passed through constructor. */
    public transient EventBus eventBus;

    /**
     * The order in which we load the tables is important for two reasons.
     * 1. We must load feed_info first so we know the feed ID before loading any other entities. This could be relaxed
     * by having entities point to the feed object rather than its ID String.
     * 2. Referenced entities must be loaded before any entities that reference them. This is because we check
     * referential integrity while the files are being loaded. This is done on the fly during loading because it allows
     * us to associate a line number with errors in objects that don't have any other clear identifier.
     *
     * Interestingly, all references are resolvable when tables are loaded in alphabetical order.
     */
    public void loadFromFile(ZipFile zip, String fid) throws Exception {
        if (this.loaded) throw new UnsupportedOperationException("Attempt to load GTFS into existing database");

        // NB we don't have a single CRC for the file, so we combine all the CRCs of the component files. NB we are not
        // simply summing the CRCs because CRCs are (I assume) uniformly randomly distributed throughout the width of a
        // long, so summing them is a convolution which moves towards a Gaussian with mean 0 (i.e. more concentrated
        // probability in the center), degrading the quality of the hash. Instead we XOR. Assuming each bit is independent,
        // this will yield a nice uniformly distributed result, because when combining two bits there is an equal
        // probability of any input, which means an equal probability of any output. At least I think that's all correct.
        // Repeated XOR is not commutative but zip.stream returns files in the order they are in the central directory
        // of the zip file, so that's not a problem.
        checksum = zip.stream().mapToLong(ZipEntry::getCrc).reduce((l1, l2) -> l1 ^ l2).getAsLong();

        db.getAtomicLong("checksum").set(checksum);

        new FeedInfo.Loader(this).loadTable(zip);
        // maybe we should just point to the feed object itself instead of its ID, and null out its stoptimes map after loading
        if (fid != null) {
            feedId = fid;
            LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID
        }
        else if (feedId == null || feedId.isEmpty()) {
            feedId = new File(zip.getName()).getName().replaceAll("\\.zip$", "");
            LOG.info("Feed ID is undefined, pester maintainers to include a feed ID. Using file name {}.", feedId); // TODO log an error, ideally feeds should include a feedID
        }
        else {
            LOG.info("Feed ID is '{}'.", feedId);
        }

        db.getAtomicString("feed_id").set(feedId);

        new Agency.Loader(this).loadTable(zip);

        // calendars and calendar dates are joined into services. This means a lot of manipulating service objects as
        // they are loaded; since mapdb keys/values are immutable, load them in memory then copy them to MapDB once
        // we're done loading them
        Map<String, Service> serviceTable = new HashMap<>();
        new Calendar.Loader(this, serviceTable).loadTable(zip);
        new CalendarDate.Loader(this, serviceTable).loadTable(zip);
        this.services.putAll(serviceTable);
        serviceTable = null; // free memory

        // Same deal
        Map<String, Fare> fares = new HashMap<>();
        new FareAttribute.Loader(this, fares).loadTable(zip);
        new FareRule.Loader(this, fares).loadTable(zip);
        this.fares.putAll(fares);
        fares = null; // free memory

        new Route.Loader(this).loadTable(zip);
        new ShapePoint.Loader(this).loadTable(zip);
        new Stop.Loader(this).loadTable(zip);
        new Transfer.Loader(this).loadTable(zip);
        new Trip.Loader(this).loadTable(zip);
        new Frequency.Loader(this).loadTable(zip);
        new StopTime.Loader(this).loadTable(zip); // comment out this line for quick testing using NL feed
        LOG.info("{} errors", errors.size());
        for (GTFSError error : errors) {
            LOG.info("{}", error);
        }
        LOG.info("Building stop to stop times index");
        Bind.histogram(stop_times, stopCountByStopTime, (key, stopTime) -> stopTime.stop_id);
        Bind.secondaryKeys(stop_times, stopStopTimeSet, (key, stopTime) -> new String[] {stopTime.stop_id});
        LOG.info("Building trips per service index");
        Bind.secondaryKeys(trips, tripsPerService, (key, trip) -> new String[] {trip.service_id});
        LOG.info("Building services per date index");
        Bind.secondaryKeys(services, servicesPerDate, (key, service) -> {

            LocalDate startDate = service.calendar != null
                    ? service.calendar.start_date
                    : service.calendar_dates.keySet().stream().sorted().findFirst().get();
            LocalDate endDate = service.calendar != null
                    ? service.calendar.end_date
                    : service.calendar_dates.keySet().stream().sorted().reduce((first, second) -> second).get();
            // end date for Period.between is not inclusive
            int daysOfService = (int) ChronoUnit.DAYS.between(startDate, endDate.plus(1, ChronoUnit.DAYS));
            return IntStream.range(0, daysOfService)
                    .mapToObj(offset -> startDate.plusDays(offset))
                    .filter(service::activeOn)
                    .map(date -> date.format(dateFormatter))
                    .toArray(size -> new String[size]);
        });

        loaded = true;
    }

    public void loadFromFile(ZipFile zip) throws Exception {
        loadFromFile(zip, null);
    }

    public FeedLoadResult toSQL (DataSource dataSource) {
        JdbcGTFSFeedConverter converter = new JdbcGTFSFeedConverter(this, dataSource);
        return converter.loadTables();
    }

    public void toFile (String file) {
        try {
            File out = new File(file);
            OutputStream os = new FileOutputStream(out);
            ZipOutputStream zip = new ZipOutputStream(os);

            // write everything
            // TODO: shapes

            // don't write empty feed_info.txt
            if (!this.feedInfo.isEmpty()) new FeedInfo.Writer(this).writeTable(zip);

            new Agency.Writer(this).writeTable(zip);
            new Calendar.Writer(this).writeTable(zip);
            new CalendarDate.Writer(this).writeTable(zip);
            new FareAttribute.Writer(this).writeTable(zip);
            new FareRule.Writer(this).writeTable(zip);
            new Frequency.Writer(this).writeTable(zip);
            new Route.Writer(this).writeTable(zip);
            new Stop.Writer(this).writeTable(zip);
            new ShapePoint.Writer(this).writeTable(zip);
            new Transfer.Writer(this).writeTable(zip);
            new Trip.Writer(this).writeTable(zip);
            new StopTime.Writer(this).writeTable(zip);

            zip.close();

            LOG.info("GTFS file written");
        } catch (Exception e) {
            LOG.error("Error saving GTFS: {}", e.getMessage());
            throw new RuntimeException(e);
        }
    }
//    public void validate (EventBus eventBus, Validator... validators) {
//        if (eventBus == null) {
//
//        }
//        for (Validator validator : validators) {
//            validator.getClass().getSimpleName();
//            validator.validate(this, false);
//        }
//    }
    public void validate (boolean repair, Validator... validators) {
        long startValidation = System.currentTimeMillis();
        for (Validator validator : validators) {
            try {
                long startValidator = System.currentTimeMillis();
//                validator.validate(this, repair);
                long endValidator = System.currentTimeMillis();
                long diff = endValidator - startValidator;
                LOG.info("{} finished in {} milliseconds.", validator.getClass().getSimpleName(), TimeUnit.NANOSECONDS.toMillis(diff));
            } catch (Exception e) {
                LOG.error("Could not run {} validator.", validator.getClass().getSimpleName());
//                LOG.error(e.toString());
                e.printStackTrace();
            }
        }
        long endValidation = System.nanoTime();
        long total = endValidation - startValidation;
        LOG.info("{} validators completed in {} milliseconds.", validators.length, TimeUnit.NANOSECONDS.toMillis(total));
    }

    // validate function call that should explicitly list each validator to run on GTFSFeed
    public void validate () {
/////////////////
    }

    public FeedStats calculateStats() {
        FeedStats feedStats = new FeedStats(this);
        return feedStats;
    }

    /**
     * Static factory method returning a new instance of GTFSFeed containing the contents of
     * the GTFS file at the supplied filesystem path.
     */
    public static GTFSFeed fromFile(String file) {
        return fromFile(file, null);
    }

    /**
     * Static factory method returning a new instance of GTFSFeed containing the contents of
     * the GTFS file at the supplied filesystem path. Forces the feedId to the
     * supplied value if the parameter is non-null.
     */
    public static GTFSFeed fromFile(String file, String feedId) {
        GTFSFeed feed = new GTFSFeed();
        ZipFile zip;
        try {
            zip = new ZipFile(file);
            if (feedId == null) {
                feed.loadFromFile(zip);
            }
            else {
                feed.loadFromFile(zip, feedId);
            }
            zip.close();
            return feed;
        } catch (Exception e) {
            LOG.error("Error loading GTFS: {}", e.getMessage());
            throw new RuntimeException(e);
        }
    }

    public boolean hasFeedInfo () {
        return !this.feedInfo.isEmpty();
    }

    public FeedInfo getFeedInfo () {
        return this.hasFeedInfo() ? this.feedInfo.values().iterator().next() : null;
    }

    /**
     * For the given trip ID, fetch all the stop times in order of increasing stop_sequence.
     * This is an efficient iteration over a tree map.
     */
    public Iterable<StopTime> getOrderedStopTimesForTrip (String trip_id) {
        Map<Fun.Tuple2, StopTime> tripStopTimes =
                stop_times.subMap(
                        Fun.t2(trip_id, null),
                        Fun.t2(trip_id, Fun.HI)
                );
        return tripStopTimes.values();
    }

    /**
     * TODO rename getStopSpatialIndex to make it clear what the index contains.
     */
    public STRtree getSpatialIndex () {
        if (this.spatialIndex == null) {
            synchronized (this) {
                if (this.spatialIndex == null) {
                    // build spatial index
                    STRtree stopIndex = new STRtree();
                    for(Stop stop : this.stops.values()) {
                        try {
                            if (Double.isNaN(stop.stop_lat) || Double.isNaN(stop.stop_lon)) {
                                continue;
                            }
                            Coordinate stopCoord = new Coordinate(stop.stop_lat, stop.stop_lon);
                            stopIndex.insert(new Envelope(stopCoord), stop);
                        } catch (Exception e) {
                            e.printStackTrace();
                        }

                    }
                    try {
                        stopIndex.build();
                        this.spatialIndex = stopIndex;
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        }
        return this.spatialIndex;
    }

    /** Get the shape for the given shape ID */
    public Shape getShape (String shape_id) {
        Shape shape = new Shape(this, shape_id);
        return shape.shape_dist_traveled.length > 0 ? shape : null;
    }

    /**
     * MapDB-based implementation to find patterns.
     *
     * FIXME: Remove and make pattern finding happen during validation? We want to share the pattern finder between the
     * two implementations (MapDB and RDBMS), apply the same validation process to both kinds of storage, and produce
     * Patterns in the same way in both cases, during validation. This prevents us from iterating over every stopTime
     * twice, since we're already iterating over all of them in validation. However, in this case it might not be costly
     * to simply retrieve the stop times from the stop_times map.
     */
    public void findPatterns () {
        PatternFinder patternFinder = new PatternFinder();
        // Iterate over trips and process each trip and its stop times.
        for (Trip trip : this.trips.values()) {
            Iterable<StopTime> orderedStopTimesForTrip = this.getOrderedStopTimesForTrip(trip.trip_id);
            patternFinder.processTrip(trip, orderedStopTimesForTrip);
        }
        Map<TripPatternKey, Pattern> patternObjects = patternFinder.createPatternObjects(this.stops, null);
        this.patterns.putAll(patternObjects.values().stream()
                .collect(Collectors.toMap(Pattern::getId, pattern -> pattern)));
    }

    /**
     * For the given trip ID, fetch all the stop times in order, and interpolate stop-to-stop travel times.
     */
    public Iterable<StopTime> getInterpolatedStopTimesForTrip (String trip_id) throws FirstAndLastStopsDoNotHaveTimes {
        // clone stop times so as not to modify base GTFS structures
        StopTime[] stopTimes = StreamSupport.stream(getOrderedStopTimesForTrip(trip_id).spliterator(), false)
                .map(st -> st.clone())
                .toArray(i -> new StopTime[i]);

        // avoid having to make sure that the array has length below.
        if (stopTimes.length == 0) return Collections.emptyList();

        // first pass: set all partially filled stop times
        for (StopTime st : stopTimes) {
            if (st.arrival_time != Entity.INT_MISSING && st.departure_time == Entity.INT_MISSING) {
                st.departure_time = st.arrival_time;
            }

            if (st.arrival_time == Entity.INT_MISSING && st.departure_time != Entity.INT_MISSING) {
                st.arrival_time = st.departure_time;
            }
        }

        // quick check: ensure that first and last stops have times.
        // technically GTFS requires that both arrival_time and departure_time be filled at both the first and last stop,
        // but we are slightly more lenient and only insist that one of them be filled at both the first and last stop.
        // The meaning of the first stop's arrival time is unclear, and same for the last stop's departure time (except
        // in the case of interlining).

        // it's fine to just check departure time, as the above pass ensures that all stop times have either both
        // arrival and departure times, or neither
        if (stopTimes[0].departure_time == Entity.INT_MISSING || stopTimes[stopTimes.length - 1].departure_time == Entity.INT_MISSING) {
            throw new FirstAndLastStopsDoNotHaveTimes();
        }

        // second pass: fill complete stop times
        int startOfInterpolatedBlock = -1;
        for (int stopTime = 0; stopTime < stopTimes.length; stopTime++) {

            if (stopTimes[stopTime].departure_time == Entity.INT_MISSING && startOfInterpolatedBlock == -1) {
                startOfInterpolatedBlock = stopTime;
            }
            else if (stopTimes[stopTime].departure_time != Entity.INT_MISSING && startOfInterpolatedBlock != -1) {
                // we have found the end of the interpolated section
                int nInterpolatedStops = stopTime - startOfInterpolatedBlock;
                double totalLengthOfInterpolatedSection = 0;
                double[] lengthOfInterpolatedSections = new double[nInterpolatedStops];

                GeodeticCalculator calc = new GeodeticCalculator();

                for (int stopTimeToInterpolate = startOfInterpolatedBlock, i = 0; stopTimeToInterpolate < stopTime; stopTimeToInterpolate++, i++) {
                    Stop start = stops.get(stopTimes[stopTimeToInterpolate - 1].stop_id);
                    Stop end = stops.get(stopTimes[stopTimeToInterpolate].stop_id);
                    calc.setStartingGeographicPoint(start.stop_lon, start.stop_lat);
                    calc.setDestinationGeographicPoint(end.stop_lon, end.stop_lat);
                    double segLen = calc.getOrthodromicDistance();
                    totalLengthOfInterpolatedSection += segLen;
                    lengthOfInterpolatedSections[i] = segLen;
                }

                // add the segment post-last-interpolated-stop
                Stop start = stops.get(stopTimes[stopTime - 1].stop_id);
                Stop end = stops.get(stopTimes[stopTime].stop_id);
                calc.setStartingGeographicPoint(start.stop_lon, start.stop_lat);
                calc.setDestinationGeographicPoint(end.stop_lon, end.stop_lat);
                totalLengthOfInterpolatedSection += calc.getOrthodromicDistance();

                int departureBeforeInterpolation = stopTimes[startOfInterpolatedBlock - 1].departure_time;
                int arrivalAfterInterpolation = stopTimes[stopTime].arrival_time;
                int totalTime = arrivalAfterInterpolation - departureBeforeInterpolation;

                double lengthSoFar = 0;
                for (int stopTimeToInterpolate = startOfInterpolatedBlock, i = 0; stopTimeToInterpolate < stopTime; stopTimeToInterpolate++, i++) {
                    lengthSoFar += lengthOfInterpolatedSections[i];

                    int time = (int) (departureBeforeInterpolation + totalTime * (lengthSoFar / totalLengthOfInterpolatedSection));
                    stopTimes[stopTimeToInterpolate].arrival_time = stopTimes[stopTimeToInterpolate].departure_time = time;
                }

                // we're done with this block
                startOfInterpolatedBlock = -1;
            }
        }

        return Arrays.asList(stopTimes);
    }

    public Collection<Frequency> getFrequencies (String trip_id) {
        // IntelliJ tells me all these casts are unnecessary, and that's also my feeling, but the code won't compile
        // without them
        return (List<Frequency>) frequencies.subSet(new Fun.Tuple2(trip_id, null), new Fun.Tuple2(trip_id, Fun.HI)).stream()
                .map(t2 -> ((Tuple2<String, Frequency>) t2).b)
                .collect(Collectors.toList());
    }

    public List<String> getOrderedStopListForTrip (String trip_id) {
        Iterable<StopTime> orderedStopTimes = getOrderedStopTimesForTrip(trip_id);
        List<String> stops = Lists.newArrayList();
        // In-order traversal of StopTimes within this trip. The 2-tuple keys determine ordering.
        for (StopTime stopTime : orderedStopTimes) {
            stops.add(stopTime.stop_id);
        }
        return stops;
    }

    public LineString getStraightLineForStops(String trip_id) {
        CoordinateList coordinates = new CoordinateList();
        LineString ls = null;
        Trip trip = trips.get(trip_id);

        Iterable<StopTime> stopTimes;
        stopTimes = getOrderedStopTimesForTrip(trip.trip_id);
        if (Iterables.size(stopTimes) > 1) {
            for (StopTime stopTime : stopTimes) {
                Stop stop = stops.get(stopTime.stop_id);
                Double lat = stop.stop_lat;
                Double lon = stop.stop_lon;
                coordinates.add(new Coordinate(lon, lat));
            }
            ls = gf.createLineString(coordinates.toCoordinateArray());
        }
        // set ls equal to null if there is only one stopTime to avoid an exception when creating linestring
        else{
            ls = null;
        }
        return ls;
    }

    /**
     * Returns a trip geometry object (LineString) for a given trip id.
     * If the trip has a shape reference, this will be used for the geometry.
     * Otherwise, the ordered stoptimes will be used.
     *
     * @param   trip_id   trip id of desired trip geometry
     * @return          the LineString representing the trip geometry.
     * @see             LineString
     */
    public LineString getTripGeometry(String trip_id){

        CoordinateList coordinates = new CoordinateList();
        LineString ls = null;
        Trip trip = trips.get(trip_id);

        // If trip has shape_id, use it to generate geometry.
        if (trip.shape_id != null) {
            Shape shape = getShape(trip.shape_id);
            if (shape != null) ls = shape.geometry;
        }

        // Use the ordered stoptimes.
        if (ls == null) {
            ls = getStraightLineForStops(trip_id);
        }

        return ls;
    }

    /** Get the length of a trip in meters. */
    public double getTripDistance (String trip_id, boolean straightLine) {
        return straightLine
                ? GeoUtils.getDistance(this.getStraightLineForStops(trip_id))
                : GeoUtils.getDistance(this.getTripGeometry(trip_id));
    }

    /** Get trip speed (using trip shape if available) in meters per second. */
    public double getTripSpeed (String trip_id) {
        return getTripSpeed(trip_id, false);
    }

    /** Get trip speed in meters per second. */
    public double getTripSpeed (String trip_id, boolean straightLine) {

        StopTime firstStopTime = this.stop_times.ceilingEntry(Fun.t2(trip_id, null)).getValue();
        StopTime lastStopTime = this.stop_times.floorEntry(Fun.t2(trip_id, Fun.HI)).getValue();

        // ensure that stopTime returned matches trip id (i.e., that the trip has stoptimes)
        if (!firstStopTime.trip_id.equals(trip_id) || !lastStopTime.trip_id.equals(trip_id)) {
            return Double.NaN;
        }

        double distance = getTripDistance(trip_id, straightLine);

        // trip time (in seconds)
        int time = lastStopTime.arrival_time - firstStopTime.departure_time;

        return distance / time; // meters per second
    }

    /** Get list of stop_times for a given stop_id. */
    public List<StopTime> getStopTimesForStop (String stop_id) {
        SortedSet<Tuple2<String, Tuple2>> index = this.stopStopTimeSet
                .subSet(new Tuple2<>(stop_id, null), new Tuple2(stop_id, Fun.HI));

        return index.stream()
                .map(tuple -> this.stop_times.get(tuple.b))
                .collect(Collectors.toList());
    }

    public List<Trip> getTripsForService (String service_id) {
        SortedSet<Tuple2<String, String>> index = this.tripsPerService
                .subSet(new Tuple2<>(service_id, null), new Tuple2(service_id, Fun.HI));

        return index.stream()
                .map(tuple -> this.trips.get(tuple.b))
                .collect(Collectors.toList());
    }

    /** Get list of services for each date of service. */
    public List<Service> getServicesForDate (LocalDate date) {
        String dateString = date.format(dateFormatter);
        SortedSet<Tuple2<String, String>> index = this.servicesPerDate
                .subSet(new Tuple2<>(dateString, null), new Tuple2(dateString, Fun.HI));

        return index.stream()
                .map(tuple -> this.services.get(tuple.b))
                .collect(Collectors.toList());
    }

    public List<LocalDate> getDatesOfService () {
        return this.servicesPerDate.stream()
                .map(tuple -> LocalDate.parse(tuple.a, dateFormatter))
                .collect(Collectors.toList());
    }

    /** Get list of distinct trips (filters out multiple visits by a trip) a given stop_id. */
    public List<Trip> getDistinctTripsForStop (String stop_id) {
        return getStopTimesForStop(stop_id).stream()
                .map(stopTime -> this.trips.get(stopTime.trip_id))
                .distinct()
                .collect(Collectors.toList());
    }

    /** Get the likely time zone for a stop using the agency of the first stop time encountered for the stop. */
    public ZoneId getAgencyTimeZoneForStop (String stop_id) {
        StopTime stopTime = getStopTimesForStop(stop_id).iterator().next();

        Trip trip = this.trips.get(stopTime.trip_id);
        Route route = this.routes.get(trip.route_id);
        Agency agency = route.agency_id != null ? this.agency.get(route.agency_id) : this.agency.get(0);

        return ZoneId.of(agency.agency_timezone);
    }

    // TODO: code review
    public Geometry getMergedBuffers() {
        if (this.mergedBuffers == null) {
//            synchronized (this) {
                Collection<Geometry> polygons = new ArrayList<>();
                for (Stop stop : this.stops.values()) {
                    if (getStopTimesForStop(stop.stop_id).isEmpty()) {
                        continue;
                    }
                    if (stop.stop_lat > -1 && stop.stop_lat < 1 || stop.stop_lon > -1 && stop.stop_lon < 1) {
                        continue;
                    }
                    Point stopPoint = gf.createPoint(new Coordinate(stop.stop_lon, stop.stop_lat));
                    Polygon stopBuffer = (Polygon) stopPoint.buffer(.01);
                    polygons.add(stopBuffer);
                }
                Geometry multiGeometry = gf.buildGeometry(polygons);
                this.mergedBuffers = multiGeometry.union();
                if (polygons.size() > 100) {
                    this.mergedBuffers = DouglasPeuckerSimplifier.simplify(this.mergedBuffers, .001);
                }
//            }
        }
        return this.mergedBuffers;
    }

    public Polygon getConvexHull() {
        if (this.convexHull == null) {
            synchronized (this) {
                List<Coordinate> coordinates = this.stops.values().stream().map(
                        stop -> new Coordinate(stop.stop_lon, stop.stop_lat)
                ).collect(Collectors.toList());
                Coordinate[] coords = coordinates.toArray(new Coordinate[coordinates.size()]);
                ConvexHull convexHull = new ConvexHull(coords, gf);
                this.convexHull = (Polygon) convexHull.getConvexHull();
            }
        }
        return this.convexHull;
    }

    /**
     * Cloning can be useful when you want to make only a few modifications to an existing feed.
     * Keep in mind that this is a shallow copy, so you'll have to create new maps in the clone for tables you want
     * to modify.
     */
    @Override
    public GTFSFeed clone() {
        try {
            return (GTFSFeed) super.clone();
        } catch (CloneNotSupportedException e) {
            throw new RuntimeException(e);
        }
    }

    protected void finalize() throws IOException {
        close();
    }

    public void close () {
        db.close();
    }

    /** Thrown when we cannot interpolate stop times because the first or last stops do not have times */
    public class FirstAndLastStopsDoNotHaveTimes extends Exception {
        /** do nothing */
    }

    /** Create a GTFS feed in a temp file */
    public GTFSFeed () {
        // calls to this must be first operation in constructor - why, Java?
        this(DBMaker.newTempFileDB()
                .transactionDisable()
                .mmapFileEnable()
                .asyncWriteEnable()
                .deleteFilesAfterClose()
                .compressionEnable()
                // .cacheSize(1024 * 1024) this bloats memory consumption
                .make()); // TODO db.close();
    }

    /** Create a GTFS feed connected to a particular DB, which will be created if it does not exist. */
    public GTFSFeed (String dbFile) throws IOException, ExecutionException {
        this(constructDB(dbFile)); // TODO db.close();
    }

    private static DB constructDB(String dbFile) {
        DB db;
        try{
            DBMaker dbMaker = DBMaker.newFileDB(new File(dbFile));
            db = dbMaker
                    .transactionDisable()
                    .mmapFileEnable()
                    .asyncWriteEnable()
                    .compressionEnable()
//                     .cacheSize(1024 * 1024) this bloats memory consumption
                    .make();
            return db;
        } catch (ExecutionError | IOError | Exception e) {
            LOG.error("Could not construct db from file.", e);
            return null;
        }
    }

    private GTFSFeed (DB db) {
        this.db = db;

        agency = db.getTreeMap("agency");
        feedInfo = db.getTreeMap("feed_info");
        routes = db.getTreeMap("routes");
        trips = db.getTreeMap("trips");
        stop_times = db.getTreeMap("stop_times");
        frequencies = db.getTreeSet("frequencies");
        transfers = db.getTreeMap("transfers");
        stops = db.getTreeMap("stops");
        fares = db.getTreeMap("fares");
        services = db.getTreeMap("services");
        shape_points = db.getTreeMap("shape_points");

        feedId = db.getAtomicString("feed_id").get();
        checksum = db.getAtomicLong("checksum").get();

        // use Java serialization because MapDB serialization is very slow with JTS as they have a lot of references.
        // nothing else contains JTS objects
        patterns = db.createTreeMap("patterns")
                .valueSerializer(Serializer.JAVA)
                .makeOrGet();

        tripPatternMap = db.getTreeMap("patternForTrip");

        stopCountByStopTime = db.getTreeMap("stopCountByStopTime");
        stopStopTimeSet = db.getTreeSet("stopStopTimeSet");
        tripsPerService = db.getTreeSet("tripsPerService");
        servicesPerDate = db.getTreeSet("servicesPerDate");

        errors = db.getTreeSet("errors");
    }
}


package com.conveyal.gtfs.model;

import com.conveyal.gtfs.GTFSFeed;
import com.conveyal.gtfs.error.DuplicateKeyError;

import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.Map;

public class FareAttribute extends Entity {

    private static final long serialVersionUID = 2157859372072056891L;
    public static final int UNLIMITED_TRANSFERS = Integer.MAX_VALUE;
    public String fare_id;
    public String agency_id;
    public double price;
    public String currency_type;
    public int payment_method;
    public int transfers;
    public int transfer_duration;
    public String feed_id;

    @Override
    public String getId () {
        return fare_id;
    }

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#FARE_ATTRIBUTES}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        statement.setString(oneBasedIndex++, fare_id);
        statement.setDouble(oneBasedIndex++, price);
        statement.setString(oneBasedIndex++, currency_type);
        setIntParameter(statement, oneBasedIndex++, payment_method);
        // FIXME Entity.INT_MISSING causing out of range error on small int
        setIntParameter(statement, oneBasedIndex++, transfers);
        statement.setString(oneBasedIndex++, agency_id);
        setIntParameter(statement, oneBasedIndex++, transfer_duration);
    }

    public static class Loader extends Entity.Loader<FareAttribute> {
        private final Map<String, Fare> fares;

        public Loader(GTFSFeed feed, Map<String, Fare> fares) {
            super(feed, "fare_attributes");
            this.fares = fares;
        }

        @Override
        protected boolean isRequired() {
            return false;
        }

        @Override
        public void loadOneRow() throws IOException {

            /* Calendars and Fares are special: they are stored as joined tables rather than simple maps. */
            String fareId = getStringField("fare_id", true);
            Fare fare = fares.computeIfAbsent(fareId, Fare::new);
            if (fare.fare_attribute != null) {
                feed.errors.add(new DuplicateKeyError(tableName, row, "fare_id"));
            } else {
                FareAttribute fa = new FareAttribute();
                fa.id = row + 1; // offset line number by 1 to account for 0-based row index
                fa.fare_id = fareId;
                fa.price = getDoubleField("price", true, 0, Integer.MAX_VALUE);
                fa.currency_type = getStringField("currency_type", true);
                fa.payment_method = getIntField("payment_method", true, 0, 1);
                fa.transfers = getIntField("transfers", false, 0, 10, UNLIMITED_TRANSFERS); // in the GTFS spec, a missing value means "unlimited", so we default to UNLIMITED_TRANSFERS (or MAX_INT) when no value is found
                fa.agency_id = getStringField("agency_id", false);
                fa.transfer_duration = getIntField("transfer_duration", false, 0, 24 * 60 * 60);
                fa.feed = feed;
                fa.feed_id = feed.feedId;
                fare.fare_attribute = fa;
            }

        }

    }

    public static class Writer extends Entity.Writer<FareAttribute> {
        public Writer(GTFSFeed feed) {
            super(feed, "fare_attributes");
        }

        @Override
        public void writeHeaders() throws IOException {
            writer.writeRecord(new String[] {"fare_id", "agency_id", "price", "currency_type", "payment_method",
                    "transfers", "transfer_duration"});
        }

        @Override
        public void writeOneRow(FareAttribute fa) throws IOException {
            writeStringField(fa.fare_id);
            writeStringField(fa.agency_id);
            writeDoubleField(fa.price);
            writeStringField(fa.currency_type);
            writeIntField(fa.payment_method);
            writeIntField(fa.transfers);
            writeIntField(fa.transfer_duration);
            endRecord();
        }

        @Override
        public Iterator<FareAttribute> iterator() {
            return feed.fares.values().stream()
                    .map(f -> f.fare_attribute)
                    .iterator();
        }
    }

}


package com.conveyal.gtfs;

import com.vividsolutions.jts.geom.*;
import com.vividsolutions.jts.geom.impl.PackedCoordinateSequence;

/**
 * Make some bounding geometries for use in cropping.
 */
public class Geometries {

    public static final GeometryFactory geometryFactory = new GeometryFactory();

    /**
     * Taken from geofabrik.de country boundary KML, because our OSM PBF files use this same boundary.
     */
    public static Geometry getNetherlands() {
        CoordinateSequence netherlandsBoundaryCoordinates = new PackedCoordinateSequence.Double(new double[]{
                        5.920645, 50.748480,
                        5.914113, 50.747530,
                        5.911786, 50.752290,
                        5.900865, 50.748480,
                        5.888615, 50.753490,
                        5.885185, 50.766310,
                        5.864401, 50.760600,
                        5.849655, 50.750500,
                        5.842990, 50.761710,
                        5.831393, 50.755630,
                        5.806889, 50.753150,
                        5.792512, 50.766680,
                        5.782796, 50.764700,
                        5.773869, 50.780210,
                        5.749096, 50.768200,
                        5.740229, 50.754260,
                        5.720635, 50.760300,
                        5.699182, 50.752280,
                        5.681583, 50.754740,
                        5.680362, 50.763910,
                        5.698492, 50.782620,
                        5.688232, 50.795590,
                        5.696357, 50.805360,
                        5.652121, 50.817740,
                        5.635965, 50.846910,
                        5.642047, 50.873480,
                        5.676125, 50.883900,
                        5.693287, 50.899480,
                        5.698111, 50.913440,
                        5.721419, 50.911900,
                        5.725882, 50.927620,
                        5.738869, 50.937550,
                        5.744784, 50.949700,
                        5.756042, 50.953140,
                        5.746100, 50.958630,
                        5.725894, 50.952350,
                        5.716823, 50.961980,
                        5.735358, 50.980980,
                        5.748154, 50.984990,
                        5.764667, 51.000650,
                        5.764155, 51.013410,
                        5.773253, 51.022760,
                        5.757832, 51.028300,
                        5.755501, 51.035840,
                        5.767694, 51.049870,
                        5.772074, 51.064960,
                        5.797835, 51.062840,
                        5.793751, 51.072700,
                        5.801836, 51.079130,
                        5.793118, 51.086550,
                        5.793639, 51.093300,
                        5.806251, 51.099130,
                        5.823016, 51.095330,
                        5.829870, 51.098960,
                        5.828523, 51.103860,
                        5.811779, 51.105140,
                        5.804669, 51.113010,
                        5.819013, 51.129230,
                        5.839765, 51.134040,
                        5.847521, 51.144960,
                        5.834189, 51.151610,
                        5.829746, 51.160620,
                        5.824630, 51.164330,
                        5.816153, 51.156060,
                        5.805567, 51.159410,
                        5.777226, 51.148330,
                        5.771871, 51.153870,
                        5.774877, 51.160270,
                        5.767215, 51.163820,
                        5.772426, 51.177410,
                        5.766246, 51.180770,
                        5.745984, 51.186290,
                        5.740733, 51.181920,
                        5.709280, 51.177420,
                        5.688692, 51.182170,
                        5.657367, 51.181820,
                        5.647189, 51.192040,
                        5.648135, 51.196140,
                        5.559201, 51.219600,
                        5.551906, 51.244010,
                        5.553776, 51.264200,
                        5.527008, 51.279270,
                        5.513870, 51.292470,
                        5.485651, 51.296920,
                        5.465753, 51.281710,
                        5.444162, 51.279340,
                        5.417866, 51.259320,
                        5.349209, 51.272180,
                        5.337141, 51.260140,
                        5.295677, 51.258520,
                        5.263199, 51.263750,
                        5.236915, 51.258520,
                        5.222979, 51.268460,
                        5.238275, 51.303600,
                        5.200519, 51.319290,
                        5.162749, 51.307320,
                        5.131953, 51.313700,
                        5.128835, 51.345240,
                        5.068350, 51.392500,
                        5.100715, 51.431820,
                        5.077345, 51.468360,
                        5.044581, 51.468360,
                        5.035358, 51.477740,
                        5.036552, 51.484140,
                        5.012929, 51.469860,
                        5.013917, 51.454840,
                        5.006537, 51.441650,
                        4.921044, 51.390690,
                        4.911727, 51.392960,
                        4.913135, 51.398490,
                        4.905760, 51.405610,
                        4.888757, 51.412180,
                        4.876084, 51.412170,
                        4.867381, 51.406110,
                        4.839025, 51.411590,
                        4.789316, 51.405990,
                        4.769536, 51.412410,
                        4.763576, 51.430300,
                        4.766441, 51.433650,
                        4.787077, 51.435690,
                        4.824689, 51.426100,
                        4.820426, 51.449230,
                        4.833136, 51.460930,
                        4.837352, 51.478910,
                        4.819916, 51.480330,
                        4.814349, 51.491900,
                        4.782196, 51.496690,
                        4.778101, 51.501420,
                        4.754141, 51.497100,
                        4.748593, 51.487300,
                        4.731148, 51.481530,
                        4.718609, 51.466400,
                        4.706865, 51.464440,
                        4.694952, 51.449370,
                        4.670006, 51.442420,
                        4.671392, 51.423890,
                        4.641470, 51.419020,
                        4.574917, 51.429810,
                        4.534364, 51.420220,
                        4.526970, 51.450180,
                        4.544128, 51.472940,
                        4.537175, 51.479270,
                        4.476857, 51.475260,
                        4.465859, 51.468230,
                        4.443222, 51.465730,
                        4.393420, 51.449040,
                        4.401744, 51.434580,
                        4.397519, 51.425000,
                        4.387858, 51.419980,
                        4.393899, 51.410180,
                        4.434043, 51.376910,
                        4.432594, 51.361120,
                        4.422241, 51.362140,
                        4.384498, 51.351350,
                        4.340371, 51.354730,
                        4.332372, 51.374530,
                        4.222024, 51.371040,
                        4.244665, 51.352500,
                        4.167273, 51.290120,
                        4.062695, 51.241690,
                        4.040278, 51.238330,
                        4.014192, 51.241800,
                        3.987056, 51.230400,
                        3.980126, 51.222620,
                        3.966785, 51.221280,
                        3.959444, 51.212940,
                        3.935476, 51.208930,
                        3.928753, 51.215900,
                        3.921538, 51.214080,
                        3.919711, 51.204970,
                        3.886517, 51.197170,
                        3.874627, 51.207940,
                        3.887993, 51.219510,
                        3.859817, 51.207840,
                        3.837323, 51.210640,
                        3.824674, 51.205960,
                        3.789446, 51.211540,
                        3.785459, 51.226510,
                        3.790882, 51.254070,
                        3.775507, 51.260170,
                        3.768807, 51.256900,
                        3.753589, 51.266830,
                        3.693150, 51.273140,
                        3.657541, 51.287200,
                        3.640011, 51.285180,
                        3.589920, 51.301560,
                        3.586048, 51.298150,
                        3.590697, 51.291810,
                        3.582435, 51.283980,
                        3.562355, 51.292800,
                        3.556428, 51.286890,
                        3.545540, 51.288170,
                        3.538944, 51.280780,
                        3.519408, 51.284680,
                        3.528826, 51.243300,
                        3.448568, 51.238510,
                        3.426458, 51.241840,
                        3.420358, 51.253670,
                        3.406010, 51.254300,
                        3.373119, 51.275680,
                        3.377565, 51.285800,
                        3.363010, 51.298030,
                        3.370756, 51.302930,
                        3.355429, 51.315530,
                        3.381920, 51.335370,
                        3.380865, 51.339460,
                        3.371422, 51.347100,
                        3.371675, 51.358150,
                        3.350258, 51.376730,
                        3.306017, 51.431030,
                        3.078404, 51.551970,
                        2.992192, 51.633610,
                        4.362889, 53.305060,
                        5.530069, 54.017860,
                        6.346609, 53.728270,
                        6.414744, 53.697610,
                        6.594984, 53.595190,
                        6.821311, 53.496490,
                        6.858844, 53.471530,
                        6.891564, 53.468070,
                        6.917659, 53.459810,
                        6.934755, 53.439730,
                        6.954578, 53.395700,
                        6.995954, 53.350730,
                        6.997671, 53.334020,
                        7.006850, 53.329430,
                        7.061512, 53.327040,
                        7.111054, 53.334810,
                        7.189534, 53.334480,
                        7.220022, 53.214900,
                        7.220549, 53.197160,
                        7.212982, 53.189670,
                        7.230455, 53.180790,
                        7.205009, 53.173520,
                        7.194018, 53.160900,
                        7.192205, 53.146470,
                        7.182299, 53.135820,
                        7.188436, 53.122690,
                        7.205696, 53.114170,
                        7.202251, 53.081510,
                        7.215711, 53.012480,
                        7.220429, 53.006720,
                        7.211986, 52.998620,
                        7.183969, 52.939860,
                        7.090966, 52.849050,
                        7.095411, 52.836830,
                        7.074470, 52.809300,
                        7.057560, 52.641040,
                        7.039568, 52.629710,
                        6.975989, 52.643210,
                        6.953745, 52.636000,
                        6.918933, 52.636860,
                        6.912264, 52.644010,
                        6.872838, 52.650240,
                        6.821407, 52.644690,
                        6.784084, 52.649540,
                        6.743337, 52.642560,
                        6.717383, 52.627770,
                        6.730256, 52.615370,
                        6.722265, 52.590200,
                        6.768056, 52.566190,
                        6.769131, 52.559890,
                        6.752295, 52.556040,
                        6.727875, 52.559800,
                        6.716428, 52.545530,
                        6.687630, 52.549380,
                        6.708209, 52.521860,
                        6.700986, 52.488160,
                        6.753652, 52.466990,
                        6.774899, 52.462600,
                        6.854735, 52.462670,
                        6.856230, 52.453520,
                        6.941407, 52.438530,
                        6.959788, 52.446690,
                        6.975400, 52.467790,
                        6.988624, 52.472590,
                        6.996353, 52.467220,
                        7.013247, 52.431210,
                        7.024209, 52.425000,
                        7.037172, 52.405650,
                        7.060917, 52.401460,
                        7.075066, 52.373840,
                        7.074867, 52.350400,
                        7.059279, 52.336640,
                        7.050042, 52.322630,
                        7.050009, 52.313690,
                        7.029410, 52.290960,
                        7.031167, 52.275610,
                        7.044544, 52.258330,
                        7.068264, 52.242930,
                        7.063317, 52.232510,
                        7.026832, 52.222750,
                        6.989159, 52.223250,
                        6.953173, 52.178760,
                        6.910407, 52.173560,
                        6.907694, 52.167660,
                        6.884791, 52.155000,
                        6.875589, 52.128170,
                        6.856423, 52.117620,
                        6.763181, 52.115770,
                        6.758851, 52.095340,
                        6.748639, 52.091730,
                        6.754072, 52.085770,
                        6.750768, 52.080660,
                        6.736213, 52.071730,
                        6.699834, 52.067700,
                        6.690838, 52.053740,
                        6.690935, 52.042880,
                        6.714943, 52.042950,
                        6.753837, 52.031280,
                        6.814332, 51.997460,
                        6.828034, 51.996170,
                        6.835752, 51.973060,
                        6.830667, 51.962000,
                        6.801478, 51.956260,
                        6.799999, 51.941300,
                        6.790685, 51.927210,
                        6.771269, 51.913460,
                        6.722739, 51.893160,
                        6.693687, 51.911890,
                        6.681975, 51.914160,
                        6.634806, 51.898070,
                        6.586462, 51.891150,
                        6.561388, 51.879320,
                        6.544918, 51.881310,
                        6.505764, 51.865820,
                        6.501361, 51.859470,
                        6.472824, 51.850840,
                        6.449746, 51.862380,
                        6.431991, 51.856420,
                        6.425879, 51.862940,
                        6.392888, 51.869980,
                        6.390379, 51.864800,
                        6.411849, 51.854130,
                        6.405673, 51.843820,
                        6.411140, 51.835980,
                        6.407495, 51.825030,
                        6.380975, 51.831870,
                        6.366882, 51.830600,
                        6.360515, 51.833680,
                        6.358099, 51.843710,
                        6.346470, 51.847750,
                        6.305316, 51.846210,
                        6.295315, 51.865260,
                        6.279113, 51.871180,
                        6.262056, 51.865150,
                        6.209305, 51.866150,
                        6.181319, 51.880520,
                        6.179351, 51.887170,
                        6.185248, 51.889760,
                        6.156526, 51.902040,
                        6.116557, 51.895290,
                        6.138796, 51.888300,
                        6.147601, 51.871940,
                        6.169398, 51.863420,
                        6.167472, 51.837860,
                        6.100394, 51.846090,
                        6.064946, 51.861680,
                        6.057291, 51.850110,
                        6.036744, 51.839840,
                        6.028512, 51.841750,
                        5.994722, 51.827930,
                        5.963244, 51.833400,
                        5.950530, 51.824140,
                        5.960760, 51.818410,
                        5.957302, 51.812950,
                        5.981246, 51.799710,
                        5.978057, 51.788780,
                        5.992548, 51.785320,
                        5.987021, 51.775330,
                        5.995037, 51.770670,
                        5.991943, 51.763650,
                        5.956302, 51.748090,
                        5.956064, 51.742220,
                        5.994732, 51.741260,
                        6.046624, 51.719400,
                        6.043521, 51.710750,
                        6.029834, 51.708290,
                        6.034777, 51.692680,
                        6.033234, 51.679120,
                        6.037929, 51.675530,
                        6.119859, 51.658390,
                        6.120054, 51.649660,
                        6.110479, 51.640020,
                        6.095011, 51.607500,
                        6.123231, 51.595190,
                        6.132643, 51.583300,
                        6.158656, 51.569100,
                        6.178861, 51.540980,
                        6.201415, 51.529990,
                        6.214704, 51.514740,
                        6.215800, 51.492330,
                        6.226676, 51.468740,
                        6.209491, 51.403570,
                        6.229287, 51.401730,
                        6.218449, 51.386690,
                        6.229390, 51.360290,
                        6.194644, 51.338740,
                        6.195852, 51.332350,
                        6.172269, 51.330150,
                        6.156020, 51.305400,
                        6.131365, 51.283730,
                        6.126709, 51.272700,
                        6.077269, 51.241140,
                        6.088301, 51.220720,
                        6.071111, 51.218870,
                        6.076000, 51.184150,
                        6.083935, 51.174430,
                        6.105466, 51.175170,
                        6.165647, 51.197330,
                        6.182602, 51.188700,
                        6.183035, 51.184450,
                        6.147734, 51.172980,
                        6.177047, 51.160980,
                        6.177587, 51.156390,
                        6.164554, 51.145950,
                        6.126782, 51.141710,
                        6.093797, 51.131760,
                        6.088709, 51.122140,
                        6.061897, 51.113350,
                        6.038192, 51.094000,
                        5.999214, 51.081580,
                        5.973847, 51.059610,
                        5.971765, 51.045220,
                        5.958617, 51.031850,
                        5.936460, 51.032630,
                        5.913405, 51.063380,
                        5.893022, 51.050310,
                        5.869790, 51.048620,
                        5.880831, 51.038650,
                        5.878007, 51.028400,
                        5.881900, 51.019230,
                        5.897435, 51.013490,
                        5.898569, 51.006870,
                        5.908300, 51.003290,
                        5.906599, 50.986330,
                        5.899490, 50.977780,
                        5.955096, 50.991420,
                        5.967899, 50.982380,
                        5.981279, 50.986220,
                        6.027451, 50.986120,
                        6.029409, 50.979820,
                        6.012523, 50.957300,
                        6.019548, 50.953960,
                        6.020674, 50.937290,
                        6.051754, 50.932770,
                        6.058370, 50.924710,
                        6.097096, 50.921280,
                        6.080364, 50.898360,
                        6.079201, 50.890810,
                        6.089250, 50.881000,
                        6.091101, 50.872330,
                        6.079903, 50.858810,
                        6.076394, 50.844520,
                        6.055450, 50.848330,
                        6.053880, 50.853070,
                        6.021010, 50.843360,
                        6.019818, 50.833840,
                        6.028167, 50.828600,
                        6.027036, 50.811900,
                        6.004115, 50.798500,
                        5.985312, 50.807300,
                        5.978653, 50.801100,
                        6.029923, 50.776130,
                        6.021710, 50.762600,
                        6.023293, 50.752340,
                        5.982748, 50.749850,
                        5.958986, 50.759260,
                        5.920645, 50.748480}, 2);
        Polygon boundary = geometryFactory.createPolygon(netherlandsBoundaryCoordinates);
        return boundary;
    }

    /**
     * The Dutch island of Texel. The Netherlands data set includes a crazy all-to-all representation of Texel's
     * on-demand transit service.
     */
    public static Geometry getTexel() {
        return geometryFactory.toGeometry(new Envelope(4.702663, 4.933548, 52.98069, 53.192047));
    }

    public static Geometry getNetherlandsWithoutTexel() {
        return getNetherlands().difference(getTexel());
    }

    public static Geometry getZuidHolland() {
        return geometryFactory.toGeometry(new Envelope(4.002074, 4.98848, 51.696796, 52.278661));
    }

}


package com.conveyal.gtfs;

import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.NewGTFSErrorType;
import com.conveyal.gtfs.error.SQLErrorStorage;
import com.conveyal.gtfs.model.Pattern;
import com.conveyal.gtfs.model.PatternStop;
import com.conveyal.gtfs.model.ShapePoint;
import com.conveyal.gtfs.model.Stop;
import com.conveyal.gtfs.model.StopTime;
import com.conveyal.gtfs.model.Trip;
import com.conveyal.gtfs.validator.service.GeoUtils;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.CoordinateList;
import com.vividsolutions.jts.geom.LineString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static com.conveyal.gtfs.util.Util.human;

/**
 * This abstracts out the logic for finding stop sequences ("journey patterns" in Transmodel parlance) based on trips.
 * Placing this logic in a separate class allows us to use it on GTFS data from multiple sources.
 * Our two specific use cases are finding patterns in stop_times that have already been loaded into an RDBMS, and
 * finding patterns while loading Java objects directly into a MapDB database.
 *
 * Created by abyrd on 2017-10-08
 */
public class PatternFinder {

    private static final Logger LOG = LoggerFactory.getLogger(PatternFinder.class);

    // A multi-map that groups trips together by their sequence of stops
    private Multimap<TripPatternKey, Trip> tripsForPattern = HashMultimap.create();

    private int nTripsProcessed = 0;

    /**
     * Bin all trips by the sequence of stops they visit.
     * @return A map from a list of stop IDs to a list of Trip IDs that visit those stops in that sequence.
     */
//    public void findPatterns(Feed feed) {
//
//        for (Trip trip : trips) {
//        }
//        feed.patterns.stream().forEach(p -> {
//            feed.patterns.put(p.pattern_id, p);
//            p.associatedTrips.stream().forEach(t -> feed.tripPatternMap.put(t, p.pattern_id));
//        });
//
//    }

    public void processTrip(Trip trip, Iterable<StopTime> orderedStopTimes) {
        if (++nTripsProcessed % 100000 == 0) {
            LOG.info("trip {}", human(nTripsProcessed));
        }
        // No need to scope the route ID here, patterns are built within the context of a single feed.
        // Create a key that might already be in the map (by semantic equality)
        TripPatternKey key = new TripPatternKey(trip.route_id);
        for (StopTime st : orderedStopTimes) {
            key.addStopTime(st);
        }
        // Add the current trip to the map, possibly extending an existing list of trips on this pattern.
        tripsForPattern.put(key, trip);
    }

    /**
     * Once all trips have been processed, call this method to produce the final Pattern objects representing all the
     * unique sequences of stops encountered. Returns map of patterns to their keys so that downstream functions can
     * make use of trip pattern keys for constructing pattern stops or other derivative objects.
     */
    public Map<TripPatternKey, Pattern> createPatternObjects(Map<String, Stop> stopById, SQLErrorStorage errorStorage) {
        // Make pattern ID one-based to avoid any JS type confusion between an ID of zero vs. null value.
        int nextPatternId = 1;
        // Create an in-memory list of Patterns because we will later rename them before inserting them into storage.
        Map<TripPatternKey, Pattern> patterns = new HashMap<>();
        // TODO assign patterns sequential small integer IDs (may include route)
        for (TripPatternKey key : tripsForPattern.keySet()) {
            Collection<Trip> trips = tripsForPattern.get(key);
            Pattern pattern = new Pattern(key.stops, trips, null);
            // Overwrite long UUID with sequential integer pattern ID
            pattern.pattern_id = Integer.toString(nextPatternId++);
            // FIXME: Should associated shapes be a single entry?
            pattern.associatedShapes = new HashSet<>();
            trips.stream().forEach(trip -> pattern.associatedShapes.add(trip.shape_id));
            if (pattern.associatedShapes.size() > 1 && errorStorage != null) {
                // Store an error if there is more than one shape per pattern. Note: error storage is null if called via
                // MapDB implementation.
                // TODO: Should shape ID be added to trip pattern key?
                errorStorage.storeError(NewGTFSError.forEntity(
                        pattern,
                        NewGTFSErrorType.MULTIPLE_SHAPES_FOR_PATTERN)
                            .setBadValue(pattern.associatedShapes.toString()));
            }
            patterns.put(key, pattern);
        }
        // Name patterns before storing in SQL database.
        renamePatterns(patterns.values(), stopById);
        LOG.info("Total patterns: {}", tripsForPattern.keySet().size());
        return patterns;
    }

    /**
     * Destructively rename the supplied collection of patterns.
     * This process requires access to all the stops in the feed.
     * Some validators already cache a map of all the stops. There's probably a cleaner way to do this.
     */
    public static void renamePatterns(Collection<Pattern> patterns, Map<String, Stop> stopById) {
        LOG.info("Generating unique names for patterns");

        Map<String, PatternNamingInfo> namingInfoForRoute = new HashMap<>();

        for (Pattern pattern : patterns) {
            if (pattern.associatedTrips.isEmpty() || pattern.orderedStops.isEmpty()) continue;

            // Each pattern within a route has a unique name (within that route, not across the entire feed)

            PatternNamingInfo namingInfo = namingInfoForRoute.get(pattern.route_id);
            if (namingInfo == null) {
                namingInfo = new PatternNamingInfo();
                namingInfoForRoute.put(pattern.route_id, namingInfo);
            }

            // Pattern names are built using stop names rather than stop IDs.
            // Stop names, unlike IDs, are not guaranteed to be unique.
            // Therefore we must track used names carefully to avoid duplicates.

            String fromName = stopById.get(pattern.orderedStops.get(0)).stop_name;
            String toName = stopById.get(pattern.orderedStops.get(pattern.orderedStops.size() - 1)).stop_name;

            namingInfo.fromStops.put(fromName, pattern);
            namingInfo.toStops.put(toName, pattern);

            for (String stopId : pattern.orderedStops) {
                Stop stop = stopById.get(stopId);
                if (fromName.equals(stop.stop_name) || toName.equals(stop.stop_name)) continue;
                namingInfo.vias.put(stop.stop_name, pattern);
            }
            namingInfo.patternsOnRoute.add(pattern);
        }

        // name the patterns on each route
        for (PatternNamingInfo info : namingInfoForRoute.values()) {
            for (Pattern pattern : info.patternsOnRoute) {
                pattern.name = null; // clear this now so we don't get confused later on
                String fromName = stopById.get(pattern.orderedStops.get(0)).stop_name;
                String toName = stopById.get(pattern.orderedStops.get(pattern.orderedStops.size() - 1)).stop_name;

                // check if combination from, to is unique
                Set<Pattern> intersection = new HashSet<>(info.fromStops.get(fromName));
                intersection.retainAll(info.toStops.get(toName));

                if (intersection.size() == 1) {
                    pattern.name = String.format(Locale.US, "from %s to %s", fromName, toName);
                    continue;
                }

                // check for unique via stop
                pattern.orderedStops.stream().map(stopById::get).forEach(stop -> {
                    Set<Pattern> viaIntersection = new HashSet<>(intersection);
                    viaIntersection.retainAll(info.vias.get(stop.stop_name));

                    if (viaIntersection.size() == 1) {
                        pattern.name = String.format(Locale.US, "from %s to %s via %s", fromName, toName, stop.stop_name);
                    }
                });

                if (pattern.name == null) {
                    // no unique via, one pattern is subset of other.
                    if (intersection.size() == 2) {
                        Iterator<Pattern> it = intersection.iterator();
                        Pattern p0 = it.next();
                        Pattern p1 = it.next();
                        if (p0.orderedStops.size() > p1.orderedStops.size()) {
                            p1.name = String.format(Locale.US, "from %s to %s express", fromName, toName);
                            p0.name = String.format(Locale.US, "from %s to %s local", fromName, toName);
                        } else if (p1.orderedStops.size() > p0.orderedStops.size()){
                            p0.name = String.format(Locale.US, "from %s to %s express", fromName, toName);
                            p1.name = String.format(Locale.US, "from %s to %s local", fromName, toName);
                        }
                    }
                }

                if (pattern.name == null) {
                    // give up
                    pattern.name = String.format(Locale.US, "from %s to %s like trip %s", fromName, toName, pattern.associatedTrips.get(0));
                }
            }

            // attach a stop and trip count to each
            for (Pattern pattern : info.patternsOnRoute) {
                pattern.name = String.format(Locale.US, "%s stops %s (%s trips)",
                        pattern.orderedStops.size(), pattern.name, pattern.associatedTrips.size());
            }
        }
    }

    /**
     * Holds information about all pattern names on a particular route,
     * modeled on https://github.com/opentripplanner/OpenTripPlanner/blob/master/src/main/java/org/opentripplanner/routing/edgetype/TripPattern.java#L379
     */
    private static class PatternNamingInfo {
        // These are all maps from ?
        // FIXME For type safety and clarity maybe we should have a parameterized ID type, i.e. EntityId<Stop> stopId.
        Multimap<String, Pattern> fromStops = HashMultimap.create();
        Multimap<String, Pattern> toStops = HashMultimap.create();
        Multimap<String, Pattern> vias = HashMultimap.create();
        List<Pattern> patternsOnRoute = new ArrayList<>();
    }

}


package com.conveyal.gtfs.model;

import com.conveyal.gtfs.GTFSFeed;

import java.io.IOException;
import java.net.URL;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Iterator;

public class Stop extends Entity {

    private static final long serialVersionUID = 464065335273514677L;
    public String stop_id;
    public String stop_code;
    public String stop_name;
    public String stop_desc;
    public double stop_lat;
    public double stop_lon;
    public String zone_id;
    public URL    stop_url;
    public int    location_type;
    public String parent_station;
    public String stop_timezone;
    // TODO should be int
    public String wheelchair_boarding;
    public String feed_id;

    @Override
    public String getId () {
        return stop_id;
    }

    /**
     * Sets the parameters for a prepared statement following the parameter order defined in
     * {@link com.conveyal.gtfs.loader.Table#STOPS}. JDBC prepared statement parameters use a one-based index.
     */
    @Override
    public void setStatementParameters(PreparedStatement statement, boolean setDefaultId) throws SQLException {
        int oneBasedIndex = 1;
        if (!setDefaultId) statement.setInt(oneBasedIndex++, id);
        int wheelchairBoarding = 0;
        try {
             wheelchairBoarding = Integer.parseInt(wheelchair_boarding);
        } catch (NumberFormatException e) {
            // Do nothing, wheelchairBoarding will remain zero.
        }
        statement.setString(oneBasedIndex++, stop_id);
        statement.setString(oneBasedIndex++, stop_code);
        statement.setString(oneBasedIndex++, stop_name);
        statement.setString(oneBasedIndex++, stop_desc);
        statement.setDouble(oneBasedIndex++, stop_lat);
        statement.setDouble(oneBasedIndex++, stop_lon);
        statement.setString(oneBasedIndex++, zone_id);
        statement.setString(oneBasedIndex++, stop_url != null ? stop_url.toString() : null);
        setIntParameter(statement, oneBasedIndex++, location_type);
        statement.setString(oneBasedIndex++, parent_station);
        statement.setString(oneBasedIndex++, stop_timezone);
        // FIXME: For some reason wheelchair boarding type is String
        setIntParameter(statement, oneBasedIndex++, wheelchairBoarding);
    }

    public static class Loader extends Entity.Loader<Stop> {

        public Loader(GTFSFeed feed) {
            super(feed, "stops");
        }

        @Override
        protected boolean isRequired() {
            return true;
        }

        @Override
        public void loadOneRow() throws IOException {
            Stop s = new Stop();
            s.id = row + 1; // offset line number by 1 to account for 0-based row index
            s.stop_id   = getStringField("stop_id", true);
            s.stop_code = getStringField("stop_code", false);
            s.stop_name = getStringField("stop_name", true);
            s.stop_desc = getStringField("stop_desc", false);
            s.stop_lat  = getDoubleField("stop_lat", true, -90D, 90D);
            s.stop_lon  = getDoubleField("stop_lon", true, -180D, 180D);
            s.zone_id   = getStringField("zone_id", false);
            s.stop_url  = getUrlField("stop_url", false);
            s.location_type  = getIntField("location_type", false, 0, 1);
            s.parent_station = getStringField("parent_station", false);
            s.stop_timezone  = getStringField("stop_timezone", false);
            s.wheelchair_boarding = getStringField("wheelchair_boarding", false);
            s.feed = feed;
            s.feed_id = feed.feedId;
            /* TODO check ref integrity later, this table self-references via parent_station */

            feed.stops.put(s.stop_id, s);
        }

    }

    public static class Writer extends Entity.Writer<Stop> {
        public Writer (GTFSFeed feed) {
            super(feed, "stops");
        }

        @Override
        public void writeHeaders() throws IOException {
            writer.writeRecord(new String[] {"stop_id", "stop_code", "stop_name", "stop_desc", "stop_lat", "stop_lon", "zone_id",
                    "stop_url", "location_type", "parent_station", "stop_timezone", "wheelchair_boarding"});
        }

        @Override
        public void writeOneRow(Stop s) throws IOException {
            writeStringField(s.stop_id);
            writeStringField(s.stop_code);
            writeStringField(s.stop_name);
            writeStringField(s.stop_desc);
            writeDoubleField(s.stop_lat);
            writeDoubleField(s.stop_lon);
            writeStringField(s.zone_id);
            writeUrlField(s.stop_url);
            writeIntField(s.location_type);
            writeStringField(s.parent_station);
            writeStringField(s.stop_timezone);
            writeStringField(s.wheelchair_boarding);
            endRecord();
        }

        @Override
        public Iterator<Stop> iterator() {
            return feed.stops.values().iterator();
        }
    }
}


package com.conveyal.gtfs.loader;

import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.NewGTFSErrorType;
import com.conveyal.gtfs.storage.StorageException;

import java.sql.JDBCType;
import java.sql.PreparedStatement;
import java.sql.SQLType;
import java.util.Set;

import static com.conveyal.gtfs.loader.Field.cleanString;

/**
 * Represents a six-digit hex color code. Should be parsed as hex text and less than OxFFFFFF.
 *
 * Created by abyrd on 2017-03-30
 */
public class ColorField extends Field {

    public ColorField(String name, Requirement requirement) {
        super(name, requirement);
    }

    /** Check that a string can be properly parsed and is in range. */
    public ValidateFieldResult<String> validateAndConvert (String string) {
        ValidateFieldResult<String> result = new ValidateFieldResult<>(string);
        try {
            if (string.length() != 6) {
                result.errors.add(NewGTFSError.forFeed(NewGTFSErrorType.COLOR_FORMAT, string));
            }
            int integer = Integer.parseInt(string, 16);
            return result; // Could also store the integer.
        } catch (Exception ex) {
            throw new StorageException(NewGTFSErrorType.COLOR_FORMAT, string);
        }
    }

    public Set<NewGTFSError> setParameter(PreparedStatement preparedStatement, int oneBasedIndex, String string) {
        try {
            ValidateFieldResult<String> result = validateAndConvert(string);
            preparedStatement.setString(oneBasedIndex, result.clean);
            return result.errors;
        } catch (Exception ex) {
            throw new StorageException(ex);
        }
    }

    /**
     * From the Postgres manual, on character types:
     * There is no performance difference among these three types, apart from increased storage space when using
     * the blank-padded type, and a few extra CPU cycles to check the length when storing into a length-constrained
     * column. While character(n) has performance advantages in some other database systems, there is no such advantage
     * in PostgreSQL; in fact character(n) is usually the slowest of the three because of its additional storage costs.
     *
     * Arguably colors are integers expressed in hex though.
     */
    @Override
    public SQLType getSqlType() {
        return JDBCType.VARCHAR;
    }

}


package com.conveyal.gtfs.validator;

import com.conveyal.gtfs.PatternFinder;
import com.conveyal.gtfs.TripPatternKey;
import com.conveyal.gtfs.error.SQLErrorStorage;
import com.conveyal.gtfs.loader.BatchTracker;
import com.conveyal.gtfs.loader.Feed;
import com.conveyal.gtfs.loader.Requirement;
import com.conveyal.gtfs.loader.Table;
import com.conveyal.gtfs.model.Pattern;
import com.conveyal.gtfs.model.PatternStop;
import com.conveyal.gtfs.model.Route;
import com.conveyal.gtfs.model.Stop;
import com.conveyal.gtfs.model.StopTime;
import com.conveyal.gtfs.model.Trip;
import org.apache.commons.dbutils.DbUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.conveyal.gtfs.loader.JdbcGtfsLoader.copyFromFile;
import static com.conveyal.gtfs.model.Entity.INT_MISSING;
import static com.conveyal.gtfs.model.Entity.setDoubleParameter;
import static com.conveyal.gtfs.model.Entity.setIntParameter;

/**
 * Groups trips together into "patterns" that share the same sequence of stops.
 * This is not a normal validator in the sense that it does not check for bad data.
 * It's taking advantage of the fact that we're already iterating over the trips one by one to build up the patterns.
 */
public class PatternFinderValidator extends TripValidator {

    private static final Logger LOG = LoggerFactory.getLogger(PatternFinderValidator.class);

    PatternFinder patternFinder;
    private File tempPatternForTripsTextFile;
    private PrintStream patternForTripsFileStream;
    private String tempPatternForTripsTable;

    public PatternFinderValidator(Feed feed, SQLErrorStorage errorStorage) {
        super(feed, errorStorage);
        patternFinder = new PatternFinder();
    }

    @Override
    public void validateTrip (Trip trip, Route route, List<StopTime> stopTimes, List<Stop> stops) {
        // As we hit each trip, accumulate them into the wrapped PatternFinder object.
        patternFinder.processTrip(trip, stopTimes);
    }

    /**
     * Store patterns and pattern stops in the database. Also, update the trips table with a pattern_id column.
     */
    @Override
    public void complete(ValidationResult validationResult) {
        LOG.info("Finding patterns...");
        // FIXME: There may be a better way to handle getting the full list of stops
        Map<String, Stop> stopById = new HashMap<>();
        for (Stop stop : feed.stops) {
            stopById.put(stop.stop_id, stop);
        }
        // FIXME In the editor we need patterns to exist separately from and before trips themselves, so me make another table.
        Map<TripPatternKey, Pattern> patterns = patternFinder.createPatternObjects(stopById, errorStorage);
        Connection connection = null;
        try {
            // TODO this assumes gtfs-lib is using an SQL database and not a MapDB.
            //   Maybe we should just create patterns in a separate step, but that would mean iterating over the
            //   stop_times twice.
            LOG.info("Creating pattern and pattern stops tables.");
            connection = feed.getConnection();
            Statement statement = connection.createStatement();
            String tripsTableName = feed.tablePrefix + "trips";
            String patternsTableName = feed.tablePrefix + "patterns";
            String patternStopsTableName = feed.tablePrefix + "pattern_stops";
            statement.execute(String.format("alter table %s add column pattern_id varchar", tripsTableName));
            // FIXME: Here we're creating a pattern table that has an integer ID field (similar to the other GTFS tables)
            //   AND a varchar pattern_id with essentially the same value cast to a string. Perhaps the pattern ID should
            //   be a UUID or something, just to better distinguish it from the int ID?
            Table patternsTable = new Table(patternsTableName, Pattern.class, Requirement.EDITOR, Table.PATTERNS.fields);
            Table patternStopsTable = new Table(patternStopsTableName, PatternStop.class, Requirement.EDITOR,
                    Table.PATTERN_STOP.fields);
            // Create pattern and pattern stops table, each with serial ID fields.
            patternsTable.createSqlTable(connection, null, true);
            patternStopsTable.createSqlTable(connection, null, true);
            // Generate prepared statements for inserts.
            String insertPatternSql = patternsTable.generateInsertSql(true);
            String insertPatternStopSql = patternStopsTable.generateInsertSql(true);
            PreparedStatement insertPatternStatement = connection.prepareStatement(insertPatternSql);
            BatchTracker patternTracker = new BatchTracker("pattern", insertPatternStatement);
            PreparedStatement insertPatternStopStatement = connection.prepareStatement(insertPatternStopSql);
            BatchTracker patternStopTracker = new BatchTracker("pattern stop", insertPatternStopStatement);
            int currentPatternIndex = 0;
            LOG.info("Storing patterns and pattern stops");
            // If using Postgres, load pattern to trips mapping into temp table for quick updating.
            boolean postgresText = (connection.getMetaData().getDatabaseProductName().equals("PostgreSQL"));
            if (postgresText) {
                // NOTE: temp table name must NOT be prefixed with schema because temp tables are prefixed with their own
                // connection-unique schema.
                tempPatternForTripsTable = "pattern_for_trips";
                tempPatternForTripsTextFile = File.createTempFile(tempPatternForTripsTable, "text");
                LOG.info("Loading via temporary text file at {}", tempPatternForTripsTextFile.getAbsolutePath());
                // Create temp table for updating trips with pattern IDs to be dropped at the end of the transaction.
                String createTempSql = String.format("create temp table %s(trip_id varchar, pattern_id varchar) on commit drop", tempPatternForTripsTable);
                LOG.info(createTempSql);
                statement.execute(createTempSql);
                patternForTripsFileStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(tempPatternForTripsTextFile)));
            }
            for (Map.Entry<TripPatternKey, Pattern> entry : patterns.entrySet()) {
                Pattern pattern = entry.getValue();
                LOG.debug("Batching pattern {}", pattern.pattern_id);
                TripPatternKey key = entry.getKey();
                pattern.setStatementParameters(insertPatternStatement, true);
                patternTracker.addBatch();
                // Construct pattern stops based on values in trip pattern key.
                // FIXME: Use pattern stops table here?
                int lastValidDeparture = key.departureTimes.get(0);
                for (int i = 0; i < key.stops.size(); i++) {
                    int travelTime = 0;
                    String stopId = key.stops.get(i);
                    int arrival = key.arrivalTimes.get(i);
                    if (i > 0) {
                        int prevDeparture = key.departureTimes.get(i - 1);
                        // Set travel time for all stops except the first.
                        if (prevDeparture != INT_MISSING) {
                            // Update the previous departure if it's not missing. Otherwise, base travel time based on the
                            // most recent valid departure.
                            lastValidDeparture = prevDeparture;
                        }
                        travelTime = arrival == INT_MISSING || lastValidDeparture == INT_MISSING
                            ? INT_MISSING
                            : arrival - lastValidDeparture;
                    }
                    int departure = key.departureTimes.get(i);
                    int dwellTime = arrival == INT_MISSING || departure == INT_MISSING
                        ? INT_MISSING
                        : departure - arrival;

                    insertPatternStopStatement.setString(1, pattern.pattern_id);
                    // Stop sequence is zero-based.
                    setIntParameter(insertPatternStopStatement, 2, i);
                    insertPatternStopStatement.setString(3, stopId);
                    setIntParameter(insertPatternStopStatement,4, travelTime);
                    setIntParameter(insertPatternStopStatement,5, dwellTime);
                    setIntParameter(insertPatternStopStatement,6, key.dropoffTypes.get(i));
                    setIntParameter(insertPatternStopStatement,7, key.pickupTypes.get(i));
                    setDoubleParameter(insertPatternStopStatement, 8, key.shapeDistances.get(i));
                    setIntParameter(insertPatternStopStatement,9, key.timepoints.get(i));
                    patternStopTracker.addBatch();
                }
                // Finally, update all trips on this pattern to reference this pattern's ID.
                String questionMarks = String.join(", ", Collections.nCopies(pattern.associatedTrips.size(), "?"));
                PreparedStatement updateTripStatement = connection.prepareStatement(
                        String.format("update %s set pattern_id = ? where trip_id in (%s)", tripsTableName, questionMarks));
                int oneBasedIndex = 1;
                updateTripStatement.setString(oneBasedIndex++, pattern.pattern_id);
                // Prepare each trip in pattern to update trips table.
                for (String tripId : pattern.associatedTrips) {
                    if (postgresText) {
                        // Add line to temp csv file if using postgres.
                        // No need to worry about null trip IDs because the trips have already been processed.
                        String[] strings = new String[]{tripId, pattern.pattern_id};
                        // Print a new line in the standard postgres text format:
                        // https://www.postgresql.org/docs/9.1/static/sql-copy.html#AEN64380
                        patternForTripsFileStream.println(String.join("\t", strings));
                    } else {
                        // Otherwise, set statement parameter.
                        updateTripStatement.setString(oneBasedIndex++, tripId);
                    }
                }
                if (!postgresText) {
                    // Execute trip update statement if not using temp text file.
                    LOG.info("Updating {} trips with pattern ID {} (%d/%d)", pattern.associatedTrips.size(), pattern.pattern_id, currentPatternIndex, patterns.size());
                    updateTripStatement.executeUpdate();
                }
                currentPatternIndex += 1;
            }
            // Send any remaining prepared statement calls to the database backend.
            patternTracker.executeRemaining();
            patternStopTracker.executeRemaining();
            LOG.info("Done storing patterns and pattern stops.");
            if (postgresText) {
                // Finally, copy the pattern for trips text file into a table, create an index on trip IDs, and update
                // the trips table.
                LOG.info("Updating trips with pattern IDs");
                patternForTripsFileStream.close();
                // Copy file contents into temp pattern for trips table.
                copyFromFile(connection, tempPatternForTripsTextFile, tempPatternForTripsTable);
                // Before updating the trips with pattern IDs, index the table on trip_id.
                String patternForTripsIndexSql = String.format("create index temp_trips_pattern_id_idx on %s (trip_id)", tempPatternForTripsTable);
                LOG.info(patternForTripsIndexSql);
                statement.execute(patternForTripsIndexSql);
                // Finally, execute the update statement.
                String updateTripsSql = String.format("update %s set pattern_id = %s.pattern_id from %s where %s.trip_id = %s.trip_id", tripsTableName, tempPatternForTripsTable, tempPatternForTripsTable, tripsTableName, tempPatternForTripsTable);
                LOG.info(updateTripsSql);
                statement.executeUpdate(updateTripsSql);
                // Delete temp file. Temp table will be dropped after the transaction is committed.
                tempPatternForTripsTextFile.delete();
                LOG.info("Updating trips complete");
            }
            LOG.info("Creating index on patterns");
            statement.executeUpdate(String.format("alter table %s add primary key (pattern_id)", patternsTableName));
            LOG.info("Creating index on pattern stops");
            statement.executeUpdate(String.format("alter table %s add primary key (pattern_id, stop_sequence)", patternStopsTableName));
            // Index new pattern_id column on trips. The other tables are already indexed because they have primary keys.
            LOG.info("Indexing trips on pattern id.");
            statement.execute(String.format("create index trips_pattern_id_idx on %s (pattern_id)", tripsTableName));
            LOG.info("Done indexing.");
            connection.commit();
        } catch (SQLException | IOException e) {
            // Rollback transaction if failure occurs on creating patterns.
            DbUtils.rollbackAndCloseQuietly(connection);
            // This exception will be stored as a validator failure.
            throw new RuntimeException(e);
        } finally {
            // Close transaction finally.
            DbUtils.closeQuietly(connection);
        }

    }

}


package com.conveyal.gtfs.loader;

import com.conveyal.gtfs.error.NewGTFSError;
import com.conveyal.gtfs.error.NewGTFSErrorType;
import com.conveyal.gtfs.storage.StorageException;

import java.net.URL;
import java.sql.JDBCType;
import java.sql.PreparedStatement;
import java.sql.SQLType;
import java.util.Set;

/**
 * Created by abyrd on 2017-03-31
 */
public class URLField extends Field {

    public URLField(String name, Requirement requirement) {
        super(name, requirement);
    }

    /** Check that a string can be properly parsed and is in range. */
    public ValidateFieldResult<String> validateAndConvert (String string) {
        ValidateFieldResult<String> result = cleanString(string);
        try {
            new URL(result.clean); // TODO call this to validate, but we can't default to zero
            return result;
        } catch (Exception ex) {
            result.errors.add(NewGTFSError.forFeed(NewGTFSErrorType.URL_FORMAT, string));
            return result;
        }
    }

    public Set<NewGTFSError> setParameter(PreparedStatement preparedStatement, int oneBasedIndex, String string) {
        try {
            ValidateFieldResult<String> result = validateAndConvert(string);
            preparedStatement.setString(oneBasedIndex, result.clean);
            return result.errors;
        } catch (Exception ex) {
            throw new StorageException(ex);
        }
    }

    @Override
    public SQLType getSqlType() {
        return JDBCType.VARCHAR;
    }

}