diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java index e457e03be..0d61f7365 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java @@ -81,8 +81,8 @@ public JdbcGtfsExporter(String feedId, String outFile, DataSource dataSource, bo public Boolean exceptionInvolvesService(ScheduleException ex, String serviceId) { return ( ex.addedService.contains(serviceId) || - ex.removedService.contains(serviceId) || - ex.customSchedule.contains(serviceId) + ex.removedService.contains(serviceId) || + ex.customSchedule.contains(serviceId) ); } @@ -114,6 +114,7 @@ public FeedLoadResult exportTables() { String whereRouteIsApproved = String.format("where %s.%s.status = 2", feedIdToExport, Table.ROUTES.name); // Export each table in turn (by placing entry in zip output stream). result.agency = export(Table.AGENCY, connection); + result.area = export(Table.AREA, connection); result.bookingRules = export(Table.BOOKING_RULES, connection); result.stopAreas = exportStopAreas(); if (fromEditor) { @@ -135,11 +136,11 @@ public FeedLoadResult exportTables() { GTFSFeed feed = new GTFSFeed(); // FIXME: The below table readers should probably just share a connection with the exporter. JDBCTableReader exceptionsReader = - new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".", - EntityPopulator.SCHEDULE_EXCEPTION); + new JDBCTableReader(Table.SCHEDULE_EXCEPTIONS, dataSource, feedIdToExport + ".", + EntityPopulator.SCHEDULE_EXCEPTION); JDBCTableReader calendarsReader = - new JDBCTableReader(Table.CALENDAR, dataSource, feedIdToExport + ".", - EntityPopulator.CALENDAR); + new JDBCTableReader(Table.CALENDAR, dataSource, feedIdToExport + ".", + EntityPopulator.CALENDAR); Iterable calendars = calendarsReader.getAll(); Iterable exceptionsIterator = exceptionsReader.getAll(); List exceptions = new ArrayList<>(); diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index 48f27905c..75d8bfa25 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -1017,7 +1017,7 @@ private int updateStopTimesForPatternLocationOrPatternStopArea( patternId, stopSequence ); - LOG.info("{} stop_time flex service arrivals/departures updated", entitiesUpdated); + LOG.info("{} stop_time flex service start/end pickup drop off window updated.", entitiesUpdated); return travelTime + dwellTime; } diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index b418081d0..dd107784a 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -43,6 +43,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -777,7 +778,9 @@ public CsvReader getCsvReader(ZipFile zipFile, SQLErrorStorage sqlErrorStorage) Enumeration entries = zipFile.entries(); while (entries.hasMoreElements()) { ZipEntry e = entries.nextElement(); - if (e.getName().endsWith(tableFileName)) { + // Include the file separator prefix to force the complete file name to be considered. + // This prevents stop_areas.txt from being loaded instead of areas.txt. + if (e.getName().endsWith(String.format("%s%s", File.separator, tableFileName))) { entry = e; if (sqlErrorStorage != null) sqlErrorStorage.storeError(NewGTFSError.forTable(this, TABLE_IN_SUBDIRECTORY)); break; diff --git a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java index 079941ba5..5aebac91c 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSFeedTest.java @@ -26,39 +26,86 @@ public class GTFSFeedTest { private static final Logger LOG = LoggerFactory.getLogger(GTFSFeedTest.class); private static String simpleGtfsZipFileName; + private static String simpleFlexGtfsZipFileName; @BeforeAll public static void setUpClass() { //executed only once, before the first test simpleGtfsZipFileName = null; + simpleFlexGtfsZipFileName = null; try { simpleGtfsZipFileName = TestUtils.zipFolderFiles("fake-agency", true); + simpleFlexGtfsZipFileName = TestUtils.zipFolderFiles("fake-agency-with-flex", true); } catch (IOException e) { e.printStackTrace(); } } /** - * Make sure a roundtrip of loading a GTFS zip file and then writing another zip file can be performed. + * Make sure a round-trip of loading a GTFS zip file and then writing another zip file can be performed. */ @Test public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { - // create a temp file for this test - File outZip = File.createTempFile("fake-agency-output", ".zip"); - - // delete file to make sure we can assert that this program created the file - outZip.delete(); - - GTFSFeed feed = GTFSFeed.fromFile(simpleGtfsZipFileName); - feed.toFile(outZip.getAbsolutePath()); - feed.close(); - assertThat(outZip.exists(), is(true)); + FileTestCase[] fileTestCases = { + // agency.txt + new FileTestCase( + "agency.txt", + new TestUtils.DataExpectation[]{ + new TestUtils.DataExpectation("agency_id", "1"), + new TestUtils.DataExpectation("agency_name", "Fake Transit") + } + ), + new FileTestCase( + "calendar.txt", + new DataExpectation[]{ + new DataExpectation("service_id", "04100312-8fe1-46a5-a9f2-556f39478f57"), + new DataExpectation("start_date", "20170915"), + new DataExpectation("end_date", "20170917") + } + ), + new FileTestCase( + "routes.txt", + new DataExpectation[]{ + new DataExpectation("agency_id", "1"), + new DataExpectation("route_id", "1"), + new DataExpectation("route_long_name", "Route 1") + } + ), + new FileTestCase( + "shapes.txt", + new DataExpectation[]{ + new DataExpectation("shape_id", "5820f377-f947-4728-ac29-ac0102cbc34e"), + new DataExpectation("shape_pt_lat", "37.0612132"), + new DataExpectation("shape_pt_lon", "-122.0074332") + } + ), + new FileTestCase( + "stop_times.txt", + new DataExpectation[]{ + new DataExpectation("trip_id", "a30277f8-e50a-4a85-9141-b1e0da9d429d"), + new DataExpectation("departure_time", "07:00:00"), + new DataExpectation("stop_id", "4u6g") + } + ), + new FileTestCase( + "trips.txt", + new DataExpectation[]{ + new DataExpectation("route_id", "1"), + new DataExpectation("trip_id", "a30277f8-e50a-4a85-9141-b1e0da9d429d"), + new DataExpectation("service_id", "04100312-8fe1-46a5-a9f2-556f39478f57") + } + ) + }; + loadAndWriteToZipFile(simpleGtfsZipFileName, fileTestCases); - // assert that rows of data were written to files within the zipfile - ZipFile zip = new ZipFile(outZip); + } + /** + * Make sure a round-trip of loading a GTFS flex zip file and then writing another zip file can be performed. + */ + @Test + void canDoRoundTripLoadAndWriteToFlexZipFile() throws IOException { FileTestCase[] fileTestCases = { - // agency.txt new FileTestCase( "agency.txt", new TestUtils.DataExpectation[]{ @@ -66,6 +113,21 @@ public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { new TestUtils.DataExpectation("agency_name", "Fake Transit") } ), + new FileTestCase( + "areas.txt", + new TestUtils.DataExpectation[]{ + new TestUtils.DataExpectation("area_id", "1"), + new TestUtils.DataExpectation("area_name", "Area referencing a stop") + } + ), + new FileTestCase( + "booking_rules.txt", + new TestUtils.DataExpectation[]{ + new TestUtils.DataExpectation("booking_rule_id", "1"), + new TestUtils.DataExpectation("booking_type", "1"), + new TestUtils.DataExpectation("pickup_message", "This is a pickup message") + } + ), new FileTestCase( "calendar.txt", new DataExpectation[]{ @@ -98,6 +160,13 @@ public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { new DataExpectation("stop_id", "4u6g") } ), + new FileTestCase( + "stop_areas.txt", + new DataExpectation[]{ + new DataExpectation("area_id", "2"), + new DataExpectation("stop_id", "area-999") + } + ), new FileTestCase( "trips.txt", new DataExpectation[]{ @@ -107,6 +176,27 @@ public void canDoRoundtripLoadAndWriteToZipFile() throws IOException { } ) }; + loadAndWriteToZipFile(simpleFlexGtfsZipFileName, fileTestCases); + } + + /** + * Load feed and then write to zip file. Once complete, perform tests. + */ + void loadAndWriteToZipFile(String zipFileName, FileTestCase[] fileTestCases) throws IOException { + // create a temp file for this test + File outZip = File.createTempFile(zipFileName, ".zip"); + + // delete file to make sure we can assert that this program created the file + outZip.delete(); + + GTFSFeed feed = GTFSFeed.fromFile(zipFileName); + feed.toFile(outZip.getAbsolutePath()); + feed.close(); + assertThat(outZip.exists(), is(true)); + + // assert that rows of data were written to files within the zip file. + ZipFile zip = new ZipFile(outZip); + TestUtils.lookThroughFiles(fileTestCases, zip); // Close the zip file so it can be deleted. zip.close(); diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java index 7f724cb79..88d5de1c1 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java @@ -1203,7 +1203,7 @@ private void assertThatPersistenceExpectationRecordWasFound( /** * Persistence expectations for use with the GTFS contained within the "fake-agency" resources folder. */ - private PersistenceExpectation[] fakeAgencyPersistenceExpectations = new PersistenceExpectation[]{ + private final PersistenceExpectation[] fakeAgencyPersistenceExpectations = new PersistenceExpectation[]{ new PersistenceExpectation( "agency", new RecordExpectation[]{ @@ -1212,6 +1212,21 @@ private void assertThatPersistenceExpectationRecordWasFound( new RecordExpectation("agency_timezone", "America/Los_Angeles") } ), + new PersistenceExpectation( + "areas", + new RecordExpectation[]{ + new RecordExpectation("area_id", "area1"), + new RecordExpectation("area_name", "This is the area name") + } + ), + new PersistenceExpectation( + "booking_rules", + new RecordExpectation[]{ + new RecordExpectation("booking_rule_id", "1"), + new RecordExpectation("booking_type", "1"), + new RecordExpectation("pickup_message", "This is a pickup message") + } + ), new PersistenceExpectation( "calendar", new RecordExpectation[]{ @@ -1315,6 +1330,13 @@ private void assertThatPersistenceExpectationRecordWasFound( new RecordExpectation("shape_dist_traveled", 0.0, 0.01) } ), + new PersistenceExpectation( + "stop_areas", + new RecordExpectation[]{ + new RecordExpectation("area_id", "area1"), + new RecordExpectation("stop_id", "123") + } + ), new PersistenceExpectation( "trips", new RecordExpectation[]{ diff --git a/src/test/java/com/conveyal/gtfs/dto/PatternStopAreaDTO.java b/src/test/java/com/conveyal/gtfs/dto/PatternStopAreaDTO.java index bd8aa4ef8..5de7e6437 100644 --- a/src/test/java/com/conveyal/gtfs/dto/PatternStopAreaDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/PatternStopAreaDTO.java @@ -50,9 +50,9 @@ public PatternStopAreaDTO( this.flex_default_zone_time = flexDefaultZoneTime; } - public PatternStopAreaDTO(String pattern_id, String area_id, int stop_sequence) { - this.pattern_id = pattern_id; - this.area_id = area_id; - this.stop_sequence = stop_sequence; + public PatternStopAreaDTO(String patternId, String areaId, int stopSequence) { + this.pattern_id = patternId; + this.area_id = areaId; + this.stop_sequence = stopSequence; } } diff --git a/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java b/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java index 22e300e6f..b2ba9c411 100644 --- a/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/StopTimeDTO.java @@ -39,4 +39,13 @@ public StopTimeDTO(String stopId, Integer arrivalTime, Integer departureTime, In departure_time = departureTime; stop_sequence = stopSequence; } + + public static StopTimeDTO flexStopTime(String stopId, Integer startPickupDropOffWindow, Integer endPickupDropOffWindow, Integer stopSequence) { + StopTimeDTO stopTimeDTO = new StopTimeDTO(); + stopTimeDTO.stop_id = stopId; + stopTimeDTO.start_pickup_drop_off_window = startPickupDropOffWindow; + stopTimeDTO.end_pickup_drop_off_window = endPickupDropOffWindow; + stopTimeDTO.stop_sequence = stopSequence; + return stopTimeDTO; + } } diff --git a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java index e540e1222..9fb9a8275 100644 --- a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java +++ b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java @@ -1266,6 +1266,141 @@ public void canNormalizePatternStopTimes() throws IOException, SQLException, Inv assertThat(index, equalTo(patternStops.length)); } + /** + * Checks that {@link JdbcTableWriter#normalizeStopTimesForPattern(int, int)} can normalize stop times for flex + * patterns. + */ + @Test + void canNormalizePatternStopTimesForFlex() throws IOException, SQLException, InvalidNamespaceException { + int startTime = 6 * 60 * 60; // 6AM + String patternId = newUUID(); + + StopDTO stopOne = createSimpleStop(newUUID(), "Stop One", 0.0, 0.0); + StopDTO stopTwo = createSimpleStop(newUUID(), "Stop Two", 0.0, 0.0); + LocationDTO locationOne = createSimpleTestLocation(newUUID()); + LocationDTO locationTwo = createSimpleTestLocation(newUUID()); + StopAreaDTO stopAreaOne = createSimpleTestStopArea(newUUID()); + StopAreaDTO stopAreaTwo = createSimpleTestStopArea(newUUID()); + + PatternStopDTO[] patternStops = new PatternStopDTO[] { + new PatternStopDTO(patternId, stopOne.stop_id, 0), + new PatternStopDTO(patternId, stopTwo.stop_id, 1) + }; + PatternLocationDTO[] patternLocations = new PatternLocationDTO[] { + new PatternLocationDTO(patternId, locationOne.location_id, 2), + new PatternLocationDTO(patternId, locationTwo.location_id, 3) + }; + PatternStopAreaDTO[] patternStopAreas = new PatternStopAreaDTO[] { + new PatternStopAreaDTO(patternId, stopAreaOne.area_id, 4), + new PatternStopAreaDTO(patternId, stopAreaTwo.area_id, 5) + }; + + int travelTime = 60; + patternStops[0].default_travel_time = 0; + patternStops[1].default_travel_time = travelTime; + patternLocations[0].flex_default_travel_time = travelTime; + patternLocations[1].flex_default_travel_time = travelTime; + patternStopAreas[0].flex_default_travel_time = travelTime; + patternStopAreas[1].flex_default_travel_time = travelTime; + PatternDTO pattern = createRouteAndPattern(newUUID(), + patternId, + "Pattern A", + null, + new ShapePointDTO[] {}, + patternStops, + patternLocations, + patternStopAreas, + 0 + ); + + int cumulativeTravelTime = startTime + travelTime; + StopTimeDTO[] stopTimes = new StopTimeDTO[] { + new StopTimeDTO(stopOne.stop_id, startTime, startTime, 0), + new StopTimeDTO(stopTwo.stop_id, startTime, startTime, 1), + StopTimeDTO.flexStopTime(locationOne.location_id, cumulativeTravelTime, cumulativeTravelTime, 2), + StopTimeDTO.flexStopTime(locationTwo.location_id, (cumulativeTravelTime += travelTime), cumulativeTravelTime, 3), + StopTimeDTO.flexStopTime(stopAreaOne.area_id, (cumulativeTravelTime += travelTime), cumulativeTravelTime, 4), + StopTimeDTO.flexStopTime(stopAreaTwo.area_id, (cumulativeTravelTime += travelTime), cumulativeTravelTime, 5) + }; + + // Create trip with travel times that match pattern stops. + TripDTO tripInput = new TripDTO(); + tripInput.pattern_id = patternId; + tripInput.route_id = pattern.route_id; + tripInput.service_id = simpleServiceId; + tripInput.stop_times = stopTimes; + tripInput.frequencies = new FrequencyDTO[] {}; + JdbcTableWriter createTripWriter = createTestTableWriter(Table.TRIPS); + String createTripOutput = createTripWriter.create(mapper.writeValueAsString(tripInput), true); + LOG.info(createTripOutput); + TripDTO createdTrip = mapper.readValue(createTripOutput, TripDTO.class); + + checkStopArrivalAndDepartures( + createdTrip.trip_id, + startTime, + 0, + travelTime, + patternStops.length + patternLocations.length + patternStopAreas.length + ); + + // Update pattern stop with new travel time. + JdbcTableWriter patternUpdater = createTestTableWriter(Table.PATTERNS); + int updatedTravelTime = 3600; // one hour + pattern.pattern_stops[1].default_travel_time = updatedTravelTime; + String updatedPatternOutput = patternUpdater.update(pattern.id, mapper.writeValueAsString(pattern), true); + LOG.info("Updated pattern output: {}", updatedPatternOutput); + // Normalize stop times. + JdbcTableWriter updateTripWriter = createTestTableWriter(Table.TRIPS); + updateTripWriter.normalizeStopTimesForPattern(pattern.id, 0); + checkStopArrivalAndDepartures( + createdTrip.trip_id, + startTime, + updatedTravelTime, + travelTime, + patternStops.length + patternLocations.length + patternStopAreas.length + ); + } + + /** + * Read stop times from the database and check that the arrivals/departures have been set correctly. + */ + private void checkStopArrivalAndDepartures( + String tripId, + int startTime, + int updatedTravelTime, + int travelTime, + int totalNumberOfPatterns + ) { + JDBCTableReader stopTimesTable = new JDBCTableReader(Table.STOP_TIMES, + testDataSource, + testNamespace + ".", + EntityPopulator.STOP_TIME + ); + int index = 0; + for (StopTime stopTime : stopTimesTable.getOrdered(tripId)) { + if (stopTime.stop_sequence < 2) { + LOG.info("stop times i={} arrival={} departure={}", + index, + stopTime.arrival_time, + stopTime.departure_time + ); + assertEquals(stopTime.arrival_time, startTime + index * updatedTravelTime); + assertEquals(stopTime.departure_time, startTime + index * updatedTravelTime); + } else { + LOG.info("stop times i={} start_pickup_drop_off_window={} end_pickup_drop_off_window={}", + index, + stopTime.start_pickup_drop_off_window, + stopTime.end_pickup_drop_off_window + ); + assertEquals(stopTime.start_pickup_drop_off_window, startTime + updatedTravelTime + (index-1) * travelTime); + assertEquals(stopTime.end_pickup_drop_off_window, startTime + updatedTravelTime + (index-1) * travelTime); + } + index++; + } + // Ensure that updated stop times equals pattern stops length + assertEquals(index, totalNumberOfPatterns); + } + /** * This test makes sure that updated the service_id will properly update affected referenced entities properly. * This test case was initially developed to prove that https://github.com/conveyal/gtfs-lib/issues/203 is @@ -1744,6 +1879,33 @@ private static PatternDTO createRouteAndPattern( ShapePointDTO[] shapes, PatternStopDTO[] patternStops, int useFrequency + ) throws InvalidNamespaceException, SQLException, IOException { + return createRouteAndPattern( + routeId, + patternId, + name, + shapeId, + shapes, + patternStops, + null, + null, + useFrequency + ); + } + + /** + * Creates a pattern by first creating a route and then a pattern for that route. + */ + private static PatternDTO createRouteAndPattern( + String routeId, + String patternId, + String name, + String shapeId, + ShapePointDTO[] shapes, + PatternStopDTO[] patternStops, + PatternLocationDTO[] patternLocations, + PatternStopAreaDTO[] patternStopAreas, + int useFrequency ) throws InvalidNamespaceException, SQLException, IOException { // Create new route createSimpleTestRoute(routeId, "RTA", "500", "Hollingsworth", 3); @@ -1756,6 +1918,8 @@ private static PatternDTO createRouteAndPattern( input.shape_id = shapeId; input.shapes = shapes; input.pattern_stops = patternStops; + if (patternLocations != null) input.pattern_locations = patternLocations; + if (patternStopAreas != null) input.pattern_stop_areas = patternStopAreas; // Write the pattern to the database JdbcTableWriter createPatternWriter = createTestTableWriter(Table.PATTERNS); String output = createPatternWriter.create(mapper.writeValueAsString(input), true);