Merge pull request #5 from TheTransitClock/develop

Merge from dev

transitclock/src/main/java/org/transitclock/configData/CoreConfig.java

@@ -47,7 +47,7 @@ public static int getDaysPopulateHistoricalCache() {
 		return daysPopulateHistoricalCache.getValue();
 	}
 	private static IntegerConfigValue daysPopulateHistoricalCache =
-			new IntegerConfigValue("transitclock.cache.daysPopulateHistoricalCache", 4,
+			new IntegerConfigValue("transitclock.cache.core.daysPopulateHistoricalCache", 0,
 					"How many days data to read in to populate historical cache on start up.");
 	/**
 	 * When in playback mode or some other situations don't want to store

transitclock/src/main/java/org/transitclock/core/AvlProcessor.java

@@ -236,6 +236,16 @@ public void makeVehicleUnpredictableAndGrabAssignment(
 		vehicleState.unsetBlock(BlockAssignmentMethod.ASSIGNMENT_GRABBED);
 	}
 
+	/**
+	 * Removes the vehicle from the VehicleDataCache.
+	 * 
+	 * @param vehicleId
+	 *            The vehicle to remove
+	 */
+	public void removeFromVehicleDataCache(String vehicleId) {
+		VehicleDataCache.getInstance().removeVehicle(vehicleId);
+	}
+
 	/**
 	 * Looks at the previous AVL reports to determine if vehicle is actually
 	 * moving. If it is not moving then the vehicle is made unpredictable. Uses
@@ -1320,7 +1330,7 @@ private void determineAndSetRealTimeSchAdh(VehicleState vehicleState) {
 		// Determine the schedule adherence for the vehicle
 		TemporalDifference scheduleAdherence = RealTimeSchedAdhProcessor
 				.generate(vehicleState);
-
+
 		// Store the schedule adherence with the vehicle
 		vehicleState.setRealTimeSchedAdh(scheduleAdherence);		
 	}

transitclock/src/main/java/org/transitclock/core/TimeoutHandlerModule.java

@@ -22,6 +22,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.transitclock.applications.Core;
+import org.transitclock.config.BooleanConfigValue;
 import org.transitclock.config.IntegerConfigValue;
 import org.transitclock.configData.AgencyConfig;
 import org.transitclock.core.dataCache.VehicleStateManager;
@@ -87,6 +88,18 @@ public class TimeoutHandlerModule extends Module {
 					+ "for long after scheduled departure time if vehicle "
 					+ "taken out of service.");
 
+	private static BooleanConfigValue removeTimedOutVehiclesFromVehicleDataCache =
+			new BooleanConfigValue(
+					"transitclock.timeout.removeTimedOutVehiclesFromVehicleDataCache", 
+					false, 
+					"When timing out vehicles, the default behavior is to make "
+					+ "the vehicle unpredictable but leave it in the "
+					+ "VehicleDataCache. When set to true, a timeout will also "
+					+ "remove the vehicle from the VehicleDataCache. This can "
+					+ "be useful in situations where it is not desirable to "
+					+ "include timed out vehicles in data feeds, e.g. the GTFS "
+					+ "Realtime vehicle positions feed.");
+
 	/********************* Logging ************************************/
 
 	private static final Logger logger = LoggerFactory
@@ -115,6 +128,19 @@ public void storeAvlReport(AvlReport avlReport) {
 			avlReportsMap.put(avlReport.getVehicleId(), avlReport);
 		}
 	}
+
+	/**
+	 * Removes the specified vehicle from the VehicleDataCache if configured to do so
+	 * 
+	 * @param vehicleId
+	 *            Vehicle to remove
+	 */
+	public void removeFromVehicleDataCache(String vehicleId) {
+		if (removeTimedOutVehiclesFromVehicleDataCache.getValue()) {
+			logger.info("Removing vehicleId={} from VehicleDataCache", vehicleId);
+			AvlProcessor.getInstance().removeFromVehicleDataCache(vehicleId);
+		}
+	}
 
 	/**
 	 * For regular predictable vehicle that is not a schedule based prediction
@@ -147,20 +173,43 @@ private void handlePredictablePossibleTimeout(VehicleState vehicleState, long no
 
 			// Remove vehicle from map for next time looking for timeouts
 			mapIterator.remove();
-
+
+			// Remove vehicle from cache if configured to do so
+			removeFromVehicleDataCache(vehicleState.getVehicleId());
 		}
 	}
 
 	/**
-	 * Don't need to worry about vehicles that are not predictable. But might as
-	 * well remove vehicle from map so don't examine vehicle every
-	 * TimeoutHandleModule polling cycle
+	 * For not predictable vehicle. If not removing vehicles from cache,
+	 * removes the vehicle from the map to avoid looking at it again. If
+	 * configured to remove timed out vehicles from cache, and haven't 
+	 * reported in too long, removes the vehicle from map and cache.
 	 * 
 	 * @param mapIterator
 	 *            So can remove AVL report from map
 	 */
-	private void handleNotPredictablePossibleTimeout(Iterator<AvlReport> mapIterator) {
-		mapIterator.remove();
+	private void handleNotPredictablePossibleTimeout(VehicleState vehicleState,
+					long now, Iterator<AvlReport> mapIterator) {
+		if (!removeTimedOutVehiclesFromVehicleDataCache.getValue()) {
+			// Remove vehicle from map for next time looking for timeouts and return
+			mapIterator.remove();
+			return;
+		}
+
+		// If haven't reported in too long...
+		long maxNoAvl = allowableNoAvlSecs.getValue() * Time.MS_PER_SEC;
+		if (now > vehicleState.getAvlReport().getTime() + maxNoAvl) {
+
+			// Log the situation
+			logger.info("For not predictable vehicleId={} generated timeout "
+					+ "event.", vehicleState.getVehicleId());
+
+			// Remove vehicle from map for next time looking for timeouts
+			mapIterator.remove();
+
+			// Remove vehicle from cache
+			removeFromVehicleDataCache(vehicleState.getVehicleId());
+		}
 	}
 
 	/**
@@ -190,7 +239,10 @@ private void handleSchedBasedPredsPossibleTimeout(VehicleState vehicleState,
 					vehicleState.getVehicleId(), shouldTimeoutEventDescription);
 
 			// Remove vehicle from map for next time looking for timeouts
-			mapIterator.remove();			
+			mapIterator.remove();
+
+			// Remove vehicle from cache if configured to do so
+			removeFromVehicleDataCache(vehicleState.getVehicleId());
 		}
 	}
 
@@ -263,6 +315,9 @@ private void handleWaitStopPossibleTimeout(VehicleState vehicleState, long now,
 
 				// Remove vehicle from map for next time looking for timeouts
 				mapIterator.remove();
+
+				// Remove vehicle from cache if configured to do so
+				removeFromVehicleDataCache(vehicleState.getVehicleId());
 			}
 		}
 	}
@@ -294,7 +349,8 @@ public void handlePossibleTimeouts() {
 				synchronized (vehicleState) {
 					if (!vehicleState.isPredictable()) {
 						// Vehicle is not predictable
-						handleNotPredictablePossibleTimeout(mapIterator);
+						handleNotPredictablePossibleTimeout(vehicleState, now,
+								mapIterator);
 					} else if (vehicleState.isForSchedBasedPreds()) {
 						// Handle schedule based predictions vehicle
 						handleSchedBasedPredsPossibleTimeout(vehicleState, now,

transitclock/src/main/java/org/transitclock/core/dataCache/VehicleDataCache.java

@@ -539,4 +539,15 @@ public void updateVehicle(VehicleState vehicleState) {
 		updateVehicleIdsByBlockMap(originalVehicle, vehicle);
 		updateVehiclesMap(vehicle);
 	}
+
+	/**
+	 * Removes a vehicle from the vehiclesMap
+	 * 
+	 * @param vehicleId
+	 *            The id of the vehicle to remove from the vehiclesMap
+	 */
+	public void removeVehicle(String vehicleId) {
+		logger.debug("Removing from VehicleDataCache vehiclesMap vehicleId={}", vehicleId);
+		vehiclesMap.remove(vehicleId);
+	}
 }

transitclock/src/main/java/org/transitclock/gtfs/GtfsData.java

@@ -33,7 +33,7 @@
 import java.util.regex.Pattern;
 
 import org.hibernate.HibernateException;
-
+
 import org.hibernate.Session;
 import org.hibernate.SessionFactory;
 import org.slf4j.Logger;
@@ -256,6 +256,14 @@ public class GtfsData {
 					+ "name of the last stop, won't disambiguate if the last "
 					+ "stops for the trips with the same headsign differ by "
 					+ "less than this amount.");
+
+	private static StringConfigValue outputPathsAndStopsForGraphingRouteIds = 
+		new StringConfigValue("transitclock.gtfs.outputPathsAndStopsForGraphingRouteIds", 
+			null, // Default of null means don't output any routes
+			"Outputs data for specified routes grouped by trip pattern."
+			+ "The resulting data can be visualized on a map by cutting"
+			+ "and pasting it in to http://www.gpsvisualizer.com/map_input"
+			+ "Separate multiple route ids with commas");
 
 	// Logging
 	public static final Logger logger = 
@@ -2546,6 +2554,16 @@ public ConfigRevision getConfigRevision() {
 
 	/*************************** Main Public Methods **********************/
 
+	public void outputRoutesForGraphing() {
+		if (outputPathsAndStopsForGraphingRouteIds.getValue() == null)
+			return;
+
+		String[] routeIds = outputPathsAndStopsForGraphingRouteIds.getValue().split(",");
+		for (String routeId : routeIds) {
+			outputPathsAndStopsForGraphing(routeId);
+		}
+	}
+
 	/**
 	 * Outputs data for specified route grouped by trip pattern.
 	 * The resulting data can be visualized on a map by cutting
@@ -2701,8 +2719,8 @@ public void processData() {
 		// MBTA commuter rail.
 		trimCalendars();
 
-		// debugging
-		//outputPathsAndStopsForGraphing("8699");
+		// Optionally output routes for debug graphing		
+		outputRoutesForGraphing();
 
 		// Now process travel times and update the Trip objects. 
 		TravelTimesProcessorForGtfsUpdates travelTimesProcesssor =

transitclock/src/main/java/org/transitclock/ipc/data/IpcRoute.java

@@ -222,7 +222,7 @@ private static IpcDirectionsForRoute createStops(Route dbRoute,
 				if (stopId != null) {
 					isUiStop = false;
 					for (TripPattern tripPattern : uiTripPatterns) {
-						if(tripPattern.getDirectionId().compareTo(currentDirectionId)==0)
+						if((tripPattern.getDirectionId()==null && currentDirectionId==null)  || tripPattern.getDirectionId().compareTo(currentDirectionId)==0)
 							currentTripPattern=tripPattern;
 						if (tripPattern.isStopAtOrAfterStop(stopId, currentStopId)) {
 							isUiStop = true;
@@ -233,7 +233,7 @@ private static IpcDirectionsForRoute createStops(Route dbRoute,
 				else
 				{
 					for (TripPattern tripPattern : uiTripPatterns) {
-						if(tripPattern.getDirectionId().compareTo(currentDirectionId)==0)
+						if((tripPattern.getDirectionId()==null && currentDirectionId==null) || tripPattern.getDirectionId().compareTo(currentDirectionId)==0)
 							currentTripPattern=tripPattern;
 					}
 				}

transitclockApi/src/main/java/org/transitclock/api/rootResources/TransitimeApi.java

@@ -779,7 +779,7 @@ public Response getRoutes(@BeanParam StandardParameters stdParameters,
 			+ "and paths such that it can be drawn in a map.",tags= {"base data","route"})
 	public Response getRouteDetails(@BeanParam StandardParameters stdParameters,
 			@Parameter(description="List of routeId or routeShortName. Example: r=1&r=2" ,required=false) 
-			 List<String> routeIdsOrShortNames, 
+	@QueryParam(value = "r")  List<String> routeIdsOrShortNames, 
 			 @Parameter(description="If set then only the shape for specified direction is marked as being for the UI." ,required=false) 
 			 @QueryParam(value = "d") String directionId,
 			 @Parameter(description="If set then only this stop and the remaining ones on "

transitclockQuickStart/README.md

@@ -5,10 +5,11 @@ to build the program, you can do this with the command
 mvn install -DskipTests
 
 This will take a few minutes to run. when complete navigate to 
-transitimeQuickStart/Target/
-from here you see the transitimeQuickStart.jar to run it from here enter the line:
+transitimeQuickStart
+from here you see the target/transitimeQuickStart.jar to run it from here enter the line:
 
-java -jar transitimeQuickStart.jar
+
+java -jar -Dtransitclock.configFiles=src/main/resources/transitclock.properties target/transitclockQuickStart.jar
 
 The gui should then pop up if done correctly.
 

transitclockQuickStart/pom.xml

@@ -9,7 +9,7 @@
 	<name>transitclockQuickStart</name>
 	<url>http://maven.apache.org</url>
 	<properties>
-		<jettyVersion>9.4.0.M0</jettyVersion>
+		<jettyVersion>9.4.15.v20190215</jettyVersion>
 	</properties>
 	<dependencies>
 		<dependency>
@@ -42,15 +42,25 @@
 			<artifactId>apache-jstl</artifactId>
 			<version>${jettyVersion}</version>
 		</dependency>
-		<dependency>
+		<!--<dependency>
 			<groupId>javax.servlet</groupId>
 			<artifactId>jstl</artifactId>
 			<version>1.2</version>
-		</dependency>
+		</dependency>-->
 		<dependency>
 			<groupId>TheTransitClock</groupId>
 			<artifactId>transitclockCore</artifactId>
 			<version>2.0.0-SNAPSHOT</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.glassfish.jersey.media</groupId>
+					<artifactId>jersey-media-multipart</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.glassfish.jersey.media</groupId>
+					<artifactId>jersey-media-json-jackson</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>junit</groupId>
@@ -79,10 +89,10 @@
 		<plugins>
 			<plugin>
 				<artifactId>maven-compiler-plugin</artifactId>
-				<version>2.5.1</version>
+				<version>3.8.0</version>
 				<configuration>
-					<source>1.7</source>
-					<target>1.7</target>
+					<source>1.8</source>
+					<target>1.8</target>
 				</configuration>
 			</plugin>
 			<plugin>

transitclockQuickStart/src/main/java/org/transitclock/gui/TransitimeQuickStart.java

@@ -267,7 +267,7 @@ public void addApi() throws QuickStartException {
 			// Set the path to the override descriptor, based on your
 			// $(jetty.home)
 			// directory
-			apiapp.setOverrideDescriptor("override-web.xml");
+			//apiapp.setOverrideDescriptor("override-web.xml");
 
 			// server.start();
 			// server.join();
@@ -293,7 +293,7 @@ public void addWebapp() throws QuickStartException {
 
 			webapp.setAttribute("org.eclipse.jetty.server.webapp.ContainerIncludeJarPattern",
 					".*/[^/]*servlet-api-[^/]*\\.jar$|.*/javax.servlet.jsp.jstl-.*\\.jar$|.*/[^/]*taglibs.*\\.jar$");
-			webapp.setOverrideDescriptor("override-web.xml");
+			//webapp.setOverrideDescriptor("override-web.xml");
 
 			System.setProperty("transitclock.apikey", apiKey.getKey());
 			// server.join();
@@ -339,7 +339,7 @@ public void webAgency() throws QuickStartException {
 			String agencyId = System.getProperties().getProperty("transitclock.core.agencyId");
 			String hostName = "127.0.0.1";
 			boolean active = true;
-			String dbName = "02";
+			String dbName = "test";
 			String dbType = "hsql";
 			String dbHost = "http://127.0.0.1:8080/";
 			String dbUserName = "sa";

transitclockWebapp/src/main/webapp/synoptic/index.jsp

@@ -291,6 +291,8 @@ function routeConfigCallback(routeDetail, status)
 			var stop=routeDetail.routes[0].direction[i].stop[j];
 			distanceOverPath+=stop.pathLength;
 			var projectionStop=distanceOverPath/routeLenght;
+			if(stop.direction==undefined)
+				stop.direction="0";
 			var directionStop=(stop.direction==routeDetail.routes[0].direction[i].id=="0")?0:1;
 			stops.push({id: stop.id, identifier: stop.name,projection:projectionStop,direction:directionStop,distance:distanceOverPath});
 		}