Merge branch 'master' into moia-deployment

# Conflicts:
#	contribs/commercialTrafficApplications/src/main/java/org/matsim/contrib/commercialTrafficApplications/jointDemand/examples/RunJointDemandDRTExample.java
#	contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/edrt/run/EDrtModeOptimizerQSimModule.java
#	contribs/drt-extensions/src/test/java/org/matsim/contrib/drt/extension/fiss/RunFissDrtScenarioIT.java
#	contribs/drt-extensions/src/test/java/org/matsim/contrib/drt/extension/operations/eshifts/run/RunEShiftDrtScenarioIT.java
#	contribs/drt-extensions/src/test/java/org/matsim/contrib/drt/extension/operations/shifts/run/RunMultiHubShiftDrtScenarioIT.java
#	contribs/drt-extensions/src/test/java/org/matsim/contrib/drt/extension/operations/shifts/run/RunShiftDrtScenarioIT.java
#	contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/DrtAnalysisControlerListener.java
#	contribs/drt/src/main/java/org/matsim/contrib/drt/estimator/impl/PessimisticDrtEstimator.java
#	contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/DrtModeOptimizerQSimModule.java
#	contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/insertion/DefaultInsertionCostCalculator.java
#	contribs/drt/src/main/java/org/matsim/contrib/drt/routing/DrtRouteCreator.java
#	contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtConfigGroup.java
#	contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtModeRoutingModule.java
#	contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/MaxDetourConstraintTest.java
#	contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/insertion/DrtPoolingParameterTest.java
#	contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/insertion/InsertionCostCalculatorTest.java
#	contribs/drt/src/test/java/org/matsim/contrib/drt/prebooking/PrebookingTestEnvironment.java
#	contribs/drt/src/test/java/org/matsim/contrib/drt/routing/DrtRoutingModuleTest.java
#	contribs/drt/src/test/java/org/matsim/contrib/drt/run/DrtConfigGroupTest.java
#	contribs/drt/src/test/java/org/matsim/contrib/drt/teleportation/DrtTeleportationTest.java
#	contribs/hybridsim/pom.xml
#	contribs/protobuf/pom.xml
#	contribs/taxi/src/main/java/org/matsim/contrib/taxi/run/MultiModeTaxiModule.java
#	contribs/vsp/src/test/java/org/matsim/integration/drtAndPt/PtAlongALine2Test.java
#	contribs/vsp/src/test/java/org/matsim/integration/drtAndPt/PtAlongALineTest.java
#	pom.xml

contribs/application/src/main/avro/network.avsc

@@ -0,0 +1,79 @@
+{
+  "namespace": "org.matsim.application.avro",
+  "type": "record",
+  "name": "AvroNetwork",
+  "fields": [
+    {
+      "name": "crs",
+      "type": "string",
+      "doc": "Coordinate reference system"
+    },
+    {
+      "name": "nodeAttributes",
+      "type": {
+        "type": "array",
+        "items": "string"
+      }
+    },
+    {
+      "name": "linkAttributes",
+      "type": {
+        "type": "array",
+        "items": "string"
+      }
+    },
+    {
+      "type": {
+        "type": "array",
+        "items": "string"
+      },
+      "name": "nodeId"
+    },
+    {
+      "type": {
+        "type": "array",
+        "items": "float"
+      },
+      "name": "nodeCoordinates"
+    },
+    {"type": {"type": "array", "items": "string" }, "name": "linkId"},
+    {"type": {"type": "array", "items": "int" }, "name": "from"},
+    {"type": {"type": "array", "items": "int"}, "name": "to"},
+    {"type": { "type": "array", "items": "string"}, "name": "modes"},
+    {
+      "type": {
+        "type": "array",
+        "items": "float"
+      },
+      "name": "length"
+    },
+    {
+      "type": {
+        "type": "array",
+        "items": "float"
+      },
+      "name": "freespeed"
+    },
+    {
+      "type": {
+        "type": "array",
+        "items": "float"
+      },
+      "name": "capacity"
+    },
+    {
+      "type": {
+        "type": "array",
+        "items": "float"
+      },
+      "name": "permlanes"
+    },
+    {
+      "type": {
+        "type": "array",
+        "items": "int"
+      },
+      "name": "allowedModes"
+    }
+  ]
+}

contribs/application/src/main/java/org/matsim/application/analysis/emissions/AirPollutionAnalysis.java

@@ -1,7 +1,18 @@
 package org.matsim.application.analysis.emissions;
 
+import it.unimi.dsi.fastutil.floats.FloatArrayList;
+import it.unimi.dsi.fastutil.floats.FloatFloatPair;
+import it.unimi.dsi.fastutil.floats.FloatList;
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 import it.unimi.dsi.fastutil.objects.Object2DoubleLinkedOpenHashMap;
 import it.unimi.dsi.fastutil.objects.Object2DoubleMap;
+import it.unimi.dsi.fastutil.objects.Object2FloatMap;
+import it.unimi.dsi.fastutil.objects.Object2FloatOpenHashMap;
+import org.apache.avro.file.CodecFactory;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.io.DatumWriter;
+import org.apache.avro.specific.SpecificDatumWriter;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.logging.log4j.LogManager;
@@ -14,6 +25,8 @@
 import org.matsim.application.ApplicationUtils;
 import org.matsim.application.CommandSpec;
 import org.matsim.application.MATSimAppCommand;
+import org.matsim.application.avro.AvroNetwork;
+import org.matsim.application.avro.XYTData;
 import org.matsim.application.options.InputOptions;
 import org.matsim.application.options.OutputOptions;
 import org.matsim.application.options.SampleOptions;
@@ -43,20 +56,19 @@
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.text.NumberFormat;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
+import java.util.*;
 
 @CommandLine.Command(
 	name = "air-pollution", description = "General air pollution analysis.",
 	mixinStandardHelpOptions = true, showDefaultValues = true
 )
 @CommandSpec(requireRunDirectory = true,
 	produces = {
-		"emissions_total.csv", "emissions_grid_per_day.csv", "emissions_per_link.csv",
+		"emissions_total.csv", "emissions_per_link.csv",
 		"emissions_per_link_per_m.csv",
-		"emissions_grid_per_hour.csv",
+		"emissions_grid_per_hour.%s",
 		"emissions_vehicle_info.csv",
+		"emissions_grid_per_day.%s"
 	}
 )
 public class AirPollutionAnalysis implements MATSimAppCommand {
@@ -128,7 +140,9 @@ public void install() {
 		writeTotal(filteredNetwork, emissionsEventHandler);
 
 		writeRaster(filteredNetwork, config, emissionsEventHandler);
+		writeAvroRaster(filteredNetwork, config, emissionsEventHandler);
 
+		writeTimeDependentAvroRaster(filteredNetwork, config, emissionsEventHandler);
 		writeTimeDependentRaster(filteredNetwork, config, emissionsEventHandler);
 
 		return 0;
@@ -241,6 +255,62 @@ private void writeTotal(Network network, EmissionsOnLinkEventHandler emissionsEv
 		}
 	}
 
+	/**
+	 * Creates the data for the XY-Time plot. The time is fixed and the data is summarized over the run.
+	 * Currently only the CO2_Total Values is printed because Simwrapper can handle only one value.
+	 */
+	private void writeAvroRaster(Network network, Config config, EmissionsOnLinkEventHandler emissionsEventHandler) {
+
+		String crs = ProjectionUtils.getCRS(network);
+		if (crs == null)
+			crs = config.network().getInputCRS();
+		if (crs == null)
+			crs = config.global().getCoordinateSystem();
+
+		XYTData avroData = new XYTData();
+		avroData.setCrs(crs);
+
+		Map<Pollutant, Raster> rasterMap = FastEmissionGridAnalyzer.processHandlerEmissions(emissionsEventHandler.getLink2pollutants(), network, gridSize, 20);
+		List<Integer> xLength = rasterMap.values().stream().map(Raster::getXLength).distinct().toList();
+		List<Integer> yLength = rasterMap.values().stream().map(Raster::getYLength).distinct().toList();
+		Raster raster = rasterMap.values().stream().findFirst().orElseThrow();
+
+		List<Float> xCoords = new ArrayList<>();
+		List<Float> yCoords = new ArrayList<>();
+		Map<CharSequence, List<Float>> values = new HashMap<>();
+		List<Float> valuesList = new ArrayList<>();
+		List<Integer> times = new ArrayList<>();
+
+		times.add(0);
+
+		for (int xi = 0; xi < xLength.get(0); xi++) {
+			for (int yi = 0; yi < yLength.get(0); yi++) {
+				Coord coord = raster.getCoordForIndex(xi, yi);
+				double value = rasterMap.get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi);
+				if (xi == 0) yCoords.add((float) coord.getY());
+				if (yi == 0) xCoords.add((float) coord.getX());
+				valuesList.add((float) value);
+			}
+		}
+
+
+		values.put(String.valueOf(Pollutant.CO2_TOTAL), valuesList);
+
+		avroData.setYCoords(yCoords);
+		avroData.setXCoords(xCoords);
+		avroData.setData(values);
+		avroData.setTimestamps(times);
+
+		DatumWriter<XYTData> datumWriter = new SpecificDatumWriter<>(XYTData.class);
+		try (DataFileWriter<XYTData> dataFileWriter = new DataFileWriter<>(datumWriter)) {
+			dataFileWriter.setCodec(CodecFactory.deflateCodec(9));
+			dataFileWriter.create(avroData.getSchema(), IOUtils.getOutputStream(IOUtils.getFileUrl(output.getPath("emissions_grid_per_day.%s", "avro").toString()), false));
+			dataFileWriter.append(avroData);
+		} catch (IOException e) {
+			throw new UncheckedIOException(e);
+		}
+	}
+
 	/**
 	 * Creates the data for the XY-Time plot. The time is fixed and the data is summarized over the run.
 	 * Currently only the CO2_Total Values is printed because Simwrapper can handle only one value.
@@ -254,7 +324,7 @@ private void writeRaster(Network network, Config config, EmissionsOnLinkEventHan
 
 		Raster raster = rasterMap.values().stream().findFirst().orElseThrow();
 
-		try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("emissions_grid_per_day.csv")),
+		try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("emissions_grid_per_day.%s", "csv")),
 			CSVFormat.DEFAULT.builder().setCommentMarker('#').build())) {
 
 			String crs = ProjectionUtils.getCRS(network);
@@ -281,8 +351,8 @@ private void writeRaster(Network network, Config config, EmissionsOnLinkEventHan
 					Coord coord = raster.getCoordForIndex(xi, yi);
 					double value = rasterMap.get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi);
 
-//					if (value == 0)
-//						continue;
+					if (value == 0)
+						continue;
 
 					printer.print(0.0);
 					printer.print(coord.getX());
@@ -310,7 +380,7 @@ private void writeTimeDependentRaster(Network network, Config config, EmissionsO
 
 		Raster raster = firstBin.values().stream().findFirst().orElseThrow();
 
-		try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("emissions_grid_per_hour.csv").toString()),
+		try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("emissions_grid_per_hour.%s", "csv").toString()),
 			CSVFormat.DEFAULT.builder().setCommentMarker('#').build())) {
 
 			String crs = ProjectionUtils.getCRS(network);
@@ -358,4 +428,65 @@ private void writeTimeDependentRaster(Network network, Config config, EmissionsO
 
 	}
 
+	private void writeTimeDependentAvroRaster(Network network, Config config, EmissionsOnLinkEventHandler emissionsEventHandler) {
+
+		TimeBinMap<Map<Pollutant, Raster>> timeBinMap = FastEmissionGridAnalyzer.processHandlerEmissionsPerTimeBin(emissionsEventHandler.getTimeBins(), network, gridSize, 20);
+
+		String crs = ProjectionUtils.getCRS(network);
+		if (crs == null)
+			crs = config.network().getInputCRS();
+		if (crs == null)
+			crs = config.global().getCoordinateSystem();
+
+		XYTData avroData = new XYTData();
+		avroData.setCrs(crs);
+
+		Map<Pollutant, Raster> rasterMap = FastEmissionGridAnalyzer.processHandlerEmissions(emissionsEventHandler.getLink2pollutants(), network, gridSize, 20);
+		List<Integer> xLength = rasterMap.values().stream().map(Raster::getXLength).distinct().toList();
+		List<Integer> yLength = rasterMap.values().stream().map(Raster::getYLength).distinct().toList();
+		Raster raster = rasterMap.values().stream().findFirst().orElseThrow();
+
+		List<Float> xCoords = new ArrayList<>();
+		List<Float> yCoords = new ArrayList<>();
+		Map<CharSequence, List<Float>> values = new HashMap<>();
+		List<Float> valuesList = new ArrayList<>();
+		List<Integer> times = new ArrayList<>();
+
+		for (TimeBinMap.TimeBin<Map<Pollutant, Raster>> timeBin : timeBinMap.getTimeBins()) {
+
+			boolean isFirst = times.isEmpty();
+
+			times.add((int) timeBin.getStartTime());
+
+			for (int xi = 0; xi < xLength.get(0); xi++) {
+				for (int yi = 0; yi < yLength.get(0); yi++) {
+					Coord coord = raster.getCoordForIndex(xi, yi);
+
+					if (xi == 0 && isFirst)
+						yCoords.add((float) coord.getY());
+					if (yi == 0 && isFirst)
+						xCoords.add((float) coord.getX());
+
+					valuesList.add((float) timeBin.getValue().get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi));
+				}
+			}
+		}
+
+		values.put(String.valueOf(Pollutant.CO2_TOTAL), valuesList);
+
+		avroData.setYCoords(yCoords);
+		avroData.setXCoords(xCoords);
+		avroData.setData(values);
+		avroData.setTimestamps(times);
+
+		DatumWriter<XYTData> datumWriter = new SpecificDatumWriter<>(XYTData.class);
+		try (DataFileWriter<XYTData> dataFileWriter = new DataFileWriter<>(datumWriter)) {
+			dataFileWriter.setCodec(CodecFactory.deflateCodec(9));
+			dataFileWriter.create(avroData.getSchema(), IOUtils.getOutputStream(IOUtils.getFileUrl(output.getPath("emissions_grid_per_hour.%s", "avro").toString()), false));
+			dataFileWriter.append(avroData);
+		} catch (IOException e) {
+			throw new UncheckedIOException(e);
+		}
+	}
+
 }

contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java

@@ -110,7 +110,8 @@ public Integer call() throws Exception {
 	private Config prepareConfig() {
 		Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString(), new NoiseConfigGroup());
 
-		config.vehicles().setVehiclesFile(ApplicationUtils.matchInput("vehicles", input.getRunDirectory()).toAbsolutePath().toString());
+		//it is important to match "output_vehicles" because otherwise dvrpVehicle files might be matched and the code crashes later
+		config.vehicles().setVehiclesFile(ApplicationUtils.matchInput("output_vehicles", input.getRunDirectory()).toAbsolutePath().toString());
 		config.network().setInputFile(ApplicationUtils.matchInput("network", input.getRunDirectory()).toAbsolutePath().toString());
 		config.transit().setTransitScheduleFile(null);
 		config.transit().setVehiclesFile(null);