From 1a75b4d9995a32bc67ad671c333e156c7c214b57 Mon Sep 17 00:00:00 2001 From: simei94 <67737999+simei94@users.noreply.github.com> Date: Wed, 15 May 2024 19:15:01 +0200 Subject: [PATCH 1/2] read first 5 lines of csv to check delimiter (#3271) --- .../application/options/CsvOptions.java | 54 +++++++++++++------ .../application/options/CsvOptionsTest.java | 4 +- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/contribs/application/src/main/java/org/matsim/application/options/CsvOptions.java b/contribs/application/src/main/java/org/matsim/application/options/CsvOptions.java index 1b2d6575450..74c25f3a96b 100644 --- a/contribs/application/src/main/java/org/matsim/application/options/CsvOptions.java +++ b/contribs/application/src/main/java/org/matsim/application/options/CsvOptions.java @@ -23,7 +23,7 @@ public final class CsvOptions { @CommandLine.Option(names = "--csv-format", description = "CSV Format", defaultValue = "Default") private CSVFormat.Predefined csvFormat; - @CommandLine.Option(names = "--csv-delimiter", description = "CSV Delimiter", required = false) + @CommandLine.Option(names = "--csv-delimiter", description = "CSV Delimiter") private Character csvDelimiter; @CommandLine.Option(names = "--csv-charset", description = "CSV input encoding", defaultValue = "UTF8") @@ -56,24 +56,48 @@ public CsvOptions(CSVFormat.Predefined csvFormat, Character csvDelimiter, Charse */ public static Character detectDelimiter(String path) throws IOException { try (BufferedReader reader = IOUtils.getBufferedReader(path)) { - String firstLine = reader.readLine(); + int[] comma = new int[5]; + int[] semicolon = new int[5]; + int[] tab = new int[5]; + String[] lines = new String[5]; + +// check five first lines for separator chars. It might be that the csv file has additional info in the first x lines (e.g. EPSG) + for (int i = 0; i < 5; i++) { + lines[i] = reader.readLine(); + if (lines[i] == null) { + comma[i] = 0; + semicolon[i] = 0; + tab[i] = 0; + } else { + comma[i] = StringUtils.countMatches(lines[i], ","); + semicolon[i] = StringUtils.countMatches(lines[i], ";"); + tab[i] = StringUtils.countMatches(lines[i], "\t"); + } + } - int comma = StringUtils.countMatches(firstLine, ","); - int semicolon = StringUtils.countMatches(firstLine, ";"); - int tab = StringUtils.countMatches(firstLine, "\t"); + Integer index = null; - if (comma == 0 && semicolon == 0 && tab == 0) { - throw new IllegalArgumentException("No delimiter found in the first line of the file."); + for (int i = 0; i < comma.length - 1; i++) { +// only check next index if line with separators was not found + if (index == null) { + if (!(comma[i] == 0 && semicolon[i] == 0 && tab[i] == 0)) { + index = i; + } + } } - // Comma is preferred as the more likely format - if (comma >= semicolon && comma >= tab) { - return ','; - } else if (tab >= semicolon) - return '\t'; - else - return ';'; - } + if (index == null) { + throw new IllegalArgumentException("No delimiter found in the first line of the file."); + } else { + // Comma is preferred as the more likely format + if (comma[index] >= semicolon[index] && comma[index] >= tab[index]) { + return ','; + } else if (tab[index] >= semicolon[index]) + return '\t'; + else + return ';'; + } + } } /** diff --git a/contribs/application/src/test/java/org/matsim/application/options/CsvOptionsTest.java b/contribs/application/src/test/java/org/matsim/application/options/CsvOptionsTest.java index a406deca398..70452d23138 100644 --- a/contribs/application/src/test/java/org/matsim/application/options/CsvOptionsTest.java +++ b/contribs/application/src/test/java/org/matsim/application/options/CsvOptionsTest.java @@ -37,10 +37,12 @@ void output() throws IOException { printer.printRecord("header", "column"); printer.printRecord("1", "2"); + printer.printRecord("3", "4"); + printer.printRecord("5", "6"); printer.close(); assertThat(tmp) - .hasContent("header" + delimiter + "column\n1" + delimiter + "2"); + .hasContent("header" + delimiter + "column\n1" + delimiter + "2" + "\n3" + delimiter + "4" + "\n5" + delimiter + "6"); assertThat(delimiter).isEqualTo(CsvOptions.detectDelimiter(tmp.toString()).toString()); } From a9cdc5b4d77d3b3211ed9567f63ae2ae2111fc46 Mon Sep 17 00:00:00 2001 From: Paul Heinrich Date: Thu, 16 May 2024 15:22:07 +0200 Subject: [PATCH 2/2] inline abstract module attributes and get rid of them --- .../matsim/core/controler/AbstractModule.java | 53 +++++++------------ 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/matsim/src/main/java/org/matsim/core/controler/AbstractModule.java b/matsim/src/main/java/org/matsim/core/controler/AbstractModule.java index d6d3eb94efa..fcf5feeefa8 100644 --- a/matsim/src/main/java/org/matsim/core/controler/AbstractModule.java +++ b/matsim/src/main/java/org/matsim/core/controler/AbstractModule.java @@ -76,25 +76,10 @@ public abstract class AbstractModule implements Module { private Binder binder; - private Multibinder eventHandlerMultibinder; - private Multibinder controlerListenerMultibinder; - - /** - * Contents retrieved (I think) by injected method QSim#addQueueSimulationListeners(...). Is not public, and therefore cannot be referenced from here. - *
- * I think that that method will be called every time the mobsim will be constructed. If the injected classes are singletons, they will - * presumably be re-used, otherwise they will be newly constructed. - */ - private Multibinder mobsimListenerMultibinder; - - private Multibinder snapshotWriterMultibinder; - private MapBinder, AttributeConverter> attributeConverterMapBinder; - private Multibinder qsimModulesMultibinder; @Inject com.google.inject.Injector bootstrapInjector; private Config config; - private Multibinder qsimOverridingModulesMultibinder; public AbstractModule() { // config will be injected later @@ -121,17 +106,13 @@ public final void configure(Binder binder) { private void initializeMultibinders() { // We do need to make these calls here in order to register the multi binders. Otherwise, guice doesn't know, that they exist. In particular, // if none of the corresponding addXXXBinding methods was called, the set binder would not be registered, and guice would complain. - this.mobsimListenerMultibinder = Multibinder.newSetBinder(this.binder, MobsimListener.class); - this.snapshotWriterMultibinder = Multibinder.newSetBinder(this.binder, SnapshotWriter.class); - this.eventHandlerMultibinder = Multibinder.newSetBinder(this.binder, EventHandler.class); - this.controlerListenerMultibinder = Multibinder.newSetBinder(this.binder, ControlerListener.class); - this.attributeConverterMapBinder = - MapBinder.newMapBinder( - this.binder, - new TypeLiteral>(){}, - new TypeLiteral>() {} ); - this.qsimModulesMultibinder = Multibinder.newSetBinder(this.binder, AbstractQSimModule.class); - this.qsimOverridingModulesMultibinder = Multibinder.newSetBinder( this.binder, AbstractQSimModule.class, Names.named( "overridesFromAbstractModule" ) ); + Multibinder.newSetBinder(this.binder, MobsimListener.class); + Multibinder.newSetBinder(this.binder, SnapshotWriter.class); + Multibinder.newSetBinder(this.binder, EventHandler.class); + Multibinder.newSetBinder(this.binder, ControlerListener.class); + MapBinder.newMapBinder(this.binder, new TypeLiteral>(){}, new TypeLiteral>() {} ); + Multibinder.newSetBinder(this.binder, AbstractQSimModule.class); + Multibinder.newSetBinder( this.binder, AbstractQSimModule.class, Names.named( "overridesFromAbstractModule" ) ); } public abstract void install(); @@ -146,21 +127,21 @@ protected final void install(Module module) { } protected final LinkedBindingBuilder addEventHandlerBinding() { - return eventHandlerMultibinder.addBinding(); + return Multibinder.newSetBinder(this.binder, EventHandler.class).addBinding(); } protected final void installQSimModule(AbstractQSimModule qsimModule) { - qsimModulesMultibinder.addBinding().toInstance(qsimModule); + Multibinder.newSetBinder(this.binder, AbstractQSimModule.class).addBinding().toInstance(qsimModule); } protected final void installOverridingQSimModule(AbstractQSimModule qsimModule) { - qsimOverridingModulesMultibinder.addBinding().toInstance(qsimModule); + Multibinder.newSetBinder( this.binder, AbstractQSimModule.class, Names.named( "overridesFromAbstractModule" ) ).addBinding().toInstance(qsimModule); } /** * @see ControlerListener */ protected final LinkedBindingBuilder addControlerListenerBinding() { - return controlerListenerMultibinder.addBinding(); + return Multibinder.newSetBinder(this.binder, ControlerListener.class).addBinding(); } /** @@ -182,16 +163,22 @@ protected final com.google.inject.binder.LinkedBindingBuilder + * I think that that method will be called every time the mobsim will be constructed. If the injected classes are singletons, they will + * presumably be re-used, otherwise they will be newly constructed. + */ protected final com.google.inject.binder.LinkedBindingBuilder addMobsimListenerBinding() { - return mobsimListenerMultibinder.addBinding(); + return Multibinder.newSetBinder(this.binder, MobsimListener.class).addBinding(); } protected final com.google.inject.binder.LinkedBindingBuilder addSnapshotWriterBinding() { - return snapshotWriterMultibinder.addBinding(); + return Multibinder.newSetBinder(this.binder, SnapshotWriter.class).addBinding(); } protected final LinkedBindingBuilder> addAttributeConverterBinding(final Class clazz ) { - return attributeConverterMapBinder.addBinding( clazz ); + return MapBinder.newMapBinder(this.binder, new TypeLiteral>(){}, new TypeLiteral>() {} ).addBinding( clazz ); } /** * @deprecated better use {@link #addTravelDisutilityFactoryBinding(String)}.