diff --git a/pom.xml b/pom.xml index 0a36d4fb3c..a58a4b38ac 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ net.sumaris sumaris-pod - 2.0.6 + 2.1.0 pom SUMARiS SUMARiS :: Maven parent diff --git a/sumaris-core-shared/pom.xml b/sumaris-core-shared/pom.xml index fdc9768d7e..60f98dba4f 100644 --- a/sumaris-core-shared/pom.xml +++ b/sumaris-core-shared/pom.xml @@ -4,7 +4,7 @@ net.sumaris sumaris-pod - 2.0.6 + 2.1.0 sumaris-core-shared diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfiguration.java b/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfiguration.java index 9c04767ed1..90e15a9772 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfiguration.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfiguration.java @@ -27,9 +27,12 @@ import com.google.common.base.Charsets; import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import net.sumaris.core.dao.technical.Daos; @@ -53,6 +56,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.*; +import java.util.stream.Collectors; import static org.nuiton.i18n.I18n.t; @@ -251,7 +255,11 @@ protected void addAlias(ApplicationConfig applicationConfig) { applicationConfig.addAlias("-d", "--option", SumarisConfigurationOption.CLI_DAEMONIZE.getKey(), "true"); applicationConfig.addAlias("--output", "--option", SumarisConfigurationOption.CLI_OUTPUT_FILE.getKey()); applicationConfig.addAlias("-f", "--option", SumarisConfigurationOption.CLI_FORCE_OUTPUT.getKey(), "true"); + applicationConfig.addAlias("--program", "--option", SumarisConfigurationOption.CLI_FILTER_PROGRAM_LABEL.getKey()); applicationConfig.addAlias("--year", "--option", SumarisConfigurationOption.CLI_FILTER_YEAR.getKey()); + applicationConfig.addAlias("--trip", "--option", SumarisConfigurationOption.CLI_FILTER_TRIP_IDS.getKey()); + applicationConfig.addAlias("--operation", "--option", SumarisConfigurationOption.CLI_FILTER_OPERATION_IDS.getKey()); + // TODO : add --sale, --observed-location --landing } @@ -906,9 +914,17 @@ public Integer getCliFilterYear() { return year == -1 ? null : year; } - public Integer getCliFilterTripId() { - int tripId = applicationConfig.getOptionAsInt(SumarisConfigurationOption.CLI_FILTER_TRIP_ID.getKey()); - return tripId == -1 ? null : tripId; + public String getCliFilterProgramLabel() { + String programLabel = applicationConfig.getOption(SumarisConfigurationOption.CLI_FILTER_PROGRAM_LABEL.getKey()); + return StringUtils.isBlank(programLabel) ? null : programLabel; + } + + public List getCliFilterTripIds() { + return getConfigurationOptionAsNumbers(SumarisConfigurationOption.CLI_FILTER_TRIP_IDS.getKey()); + } + + public List getCliFilterOperationIds() { + return getConfigurationOptionAsNumbers(SumarisConfigurationOption.CLI_FILTER_OPERATION_IDS.getKey()); } /** @@ -1090,4 +1106,39 @@ public String getColumnDefaultValue(String tableName, String columnName) { return applicationConfig.getOption("sumaris." + tableName.toUpperCase() + "." + columnName.toUpperCase() + ".defaultValue"); } + + public List getConfigurationOptionAsNumbers(String optionKey) { + List result = (List) complexOptionsCache.getIfPresent(optionKey); + + // Not exists in cache + if (result == null) { + String ids = applicationConfig.getOption(optionKey); + if (StringUtils.isBlank(ids)) { + result = ImmutableList.of(); + } else { + final List invalidIds = Lists.newArrayList(); + result = Splitter.on(",").omitEmptyStrings().trimResults() + .splitToList(ids) + .stream() + .map(id -> { + try { + return Integer.parseInt(id); + } catch (Exception e) { + invalidIds.add(id); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(invalidIds)) { + log.error("Skipping invalid values found in configuration option '{}': {}", optionKey, invalidIds); + } + } + + // Add to cache + complexOptionsCache.put(optionKey, result); + } + return result; + } } diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfigurationOption.java b/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfigurationOption.java index 3c394f6b37..58b94e9418 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfigurationOption.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/config/SumarisConfigurationOption.java @@ -528,13 +528,24 @@ public enum SumarisConfigurationOption implements ConfigOptionDef { Integer.class, false), - CLI_FILTER_TRIP_ID( - "sumaris.cli.filter.tripId", - n("sumaris.config.option.cli.filter.tripId.description"), - "-1", + CLI_FILTER_PROGRAM_LABEL( + "sumaris.cli.filter.programLabel", + n("sumaris.config.option.cli.filter.programLabel.description"), + "", + String.class, + false), + CLI_FILTER_TRIP_IDS( + "sumaris.cli.filter.tripIds", + n("sumaris.config.option.cli.filter.tripIds.description"), + "", + Integer.class, + false), + CLI_FILTER_OPERATION_IDS( + "sumaris.cli.filter.operationIds", + n("sumaris.config.option.cli.filter.operationIds.description"), + "", Integer.class, false), - CSV_SEPARATOR( "sumaris.csv.separator", n("sumaris.config.option.csv.separator.description"), @@ -663,26 +674,6 @@ public enum SumarisConfigurationOption implements ConfigOptionDef { Boolean.class, false), - ENABLE_BATCH_TAXON_NAME( - "sumaris.trip.operation.batch.taxonName.enable", - n("sumaris.config.option.trip.operation.batch.taxonName.enable.description"), - Boolean.TRUE.toString(), - Boolean.class, - false), - - ENABLE_BATCH_TAXON_GROUP( - "sumaris.trip.operation.batch.taxonGroup.enable", - n("sumaris.config.option.trip.operation.batch.taxonGroup.enable.description"), - Boolean.TRUE.toString(), - Boolean.class, - false), - - BATCH_TAXON_GROUP_LABELS_NO_WEIGHT( - "sumaris.trip.operation.batch.taxonGroups.noWeight", - n("sumaris.config.option.trip.operation.batch.taxonGroups.noWeight.description"), - "", - String.class, - false), DB_ADAGIO_SCHEMA( "sumaris.persistence.adagio.schema", diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/data/IPosition.java b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/data/IPosition.java new file mode 100644 index 0000000000..3186ea9435 --- /dev/null +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/data/IPosition.java @@ -0,0 +1,36 @@ +package net.sumaris.core.dao.data; + +/*- + * #%L + * SUMARiS:: Core + * %% + * Copyright (C) 2018 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +public interface IPosition { + + + Double getLatitude(); + + void setLatitude(Double latitude); + + Double getLongitude(); + + void setLongitude(Double longitude); + +} diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/data/Positions.java b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/data/Positions.java new file mode 100644 index 0000000000..dae5f37dde --- /dev/null +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/data/Positions.java @@ -0,0 +1,48 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.dao.data; + +public abstract class Positions { + + protected Positions() { + // Helper class + } + + public static boolean isNotNullAndValid(IPosition position) { + + if (position == null || position.getLatitude() == null || position.getLongitude() == null) return false; + + // Invalid lat/lon + if (position.getLatitude() < -90 || position.getLatitude() > 90 + || position.getLongitude() < -180 || position.getLongitude() > 180) { + return false; + } + + // OK: valid + return true; + } + + public static boolean isNullOrInvalid(IPosition position) { + return !isNotNullAndValid(position); + } +} diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/referential/location/Locations.java b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/referential/location/Locations.java index de926d156e..300747a0cf 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/referential/location/Locations.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/referential/location/Locations.java @@ -220,12 +220,11 @@ public static Geometry getGeometryFromMinuteSquareLabel(String label, int minute * Compute the statistical rectangle from the 10x10 square. * (See doc: square_10.md) * @param squareLabel 10x10 square + * @return null if invalid square label */ public static String convertMinuteSquareToRectangle(final String squareLabel, final int minute) { - String calculRectangle = ""; - if (squareLabel == null || squareLabel.length() != 8) { - return calculRectangle; + return null; } int cadran = Integer.parseInt(squareLabel.substring(0, 1)); diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/schema/DatabaseSchemaDaoImpl.java b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/schema/DatabaseSchemaDaoImpl.java index 66d0122c8b..68864e509f 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/schema/DatabaseSchemaDaoImpl.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/schema/DatabaseSchemaDaoImpl.java @@ -51,10 +51,12 @@ import org.hibernate.HibernateException; import org.hibernate.Session; import org.hibernate.SessionFactory; +import org.hibernate.annotations.common.reflection.MetadataProvider; import org.hibernate.boot.Metadata; import org.hibernate.boot.MetadataSources; import org.hibernate.boot.registry.StandardServiceRegistryBuilder; import org.hibernate.cfg.Environment; +import org.hibernate.dialect.Dialect; import org.hibernate.dialect.Oracle10gDialect; import org.hibernate.tool.hbm2ddl.SchemaExport; import org.hibernate.tool.hbm2ddl.SchemaUpdate; @@ -70,6 +72,7 @@ import org.springframework.jdbc.datasource.DataSourceUtils; import org.springframework.stereotype.Repository; +import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.persistence.*; import javax.persistence.metamodel.Attribute; @@ -181,17 +184,18 @@ public void generateCreateSchemaFile(String filename) { /** {@inheritDoc} */ @Override public void generateCreateSchemaFile(String filename, boolean doExecute, boolean withDrop, boolean withCreate) { + Metadata metadata = getMetadata(); new SchemaExport() .setDelimiter(";") .setOutputFile(filename) .execute(EnumSet.of(TargetType.SCRIPT), withDrop ? SchemaExport.Action.BOTH : SchemaExport.Action.CREATE, - getMetadata() + metadata ); // Add table and columns comment try { - appendRemarks(filename); + appendRemarks(filename, metadata); } catch (SQLException | IOException e) { throw new SumarisTechnicalException("Error when appending comments on file", e); } @@ -671,38 +675,51 @@ public boolean test(String input) { return result; } - protected Metadata getMetadata() { - - SumarisConfiguration config = getConfig(); - - Map sessionSettings; - SessionFactory session = null; + protected Map getSessionSettings(boolean configureHibernateConnectionProvider) { if (getEntityManager() != null) { - session = getEntityManager().unwrap(Session.class).getSessionFactory(); + SessionFactory session = getEntityManager().unwrap(Session.class).getSessionFactory(); + + if (session != null) { + // Allow Hibernate to get the connection + if (configureHibernateConnectionProvider) { + HibernateConnectionProvider.setDataSource(getDataSource()); + } + return session.getProperties(); + } } - if (session == null) { + + return getSessionSettings(getConfig().getConnectionProperties(), configureHibernateConnectionProvider); + } + protected Map getSessionSettings(@Nullable Properties connectionProperties, boolean configureHibernateConnectionProvider) { + + + // Allow Hibernate to get the connection + if (configureHibernateConnectionProvider) { try { - // To be able to retrieve connection from datasource - Connection conn = Daos.createConnection(config.getConnectionProperties()); + Connection conn = Daos.createConnection(connectionProperties); HibernateConnectionProvider.setConnection(conn); } catch (SQLException e) { - throw new SumarisTechnicalException("Could not open connection: " + config.getJdbcURL()); + throw new SumarisTechnicalException("Could not open connection: " + connectionProperties.get(Environment.URL)); } + } - sessionSettings = Maps.newHashMap(); - sessionSettings.put(Environment.DIALECT, config.getHibernateDialect()); - sessionSettings.put(Environment.DRIVER, config.getJdbcDriver()); - sessionSettings.put(Environment.URL, config.getJdbcURL()); - sessionSettings.put(Environment.IMPLICIT_NAMING_STRATEGY, HibernateImplicitNamingStrategy.class.getName()); + Map sessionSettings = Maps.newHashMap(); + sessionSettings.put(Environment.DIALECT, connectionProperties.get(Environment.DIALECT)); + sessionSettings.put(Environment.DRIVER, connectionProperties.get(Environment.DRIVER)); + sessionSettings.put(Environment.URL, connectionProperties.get(Environment.URL)); + sessionSettings.put(Environment.IMPLICIT_NAMING_STRATEGY, HibernateImplicitNamingStrategy.class.getName()); - sessionSettings.put(Environment.PHYSICAL_NAMING_STRATEGY, HibernatePhysicalNamingStrategy.class.getName()); - //sessionSettings.put(Environment.PHYSICAL_NAMING_STRATEGY, "org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy"); - } - else { - // To be able to retrieve connection from datasource - HibernateConnectionProvider.setDataSource(getDataSource()); - sessionSettings = session.getProperties(); - } + sessionSettings.put(Environment.PHYSICAL_NAMING_STRATEGY, HibernatePhysicalNamingStrategy.class.getName()); + //sessionSettings.put(Environment.PHYSICAL_NAMING_STRATEGY, "org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy"); + + return sessionSettings; + } + + protected Metadata getMetadata() { + return getMetadata(getSessionSettings(true)); + } + + protected Metadata getMetadata(Map sessionSettings) { MetadataSources metadata = new MetadataSources(new StandardServiceRegistryBuilder() .applySettings(sessionSettings) @@ -754,37 +771,53 @@ private String getTimezoneQuery(Connection connection) { throw new SumarisTechnicalException("Cannot generate Timezone query : not implemented for this database type"); } - private void appendRemarks(String filename) throws SQLException, IOException { + private void appendRemarks(String filename, Metadata metadata) throws SQLException, IOException { List linesToAppend = new ArrayList<>(); - String schemaName = dataSource.getConnection().getSchema(); - getEntityManager().getEntityManagerFactory().getMetamodel().getEntities().stream() - .sorted(Comparator.comparing(EntityType::getName)) - .forEach(entityType -> { - Table table = entityType.getJavaType().getAnnotation(Table.class); - Comment tableComment = entityType.getJavaType().getAnnotation(Comment.class); - if (table != null) { - if (tableComment != null) { - Optional.ofNullable(getTableCommentQuery(schemaName, table.name(), tableComment.value())).ifPresent(linesToAppend::add); - } - // iterate attributes - entityType.getAttributes().stream() - .sorted(Comparator.comparing(Attribute::getName)) - .forEach(attribute -> { - if (attribute.getJavaMember() instanceof Field) { - Field field = (Field) attribute.getJavaMember(); - Column column = field.getAnnotation(Column.class); - JoinColumn joinColumn = field.getAnnotation(JoinColumn.class); - Comment columnComment = field.getAnnotation(Comment.class); - String columnName = Optional.ofNullable(column).map(Column::name).orElse( - Optional.ofNullable(joinColumn).map(JoinColumn::name).orElse(null) - ); - if (columnName != null && columnComment != null) { - Optional.ofNullable(getColumnCommentQuery(schemaName, table.name(), columnName, columnComment.value())).ifPresent(linesToAppend::add); + + // Prepare hibernate connection (will be used by buildSessionFactory() ) + Connection connection; + if (dataSource != null) { + connection = DataSourceUtils.getConnection(dataSource); + HibernateConnectionProvider.setDataSource(dataSource); + } + else { + connection = Daos.createConnection(getConfig().getConnectionProperties()); + HibernateConnectionProvider.setConnection(connection); + } + + try { + String schemaName = connection.getSchema(); + metadata.buildSessionFactory().getMetamodel().getEntities().stream() + .sorted(Comparator.comparing(EntityType::getName)) + .forEach(entityType -> { + Table table = entityType.getJavaType().getAnnotation(Table.class); + Comment tableComment = entityType.getJavaType().getAnnotation(Comment.class); + if (table != null) { + if (tableComment != null) { + Optional.ofNullable(getTableCommentQuery(schemaName, table.name(), tableComment.value())).ifPresent(linesToAppend::add); + } + // iterate attributes + entityType.getAttributes().stream() + .sorted(Comparator.comparing(Attribute::getName)) + .forEach(attribute -> { + if (attribute.getJavaMember() instanceof Field) { + Field field = (Field) attribute.getJavaMember(); + Column column = field.getAnnotation(Column.class); + JoinColumn joinColumn = field.getAnnotation(JoinColumn.class); + Comment columnComment = field.getAnnotation(Comment.class); + String columnName = Optional.ofNullable(column).map(Column::name).orElse( + Optional.ofNullable(joinColumn).map(JoinColumn::name).orElse(null) + ); + if (columnName != null && columnComment != null) { + Optional.ofNullable(getColumnCommentQuery(schemaName, table.name(), columnName, columnComment.value())).ifPresent(linesToAppend::add); + } } - } - }); - } - }); + }); + } + }); + } finally { + DataSourceUtils.releaseConnection(connection, dataSource); + } if (!linesToAppend.isEmpty()) { Files.write(Paths.get(filename), linesToAppend, StandardOpenOption.APPEND); @@ -792,16 +825,40 @@ private void appendRemarks(String filename) throws SQLException, IOException { } private String getTableCommentQuery(String schemaName, String tableName, String comment) { - if (Daos.getDialect(getEntityManager()) instanceof Oracle10gDialect) { + if (isOracleDialect()) { return String.format("comment on table %s.%s is '%s';", schemaName, tableName, comment.replaceAll("'", "''")); } return null; } private String getColumnCommentQuery(String schemaName, String tableName, String columnName, String comment) { - if (Daos.getDialect(getEntityManager()) instanceof Oracle10gDialect) { + if (isOracleDialect()) { return String.format("comment on column %s.%s.%s is '%s';", schemaName, tableName, columnName, comment.replaceAll("'", "''")); } return null; } + + private boolean isOracleDialect() { + try { + return getHibernateDialect() instanceof Oracle10gDialect; + } + catch (InstantiationException e) { + throw new SumarisTechnicalException(e); + } + } + + private Dialect getHibernateDialect() throws InstantiationException { + EntityManager em = getEntityManager(); + if (em != null) { + return Daos.getDialect(em); + } + String dialectClassName = getConfig().getHibernateDialect(); + if (StringUtils.isBlank(dialectClassName)) return null; + try { + Class dialectClass = Class.forName(getConfig().getHibernateDialect()); + return ((Dialect) dialectClass.getConstructor().newInstance()); + } catch (Exception e) { + throw new InstantiationException(String.format("Cannot instantiate class %s: %s", dialectClassName, e.getMessage())); + } + } } diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/technical/cache/CacheTTL.java b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/technical/cache/CacheTTL.java index 5c959b0a3b..46014c8c6a 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/dao/technical/cache/CacheTTL.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/dao/technical/cache/CacheTTL.java @@ -47,7 +47,13 @@ public enum CacheTTL { MEDIUM(60 * 60), // 1 h LONG(12 * 60 * 60), // 12 h - ETERNAL(24 * 60 * 60) // 1 day + DAY(24 * 60 * 60), // 1 day + + // 2 days ~ almost eternal! :) + // - We do not use infinite duration, because some SQL operations can have been done in the DB (SQL scripts, etc.) + // - We need more than one day, to avoid cache reload/timeout each morning, when user comme back to the office + ETERNAL(24 * 60 * 60 * 2) + ; private Duration value; diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/model/ITreeNodeEntity.java b/sumaris-core-shared/src/main/java/net/sumaris/core/model/ITreeNodeEntity.java index 612c65d5d0..3a2554f9c2 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/model/ITreeNodeEntity.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/model/ITreeNodeEntity.java @@ -50,6 +50,11 @@ default boolean hasChildren() { return CollectionUtils.isNotEmpty(getChildren()); } + @JsonIgnore + default boolean isLeaf() { + return CollectionUtils.isEmpty(getChildren()); + } + @JsonIgnore default boolean hasParent() { return getParent() != null; diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/model/annotation/EntityEnums.java b/sumaris-core-shared/src/main/java/net/sumaris/core/model/annotation/EntityEnums.java index bc73b47b71..d3409f35f0 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/model/annotation/EntityEnums.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/model/annotation/EntityEnums.java @@ -22,14 +22,18 @@ package net.sumaris.core.model.annotation; +import com.google.common.base.Joiner; import com.google.common.collect.Lists; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import net.sumaris.core.config.SumarisConfiguration; import net.sumaris.core.util.Beans; import net.sumaris.core.util.StringUtils; +import org.apache.commons.collections4.CollectionUtils; import org.nuiton.config.ConfigOptionDef; +import org.nuiton.i18n.I18n; import org.reflections.Reflections; import java.util.List; @@ -98,6 +102,37 @@ public static ConfigOptionDef[] getEntityEnumAsOptions(SumarisConfiguration conf return options.toArray(new ConfigOptionDef[options.size()]); } + /** + * Check if an entity enumeration (@EntityEnum) has been resolved. Typically, if id!=-1 + * @param enumerations + */ + public static void checkResolved(String i18nMessageKey, @NonNull IEntityEnum... enumerations) { + List invalidEnumerationNames = Beans.getStream(enumerations) + .filter(EntityEnums::isUnresolved) + .map(EntityEnums::name) + .toList(); + + if (CollectionUtils.isNotEmpty(invalidEnumerationNames)) { + throw new IllegalArgumentException(I18n.t(i18nMessageKey, Joiner.on(",").join(invalidEnumerationNames))); + } + } + + public static void checkResolved(@NonNull IEntityEnum... enumerations) { + checkResolved("sumaris.error.enumeration.unresolved", enumerations); + } + + public static String name(IEntityEnum enumeration) { + return Beans.getProperty((Object)enumeration, "name").toString(); + } + + public static boolean isUnresolved(IEntityEnum enumeration) { + try { + Object id = Beans.getProperty((Object)enumeration, "id"); + return id == null || ((id instanceof Integer) && ((Integer)id) == UNRESOLVED_ENUMERATION_ID); + } catch (Exception e) { + return false; + } + } @Data @AllArgsConstructor diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/model/annotation/IEntityEnum.java b/sumaris-core-shared/src/main/java/net/sumaris/core/model/annotation/IEntityEnum.java new file mode 100644 index 0000000000..7efeffa781 --- /dev/null +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/model/annotation/IEntityEnum.java @@ -0,0 +1,27 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.model.annotation; + +public interface IEntityEnum { + +} \ No newline at end of file diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/service/referential/location/LocationByPositionService.java b/sumaris-core-shared/src/main/java/net/sumaris/core/service/referential/location/LocationByPositionService.java index c36090c2bb..e15a1f0122 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/service/referential/location/LocationByPositionService.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/service/referential/location/LocationByPositionService.java @@ -26,6 +26,8 @@ import org.springframework.transaction.annotation.Transactional; +import java.util.Optional; + /** * Service used to access locations * @@ -39,9 +41,9 @@ public interface LocationByPositionService { * * @param latitude a latitude (in decimal degrees - WG84) * @param longitude a longitude (in decimal degrees - WG84) - * @return A location label (corresponding to a statistical rectangle), or null if no statistical rectangle exists for this position + * @return A location label (corresponding to a statistical rectangle), or empty if no statistical rectangle exists for this position */ - String getLocationLabelByLatLong(Number latitude, Number longitude); + Optional getStatisticalRectangleLabelByLatLong(Number latitude, Number longitude); /** * Return a location Id, from a longitude and a latitude (in decimal degrees - WG84). @@ -49,7 +51,7 @@ public interface LocationByPositionService { * * @param latitude a latitude (in decimal degrees - WG84) * @param longitude a longitude (in decimal degrees - WG84) - * @return A location Id (corresponding to a statistical rectangle), or null if no statistical rectangle exists for this position + * @return A location Id (corresponding to a statistical rectangle), or empty if no statistical rectangle exists for this position */ - Integer getLocationIdByLatLong(Number latitude, Number longitude); + Optional getStatisticalRectangleIdByLatLong(Number latitude, Number longitude); } diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Beans.java b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Beans.java index d78cfb7316..54de91eb17 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Beans.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Beans.java @@ -462,14 +462,14 @@ public static Comparator unsortedComparator() { return (o1, o2) -> 0; } - public static T clone(T source, Class sourceClass) { - T target = newInstance(sourceClass); + public static R clone(T source, Class resultClass) { + R target = newInstance(resultClass); copyProperties(source, target); return target; } - public static T clone(T source, Class sourceClass, String... excludedPropertyNames) { - T target = newInstance(sourceClass); + public static R clone(T source, Class resultClass, String... excludedPropertyNames) { + R target = newInstance(resultClass); copyProperties(source, target, excludedPropertyNames); return target; } @@ -509,7 +509,7 @@ public static void copyProperties(S source, T target, String... exceptPro PropertyDescriptor targetDescriptor = targetProperties.get(pd.getName()); boolean ignored = targetDescriptor == null || !targetDescriptor.getPropertyType().isAssignableFrom(pd.getPropertyType()) - || Collection.class.isAssignableFrom(pd.getPropertyType()) + || Collection.class.isAssignableFrom(pd.getPropertyType()) // Ignore List, Collection, etc || targetDescriptor.getWriteMethod() == null; return ignored; }) diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Dates.java b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Dates.java index f182afe4f4..472866218c 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Dates.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Dates.java @@ -25,6 +25,7 @@ */ import com.google.common.base.Preconditions; +import lombok.NonNull; import net.sumaris.core.exception.SumarisTechnicalException; import org.apache.commons.lang3.StringUtils; import org.nuiton.util.DateUtil; @@ -56,8 +57,7 @@ public class Dates extends org.apache.commons.lang3.time.DateUtils{ * @param amount the amount to remove, in month * @return a new date (= the given date - amount in month) */ - public static Date removeMonth(Date date, int amount) { - Preconditions.checkNotNull(date); + public static Date removeMonth(@NonNull Date date, int amount) { Preconditions.checkArgument(amount > 0); // Compute the start date @@ -69,6 +69,28 @@ public static Date removeMonth(Date date, int amount) { return calendar.getTime(); } + /** + * Extract month of a date + * @param date a not null date + * @return 0 for january, 11 for december + */ + public static Integer getMonth(@NonNull Date date) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(date.getTime()); + return calendar.get(Calendar.MONTH); + } + + /** + * Extract month of a date + * @param date a not null date + * @return 1 for january, 12 for december + */ + public static Integer getYear(@NonNull Date date) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(date.getTime()); + return calendar.get(Calendar.YEAR); + } + /** * Get the number of days between two dates * diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Geometries.java b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Geometries.java index f45a71b51f..13d87baf00 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Geometries.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Geometries.java @@ -261,4 +261,6 @@ public static String getDistanceInMilles(Number distance) { } return distanceText; } + + } diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Numbers.java b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Numbers.java index ef1ec60525..372175ad8b 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/util/Numbers.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/util/Numbers.java @@ -22,6 +22,9 @@ package net.sumaris.core.util; +import javax.annotation.Nullable; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.Locale; @@ -48,4 +51,31 @@ public static String format(Number value) { formatter.setDecimalFormatSymbols(new DecimalFormatSymbols(Locale.FRANCE)); return formatter.format(value); } + + public static BigDecimal firstNotNullAsBigDecimal(Number... values) { + for (Number v: values) { + if (v != null) { + if (v instanceof BigDecimal) return (BigDecimal)v; + return new BigDecimal(v.doubleValue()); + } + } + return null; + } + + public static Double asDouble(@Nullable BigDecimal value) { + if (value != null) return value.doubleValue(); + return null; + } + + public static double doubleValue(@Nullable BigDecimal value, double defaultValue) { + if (value != null) return value.doubleValue(); + return defaultValue; + } + + public static Double round(BigDecimal value, int scale) { + if (value != null) return value + .divide(new BigDecimal(1d), scale, RoundingMode.HALF_UP) + .doubleValue(); + return null; + } } diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/util/UnicodeChars.java b/sumaris-core-shared/src/main/java/net/sumaris/core/util/UnicodeChars.java index 09361901ae..f24eaa6698 100644 --- a/sumaris-core-shared/src/main/java/net/sumaris/core/util/UnicodeChars.java +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/util/UnicodeChars.java @@ -32,4 +32,6 @@ public class UnicodeChars { public static final String SUM = "\u2211"; public static final String ARROW_DOWN = "\uA71C"; + + public static final String ARROW_UP = "\uA71B"; } diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/util/conversion/UnitConversions.java b/sumaris-core-shared/src/main/java/net/sumaris/core/util/conversion/UnitConversions.java new file mode 100644 index 0000000000..240095ef8a --- /dev/null +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/util/conversion/UnitConversions.java @@ -0,0 +1,53 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.util.conversion; + +import lombok.NonNull; + +public abstract class UnitConversions { + protected UnitConversions() { + // helper class + } + + public static double weightToKgConversion(@NonNull String unitSymbol) { + return switch (unitSymbol) { + case "t" -> 1000; + case "kg" -> 1; + case "g" -> 1d/1000; + case "mg" -> 1d/1000/1000; + default -> throw new IllegalStateException("Unexpected value: " + unitSymbol); + }; + } + + public static double lengthToMeterConversion(@NonNull String unitSymbol) { + return switch (unitSymbol) { + case "km" -> 1000; + case "m" -> 1; + case "dm" -> 1d/10; + case "cm" -> 1d/100; + case "mm" -> 1d/1000; + case "μm" -> 1d/1000/1000; + default -> throw new IllegalStateException("Unexpected value: " + unitSymbol); + }; + } +} diff --git a/sumaris-core-shared/src/main/java/net/sumaris/core/util/sound/SoundUtils.java b/sumaris-core-shared/src/main/java/net/sumaris/core/util/sound/SoundUtils.java new file mode 100644 index 0000000000..b658f615ea --- /dev/null +++ b/sumaris-core-shared/src/main/java/net/sumaris/core/util/sound/SoundUtils.java @@ -0,0 +1,152 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.util.sound; + +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; + +import javax.sound.sampled.*; +import java.io.File; +import java.io.IOException; + +@Slf4j +public class SoundUtils { + + public static float SAMPLE_RATE = 8000f; + + public static void tone(int hz, int msecs) + throws LineUnavailableException + { + tone(hz, msecs, 1.0); + } + + public static void tone(int hz, int msecs, double vol) + throws LineUnavailableException + { + byte[] buf = new byte[1]; + AudioFormat af = + new AudioFormat( + SAMPLE_RATE, // sampleRate + 8, // sampleSizeInBits + 1, // channels + true, // signed + false); // bigEndian + SourceDataLine sdl = AudioSystem.getSourceDataLine(af); + sdl.open(af); + sdl.start(); + for (int i=0; i < msecs*8; i++) { + double angle = i / (SAMPLE_RATE / hz) * 2.0 * Math.PI; + buf[0] = (byte)(Math.sin(angle) * 127.0 * vol); + sdl.write(buf,0,1); + } + sdl.drain(); + sdl.stop(); + sdl.close(); + } + + public static void playError(int count) { + + int counter = 0; + try { + while (counter < count) { + SoundUtils.tone(450, 800, 0.7); + Thread.sleep(800); + SoundUtils.tone(400, 800, 0.7); + Thread.sleep(800); + SoundUtils.tone(400, 2000); + Thread.sleep(3000); + counter++; + } + } + catch (Exception e) { + log.debug("Cannot play error sound: " + e.getMessage()); + } + } + + public static void playWaiting(int count) { + int counter = 0; + try { + while (counter < count) { + SoundUtils.tone(100, 100); + Thread.sleep(200); + SoundUtils.tone(500, 700); + Thread.sleep(800); + SoundUtils.tone(500, 1000, 0.4); + Thread.sleep(3000); + counter++; + } + } + catch (Exception e) { + log.debug("Cannot play error sound: " + e.getMessage()); + } + } + + private void playSound(File f) { + Preconditions.checkArgument(f.exists()); + + Runnable r = new Runnable() { + private File f; + + public void run() { + playSoundInternal(this.f); + } + + public Runnable setFile(File f) { + this.f = f; + return this; + } + }.setFile(f); + + new Thread(r).start(); + + } + + private void playSoundInternal(File f) { + + try { + AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(f); + try { + Clip clip = AudioSystem.getClip(); + clip.open(audioInputStream); + try { + clip.start(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + clip.drain(); + } finally { + clip.close(); + } + } catch (LineUnavailableException | IOException e) { + throw new RuntimeException(e); + } finally { + audioInputStream.close(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} \ No newline at end of file diff --git a/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_en_GB.properties b/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_en_GB.properties index 726eed6f7e..60a77de3ef 100644 --- a/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_en_GB.properties +++ b/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_en_GB.properties @@ -18,7 +18,10 @@ sumaris.config.option.attribute.separator.description= sumaris.config.option.basedir.description= sumaris.config.option.cache.directory.description= sumaris.config.option.cli.daemon.description= +sumaris.config.option.cli.filter.operationIds.description= +sumaris.config.option.cli.filter.programLabel.description= sumaris.config.option.cli.filter.tripId.description= +sumaris.config.option.cli.filter.tripIds.description= sumaris.config.option.cli.filter.year.description= sumaris.config.option.cli.output.file.description= sumaris.config.option.cli.output.force.description= @@ -107,7 +110,8 @@ sumaris.config.option.trip.operation.batch.taxonGroups.noWeight.description= sumaris.config.option.trip.operation.batch.taxonName.enable.description= sumaris.config.option.value.separator.description= sumaris.config.parse.error= -sumaris.error.account.unauthorized= +sumaris.error.account.unauthorized=Unauthorized +sumaris.error.enumeration.unresolved=Enumeration {%s} not resolved. Please define it using configuration option. sumaris.persistence.bindingQuery.error= sumaris.persistence.bindingQuery.error.log= sumaris.persistence.compactDatabase.error= diff --git a/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_fr_FR.properties b/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_fr_FR.properties index f72742313a..2c97ef3a09 100644 --- a/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_fr_FR.properties +++ b/sumaris-core-shared/src/main/resources/i18n/sumaris-core-shared_fr_FR.properties @@ -18,7 +18,10 @@ sumaris.config.option.attribute.separator.description= sumaris.config.option.basedir.description= sumaris.config.option.cache.directory.description= sumaris.config.option.cli.daemon.description= +sumaris.config.option.cli.filter.operationIds.description= +sumaris.config.option.cli.filter.programLabel.description= sumaris.config.option.cli.filter.tripId.description= +sumaris.config.option.cli.filter.tripIds.description= sumaris.config.option.cli.filter.year.description= sumaris.config.option.cli.output.file.description= sumaris.config.option.cli.output.force.description= @@ -139,7 +142,8 @@ sumaris.config.option.trip.operation.batch.taxonGroups.noWeight.description= sumaris.config.option.trip.operation.batch.taxonName.enable.description= sumaris.config.option.value.separator.description= sumaris.config.parse.error=Erreur lors de la lecture de la ligne de commande -sumaris.error.account.unauthorized= +sumaris.error.account.unauthorized=Non autorisé +sumaris.error.enumeration.unresolved=Enumération {%s} non résolue, à partir de la base de données. Veuillez la définir par une option de configuration. sumaris.persistence.bindingQuery.error= sumaris.persistence.bindingQuery.error.log= sumaris.persistence.compactDatabase.error= diff --git a/sumaris-core-shared/src/test/java/net/sumaris/core/dao/referential/location/LocationsTest.java b/sumaris-core-shared/src/test/java/net/sumaris/core/dao/referential/location/LocationsTest.java index 1d30fe5d43..c9d31ffc77 100644 --- a/sumaris-core-shared/src/test/java/net/sumaris/core/dao/referential/location/LocationsTest.java +++ b/sumaris-core-shared/src/test/java/net/sumaris/core/dao/referential/location/LocationsTest.java @@ -48,6 +48,14 @@ public void getRectangleLabelByLatLong() { String label = Locations.getRectangleLabelByLatLong(47.6f, -5.05f); assertEquals("24E4", label); + // Check label = 25E5 + label = Locations.getRectangleLabelByLatLong(48f, -5.01f); + assertEquals("25E5", label); + + // Check label = 25E + label = Locations.getRectangleLabelByLatLong(48.001f, -5.0547f); + assertEquals("25E4", label); + // Check label with a position inside the Mediterranean sea label = Locations.getRectangleLabelByLatLong(42.27f, 5.4f); assertEquals("M24C2", label); diff --git a/sumaris-core-shared/src/test/java/net/sumaris/core/util/SoundUtilsTest.java b/sumaris-core-shared/src/test/java/net/sumaris/core/util/SoundUtilsTest.java new file mode 100644 index 0000000000..f134c3a8ca --- /dev/null +++ b/sumaris-core-shared/src/test/java/net/sumaris/core/util/SoundUtilsTest.java @@ -0,0 +1,36 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.util; + +import net.sumaris.core.util.sound.SoundUtils; +import org.junit.Test; + +public class SoundUtilsTest { + + + @Test + public void playError() { + SoundUtils.playError(10); + } + +} \ No newline at end of file diff --git a/sumaris-core/pom.xml b/sumaris-core/pom.xml index 0157d61f8c..c9741dbba5 100644 --- a/sumaris-core/pom.xml +++ b/sumaris-core/pom.xml @@ -4,7 +4,7 @@ net.sumaris sumaris-pod - 2.0.6 + 2.1.0 sumaris-core diff --git a/sumaris-core/src/main/java/net/sumaris/cli/action/data/DenormalizeTripsAction.java b/sumaris-core/src/main/java/net/sumaris/cli/action/data/DenormalizeTripsAction.java index 0afb113210..ed9b67f9cd 100644 --- a/sumaris-core/src/main/java/net/sumaris/cli/action/data/DenormalizeTripsAction.java +++ b/sumaris-core/src/main/java/net/sumaris/cli/action/data/DenormalizeTripsAction.java @@ -27,8 +27,9 @@ import net.sumaris.core.model.IProgressionModel; import net.sumaris.core.model.ProgressionModel; import net.sumaris.core.service.ServiceLocator; -import net.sumaris.core.service.data.denormalize.DenormalizeTripService; +import net.sumaris.core.service.data.denormalize.DenormalizedTripService; import net.sumaris.core.util.Dates; +import net.sumaris.core.util.sound.SoundUtils; import net.sumaris.core.vo.filter.TripFilterVO; @Slf4j @@ -39,11 +40,14 @@ public class DenormalizeTripsAction { */ public void run() { SumarisConfiguration config = SumarisConfiguration.getInstance(); - DenormalizeTripService tripService = ServiceLocator.instance().getService("denormalizeTripService", DenormalizeTripService.class); + DenormalizedTripService tripService = ServiceLocator.instance().getService("denormalizeTripService", DenormalizedTripService.class); // Create filter TripFilterVO.TripFilterVOBuilder filterBuilder = TripFilterVO.builder() - .tripId(config.getCliFilterTripId()); + .includedIds(config.getCliFilterTripIds().toArray(Integer[]::new)) + .operationIds(config.getCliFilterOperationIds().toArray(Integer[]::new)) + .programLabel(config.getCliFilterProgramLabel()); + Integer year = config.getCliFilterYear(); if (year != null && year > 1970) { filterBuilder.startDate(Dates.getFirstDayOfYear(year)) @@ -55,5 +59,7 @@ public void run() { progression.addPropertyChangeListener(IProgressionModel.Fields.MESSAGE, (event) -> log.info(progression.getMessage())); tripService.denormalizeByFilter(filterBuilder.build(), progression); + // Play a beep + SoundUtils.playWaiting(2); } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/config/CacheConfiguration.java b/sumaris-core/src/main/java/net/sumaris/core/config/CacheConfiguration.java index 60223faec6..28508b45b7 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/config/CacheConfiguration.java +++ b/sumaris-core/src/main/java/net/sumaris/core/config/CacheConfiguration.java @@ -43,7 +43,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.orm.jpa.HibernatePropertiesCustomizer; import org.springframework.cache.annotation.CachingConfigurerSupport; -import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.interceptor.SimpleKey; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -72,6 +72,8 @@ public interface Names { // Person String PERSON_BY_ID = "net.sumaris.core.dao.administration.user.personById"; String PERSON_BY_PUBKEY = "net.sumaris.core.dao.administration.user.personByPubkey"; + String PERSON_BY_USERNAME = "net.sumaris.core.dao.administration.user.personByUsername"; + String PERSON_AVATAR_BY_PUBKEY = "net.sumaris.core.dao.administration.user.personAvatarByPubkey"; // Location @@ -81,12 +83,11 @@ public interface Names { // Program String PROGRAM_BY_ID = "net.sumaris.core.dao.administration.programStrategy.programById"; String PROGRAM_BY_LABEL = "net.sumaris.core.dao.administration.programStrategy.programByLabel"; + String PROGRAM_BY_LABEL_AND_OPTIONS = "net.sumaris.core.dao.administration.programStrategy.programByLabelAndOptions"; String PROGRAM_IDS_BY_USER_ID = "net.sumaris.core.dao.administration.programStrategy.programIdsByUserId"; // Program privilege String PROGRAM_PRIVILEGE_BY_ID = "net.sumaris.core.dao.administration.programStrategy.programPrivilegeById"; - // Program property - String PROGRAM_PROPERTY_BY_LABEL = "net.sumaris.core.dao.administration.programStrategy.programByLabel"; // Strategy String STRATEGY_BY_ID = "net.sumaris.core.dao.administration.programStrategy.strategyById"; @@ -114,6 +115,13 @@ public interface Names { String TAXONONOMIC_LEVEL_BY_ID = "net.sumaris.core.dao.referential.taxon.taxonomicLevelById"; String REFERENCE_TAXON_ID_BY_TAXON_NAME_ID = "net.sumaris.core.dao.referential.taxon.referenceTaxonIdByTaxonNameId"; + // Weight length conversion + String WEIGHT_LENGTH_CONVERSION_FIRST_BY_FILTER = "net.sumaris.core.service.referential.conversion.weightLengthConversion.findFirstByFilter"; + String WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PARAMETER_ID = "net.sumaris.core.service.referential.conversion.weightLengthConversion.isLengthParameterId"; + String WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PMFM_ID = "net.sumaris.core.service.referential.conversion.weightLengthConversion.isLengthPmfmId"; + + String ROUND_WEIGHT_CONVERSION_FIRST_BY_FILTER = "net.sumaris.core.service.referential.conversion.roundWeightConversion.findFirstByFilter"; + // Vessel String VESSEL_SNAPSHOT_BY_ID_AND_DATE = "net.sumaris.core.service.data.vessel.vesselSnapshotByIdAndDate"; String VESSEL_SNAPSHOTS_BY_FILTER = "net.sumaris.core.service.data.vessel.vesselSnapshotByFilter"; @@ -128,8 +136,10 @@ public interface Names { String GEAR_BY_ID = "net.sumaris.core.dao.referential.gear.gearById"; String ANALYTIC_REFERENCES_BY_FILTER = "net.sumaris.core.dao.referential.analyticReferenceByFilter"; + // Data String MAIN_UNDEFINED_OPERATION_GROUP_BY_TRIP_ID = "net.sumaris.core.dao.data.operation.mainUndefinedOperationGroupId"; + } @Bean @@ -159,6 +169,7 @@ public JCacheManagerCustomizer cacheManagerCustomizer(SumarisConfiguration confi // Person Caches.createHeapCache(cacheManager, Names.PERSON_BY_ID, Integer.class, PersonVO.class, CacheTTL.DEFAULT.asDuration(), 600); Caches.createHeapCache(cacheManager, Names.PERSON_BY_PUBKEY, String.class, PersonVO.class, CacheTTL.DEFAULT.asDuration(), 600); + Caches.createHeapCache(cacheManager, Names.PERSON_BY_USERNAME, String.class, PersonVO.class, CacheTTL.DEFAULT.asDuration(), 600); Caches.createHeapCache(cacheManager, Names.PERSON_AVATAR_BY_PUBKEY, ImageAttachmentVO.class, CacheTTL.DEFAULT.asDuration(), 600); // Location @@ -171,6 +182,7 @@ public JCacheManagerCustomizer cacheManagerCustomizer(SumarisConfiguration confi // Program Caches.createHeapCache(cacheManager, Names.PROGRAM_BY_ID, Integer.class, ProgramVO.class, CacheTTL.DEFAULT.asDuration(), 100); Caches.createEternalHeapCache(cacheManager, Names.PROGRAM_BY_LABEL, String.class, ProgramVO.class, 100); + Caches.createEternalHeapCache(cacheManager, Names.PROGRAM_BY_LABEL_AND_OPTIONS, SimpleKey.class, ProgramVO.class, 100); Caches.createEternalHeapCache(cacheManager, Names.PROGRAM_PRIVILEGE_BY_ID, Integer.class, ReferentialVO.class, 10); Caches.createCollectionHeapCache(cacheManager, Names.PROGRAM_IDS_BY_USER_ID, Integer.class, Integer.class, CacheTTL.MEDIUM.asDuration(), 500); @@ -208,9 +220,12 @@ public JCacheManagerCustomizer cacheManagerCustomizer(SumarisConfiguration confi Caches.createEternalCollectionHeapCache(cacheManager, Names.PRODUCTS_BY_FILTER, ExtractionProductVO.class, 100); Caches.createHeapCache(cacheManager, Names.TABLE_META_BY_NAME, String.class, SumarisTableMetadata.class, CacheTTL.DEFAULT.asDuration(), 500); - - // Other entities + // Other referential Caches.createEternalCollectionHeapCache(cacheManager, Names.ANALYTIC_REFERENCES_BY_FILTER, ReferentialVO.class, 100); + Caches.createHeapCache(cacheManager, Names.WEIGHT_LENGTH_CONVERSION_FIRST_BY_FILTER, Integer.class, Object.class, CacheTTL.DEFAULT.asDuration(), 200); + Caches.createEternalHeapCache(cacheManager, Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PARAMETER_ID, Integer.class, Boolean.class, 1000); + Caches.createEternalHeapCache(cacheManager, Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PMFM_ID, Integer.class, Boolean.class, 1000); + Caches.createHeapCache(cacheManager, Names.ROUND_WEIGHT_CONVERSION_FIRST_BY_FILTER, Integer.class, Object.class, CacheTTL.DEFAULT.asDuration(), 200); // Data Caches.createHeapCache(cacheManager, Names.MAIN_UNDEFINED_OPERATION_GROUP_BY_TRIP_ID, Integer.class, Integer.class, CacheTTL.DATA_DEFAULT.asDuration(), 100); diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/PmfmStrategyRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/PmfmStrategyRepositoryImpl.java index fc5eba2422..5858f0975a 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/PmfmStrategyRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/PmfmStrategyRepositoryImpl.java @@ -325,7 +325,7 @@ private void loadAcquisitionLevels() { acquisitionLevelIdByLabel.clear(); // Fill acquisition levels map - List items = referentialDao.findByFilter(AcquisitionLevel.class.getSimpleName(), new ReferentialFilterVO(), 0, 1000, null, null); + List items = referentialDao.findByFilter(AcquisitionLevel.class.getSimpleName(), new ReferentialFilterVO(), 0, 1000, null, null, null); items.forEach(item -> acquisitionLevelIdByLabel.put(item.getLabel(), item.getId())); } diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/ProgramRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/ProgramRepositoryImpl.java index c77157e560..89b9053638 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/ProgramRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/administration/programStrategy/ProgramRepositoryImpl.java @@ -171,13 +171,18 @@ public ProgramVO getByLabel(String label) { return super.getByLabel(label); } + @Override + @Cacheable(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL_AND_OPTIONS) + public ProgramVO getByLabel(String label, ProgramFetchOptions fetchOptions) { + return super.getByLabel(label, fetchOptions); + } + @Override protected Specification toSpecification(ProgramFilterVO filter, ProgramFetchOptions fetchOptions) { return super.toSpecification(filter, fetchOptions) .and(newerThan(filter.getMinUpdateDate())) .and(hasAcquisitionLevelLabels(filter.getAcquisitionLevelLabels())) - .and(hasProperty(filter.getWithProperty())) - ; + .and(hasProperty(filter.getWithProperty())); } @Override @@ -267,6 +272,7 @@ else if (fetchOptions != null && fetchOptions.isWithLocationClassifications()) { @Caching(evict = { @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_ID, allEntries = true), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL, allEntries = true), + @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL_AND_OPTIONS, allEntries = true), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_IDS_BY_USER_ID, allEntries = true) }) public void clearCache() { @@ -278,6 +284,7 @@ public void clearCache() { evict = { @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_ID, key = "#source.id", condition = "#source.id != null"), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL, key = "#source.label", condition = "#source.label != null"), + @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL_AND_OPTIONS, allEntries = true), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_IDS_BY_USER_ID, allEntries = true) }, put = { @@ -350,6 +357,7 @@ public void toEntity(ProgramVO source, Program target, boolean copyIfNull) { evict = { @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_ID, key = "#id", condition = "#id != null"), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL, allEntries = true), + @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL_AND_OPTIONS, allEntries = true), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_IDS_BY_USER_ID, allEntries = true) } ) @@ -680,7 +688,7 @@ protected void toPersonEntity(@NonNull ProgramPersonVO source, @Override public boolean hasPropertyValueByProgramId(@NonNull Integer id, @NonNull ProgramPropertyEnum property, @NonNull String expectedValue) { String value = findVOById(id) - .map(program -> program.getProperties().get(property.getLabel())) + .map(program -> program.getProperties().get(property.getKey())) .orElse(property.getDefaultValue()); // If boolean: true = TRUE @@ -694,7 +702,7 @@ public boolean hasPropertyValueByProgramId(@NonNull Integer id, @NonNull Program @Override public boolean hasPropertyValueByProgramLabel(@NonNull String label, @NonNull ProgramPropertyEnum property, @NonNull String expectedValue) { String value = findByLabel(label) - .map(program -> program.getProperties().get(property.getLabel())) + .map(program -> program.getProperties().get(property.getKey())) .orElse(property.getDefaultValue()); return expectedValue.equals(value); @@ -704,7 +712,7 @@ public boolean hasPropertyValueByProgramLabel(@NonNull String label, @NonNull Pr @Override public String getPropertyValueByProgramLabel(@NonNull String label, @NonNull ProgramPropertyEnum property) { return findByLabel(label) - .map(program -> program.getProperties().get(property.getLabel())) + .map(program -> program.getProperties().get(property.getKey())) .orElse(property.getDefaultValue()); } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/administration/user/PersonRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/administration/user/PersonRepositoryImpl.java index 3bef6a7006..bd1d586fd8 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/administration/user/PersonRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/administration/user/PersonRepositoryImpl.java @@ -115,9 +115,11 @@ public Optional findByPubkey(@NonNull String pubkey) { } @Override + @Cacheable(cacheNames = CacheConfiguration.Names.PERSON_BY_USERNAME, key = "#username", unless="#result==null") public Optional findByUsername(String username) { - return findAll(hasUsername(username)).stream().filter(p -> StatusEnum.ENABLE.getId().equals(p.getStatus().getId())) - .findFirst().map(this::toVO); + return findAll(hasUsername(username)).stream() + .filter(p -> StatusEnum.ENABLE.getId().equals(p.getStatus().getId())) + .findFirst().map(this::toVO); } @Override diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/DataSpecifications.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/DataSpecifications.java index 714b2dcc13..7576722878 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/DataSpecifications.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/DataSpecifications.java @@ -37,7 +37,6 @@ import javax.persistence.criteria.Predicate; import java.io.Serializable; import java.util.Arrays; -import java.util.Collection; import java.util.Objects; /** @@ -73,13 +72,25 @@ default Specification isNotControlled() { */ default Specification isControlled() { return (root, query, cb) -> - cb.isNotNull(root.get(IDataEntity.Fields.CONTROL_DATE)); + cb.and( + // Control date not null + cb.isNotNull(root.get(IDataEntity.Fields.CONTROL_DATE)), + // Not validated + cb.isNull(root.get(IWithDataQualityEntity.Fields.VALIDATION_DATE)) + ); } default Specification isValidated() { return (root, query, cb) -> - // Validation date not null - cb.isNotNull(root.get(IWithDataQualityEntity.Fields.VALIDATION_DATE)); + cb.and( + // Validation date not null + cb.isNotNull(root.get(IWithDataQualityEntity.Fields.VALIDATION_DATE)), + // Not qualified + cb.or( + cb.isNull(root.get(IWithDataQualityEntity.Fields.QUALIFICATION_DATE)), + cb.equal(cb.coalesce(root.get(IDataEntity.Fields.QUALITY_FLAG).get(QualityFlag.Fields.ID), QualityFlagEnum.NOT_QUALIFIED.getId()), QualityFlagEnum.NOT_QUALIFIED.getId()) + ) + ); } default Specification isQualified() { diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/MeasurementDaoImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/MeasurementDaoImpl.java index 760c4ce632..55eba2c55e 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/MeasurementDaoImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/MeasurementDaoImpl.java @@ -1134,7 +1134,10 @@ protected void toEntity(String value, PmfmVO pmfm, IMeasurementEntity target) { PmfmValueType type = PmfmValueType.fromString(pmfm.getType()); switch (type) { - case BOOLEAN -> target.setNumericalValue(Boolean.parseBoolean(value) || "1".equals(value) ? 1d : 0d); + case BOOLEAN -> { + Double boolValue = ("1".equals(value) || Boolean.parseBoolean(value)) ? 1d : 0d; + target.setNumericalValue(boolValue); + } case QUALITATIVE_VALUE -> { // If find a object structure (e.g. ReferentialVO), try to find the id try { @@ -1147,9 +1150,8 @@ protected void toEntity(String value, PmfmVO pmfm, IMeasurementEntity target) { case STRING -> target.setAlphanumericalValue(value); case DATE -> target.setAlphanumericalValue(Dates.checkISODateTimeString(value)); case INTEGER, DOUBLE -> target.setNumericalValue(Double.parseDouble(value)); - default -> - // Unknown type - throw new SumarisTechnicalException(String.format("Unable to set measurement value {%s} for the type {%s}", value, type.name().toLowerCase())); + // Unknown type + default -> throw new SumarisTechnicalException(String.format("Unable to set measurement value {%s} for the type {%s}", value, type.name().toLowerCase())); } } @@ -1222,7 +1224,9 @@ protected Object getEntityValue(IMeasurementEntity source) { (source.getNumericalValue() != null) ? (source.getNumericalValue() == 1d ? Boolean.TRUE : Boolean.FALSE) : null; case QUALITATIVE_VALUE -> // If find a object structure (e.g. ReferentialVO), try to find the id - ((source.getQualitativeValue() != null && source.getQualitativeValue().getId() != null) ? source.getQualitativeValue().getId() : null); + ((source.getQualitativeValue() != null && source.getQualitativeValue().getId() != null) + ? source.getQualitativeValue().getId() + : null); case STRING, DATE -> source.getAlphanumericalValue(); case INTEGER -> (source.getNumericalValue() != null) ? source.getNumericalValue().intValue() : null; case DOUBLE -> source.getNumericalValue(); diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/BatchRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/BatchRepositoryImpl.java index 9f2208018c..58710c8deb 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/BatchRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/BatchRepositoryImpl.java @@ -272,7 +272,7 @@ public BatchVO toTree(List sources) { // Get root BatchVO rootBatch = roots.get(0); - // Fill children + // Fill children, and children of children (=recursively) fillRecursiveChildren(rootBatch, sources); return rootBatch; @@ -645,6 +645,7 @@ protected List fillRecursiveChildren(int parentId, List source List children = sources.stream() .filter(batch -> Objects.equals(batch.getParentId(), parentId)) + .sorted(Comparator.comparing(BatchVO::getRankOrder)) .collect(Collectors.toList()); children.forEach(batch -> batch.setChildren(fillRecursiveChildren(batch.getId(), sources))); return children; diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchRepositoryImpl.java index 86d6f63004..4ecb1e8f65 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchRepositoryImpl.java @@ -31,6 +31,7 @@ import net.sumaris.core.dao.technical.jpa.SumarisJpaRepositoryImpl; import net.sumaris.core.exception.SumarisTechnicalException; import net.sumaris.core.model.data.DenormalizedBatch; +import net.sumaris.core.model.data.DenormalizedBatchSortingValue; import net.sumaris.core.model.data.Operation; import net.sumaris.core.model.data.Sale; import net.sumaris.core.model.referential.QualityFlag; @@ -59,10 +60,7 @@ import javax.annotation.Nonnull; import javax.persistence.EntityManager; -import java.util.Collection; -import java.util.List; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; /** @@ -82,17 +80,21 @@ public class DenormalizedBatchRepositoryImpl private final ApplicationContext applicationContext; + private final DenormalizedBatchSortingValueRepository sortingValueRepository; + public DenormalizedBatchRepositoryImpl(EntityManager entityManager, SumarisConfiguration config, PmfmRepository pmfmRepository, ParameterRepository parameterRepository, TaxonNameRepository taxonNameRepository, + DenormalizedBatchSortingValueRepository sortingValueRepository, ApplicationContext applicationContext) { super(DenormalizedBatch.class, entityManager); this.config = config; this.pmfmRepository = pmfmRepository; this.parameterRepository = parameterRepository; this.taxonNameRepository = taxonNameRepository; + this.sortingValueRepository = sortingValueRepository; this.applicationContext = applicationContext; } @@ -114,6 +116,11 @@ public void toVO(DenormalizedBatch source, DenormalizedBatchVO target, boolean c if (copyIfNull || saleId != null) { target.setSaleId(saleId); } + + Integer parentId = source.getParent() != null ? source.getParent().getId() : null; + if (copyIfNull || parentId != null) { + target.setParentId(parentId); + } } @Override @@ -207,6 +214,48 @@ public void toEntity(DenormalizedBatchVO source, DenormalizedBatch target, boole } } } + + // Calculated taxon group + { + Integer calculatedTaxonGroupId = source.getCalculatedTaxonGroup() != null ? source.getCalculatedTaxonGroup().getId() : null; + if (copyIfNull || calculatedTaxonGroupId != null) { + if (calculatedTaxonGroupId == null) { + target.setCalculatedTaxonGroup(null); + } else { + target.setCalculatedTaxonGroup(getReference(TaxonGroup.class, calculatedTaxonGroupId)); + } + } + } + + // Parent name + { + Integer parentBatchId = source.getParent() != null ? source.getParent().getId() : source.getParentId(); + if (copyIfNull || parentBatchId != null) { + if (parentBatchId == null) { + target.setParent(null); + } else { + target.setParent(getReference(DenormalizedBatch.class, parentBatchId)); + } + } + } + } + + @Override + protected void onAfterSaveEntity(DenormalizedBatchVO vo, DenormalizedBatch savedEntity, boolean isNew) { + super.onAfterSaveEntity(vo, savedEntity, isNew); + + List existingSvIds = Beans.collectIds(savedEntity.getSortingValues()); + + Beans.getStream(vo.getSortingValues()) + .forEach(source -> { + source.setBatchId(vo.getId()); + sortingValueRepository.save(source); + existingSvIds.remove(source.getId()); + }); + + if (CollectionUtils.isNotEmpty(existingSvIds)) { + sortingValueRepository.deleteAllById(existingSvIds); + } } @Override @@ -231,7 +280,7 @@ public List saveAllByOperationId(int operationId, @Nonnull // Set parent link sources.forEach(b -> b.setOperationId(operationId)); - // Get existing fishing areas + // Get existing ids Set existingIds = getRepository().getAllIdByOperationId(operationId); // Save @@ -267,11 +316,17 @@ public List saveAllBySaleId(int saleId, @Nonnull List. + * #L% + */ + +package net.sumaris.core.dao.data.batch; + +import net.sumaris.core.dao.technical.jpa.SumarisJpaRepository; +import net.sumaris.core.model.data.DenormalizedBatch; +import net.sumaris.core.model.data.DenormalizedBatchSortingValue; +import net.sumaris.core.vo.data.batch.DenormalizedBatchSortingValueVO; +import net.sumaris.core.vo.data.batch.DenormalizedBatchVO; +import org.springframework.data.jpa.repository.Query; + +import java.util.Set; + +public interface DenormalizedBatchSortingValueRepository + extends SumarisJpaRepository, + DenormalizedBatchSortingValueSpecifications { + + +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchSortingValueRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchSortingValueRepositoryImpl.java new file mode 100644 index 0000000000..707ebf4ebc --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchSortingValueRepositoryImpl.java @@ -0,0 +1,130 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.dao.data.batch; + +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; +import net.sumaris.core.config.SumarisConfiguration; +import net.sumaris.core.dao.referential.pmfm.ParameterRepository; +import net.sumaris.core.dao.referential.pmfm.PmfmRepository; +import net.sumaris.core.dao.referential.taxon.TaxonNameRepository; +import net.sumaris.core.dao.technical.jpa.SumarisJpaRepositoryImpl; +import net.sumaris.core.exception.SumarisTechnicalException; +import net.sumaris.core.model.data.DenormalizedBatch; +import net.sumaris.core.model.data.DenormalizedBatchSortingValue; +import net.sumaris.core.model.data.Operation; +import net.sumaris.core.model.data.Sale; +import net.sumaris.core.model.referential.QualityFlag; +import net.sumaris.core.model.referential.QualityFlagEnum; +import net.sumaris.core.model.referential.pmfm.*; +import net.sumaris.core.model.referential.taxon.ReferenceTaxon; +import net.sumaris.core.model.referential.taxon.TaxonGroup; +import net.sumaris.core.util.Beans; +import net.sumaris.core.util.Numbers; +import net.sumaris.core.util.StringUtils; +import net.sumaris.core.vo.data.MeasurementVO; +import net.sumaris.core.vo.data.QuantificationMeasurementVO; +import net.sumaris.core.vo.data.batch.BatchVO; +import net.sumaris.core.vo.data.batch.DenormalizedBatchSortingValueVO; +import net.sumaris.core.vo.data.batch.DenormalizedBatchVO; +import net.sumaris.core.vo.referential.ParameterVO; +import net.sumaris.core.vo.referential.PmfmVO; +import net.sumaris.core.vo.referential.PmfmValueType; +import net.sumaris.core.vo.referential.ReferentialVO; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.mutable.MutableInt; +import org.springframework.context.ApplicationContext; +import org.springframework.dao.DataRetrievalFailureException; + +import javax.annotation.Nonnull; +import javax.persistence.EntityManager; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * @author peck7 on 09/06/2020. + */ +@Slf4j +public class DenormalizedBatchSortingValueRepositoryImpl + extends SumarisJpaRepositoryImpl + implements DenormalizedBatchSortingValueSpecifications { + + public DenormalizedBatchSortingValueRepositoryImpl(EntityManager entityManager) { + super(DenormalizedBatchSortingValue.class, entityManager); + } + + @Override + public Class getVOClass() { + return DenormalizedBatchSortingValueVO.class; + } + + @Override + public void toVO(DenormalizedBatchSortingValue source, DenormalizedBatchSortingValueVO target, boolean copyIfNull) { + super.toVO(source, target, copyIfNull); + + } + + @Override + public void toEntity(DenormalizedBatchSortingValueVO source, DenormalizedBatchSortingValue target, boolean copyIfNull) { + super.toEntity(source, target, copyIfNull); + + // Pmfm + Integer pmfmId = source.getPmfmId() != null ? source.getPmfmId() : (source.getPmfm() != null ? source.getPmfm().getId() : null); + if (pmfmId != null || copyIfNull) { + if (pmfmId != null) target.setPmfm(getReference(Pmfm.class, pmfmId)); + else target.setPmfm(null); + } + + // Parameter + Integer parameterId = source.getParameter() != null ? source.getParameter().getId() : null; + if (parameterId != null || copyIfNull) { + if (parameterId != null) target.setParameter(getReference(Parameter.class, parameterId)); + else target.setParameter(null); + } + + // Qualitative_value + Integer qvId = source.getQualitativeValue() != null ? source.getQualitativeValue().getId() : null; + if (qvId != null || copyIfNull) { + if (qvId != null) target.setQualitativeValue(getReference(QualitativeValue.class, qvId)); + else target.setQualitativeValue(null); + } + + // Unit + Integer unitId = source.getUnit() != null ? source.getUnit().getId() : null; + if (unitId != null || copyIfNull) { + if (unitId != null) target.setUnit(getReference(Unit.class, unitId)); + else target.setUnit(null); + } + + // Link to parent + Integer batchId = source.getBatchId() != null ? source.getBatchId() : (source.getBatch() != null ? source.getBatch().getId() : null); + if (batchId != null) { + target.setBatch(getReference(DenormalizedBatch.class, batchId)); + } + } + + /* -- protected methods -- */ +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchSortingValueSpecifications.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchSortingValueSpecifications.java new file mode 100644 index 0000000000..7f21f87cf8 --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/batch/DenormalizedBatchSortingValueSpecifications.java @@ -0,0 +1,30 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.dao.data.batch; + +import net.sumaris.core.model.data.DenormalizedBatchSortingValue; +import net.sumaris.core.vo.data.batch.DenormalizedBatchSortingValueVO; + +public interface DenormalizedBatchSortingValueSpecifications { + +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationRepositoryImpl.java index 4ced055f7e..213676e199 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationRepositoryImpl.java @@ -132,29 +132,36 @@ public void toVO(Operation source, OperationVO target, OperationFetchOptions fet // Load children entities (not loaded by default) Integer operationId = source.getId(); - if (fetchOptions != null && fetchOptions.isWithChildrenEntities() && operationId != null) { - + if (operationId != null) { // Positions - target.setPositions(vesselPositionDao.getAllByOperationId(operationId)); + if (fetchOptions != null && (fetchOptions.isWithChildrenEntities() || fetchOptions.isWithPositions())) { + target.setPositions(vesselPositionDao.getAllByOperationId(operationId)); + } // Fishing Areas - target.setFishingAreas(fishingAreaRepository.findAllVO(fishingAreaRepository.hasOperationId(operationId))); + if (fetchOptions != null && (fetchOptions.isWithChildrenEntities() || fetchOptions.isWithFishingAreas())) { + target.setFishingAreas(fishingAreaRepository.findAllVO(fishingAreaRepository.hasOperationId(operationId))); + } // Batches - target.setBatches(batchRepository.findAllVO(batchRepository.hasOperationId(operationId), + if (fetchOptions != null && (fetchOptions.isWithChildrenEntities() || fetchOptions.isWithBatches())) { + target.setBatches(batchRepository.findAllVO(batchRepository.hasOperationId(operationId), BatchFetchOptions.builder() - .withChildrenEntities(false) // Use flat list, not a tree - .withRecorderDepartment(false) - .withMeasurementValues(true) - .build())); + .withChildrenEntities(false) // Use flat list, not a tree + .withRecorderDepartment(false) + .withMeasurementValues(true) + .build())); + } // Samples - target.setSamples(sampleRepository.findAllVO(sampleRepository.hasOperationId(operationId), + if (fetchOptions != null && (fetchOptions.isWithChildrenEntities() || fetchOptions.isWithSamples())) { + target.setSamples(sampleRepository.findAllVO(sampleRepository.hasOperationId(operationId), SampleFetchOptions.builder() - .withChildrenEntities(false) // Use flat list, not a tree - .withRecorderDepartment(false) - .withMeasurementValues(true) - .build())); + .withChildrenEntities(false) // Use flat list, not a tree + .withRecorderDepartment(false) + .withMeasurementValues(true) + .build())); + } } // Measurements @@ -322,7 +329,9 @@ protected Specification toSpecification(OperationFilterVO filter, Ope .and(inPhysicalGearIds(filter.getPhysicalGearIds())) .and(inTaxonGroupLabels(filter.getTaxonGroupLabels())) .and(hasQualityFlagIds(filter.getQualityFlagIds())) - .and(inDataQualityStatus(filter.getDataQualityStatus())); + .and(inDataQualityStatus(filter.getDataQualityStatus())) + .and(needBatchDenormalization(filter.getNeedBatchDenormalization())) + ; } @Override diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationSpecifications.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationSpecifications.java index 91c4beb962..6851e065db 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationSpecifications.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/operation/OperationSpecifications.java @@ -27,10 +27,7 @@ import net.sumaris.core.dao.technical.jpa.BindableSpecification; import net.sumaris.core.model.IEntity; import net.sumaris.core.model.administration.programStrategy.Program; -import net.sumaris.core.model.data.Operation; -import net.sumaris.core.model.data.PhysicalGear; -import net.sumaris.core.model.data.Trip; -import net.sumaris.core.model.data.Vessel; +import net.sumaris.core.model.data.*; import net.sumaris.core.model.referential.QualityFlag; import net.sumaris.core.model.referential.metier.Metier; import net.sumaris.core.model.referential.taxon.TaxonGroup; @@ -39,9 +36,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.springframework.data.jpa.domain.Specification; -import javax.persistence.criteria.Join; -import javax.persistence.criteria.JoinType; -import javax.persistence.criteria.ParameterExpression; +import javax.persistence.criteria.*; import java.util.Arrays; import java.util.Collection; import java.util.Date; @@ -222,6 +217,41 @@ default Specification hasQualityFlagIds(Integer[] qualityFlagIds) { .addBind(QUALITY_FLAG_ID_PARAM, Arrays.asList(qualityFlagIds)); } + default Specification needBatchDenormalization(Boolean needBatchDenormalization) { + if (!Boolean.TRUE.equals(needBatchDenormalization)) return null; + + return BindableSpecification.where((root, query, cb) -> { + + Join catchBatch = Daos.composeJoin(root, Operation.Fields.BATCHES, JoinType.INNER); + + // Sub select that return the update to date denormalized catch batch + Subquery subQuery = query.subquery(Integer.class); + Root denormalizedBatchRoot = subQuery.from(DenormalizedBatch.class); + subQuery.select(denormalizedBatchRoot.get(DenormalizedBatch.Fields.ID)); + subQuery.where( + cb.and( + // Catch batch + cb.isNull(denormalizedBatchRoot.get(DenormalizedBatch.Fields.PARENT)), + // Same operation + cb.equal(denormalizedBatchRoot.get(DenormalizedBatch.Fields.OPERATION), root), + // Same catch batch + cb.equal(denormalizedBatchRoot.get(DenormalizedBatch.Fields.ID), catchBatch.get(Batch.Fields.ID)), + // Same date + cb.equal(denormalizedBatchRoot.get(DenormalizedBatch.Fields.UPDATE_DATE), catchBatch.get(Batch.Fields.UPDATE_DATE)) + ) + ); + + return cb.and( + // Operation with a catch batch + cb.isNull(catchBatch.get(Batch.Fields.PARENT)), + // And without an update to date denormalization + cb.not(cb.exists(subQuery)) + ); + }); + } + + + // Override the default function, because operation has no validation date default Specification isValidated() { return (root, query, cb) -> diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripRepositoryImpl.java index 325a2c5185..d2c8afd280 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripRepositoryImpl.java @@ -79,6 +79,7 @@ public Specification toSpecification(TripFilterVO filter, TripFetchOptions .and(hasObserverPersonIds(filter.getObserverPersonIds())) .and(hasQualityFlagIds(filter.getQualityFlagIds())) .and(inDataQualityStatus(filter.getDataQualityStatus())) + .and(withOperationIds(filter.getOperationIds())) ; } diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripSpecifications.java b/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripSpecifications.java index 2abe85f550..5c65e245cb 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripSpecifications.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/data/trip/TripSpecifications.java @@ -27,14 +27,17 @@ import net.sumaris.core.dao.technical.jpa.BindableSpecification; import net.sumaris.core.model.IEntity; import net.sumaris.core.model.data.Landing; +import net.sumaris.core.model.data.Operation; import net.sumaris.core.model.data.Trip; import net.sumaris.core.model.referential.QualityFlag; +import net.sumaris.core.model.referential.conversion.WeightLengthConversion; +import net.sumaris.core.model.referential.location.Location; +import net.sumaris.core.model.referential.location.LocationHierarchy; +import net.sumaris.core.util.StringUtils; import org.apache.commons.lang3.ArrayUtils; import org.springframework.data.jpa.domain.Specification; -import javax.persistence.criteria.JoinType; -import javax.persistence.criteria.ListJoin; -import javax.persistence.criteria.ParameterExpression; +import javax.persistence.criteria.*; import java.util.Arrays; import java.util.Collection; import java.util.Date; @@ -50,6 +53,7 @@ public interface TripSpecifications extends RootDataSpecifications { String OBSERVED_LOCATION_ID_PARAM = "observedLocationId"; + String OPERATION_IDS_PARAM = "operationIds"; default Specification hasLocationId(Integer locationId) { if (locationId == null) return null; return BindableSpecification.where((root, query, cb) -> { @@ -144,4 +148,18 @@ default Specification hasQualityFlagIds(Integer[] qualityFlagIds) { }) .addBind(QUALITY_FLAG_ID_PARAM, Arrays.asList(qualityFlagIds)); } + + + default Specification withOperationIds(Integer[] operationIds) { + if (ArrayUtils.isEmpty(operationIds)) return null; + + return BindableSpecification.where((root, query, cb) -> { + ParameterExpression param = cb.parameter(Collection.class, OPERATION_IDS_PARAM); + Subquery subQuery = query.subquery(Operation.class); + Root operation = subQuery.from(Operation.class); + subQuery.select(operation.get(Operation.Fields.TRIP).get(Operation.Fields.ID)); + subQuery.where(cb.in(operation.get(Operation.Fields.ID)).value(param)); + return cb.in(root.get(IEntity.Fields.ID)).value(subQuery); + }).addBind(OPERATION_IDS_PARAM, Arrays.asList(operationIds)); + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDao.java b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDao.java index cdc3e40b1d..c1f89706d4 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDao.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDao.java @@ -26,9 +26,11 @@ import net.sumaris.core.model.referential.IReferentialEntity; import net.sumaris.core.vo.filter.IReferentialFilter; import net.sumaris.core.vo.referential.IReferentialVO; +import net.sumaris.core.vo.referential.ReferentialFetchOptions; import net.sumaris.core.vo.referential.ReferentialTypeVO; import net.sumaris.core.vo.referential.ReferentialVO; +import javax.annotation.Nullable; import java.util.Collection; import java.util.Date; import java.util.List; @@ -57,7 +59,8 @@ List findByFilter(String entityName, int offset, int size, String sortAttribute, - SortDirection sortDirection); + SortDirection sortDirection, + @Nullable ReferentialFetchOptions fetchOptions); Long countByFilter(final String entityName, IReferentialFilter filter); diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDaoImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDaoImpl.java index 51d1bfa66e..f0a5953358 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDaoImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialDaoImpl.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import lombok.extern.slf4j.Slf4j; import net.sumaris.core.config.CacheConfiguration; import net.sumaris.core.dao.technical.Daos; @@ -34,12 +35,15 @@ import net.sumaris.core.model.IUpdateDateEntity; import net.sumaris.core.exception.SumarisTechnicalException; import net.sumaris.core.model.referential.*; +import net.sumaris.core.model.referential.pmfm.Method; import net.sumaris.core.util.Beans; import net.sumaris.core.vo.filter.IReferentialFilter; import net.sumaris.core.vo.filter.ReferentialFilterVO; import net.sumaris.core.vo.referential.IReferentialVO; +import net.sumaris.core.vo.referential.ReferentialFetchOptions; import net.sumaris.core.vo.referential.ReferentialTypeVO; import net.sumaris.core.vo.referential.ReferentialVO; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.nuiton.i18n.I18n; @@ -92,14 +96,15 @@ public List findByFilter(final String entityName, int offset, int size, String sortAttribute, - SortDirection sortDirection) { + SortDirection sortDirection, + ReferentialFetchOptions fetchOptions) { Preconditions.checkNotNull(entityName, "Missing entityName argument"); // Get entity class from entityName Class entityClass = ReferentialEntities.getEntityClass(entityName); return streamByFilter(entityClass, filter, offset, size, sortAttribute, sortDirection) - .map(s -> toVO(entityName, s)) + .map(s -> toVO(entityName, s, fetchOptions)) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -130,7 +135,7 @@ public Optional findByUniqueLabel(String entityName, String label } catch (NoResultException e) { // let result to null } - return result == null ? Optional.empty() : Optional.of(toVO(entityName, result)); + return result == null ? Optional.empty() : Optional.of(toVO(entityName, result, null)); } @Override @@ -174,7 +179,7 @@ public List getAllLevels(final String entityName) { return ReferentialEntities.getLevelProperty(entityName) .map(levelDescriptor -> { String levelEntityName = levelDescriptor.getPropertyType().getSimpleName(); - return findByFilter(levelEntityName, ReferentialFilterVO.builder().build(), 0, 1000, IItemReferentialEntity.Fields.NAME, SortDirection.ASC); + return findByFilter(levelEntityName, ReferentialFilterVO.builder().build(), 0, 1000, IItemReferentialEntity.Fields.NAME, SortDirection.ASC, null); }) .orElseGet(ImmutableList::of); } @@ -191,7 +196,7 @@ public ReferentialVO getLevelById(String entityName, int levelId) { throw new DataRetrievalFailureException("Unable to convert class=" + levelClass.getName() + " to a referential bean"); } - return toVO(levelClass.getSimpleName(), (IReferentialEntity) getById(levelClass, levelId)); + return toVO(levelClass.getSimpleName(), (IReferentialEntity) getById(levelClass, levelId), null); } @Override @@ -199,7 +204,7 @@ public ReferentialVO toVO(T source) { if (source == null) throw new EntityNotFoundException(); - return toVO(getEntityName(source), source); + return toVO(getEntityName(source), source, null); } @Override @@ -231,6 +236,7 @@ public void clearCache() { @CacheEvict(cacheNames = CacheConfiguration.Names.REFERENTIAL_MAX_UPDATE_DATE_BY_TYPE, key = "#entityName"), @CacheEvict(cacheNames = CacheConfiguration.Names.PERSON_BY_ID, allEntries = true, condition = "#entityName == 'Person'"), @CacheEvict(cacheNames = CacheConfiguration.Names.PERSON_BY_PUBKEY, allEntries = true, condition = "#entityName == 'Person'"), + @CacheEvict(cacheNames = CacheConfiguration.Names.PERSON_BY_USERNAME, allEntries = true, condition = "#entityName == 'Person'"), @CacheEvict(cacheNames = CacheConfiguration.Names.DEPARTMENT_BY_ID, allEntries = true, condition = "#entityName == 'Department'"), @CacheEvict(cacheNames = CacheConfiguration.Names.DEPARTMENT_BY_LABEL, allEntries = true, condition = "#entityName == 'Department'"), @CacheEvict(cacheNames = CacheConfiguration.Names.PMFM_BY_ID, allEntries = true, condition = "#entityName == 'Pmfm'"), @@ -259,6 +265,7 @@ public void clearCache(String entityName) { @CacheEvict(cacheNames = CacheConfiguration.Names.REFERENTIAL_MAX_UPDATE_DATE_BY_TYPE, key = "#entityName"), @CacheEvict(cacheNames = CacheConfiguration.Names.PERSON_BY_ID, key = "#id", condition = "#entityName == 'Person'"), @CacheEvict(cacheNames = CacheConfiguration.Names.PERSON_BY_PUBKEY, allEntries = true, condition = "#entityName == 'Person'"), + @CacheEvict(cacheNames = CacheConfiguration.Names.PERSON_BY_USERNAME, allEntries = true, condition = "#entityName == 'Person'"), @CacheEvict(cacheNames = CacheConfiguration.Names.DEPARTMENT_BY_ID, key = "#id", condition = "#entityName == 'Department'"), @CacheEvict(cacheNames = CacheConfiguration.Names.DEPARTMENT_BY_LABEL, allEntries = true, condition = "#entityName == 'Department'"), @CacheEvict(cacheNames = CacheConfiguration.Names.PMFM_BY_ID, key = "#id", condition = "#entityName == 'Pmfm'"), @@ -269,6 +276,7 @@ public void clearCache(String entityName) { @CacheEvict(cacheNames = CacheConfiguration.Names.PMFM_HAS_PARAMETER_GROUP, allEntries = true, condition = "#entityName == 'Pmfm'"), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_ID, key = "#id", condition = "#entityName == 'Program'"), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL, allEntries = true, condition = "#entityName == 'Program'"), + @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_BY_LABEL_AND_OPTIONS, allEntries = true, condition = "#entityName == 'Program'"), @CacheEvict(cacheNames = CacheConfiguration.Names.PROGRAM_PRIVILEGE_BY_ID, key = "#id", condition = "#entityName == 'ProgramPrivilege'"), @CacheEvict(cacheNames = CacheConfiguration.Names.LOCATION_BY_ID, key = "#id", condition = "#entityName == 'Location'"), @CacheEvict(cacheNames = CacheConfiguration.Names.LOCATION_LEVEL_BY_LABEL, allEntries = true, condition = "#entityName == 'LocationLevel'"), @@ -468,7 +476,8 @@ protected ReferentialTypeVO getTypeByEntityName(final String entityName) { return type; } - protected ReferentialVO toVO(final String entityName, T source) { + protected ReferentialVO toVO(final String entityName, T source, + ReferentialFetchOptions fetchOptions) { Preconditions.checkNotNull(entityName); Preconditions.checkNotNull(source); @@ -515,6 +524,10 @@ protected ReferentialVO toVO(final String entityN target.setParentId(null); } + if (fetchOptions != null && fetchOptions.isWithProperties()) { + copyProperties(source, target); + } + // EntityName (as metadata) target.setEntityName(entityName); @@ -777,12 +790,12 @@ protected TypedQuery createFilteredQuery(CriteriaBuilder builder, private TypedQuery createFindByUniqueLabelQuery(Class entityClass, String label) { CriteriaBuilder builder = getEntityManager().getCriteriaBuilder(); CriteriaQuery query = builder.createQuery(entityClass); - Root tripRoot = query.from(entityClass); - query.select(tripRoot).distinct(true); + Root root = query.from(entityClass); + query.select(root).distinct(true); // Filter on text ParameterExpression labelParam = builder.parameter(String.class); - query.where(builder.equal(tripRoot.get(IItemReferentialEntity.Fields.LABEL), labelParam)); + query.where(builder.equal(root.get(IItemReferentialEntity.Fields.LABEL), labelParam)); return getEntityManager().createQuery(query) .setParameter(labelParam, label); @@ -816,14 +829,14 @@ protected void toEntity(final ReferentialVO source, IReferentialEntity target, b } // Level - Integer levelID = source.getLevelId(); - if (copyIfNull || levelID != null) { + Integer levelId = source.getLevelId() != null ? source.getLevelId() : (source.getLevel() != null ? source.getLevel().getId() : null); + if (copyIfNull || levelId != null) { ReferentialEntities.getLevelProperty(source.getEntityName()).ifPresent(levelDescriptor -> { try { - if (levelID == null) { + if (levelId == null) { levelDescriptor.getWriteMethod().invoke(target, new Object[]{null}); } else { - Object level = getReference(levelDescriptor.getPropertyType().asSubclass(Serializable.class), levelID); + Object level = getReference(levelDescriptor.getPropertyType().asSubclass(Serializable.class), levelId); levelDescriptor.getWriteMethod().invoke(target, level); } } catch (Exception e) { @@ -841,6 +854,57 @@ protected void toEntity(final ReferentialVO source, IReferentialEntity target, b ((IWithValidityStatusEntity) target).setValidityStatus(getReference(ValidityStatus.class, source.getValidityStatusId())); } } + + // Parent + if (target instanceof ITreeNodeEntity) { + Integer parentId = source.getParentId() != null ? source.getParentId() + : (source.getParent() != null ? source.getParent().getId() : null); + if (parentId != null || copyIfNull) { + if (parentId != null) { + IReferentialEntity parent = getReference(target.getClass(), parentId); + ((ITreeNodeEntity>) target).setParent(parent); + } + else { + ((ITreeNodeEntity) target).setParent(null); + } + } + } + + // Properties + if (MapUtils.isNotEmpty(source.getProperties())) { + copyProperties(source, target, copyIfNull); + } + + } + + protected void copyProperties(final ReferentialVO source, IReferentialEntity target, boolean copyIfNull) { + if (MapUtils.isNotEmpty(source.getProperties())) { + source.getProperties().entrySet().forEach(entry -> { + try { + Object value = entry.getValue(); + if (value != null || copyIfNull) { + Beans.setProperty(target, entry.getKey(), value); + } + } catch (Exception e) { + throw new SumarisTechnicalException(String.format("Cannot set %s.%s using value: %s", + target.getClass().getSimpleName(), + entry.getKey(), + entry.getValue())); + } + }); + } } + protected void copyProperties(final IReferentialEntity source, ReferentialVO target) { + + // TODO use EntitiesUtils + switch (source.getClass().getSimpleName()) { + case "Method" -> { + target.setProperties(ImmutableMap.of( + Method.Fields.IS_CALCULATED, ((Method)source).getIsCalculated(), + Method.Fields.IS_ESTIMATED, ((Method)source).getIsEstimated() + )); + } + } + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialEntities.java b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialEntities.java index 9b69df84c3..958ebc3520 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialEntities.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/ReferentialEntities.java @@ -133,6 +133,7 @@ public class ReferentialEntities { UserProfile.class, SaleType.class, VesselType.class, + ObjectType.class, // Taxon group TaxonGroupType.class, TaxonGroup.class, @@ -235,6 +236,7 @@ protected static final Map createLevelPropertyNameMa // Other level (not having "level" in id) result.put(Pmfm.class.getSimpleName(), BeanUtils.getPropertyDescriptor(Pmfm.class, Pmfm.Fields.PARAMETER)); + result.put(Parameter.class.getSimpleName(), BeanUtils.getPropertyDescriptor(Parameter.class, Parameter.Fields.PARAMETER_GROUP)); result.put(Fraction.class.getSimpleName(), BeanUtils.getPropertyDescriptor(Fraction.class, Fraction.Fields.MATRIX)); result.put(QualitativeValue.class.getSimpleName(), BeanUtils.getPropertyDescriptor(QualitativeValue.class, QualitativeValue.Fields.PARAMETER)); result.put(TaxonGroup.class.getSimpleName(), BeanUtils.getPropertyDescriptor(TaxonGroup.class, TaxonGroup.Fields.TAXON_GROUP_TYPE)); @@ -247,6 +249,10 @@ protected static final Map createLevelPropertyNameMa result.put(ExtractionProductTable.class.getSimpleName(), BeanUtils.getPropertyDescriptor(ExtractionProductTable.class, ExtractionProductTable.Fields.PRODUCT)); result.put(LocationLevel.class.getSimpleName(), BeanUtils.getPropertyDescriptor(LocationLevel.class, LocationLevel.Fields.LOCATION_CLASSIFICATION)); result.put(Gear.class.getSimpleName(), BeanUtils.getPropertyDescriptor(Gear.class, Gear.Fields.GEAR_CLASSIFICATION)); + result.put(TranscribingItemType.class.getSimpleName(), BeanUtils.getPropertyDescriptor(TranscribingItemType.class, TranscribingItemType.Fields.OBJECT_TYPE)); + result.put(TranscribingItem.class.getSimpleName(), BeanUtils.getPropertyDescriptor(TranscribingItem.class, TranscribingItem.Fields.TYPE)); + + // TODO remove the following put ? Too many put for the same Program class result.put(Program.class.getSimpleName(), BeanUtils.getPropertyDescriptor(Program.class, Program.Fields.GEAR_CLASSIFICATION)); result.put(Program.class.getSimpleName(), BeanUtils.getPropertyDescriptor(Program.class, Program.Fields.TAXON_GROUP_TYPE)); diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/RoundWeightConversionSpecifications.java b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/RoundWeightConversionSpecifications.java index 8251a8910d..1590e95fa0 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/RoundWeightConversionSpecifications.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/RoundWeightConversionSpecifications.java @@ -27,8 +27,8 @@ import net.sumaris.core.dao.technical.DatabaseType; import net.sumaris.core.dao.technical.Page; import net.sumaris.core.dao.technical.jpa.BindableSpecification; -import net.sumaris.core.model.data.VesselFeatures; import net.sumaris.core.model.referential.conversion.RoundWeightConversion; +import net.sumaris.core.util.Dates; import net.sumaris.core.vo.referential.conversion.RoundWeightConversionFetchOptions; import net.sumaris.core.vo.referential.conversion.RoundWeightConversionFilterVO; import net.sumaris.core.vo.referential.conversion.RoundWeightConversionVO; @@ -66,10 +66,10 @@ default Specification atDate(Date aDate) { return cb.not( cb.or( cb.lessThan(Daos.nvlEndDate(root, cb, RoundWeightConversion.Fields.END_DATE, getDatabaseType()), dateParam), - cb.greaterThan(root.get(VesselFeatures.Fields.START_DATE), dateParam) + cb.greaterThan(root.get(RoundWeightConversion.Fields.START_DATE), dateParam) ) ); - }).addBind(DATE_PARAMETER, aDate); + }).addBind(DATE_PARAMETER, Dates.resetTime(aDate)); } diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionRepositoryImpl.java b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionRepositoryImpl.java index 2072932791..d448b5ab6c 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionRepositoryImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionRepositoryImpl.java @@ -208,6 +208,7 @@ protected Specification toSpecification(@NonNull WeightL .and(atYear(filter.getYear())) .and(hasReferenceTaxonIds(filter.getReferenceTaxonIds())) .and(hasLocationIds(filter.getLocationIds())) + .and(hasChildLocationIds(filter.getChildLocationIds())) .and(hasRectangleLabels(filter.getRectangleLabels())) .and(hasLengthParameterIds(filter.getLengthParameterIds())) .and(hasLengthUnitIds(filter.getLengthUnitIds())) diff --git a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionSpecifications.java b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionSpecifications.java index ae3c16cc01..c22f363605 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionSpecifications.java +++ b/sumaris-core/src/main/java/net/sumaris/core/dao/referential/conversion/WeightLengthConversionSpecifications.java @@ -48,6 +48,7 @@ public interface WeightLengthConversionSpecifications extends IEntityWithStatusSpecifications { String MONTH_PARAMETER = "month"; + String CHILD_LOCATION_IDS_PARAMETER = "childLocationIds"; String RECTANGLE_LABELS_PARAMETER = "rectangleLabels"; String RECTANGLE_LEVEL_IDS_PARAMETER = "rectangleLevelIds"; String LEANGTH_PMFM_IDS_PARAMETER = "lengthPmfmIds"; @@ -60,6 +61,27 @@ default Specification hasLocationIds(Integer... ids) { return hasInnerJoinIds(WeightLengthConversion.Fields.LOCATION, ids); } + default Specification hasChildLocationIds(Integer... childLocationIds) { + if (ArrayUtils.isEmpty(childLocationIds)) return null; + + return BindableSpecification.where((root, query, cb) -> { + ParameterExpression idsParam = cb.parameter(Collection.class, CHILD_LOCATION_IDS_PARAMETER); + + Subquery subQuery = query.subquery(LocationHierarchy.class); + Root lh = subQuery.from(LocationHierarchy.class); + subQuery.select(lh.get(LocationHierarchy.Fields.PARENT_LOCATION)); + + subQuery.where( + cb.and( + cb.equal(lh.get(LocationHierarchy.Fields.PARENT_LOCATION), Daos.composeJoin(root, WeightLengthConversion.Fields.LOCATION, JoinType.INNER)), + Daos.composePath(lh, StringUtils.doting(LocationHierarchy.Fields.CHILD_LOCATION, Location.Fields.ID), JoinType.INNER).in(idsParam) + ) + ); + + return cb.exists(subQuery); + }) + .addBind(CHILD_LOCATION_IDS_PARAMETER, Arrays.asList(childLocationIds)); + } default Specification hasRectangleLabels(String... rectangleLabels) { if (ArrayUtils.isEmpty(rectangleLabels)) return null; Integer[] rectangleLocationLevelIds = LocationLevels.getStatisticalRectangleLevelIds(); diff --git a/sumaris-core/src/main/java/net/sumaris/core/event/data/BatchEventListener.java b/sumaris-core/src/main/java/net/sumaris/core/event/data/BatchEventListener.java index 16a264ccfb..ce41a0c209 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/event/data/BatchEventListener.java +++ b/sumaris-core/src/main/java/net/sumaris/core/event/data/BatchEventListener.java @@ -27,7 +27,7 @@ import net.sumaris.core.event.entity.EntityEventService; import net.sumaris.core.event.entity.EntityUpdateEvent; import net.sumaris.core.model.data.Batch; -import net.sumaris.core.service.data.DenormalizedBatchService; +import net.sumaris.core.service.data.denormalize.DenormalizedBatchService; import net.sumaris.core.vo.data.batch.BatchVO; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/administration/programStrategy/ProgramPropertyEnum.java b/sumaris-core/src/main/java/net/sumaris/core/model/administration/programStrategy/ProgramPropertyEnum.java index c3ba376623..fb1a3e005f 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/administration/programStrategy/ProgramPropertyEnum.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/administration/programStrategy/ProgramPropertyEnum.java @@ -22,6 +22,8 @@ * #L% */ +import lombok.NonNull; + import java.io.Serializable; import java.util.Arrays; @@ -32,6 +34,13 @@ public enum ProgramPropertyEnum implements Serializable { TRIP_BATCH_TAXON_NAME_ENABLE("sumaris.trip.operation.batch.taxonName.enable", Boolean.class, Boolean.TRUE.toString()), + TRIP_BATCH_TAXON_GROUP_ENABLE("sumaris.trip.operation.batch.taxonGroup.enable", Boolean.class, Boolean.TRUE.toString()), + + TRIP_BATCH_TAXON_GROUPS_NO_WEIGHT("sumaris.trip.operation.batch.taxonGroups.noWeight", String.class, ""), + + TRIP_BATCH_LENGTH_WEIGHT_CONVERSION_ENABLE("sumaris.trip.operation.batch.lengthWeightConversion.enable", Boolean.class, Boolean.FALSE.toString()), + TRIP_BATCH_ROUND_WEIGHT_CONVERSION_COUNTRY_ID("sumaris.trip.operation.batch.roundWeightConversion.country.id", Integer.class, null), + TRIP_BATCH_MEASURE_INDIVIDUAL_TAXON_NAME_ENABLE("sumaris.trip.operation.batch.individual.taxonName.enable", Boolean.class, Boolean.TRUE.toString()), @@ -41,27 +50,27 @@ public enum ProgramPropertyEnum implements Serializable { ; - private String label; + private String key; private String defaultValue; private Class type; - ProgramPropertyEnum(String label, Class type, String defaultValue) { - this.label = label; + ProgramPropertyEnum(String key, Class type, String defaultValue) { + this.key = key; this.type = type; this.defaultValue = defaultValue; } - public static ProgramPropertyEnum findByLabel(final String label) { - return Arrays.stream(values()).filter(item -> item.name().equalsIgnoreCase(label)).findFirst() - .orElseThrow(() -> new IllegalArgumentException("Unknown ProgramPropertyEnum label: " + label)); + public static ProgramPropertyEnum findByKey(@NonNull final String key) { + return Arrays.stream(values()).filter(item -> key.equalsIgnoreCase(item.getKey())).findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown ProgramPropertyEnum with key: " + key)); } - public String getLabel() { - return label; + public String getKey() { + return key; } - public void setLabel(String label) { - this.label = label; + public void setKey(String key) { + this.key = key; } diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/data/DenormalizedBatch.java b/sumaris-core/src/main/java/net/sumaris/core/model/data/DenormalizedBatch.java index 36143c0678..0dc3129ea5 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/data/DenormalizedBatch.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/data/DenormalizedBatch.java @@ -94,6 +94,37 @@ public class DenormalizedBatch implements IEntity { @Column(name = "elevate_individual_count") private Integer elevateIndividualCount; + /** + * Nombre d'individus élevé à l'échelle du groupe de taxon, ou par défaut du taxon (cf mantis + * #37645) + * Calculé par le traitement de dénormalisation. + * @return this.taxonElevateIndivCount Integer + */ + @Column(name = "taxon_elevate_indiv_count") + private Double taxonElevateIndividualCount; + + /** + * Poids contextuel élevé à l'échelle du groupe de taxon, ou par défaut du taxon (cf mantis + * #37645). + * Calculé par le traitement de dénormalisation. + */ + @Column(name = "taxon_elevate_context_weight") + private Double taxonElevateContextWeight; + + /** + * Poids vif sans élévation reconstitué à partir du poids RTP (cf mantis #30088). + * Calculé par le traitement de dénormalisation. + */ + @Column(name = "indirect_rtp_weight") + private Double indirectRtpWeight; + + /** + * Poids vif élevé et reconstitué à partir du poids RTP (cf mantis #30088). + * Calculé par le traitement de dénormalisation. + */ + @Column(name = "elevate_rtp_weight") + private Double elevateRtpWeight; + @Column(name = "sampling_ratio") private Double samplingRatio; @@ -137,6 +168,20 @@ public class DenormalizedBatch implements IEntity { @JoinColumn(name = "inherited_taxon_group_fk") private TaxonGroup inheritedTaxonGroup; + /** + * L'espèce commerciale déterminée à partir de l'espèce scientifique. + * Calculé par le traitement de dénormalisation. + * Dans le cas où le lot ni aucun de ses lots pères indique une espèce commerciale, le + * traitement détermine la plus forte probalité d'appartenance de l'espèce scientifique à une + * espèce ciommerciale. + * Pour cela, les correspondances existantes entre TAXON_GROUP et REFERENCE_TAXON sont + * exploitées (cf table TAXON_GROUP_HISTORICAL_RECORD), ou le cas échéant les correspodnances + * trouvées dans l'arbre d'achantillonnage courant (typiquement dans la "partie retenue" PR, ou + * les deux types d'espèces une chance d'avoir été déjà saisis). + * Ce champ sert à afficher le nom commerciale probable, à côté de chaque informations relatifs + * à une espèce scientifique. C'est le cas par exemple dans les rapports de restitution OBSMER + * aux professionnels. + **/ @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "calculated_taxon_group_fk") private TaxonGroup calculatedTaxonGroup; diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/data/VesselPosition.java b/sumaris-core/src/main/java/net/sumaris/core/model/data/VesselPosition.java index b22754c777..c762bdf495 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/data/VesselPosition.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/data/VesselPosition.java @@ -25,6 +25,7 @@ import lombok.*; import lombok.ToString; import lombok.experimental.FieldNameConstants; +import net.sumaris.core.dao.data.IPosition; import net.sumaris.core.model.administration.user.Department; import net.sumaris.core.model.referential.QualityFlag; @@ -37,7 +38,7 @@ @EqualsAndHashCode(onlyExplicitlyIncluded = true) @FieldNameConstants @Entity(name = "vessel_position") -public class VesselPosition implements IDataEntity { +public class VesselPosition implements IDataEntity, IPosition { @Id @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "VESSEL_POSITION_SEQ") diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/referential/QualityFlags.java b/sumaris-core/src/main/java/net/sumaris/core/model/referential/QualityFlags.java new file mode 100644 index 0000000000..2946a645b1 --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/model/referential/QualityFlags.java @@ -0,0 +1,77 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.model.referential; + +import lombok.NonNull; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.Objects; + +public abstract class QualityFlags { + + protected QualityFlags(){ + // helper class + } + + public static boolean isInvalid(int qualityFlagId) { + return isInvalid(QualityFlagEnum.valueOf(qualityFlagId)); + } + + public static boolean isInvalid(@NonNull QualityFlagEnum qualityFlag) { + switch (qualityFlag) { + case BAD: + case MISSING: + case NOT_COMPLETED: + return true; + default: + return false; + } + } + + public static boolean isValid(int qualityFlagId) { + return isValid(QualityFlagEnum.valueOf(qualityFlagId)); + } + + public static boolean isValid(@NonNull QualityFlagEnum qualityFlag) { + return !isInvalid(qualityFlag); + } + + public static Integer worst(Integer... qualityFlags) { + return Arrays.stream(qualityFlags) + .filter(Objects::nonNull) + // Sort (invalid first, with a negative id) + .sorted(Comparator.comparingInt(qualityFlagId -> isInvalid(qualityFlagId) ? -1 * qualityFlagId : (10 - qualityFlagId))) + .findFirst() + .orElse(null); + } + + public static QualityFlagEnum worst(QualityFlagEnum... qualityFlags) { + return Arrays.stream(qualityFlags) + .filter(Objects::nonNull) + // Sort (invalid first, with a negative id) + .sorted(Comparator.comparingInt(qualityFlag -> isInvalid(qualityFlag) ? -1 * qualityFlag.getId() : (10 - qualityFlag.getId()))) + .findFirst() + .orElse(null); + } +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/ParameterEnum.java b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/ParameterEnum.java index a1c4bab05f..6dca7a63f2 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/ParameterEnum.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/ParameterEnum.java @@ -24,14 +24,17 @@ import net.sumaris.core.model.IEntity; import net.sumaris.core.model.annotation.EntityEnum; +import net.sumaris.core.model.annotation.IEntityEnum; import java.io.Serializable; import java.util.Arrays; @EntityEnum(entity = Parameter.class, joinAttributes = {ParameterGroup.Fields.LABEL, IEntity.Fields.ID}) -public enum ParameterEnum implements Serializable { +public enum ParameterEnum implements Serializable, IEntityEnum { - HULL_MATERIAL(420, "HULL_MATERIAL") + HULL_MATERIAL(420, "HULL_MATERIAL"), + + SEX(80, "SEX") ; diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/PmfmEnum.java b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/PmfmEnum.java index 3e3b98e497..2c93a7c187 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/PmfmEnum.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/PmfmEnum.java @@ -23,12 +23,13 @@ */ import net.sumaris.core.model.annotation.EntityEnum; +import net.sumaris.core.model.annotation.IEntityEnum; import java.io.Serializable; import java.util.Arrays; @EntityEnum(entity = Pmfm.class, joinAttributes = Pmfm.Fields.LABEL, required = false) -public enum PmfmEnum implements Serializable { +public enum PmfmEnum implements IEntityEnum, Serializable { SMALLER_MESH_GAUGE_MM(3, "SMALLER_MESH_GAUGE_MM"), HEADLINE_CUMULATIVE_LENGTH(12, "HEADLINE_CUMULATIVE_LENGTH"), diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/QualitativeValueEnum.java b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/QualitativeValueEnum.java index 94d188de69..e6d3b7a846 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/QualitativeValueEnum.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/QualitativeValueEnum.java @@ -22,18 +22,21 @@ * #L% */ +import net.sumaris.core.model.administration.programStrategy.Program; import net.sumaris.core.model.annotation.EntityEnum; +import net.sumaris.core.model.annotation.IEntityEnum; import java.io.Serializable; import java.util.Arrays; -@EntityEnum(entity = QualitativeValue.class) -public enum QualitativeValueEnum implements Serializable { +@EntityEnum(entity = QualitativeValue.class, joinAttributes = {QualitativeValue.Fields.LABEL, QualitativeValue.Fields.ID}) +public enum QualitativeValueEnum implements Serializable, IEntityEnum { SORTING_BULK(390, "VRAC"), // Adagio => 311 SORTING_NON_BULK(391, "H-VRAC"), // Adagio => 310 SORTING_UNSORTED(392, "NONE"), // Adagio => 2146 - DRESSING_WHOLE(381, "WHL"), + DRESSING_WHOLE(381, "WHL"), // Entier - Adagio => 139 + DRESSING_GUTTED(381, "GUT"), // Eviscéré - Adagio => 120 PRESERVATION_FRESH(332, "FRE"), SIZE_CATEGORY_NONE(435, "UNS"), @@ -47,6 +50,9 @@ public enum QualitativeValueEnum implements Serializable { // LANDING_OR_DISCARD LANDING(190, "LAN"), DISCARD(191, "DIS"), + + // SEX + SEX_UNSEXED(188, "NS"), // Adagio => 302 ; public static QualitativeValueEnum valueOf(final int id) { diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/UnitEnum.java b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/UnitEnum.java index 9b5d0b3f71..ef5461cf85 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/UnitEnum.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/referential/pmfm/UnitEnum.java @@ -22,18 +22,20 @@ package net.sumaris.core.model.referential.pmfm; +import net.sumaris.core.model.administration.programStrategy.Program; import net.sumaris.core.model.annotation.EntityEnum; import java.io.Serializable; import java.util.Arrays; -@EntityEnum(entity = Unit.class) +@EntityEnum(entity = Unit.class, joinAttributes = {Unit.Fields.LABEL}) public enum UnitEnum implements Serializable { NONE(0, "None"), - MM(1, "mm"), KG(3, "kg"), - CM(12, "cm"); + MM(1, "mm"), + CM(12, "cm") + ; public static UnitEnum valueOf(final int id) { return Arrays.stream(values()) diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbLanding.java b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbLanding.java index 2d2d8b9934..b239373077 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbLanding.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbLanding.java @@ -33,7 +33,6 @@ @Getter @Setter - @EqualsAndHashCode(onlyExplicitlyIncluded = true) @FieldNameConstants @Entity @@ -73,7 +72,7 @@ public class ProductRdbLanding implements Serializable, IEntity { private Integer id; @Column(nullable = false, length = 2, name = COLUMN_RECORD_TYPE) - @ColumnDefault(SHEET_NAME) + @ColumnDefault("'" + SHEET_NAME + "'") private String recordType; @Column(nullable = false, length = 3, name = COLUMN_LANDING_COUNTRY) diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesLength.java b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesLength.java index fb1303515d..06bd888607 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesLength.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesLength.java @@ -70,7 +70,7 @@ public class ProductRdbSpeciesLength implements Serializable, IEntity { private Integer id; @Column(nullable = false, length = 2, name = COLUMN_RECORD_TYPE) - @ColumnDefault(SHEET_NAME) + @ColumnDefault("'" + SHEET_NAME + "'") private String recordType; @Column(nullable = false, length = 2, name = COLUMN_SAMPLING_TYPE) diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesList.java b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesList.java index 5e714b17a3..fdeb39aa8a 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesList.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbSpeciesList.java @@ -70,7 +70,7 @@ public class ProductRdbSpeciesList implements Serializable, IEntity { private Integer id; @Column(nullable = false, length = 2, name = COLUMN_RECORD_TYPE) - @ColumnDefault(SHEET_NAME) + @ColumnDefault("'" + SHEET_NAME + "'") private String recordType; @Column(nullable = false, length = 2, name = COLUMN_SAMPLING_TYPE) diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbStation.java b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbStation.java index 487e04921b..e274473c63 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbStation.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbStation.java @@ -83,7 +83,7 @@ public class ProductRdbStation implements Serializable, IEntity { private Integer id; @Column(nullable = false, length = 2, name = COLUMN_RECORD_TYPE) - @ColumnDefault(SHEET_NAME) + @ColumnDefault("'" + SHEET_NAME + "'") private String recordType; @Column(nullable = false, length = 2, name = COLUMN_SAMPLING_TYPE) diff --git a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbTrip.java b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbTrip.java index 65ec14eecd..4fcf463ce3 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbTrip.java +++ b/sumaris-core/src/main/java/net/sumaris/core/model/technical/extraction/rdb/ProductRdbTrip.java @@ -67,7 +67,7 @@ public class ProductRdbTrip implements Serializable, IEntity { private Integer id; @Column(nullable = false, length = 2, name = COLUMN_RECORD_TYPE) - @ColumnDefault(SHEET_NAME) + @ColumnDefault("'" + SHEET_NAME + "'") private String recordType; @Column(nullable = false, length = 2, name = COLUMN_SAMPLING_TYPE) diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramService.java b/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramService.java index a3f88fd7e7..8b46ac9846 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramService.java @@ -56,6 +56,7 @@ public interface ProgramService { @Transactional(readOnly = true) ProgramVO getByLabel(String label); + @Transactional(readOnly = true) ProgramVO getByLabel(String label, ProgramFetchOptions fetchOptions); @Transactional(readOnly = true) diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramServiceImpl.java index e7e9247198..72933d928d 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/administration/programStrategy/ProgramServiceImpl.java @@ -25,7 +25,9 @@ import com.google.common.base.Preconditions; import lombok.NonNull; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import net.sumaris.core.config.CacheConfiguration; import net.sumaris.core.dao.administration.programStrategy.AcquisitionLevelRepository; import net.sumaris.core.dao.administration.programStrategy.ProgramRepository; import net.sumaris.core.dao.technical.Page; @@ -41,6 +43,7 @@ import net.sumaris.core.vo.referential.ReferentialVO; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import javax.annotation.Nullable; @@ -51,14 +54,13 @@ import java.util.stream.Stream; @Service("programService") +@RequiredArgsConstructor @Slf4j public class ProgramServiceImpl implements ProgramService { - @Autowired - protected ProgramRepository programRepository; + protected final ProgramRepository programRepository; - @Autowired - protected StrategyService strategyService; + protected final StrategyService strategyService; @Override public List getAll() { diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/DenormalizedBatchServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/DenormalizedBatchServiceImpl.java deleted file mode 100644 index 1d4b84d1fd..0000000000 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/DenormalizedBatchServiceImpl.java +++ /dev/null @@ -1,648 +0,0 @@ -package net.sumaris.core.service.data; - -/*- - * #%L - * SUMARiS:: Core - * %% - * Copyright (C) 2018 SUMARiS Consortium - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ - - -import com.google.common.base.Preconditions; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import net.sumaris.core.config.SumarisConfigurationOption; -import net.sumaris.core.dao.data.batch.BatchRepository; -import net.sumaris.core.dao.data.batch.DenormalizedBatchRepository; -import net.sumaris.core.dao.data.batch.InvalidSamplingBatchException; -import net.sumaris.core.model.TreeNodeEntities; -import net.sumaris.core.exception.SumarisTechnicalException; -import net.sumaris.core.model.referential.QualityFlagEnum; -import net.sumaris.core.model.referential.StatusEnum; -import net.sumaris.core.model.referential.taxon.TaxonGroupTypeEnum; -import net.sumaris.core.service.administration.programStrategy.ProgramService; -import net.sumaris.core.service.referential.taxon.TaxonGroupService; -import net.sumaris.core.util.Beans; -import net.sumaris.core.util.StringUtils; -import net.sumaris.core.util.TimeUtils; -import net.sumaris.core.vo.administration.programStrategy.ProgramFetchOptions; -import net.sumaris.core.vo.administration.programStrategy.ProgramVO; -import net.sumaris.core.vo.administration.programStrategy.Programs; -import net.sumaris.core.vo.data.batch.*; -import net.sumaris.core.vo.filter.ReferentialFilterVO; -import net.sumaris.core.vo.referential.TaxonGroupVO; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.commons.lang3.mutable.MutableShort; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import javax.annotation.Nullable; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.*; -import java.util.stream.Collectors; - -@Service("denormalizedBatchService") -@Slf4j -public class DenormalizedBatchServiceImpl implements DenormalizedBatchService { - - @Autowired - protected DenormalizedBatchRepository denormalizedBatchRepository; - - @Autowired - protected BatchRepository batchRepository; - - @Autowired - protected ProgramService programService; - - @Autowired - protected OperationService operationService; - - @Autowired - protected SaleService saleService; - - @Autowired - protected TaxonGroupService taxonGroupService; - - @Override - public List denormalize(@NonNull BatchVO catchBatch, @NonNull final DenormalizedBatchOptions options) { - - boolean trace = log.isTraceEnabled(); - long startTime = System.currentTimeMillis(); - final MutableShort flatRankOrder = new MutableShort(0); - - List result = TreeNodeEntities.streamAllAndMap(catchBatch, (source, parent) -> { - TempDenormalizedBatchVO target = createTempVO(source); - - // Add to parent's children - if (parent != null) parent.addChildren(target); - - // Depth level - if (parent == null) { - target.setTreeLevel((short)1); // First level - if (target.getIsLanding() == null) target.setIsLanding(false); - if (target.getIsDiscard() == null) target.setIsDiscard(false); - } - else { - target.setTreeLevel((short)(parent.getTreeLevel() + 1)); - // Inherit taxon group - if (target.getInheritedTaxonGroup() == null && parent.getInheritedTaxonGroup() != null) { - target.setInheritedTaxonGroup(parent.getInheritedTaxonGroup()); - } - // Inherit taxon name - if (target.getInheritedTaxonName() == null && parent.getInheritedTaxonName() != null) { - target.setInheritedTaxonName(parent.getInheritedTaxonName()); - } - // Exhaustive inventory - if (target.getExhaustiveInventory() == null) { - // Always true, when: - // - taxon name is defined - // - taxon group is defined and taxon Name disable (in options) - if (target.getInheritedTaxonName() != null) { - target.setExhaustiveInventory(Boolean.TRUE); - } - else if (target.getInheritedTaxonGroup() != null && !options.isEnableTaxonName()) { - target.setExhaustiveInventory(Boolean.TRUE); - } - else if (parent.getExhaustiveInventory() != null) { - target.setExhaustiveInventory(parent.getExhaustiveInventory()); - } - } - // Inherit location - if (parent.getLocationId() != null) { - target.setLocationId(parent.getLocationId()); - } - // Inherit landing / discard - if (target.getIsLanding() == null) { - target.setIsLanding(parent.getIsLanding()); - } - if (target.getIsDiscard() == null) { - target.setIsDiscard(parent.getIsDiscard()); - } - - // Inherit quality flag (keep the worst value) - if (parent.getQualityFlagId() > target.getQualityFlagId()) { - target.setQualityFlagId(parent.getQualityFlagId()); - } - - // If current quality is out of stats - if (target.getQualityFlagId() >= QualityFlagEnum.OUT_STATS.getId()) { - // Force both parent and current parent exhaustive inventory to FALSE - parent.setExhaustiveInventory(Boolean.FALSE); - target.setExhaustiveInventory(Boolean.FALSE); - } - - // Inherit sorting values - Beans.getStream(parent.getSortingValues()).forEach(svSource -> { - DenormalizedBatchSortingValueVO svTarget = new DenormalizedBatchSortingValueVO(); - Beans.copyProperties(svSource, svTarget); - svTarget.setIsInherited(true); - svTarget.setRankOrder(svSource.getRankOrder() / 10); - target.addSortingValue(svTarget); - }); - - } - - return target; - }) - - // Sort - .sorted(Comparator.comparing(DenormalizedBatches::computeFlatOrder)) - - .map(target -> { - // Compute flat rank order - flatRankOrder.increment(); - target.setFlatRankOrder(flatRankOrder.getValue()); - - // Compute tree indent (run once, on the root batch) - if (target.getParent() == null) computeTreeIndent(target); - - return target; - }) - .collect(Collectors.toList()); - - if (CollectionUtils.size(result) == 1) { - DenormalizedBatchVO target = result.get(0); - target.setElevateWeight(target.getWeight()); - } - else { - // Compute indirect values - computeIndirectValues(result); - - // Elevate weight - computeElevatedValues(result); - } - - // Log - if (trace) { - log.trace("Successfully denormalized batches, in {}:\n{}", - TimeUtils.printDurationFrom(startTime), - DenormalizedBatches.dumpAsString(result, true, true)); - } - else { - log.debug("Successfully denormalized batches, in {}", TimeUtils.printDurationFrom(startTime)); - } - - return result; - } - - @Override - public List denormalizeAndSaveByOperationId(int operationId, @Nullable DenormalizedBatchOptions options) { - BatchVO catchBatch = batchRepository.getCatchBatchByOperationId(operationId, BatchFetchOptions.builder() - .withChildrenEntities(true) - .withMeasurementValues(true) - .withRecorderDepartment(false) - .build()); - if (catchBatch == null) return null; - - long startTime = System.currentTimeMillis(); - log.debug("Denormalize batches of operation {id: {}}...", operationId); - - // Compute options, for the operation's program - if (options == null) { - int programId = operationService.getProgramIdById(operationId); - options = createOptionsByProgramId(programId); - } - - // Denormalize batches - List batches = denormalize(catchBatch, options); - - // Save denormalized batches - batches = denormalizedBatchRepository.saveAllByOperationId(operationId, batches); - - log.debug("Denormalize batches of operation {id: {}} [OK] in {}", operationId, TimeUtils.printDurationFrom(startTime)); - return batches; - } - - @Override - public List denormalizeAndSaveBySaleId(int saleId, @Nullable DenormalizedBatchOptions options) { - BatchVO catchBatch = batchRepository.getCatchBatchBySaleId(saleId, BatchFetchOptions.builder() - .withChildrenEntities(true) - .withMeasurementValues(true) - .withRecorderDepartment(false) - .build()); - if (catchBatch == null) return null; - - long startTime = System.currentTimeMillis(); - log.debug("Denormalize batches of sale {id: {}}...", saleId); - - // Compute options, for the sale's program - if (options == null) { - int programId = saleService.getProgramIdById(saleId); - options = createOptionsByProgramId(programId); - } - - // Denormalize batches - List denormalizedBatches = denormalize(catchBatch, options); - - // Save denormalized batches - List result = denormalizedBatchRepository.saveAllBySaleId(saleId, denormalizedBatches); - - log.debug("Denormalize batches of sale {id: {}} [OK] in {}", saleId, TimeUtils.printDurationFrom(startTime)); - return result; - } - - @Override - public DenormalizedBatchOptions createOptionsByProgramId(int programId) { - - ProgramVO program = programService.get(programId, ProgramFetchOptions.builder() - .withProperties(true) - .withLocations(false) - .withStrategies(false) - .build()); - - String taxonGroupsNoWeight = Programs.getProperty(program, SumarisConfigurationOption.BATCH_TAXON_GROUP_LABELS_NO_WEIGHT); - List taxonGroupIdsNoWeight = Arrays.stream(taxonGroupsNoWeight.split(",")) - .map(String::trim) - .map(label -> taxonGroupService.findAllByFilter(ReferentialFilterVO.builder() - .label(label) - .levelIds(new Integer[]{TaxonGroupTypeEnum.FAO.getId()}) - .statusIds(new Integer[]{ StatusEnum.ENABLE.getId() }) - .build()).stream().findFirst().orElse(null)) - .filter(Objects::nonNull) - .map(TaxonGroupVO::getId) - .collect(Collectors.toList()); - - return DenormalizedBatchOptions.builder() - .enableTaxonName(Programs.getPropertyAsBoolean(program, SumarisConfigurationOption.ENABLE_BATCH_TAXON_NAME)) - .enableTaxonGroup(Programs.getPropertyAsBoolean(program, SumarisConfigurationOption.ENABLE_BATCH_TAXON_GROUP)) - .taxonGroupIdsNoWeight(taxonGroupIdsNoWeight) - .build(); - } - - /* -- protected methods -- */ - - protected void computeIndirectValues(List batches) { - - List revertBatches = batches.stream() - .map(target -> (TempDenormalizedBatchVO)target) - // Reverse order (start from leaf) - .sorted(Collections.reverseOrder(Comparator.comparing(DenormalizedBatchVO::getFlatRankOrder))) - .collect(Collectors.toList()); - - // Compute indirect values (from children to parent) - MutableInt changesCount = new MutableInt(0); - MutableInt loopCounter = new MutableInt(0); - do { - changesCount.setValue(0); - loopCounter.increment(); - log.debug("Computing indirect values (pass #{}) ...", loopCounter); - - revertBatches.forEach(batch -> { - boolean changed = false; - - // Indirect weight - Double indirectWeight = computeIndirectWeight(batch); - changed = changed || !Objects.equals(indirectWeight, batch.getIndirectWeight()); - batch.setIndirectWeight(indirectWeight); - - // Indirect individual count - Integer indirectIndividualCount = computeIndirectIndividualCount(batch); - changed = changed || !Objects.equals(indirectIndividualCount, batch.getIndirectIndividualCount()); - batch.setIndirectIndividualCount(indirectIndividualCount); - - - // Contextual weight - //Double contextWeight = computeContextWeight(target); - //changed = changed || !Objects.equals(contextWeight, target.getContextWeight()); - //target.setContextWeight(contextWeight); - - // Compute Round weight weight - //Double sumChildRoundWeight = computeSumChildRoundWeight(target); - //changed = changed || !Objects.equals(sumChildRoundWeight, target.getSumChildRoundWeight()); - - // Compute RTP weight - //Double sumChildRtpWeight = computeSumChildRTPWeight(target); - //changed = changed || !Objects.equals(sumChildRtpWeight, target.getSumChildRTPWeight()); - - if (changed) changesCount.increment(); - }); - - log.trace("Computing indirect values (pass #{}) [OK] - {} changes", loopCounter, changesCount); - } - - // Continue while changes has been applied on tree - while (changesCount.intValue() > 0); - } - - /** - * Compute elevated values - */ - protected void computeElevatedValues(List batches) { - MutableInt changesCount = new MutableInt(0); - MutableInt loopCounter = new MutableInt(0); - - do { - changesCount.setValue(0); - loopCounter.increment(); - log.debug("Computing elevated values (pass #{}) ...", loopCounter); - - batches.stream() - .map(target -> (TempDenormalizedBatchVO) target) - .forEach(target -> { - boolean changed = false; - - log.trace("{} {}", target.getTreeIndent(), target.getLabel()); - BigDecimal elevateFactor = target.getElevateFactor(); - if (elevateFactor == null) { - elevateFactor = new BigDecimal(1); - if (target.getParent() != null) { - elevateFactor = elevateFactor.multiply(((TempDenormalizedBatchVO) target.getParent()).getElevateFactor()); - } - } - // Remember it, for children - target.setElevateFactor(elevateFactor); - - Double weight = target.getWeight() != null ? target.getWeight() : target.getIndirectWeight(); - if (weight != null) { - Double elevateWeight = elevateFactor.multiply(new BigDecimal(weight)).doubleValue(); - changed = changed || !Objects.equals(elevateWeight, target.getElevateWeight()); - target.setElevateWeight(elevateWeight); - } - - Integer individualCount = target.getIndividualCount() != null ? target.getIndividualCount() : target.getIndirectIndividualCount(); - if (individualCount != null) { - Integer elevateIndividualCount = new BigDecimal(individualCount).multiply(elevateFactor).intValue(); - changed = changed || !Objects.equals(elevateIndividualCount, target.getElevateIndividualCount()); - target.setElevateIndividualCount(elevateIndividualCount); - } - - if (changed) changesCount.increment(); - }); - - log.trace("Computing elevated values (pass #{}) [OK] - {} changes", loopCounter, changesCount); - } while (changesCount.intValue() > 0); - - } - - protected Double computeIndirectWeight(TempDenormalizedBatchVO batch) { - // Already computed: skip - if (batch.getIndirectWeight() != null) return batch.getIndirectWeight(); - - // Sampling batch - if (DenormalizedBatches.isSamplingBatch(batch)) { - try { - Double samplingWeight = computeSamplingWeightAndRatio(batch, false); - if (samplingWeight != null) return samplingWeight; - } - catch (InvalidSamplingBatchException e) { - // May be not a sampling batch ? (e.g. a species batch) - Double indirectWeight = computeSumChildrenWeight(batch); - if (indirectWeight != null) return indirectWeight; - throw e; - } - // Invalid sampling batch: Continue if not set - } - - // Child batch is a sampling batch - if (DenormalizedBatches.isParentOfSamplingBatch(batch)) { - return computeParentSamplingWeight(batch, false); - } - - Double indirectWeight = computeSumChildrenWeight(batch); - return indirectWeight; - } - - - protected Double computeSamplingWeightAndRatio(TempDenormalizedBatchVO batch, boolean checkArgument) { - if (checkArgument) - Preconditions.checkArgument(DenormalizedBatches.isSamplingBatch(batch)); - - DenormalizedBatchVO parent = batch.getParent(); - boolean parentExhaustiveInventory = DenormalizedBatches.isExhaustiveInventory(parent); - Double samplingWeight = null; - Double samplingRatio = null; - BigDecimal elevateFactor = null; - - if (batch.getSamplingRatio() != null) { - samplingRatio = batch.getSamplingRatio(); - elevateFactor = new BigDecimal(1).divide(new BigDecimal(samplingRatio), RoundingMode.HALF_UP); - - // Try to use the sampling ratio text (more accuracy) - if (StringUtils.isNotBlank(batch.getSamplingRatioText()) && batch.getSamplingRatioText().contains("/")) { - String[] parts = batch.getSamplingRatioText().split("/", 2); - try { - double d0 = Double.parseDouble(parts[0]); - double d1 = Double.parseDouble(parts[1]); - samplingRatio = d0 / d1; - elevateFactor = new BigDecimal(d1).divide(new BigDecimal(d0), RoundingMode.HALF_UP); - } catch (Exception e) { - log.warn("Cannot parse samplingRatioText on batch {id: {}}, label: '{}', saplingRatioText: '{}'} : {}", - batch.getId(), - batch.getLabel(), - batch.getSamplingRatioText(), - e.getMessage()); - } - } - } - else if (parentExhaustiveInventory && parent.getWeight() != null && batch.getWeight() != null) { - samplingRatio = batch.getWeight() / parent.getWeight(); - elevateFactor = new BigDecimal(parent.getWeight()).divide(new BigDecimal(batch.getWeight()), RoundingMode.HALF_UP); - } - - else if (parentExhaustiveInventory && parent.getWeight() != null && batch.hasChildren()) { - samplingWeight = computeSumChildrenWeight(batch); - if (samplingWeight != null) { - samplingRatio = samplingWeight / parent.getWeight(); - elevateFactor = new BigDecimal(parent.getWeight()).divide(new BigDecimal(samplingWeight), RoundingMode.HALF_UP); - } - } - - else if ((!parentExhaustiveInventory || parent.getWeight() == null) && batch.hasChildren()) { - samplingWeight = computeSumChildrenWeight(batch); - if (samplingWeight != null) { - samplingRatio = 1d; - elevateFactor = new BigDecimal(1); - } - } - - if (samplingRatio == null || elevateFactor == null) { - // Use default value (samplingRatio=1) if: - // - batch has no children - // - batch is parent of a sampling batch - if (CollectionUtils.isEmpty(batch.getChildren())) { - samplingRatio = 1d; - elevateFactor = new BigDecimal(1); - } - else { - throw new InvalidSamplingBatchException(String.format("Invalid sampling batch {id: %s, label: '%s'}: cannot get or compute the sampling ratio", - batch.getId(), batch.getLabel())); - } - } - - // Remember values - batch.setSamplingRatio(samplingRatio); - batch.setElevateFactor(elevateFactor); - - if (samplingWeight == null) { - if (batch.getWeight() != null) { - samplingWeight = batch.getWeight(); - } else if (parentExhaustiveInventory && parent.getWeight() != null) { - samplingWeight = parent.getWeight() * samplingRatio; - } - } - - return samplingWeight; - } - - - protected Double computeParentSamplingWeight(TempDenormalizedBatchVO parent, boolean checkArgument) { - if (checkArgument) Preconditions.checkArgument(DenormalizedBatches.isParentOfSamplingBatch(parent)); - - // Use reference weight, if any - if (parent.getWeight() != null) { - return parent.getWeight(); - } - - TempDenormalizedBatchVO batch = (TempDenormalizedBatchVO)CollectionUtils.extractSingleton(parent.getChildren()); - - Double parentWeight = null; - Double samplingRatio = null; - Double elevateFactor = null; - if (batch.getSamplingRatio() != null) { - samplingRatio = batch.getSamplingRatio(); - elevateFactor = 1 / samplingRatio; - - // Try to use the sampling ratio text (more accuracy) - if (StringUtils.isNotBlank(batch.getSamplingRatioText()) && batch.getSamplingRatioText().contains("/")) { - String[] parts = batch.getSamplingRatioText().split("/", 2); - try { - Double shouldBeSamplingWeight = Double.parseDouble(parts[0]); - Double shouldBeParentWeight = Double.parseDouble(parts[1]); - // If ratio text use the sampling weight, we have the parent weight - if (Objects.equals(shouldBeSamplingWeight, batch.getWeight()) - || Objects.equals(shouldBeSamplingWeight, batch.getIndirectWeight())) { - parentWeight = shouldBeParentWeight; - } - samplingRatio = shouldBeSamplingWeight / shouldBeParentWeight; - elevateFactor = shouldBeParentWeight / shouldBeSamplingWeight; - } catch (Exception e) { - log.warn(String.format("Cannot parse samplingRatioText on batch {id: %s, label: '%s', saplingRatioText: '%s'} : %s", - batch.getId(), - batch.getLabel(), - batch.getSamplingRatioText(), - e.getMessage())); - } - } - } - - if (samplingRatio == null) - throw new SumarisTechnicalException(String.format("Invalid fraction batch {id: %s, label: '%s'}: cannot get or compute the sampling ratio", - batch.getId(), batch.getLabel())); - - if (parentWeight == null) { - if (batch.getWeight() != null) { - parentWeight = batch.getWeight() * elevateFactor; - } else if (batch.getIndirectWeight() != null) { - parentWeight = batch.getIndirectWeight() * elevateFactor; - } - } - - return parentWeight; - } - - protected Double computeSumChildrenWeight(DenormalizedBatchVO batch) { - // Cannot compute children sum, when: - // - Not exhaustive inventory - // - No children - if (!DenormalizedBatches.isExhaustiveInventory(batch) - || !batch.hasChildren()) { - return null; - } - - try { - return Beans.getStream(batch.getChildren()) - .map(child -> (TempDenormalizedBatchVO)child) - .mapToDouble(child -> { - if (child.getWeight() != null) return child.getWeight(); - if (child.getIndirectWeight() != null) return child.getIndirectWeight(); - if (child.hasChildren()) { - Double indirectWeight = computeIndirectWeight(child); - if (indirectWeight != null) { - return indirectWeight; - } - } - throw new SumarisTechnicalException(String.format("Cannot compute indirect weight," - + " because some child batch has no weight {id: %s, label: '%s'}", child.getId(), child.getLabel())); - }).sum(); - } - catch(SumarisTechnicalException e) { - log.trace(e.getMessage()); - return null; - } - } - - protected Integer computeIndirectIndividualCount(TempDenormalizedBatchVO batch) { - // Already computed: skip - if (batch.getIndirectIndividualCount() != null) return batch.getIndirectIndividualCount(); - - // Cannot compute when: - // - Not exhaustive inventory - // - No children - if (!DenormalizedBatches.isExhaustiveInventory(batch) - || !batch.hasChildren()) { - return null; - } - - try { - return Beans.getStream(batch.getChildren()) - .map(child -> (TempDenormalizedBatchVO) child) - .mapToInt(child -> { - if (child.getIndividualCount() != null) return child.getIndividualCount(); - if (child.hasChildren()) { - Integer indirectIndividualCount = computeIndirectIndividualCount(child); - if (indirectIndividualCount != null) { - return indirectIndividualCount; - } - } - throw new SumarisTechnicalException(String.format("Cannot compute indirect individual count," - + " because some child batch has no individual count {id: %s, label: '%s'}", child.getId(), child.getLabel())); - }).sum(); - } catch (SumarisTechnicalException e) { - log.trace(e.getMessage()); - return null; - } - } - - protected void computeTreeIndent(DenormalizedBatchVO target) { - computeTreeIndent(target, "", true); - } - - protected void computeTreeIndent(DenormalizedBatchVO target, String inheritedTreeIndent, boolean isLast) { - if (target.getParent() == null) { - target.setTreeIndent("-"); - } else { - target.setTreeIndent(inheritedTreeIndent + (isLast? "|_" : "|-")); - } - - List children = target.getChildren(); - if (CollectionUtils.isNotEmpty(children)) { - String childrenTreeIndent = inheritedTreeIndent + (isLast? " " : "| "); - for (int i = 0; i < children.size(); i++) { - computeTreeIndent(children.get(i), childrenTreeIndent, i == children.size() - 1); - } - } - } - - protected TempDenormalizedBatchVO createTempVO(BatchVO source) { - TempDenormalizedBatchVO target = new TempDenormalizedBatchVO(); - denormalizedBatchRepository.copy(source, target, true); - return target; - } -} diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationService.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationService.java index eb470a10f7..066d71a4e8 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationService.java @@ -23,6 +23,7 @@ */ +import lombok.NonNull; import net.sumaris.core.dao.technical.SortDirection; import net.sumaris.core.vo.data.OperationFetchOptions; import net.sumaris.core.vo.data.OperationVO; @@ -47,6 +48,9 @@ public interface OperationService { @Transactional(readOnly = true) List findAllByTripId(int tripId, int offset, int size, String sortAttribute, SortDirection sortDirection, OperationFetchOptions fetchOptions); + @Transactional(readOnly = true) + List findAllByFilter(@NonNull OperationFilterVO filter, @NonNull OperationFetchOptions fetchOptions); + @Transactional(readOnly = true) List findAllByFilter(OperationFilterVO filter, int offset, int size, String sortAttribute, SortDirection sortDirection, OperationFetchOptions fetchOptions); diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationServiceImpl.java index e2b10ade6b..aca7c28f31 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/OperationServiceImpl.java @@ -99,9 +99,15 @@ public List findAllByTripId(int tripId, @NonNull OperationFetchOpti public List findAllByTripId(int tripId, int offset, int size, String sortAttribute, SortDirection sortDirection, @NonNull OperationFetchOptions fetchOptions) { - return operationRepository.findAllVO(operationRepository.hasTripId(tripId), - Pageables.create(offset, size, sortAttribute, sortDirection), - fetchOptions).getContent(); + return operationRepository.findAll(OperationFilterVO.builder().tripId(tripId).build(), + offset, size, sortAttribute, sortDirection, + fetchOptions); + } + + @Override + public List findAllByFilter(@NonNull OperationFilterVO filter, + @NonNull OperationFetchOptions fetchOptions) { + return operationRepository.findAll(filter, fetchOptions); } @Override diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/PacketServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/PacketServiceImpl.java index b0f1a56bd6..cf24352f6a 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/PacketServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/PacketServiceImpl.java @@ -33,6 +33,7 @@ import net.sumaris.core.event.config.ConfigurationReadyEvent; import net.sumaris.core.event.config.ConfigurationUpdatedEvent; import net.sumaris.core.exception.SumarisTechnicalException; +import net.sumaris.core.model.annotation.EntityEnums; import net.sumaris.core.model.data.BatchQuantificationMeasurement; import net.sumaris.core.model.data.BatchSortingMeasurement; import net.sumaris.core.model.data.IMeasurementEntity; @@ -82,6 +83,19 @@ public void onConfigurationReady(ConfigurationEvent event) { this.measuredWeightPmfmId = PmfmEnum.BATCH_MEASURED_WEIGHT.getId(); this.estimatedRatioPmfmId = PmfmEnum.BATCH_ESTIMATED_RATIO.getId(); this.sortingPmfmId = PmfmEnum.BATCH_SORTING.getId(); + + try { + + EntityEnums.checkResolved( + // Check pmfm enums + PmfmEnum.BATCH_CALCULATED_WEIGHT, PmfmEnum.BATCH_MEASURED_WEIGHT, PmfmEnum.BATCH_ESTIMATED_RATIO, PmfmEnum.BATCH_SORTING, + // Check QV enums + QualitativeValueEnum.SORTING_BULK + ); + } + catch (Exception e) { + log.error(e.getMessage()); + } } @Override diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeOperationServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeOperationServiceImpl.java new file mode 100644 index 0000000000..c46c797070 --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeOperationServiceImpl.java @@ -0,0 +1,222 @@ +package net.sumaris.core.service.data.denormalize; + +/*- + * #%L + * SUMARiS:: Core + * %% + * Copyright (C) 2018 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Lists; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.sumaris.core.dao.data.Positions; +import net.sumaris.core.dao.technical.SortDirection; +import net.sumaris.core.exception.SumarisBusinessException; +import net.sumaris.core.service.data.OperationService; +import net.sumaris.core.service.referential.LocationService; +import net.sumaris.core.util.Beans; +import net.sumaris.core.util.Dates; +import net.sumaris.core.vo.data.*; +import net.sumaris.core.vo.data.batch.DenormalizedBatchOptions; +import net.sumaris.core.vo.filter.OperationFilterVO; +import net.sumaris.core.vo.referential.LocationVO; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.mutable.MutableInt; +import org.springframework.stereotype.Service; + +import javax.annotation.Nullable; +import javax.annotation.PostConstruct; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +@Service("denormalizeOperationService") +@RequiredArgsConstructor +@Slf4j +public class DenormalizeOperationServiceImpl implements DenormalizedOperationService { + + private final DenormalizedBatchService denormalizedBatchService; + + private final OperationService operationService; + + private final LocationService locationService; + + // Create a cache for denormalized options, by programId + private LoadingCache optionsByProgramIdCache; + + private LoadingCache optionsByProgramLabelCache; + + @PostConstruct + protected void init() { + // Create a cache for denormalized options, by programId + optionsByProgramIdCache = CacheBuilder.newBuilder() + .expireAfterWrite(5, TimeUnit.MINUTES) // 5 min (if job is very long, the options will be reload) + .build(CacheLoader.from(denormalizedBatchService::createOptionsByProgramId)); + + // Create a cache for denormalized options, by programLabel + optionsByProgramLabelCache = CacheBuilder.newBuilder() + .expireAfterWrite(5, TimeUnit.MINUTES) // 5 min (if job is very long, the options will be reload) + .build(CacheLoader.from(denormalizedBatchService::createOptionsByProgramLabel)); + + } + @Override + public DenormalizedBatchOptions createOptionsByProgramId(int programId) { + return optionsByProgramIdCache.getUnchecked(programId); + } + @Override + public DenormalizedBatchOptions createOptionsByProgramLabel(String programLabel) { + return optionsByProgramLabelCache.getUnchecked(programLabel); + } + + @Override + public DenormalizedTripResultVO denormalizeByFilter(@NonNull OperationFilterVO operationFilter, + @NonNull DenormalizedBatchOptions baseOptions) { + long startTime = System.currentTimeMillis(); + + operationFilter = operationFilter.clone(); + + // Make sure to exclude parent operation, because should not have batches + // (see "filage" operation in ACOST program) + operationFilter.setHasNoChildOperation(true); + + // Select only operation that should be update (if not force) + operationFilter.setNeedBatchDenormalization(!baseOptions.isForce()); + + long operationTotal = operationService.countByFilter(operationFilter); + + boolean hasMoreData; + int offset = 0; + int pageSize = 10; + int operationCount = 0; + MutableInt batchesCount = new MutableInt(0); + MutableInt errorCount = new MutableInt(0); + List messages = Lists.newArrayList(); + + if (operationTotal > 0) { + do { + // Fetch some operations + List operations = operationService.findAllByFilter(operationFilter, + offset, pageSize, // Page + OperationVO.Fields.ID, SortDirection.ASC, // Sort by id, to keep continuity between pages + OperationFetchOptions.builder() + .withChildrenEntities(false) + .withMeasurementValues(false) + // Fetch position and fishing area, to be able to compute fishing area id, need by conversion + .withPositions(true) + .withFishingAreas(true) + .build()); + + operations.forEach(operation -> { + try { + // Prepare options (add fishing area, date, etc.) + DenormalizedBatchOptions options = createOptionsByOperation(operation, baseOptions); + + List batches = denormalizedBatchService.denormalizeAndSaveByOperationId(operation.getId(), options); + batchesCount.add(CollectionUtils.size(batches)); + } catch (SumarisBusinessException be) { + log.error(be.getMessage()); + messages.add(be.getMessage()); + errorCount.increment(); + } catch (Exception e) { + log.error(e.getMessage(), e); + messages.add(e.getMessage()); + errorCount.increment(); + } + }); + + offset += pageSize; + operationCount += operations.size(); + hasMoreData = operations.size() >= pageSize; + if (operationCount > operationTotal) { + operationTotal = operationCount; + } + } while (hasMoreData); + } + + return DenormalizedTripResultVO.builder() + .operationCount(operationCount) + .batchCount(batchesCount.intValue()) + .invalidBatchCount(errorCount.intValue()) + .message(CollectionUtils.isNotEmpty(messages) ? String.join("\n", messages) : null) + .executionTime(System.currentTimeMillis() - startTime) + .build(); + } + + public DenormalizedBatchOptions createOptionsByOperation(@NonNull OperationVO operation, + @Nullable DenormalizedBatchOptions inheritedOptions) { + + if (inheritedOptions == null) { + int programId = operationService.getProgramIdById(operation.getId()); + inheritedOptions = denormalizedBatchService.createOptionsByProgramId(programId); + } + + Optional fishingAreaLocationIds = getOperationFishingAreaIds(operation); + if (fishingAreaLocationIds.isEmpty()) { + log.warn("Cannot found the statistical rectangle for Operation #{}, neither in positions nor in fishing areas", operation.getId()); + } + + DenormalizedBatchOptions options = inheritedOptions.clone(); // Copy, to keep original options unchanged + options.setFishingAreaLocationIds(fishingAreaLocationIds.orElse(null)); + options.setDateTime(Dates.resetTime(getFishingStartDateTime(operation))); + + return options; + } + + public Optional getOperationFishingAreaIds(@NonNull OperationVO operation) { + // Get the fishing area: first, search from last position + Integer[] result = Beans.getStream(operation.getPositions()) + .filter(Positions::isNotNullAndValid) + .map(position -> locationService.getStatisticalRectangleIdByLatLong(position.getLatitude(), position.getLongitude())) + .filter(Optional::isPresent) + .map(Optional::get) + .distinct() + .toArray(Integer[]::new); + if (ArrayUtils.isNotEmpty(result)) return Optional.of(result); + + // Try to get location from fishing areas + result = Beans.getStream(operation.getFishingAreas()) + .filter(fa -> fa.getLocation() != null) + .map(FishingAreaVO::getLocation) + .map(LocationVO::getId) + .filter(Objects::nonNull) + .distinct() + .toArray(Integer[]::new); + if (ArrayUtils.isNotEmpty(result)) return Optional.of(result); + + return Optional.empty(); // Not found + } + + /** + * Get the start fishing date (or the start date if no found) + * @param operation + * @return + */ + public Date getFishingStartDateTime(@NonNull OperationVO operation) { + return operation.getFishingStartDateTime() != null + ? operation.getFishingStartDateTime() + : operation.getStartDateTime(); + } +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripServiceImpl.java deleted file mode 100644 index 4afbf1e080..0000000000 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripServiceImpl.java +++ /dev/null @@ -1,201 +0,0 @@ -package net.sumaris.core.service.data.denormalize; - -/*- - * #%L - * SUMARiS:: Core - * %% - * Copyright (C) 2018 SUMARiS Consortium - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ - -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import net.sumaris.core.config.SumarisConfiguration; -import net.sumaris.core.dao.technical.SortDirection; -import net.sumaris.core.exception.SumarisBusinessException; -import net.sumaris.core.model.IProgressionModel; -import net.sumaris.core.model.ProgressionModel; -import net.sumaris.core.service.data.DenormalizedBatchService; -import net.sumaris.core.service.data.OperationService; -import net.sumaris.core.service.data.TripService; -import net.sumaris.core.util.TimeUtils; -import net.sumaris.core.vo.data.*; -import net.sumaris.core.vo.data.batch.DenormalizedBatchOptions; -import net.sumaris.core.vo.filter.TripFilterVO; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.mutable.MutableInt; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.task.TaskExecutor; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service("denormalizeTripService") -@Slf4j -public class DenormalizeTripServiceImpl implements DenormalizeTripService { - - @Autowired - private SumarisConfiguration configuration; - - @Autowired - private TripService tripService; - - @Autowired - private DenormalizedBatchService denormalizedBatchService; - - @Autowired - private OperationService operationService; - - @Autowired(required = false) - private TaskExecutor taskExecutor; - - @Override - public DenormalizeTripResultVO denormalizeByFilter(@NonNull TripFilterVO filter) { - return denormalizeByFilter(filter, new ProgressionModel()); - } - - @Override - public DenormalizeTripResultVO denormalizeByFilter(@NonNull TripFilterVO filter, @NonNull IProgressionModel progression) { - long startTime = System.currentTimeMillis(); - - progression.setCurrent(0); - progression.setMessage(String.format("Starting trips denormalization... filter: %s", filter)); - log.debug(progression.getMessage()); - - TripFetchOptions tripFetchOptions = TripFetchOptions.builder() - .withChildrenEntities(false) - .withMeasurementValues(false) - .withRecorderPerson(false) - .build(); - - long tripTotal = tripService.countByFilter(filter); - progression.setTotal(tripTotal); - - boolean hasMoreData; - int offset = 0; - int pageSize = 10; - int tripCount = 0; - MutableInt operationCount = new MutableInt(0); - MutableInt batchCount = new MutableInt(0); - MutableInt invalidBatchCount = new MutableInt(0); - do { - // Fetch some trips - List trips = tripService.findAll(filter, - offset, pageSize, // Page - TripVO.Fields.ID, SortDirection.ASC, // Sort by id, to keep continuity between pages - tripFetchOptions); - - if (offset > 0 && offset % (pageSize * 2) == 0) { - progression.setCurrent(offset); - progression.setMessage(String.format("Processing trips denormalization... %s/%s", offset, tripTotal)); - log.debug(progression.getMessage()); - } - - // Denormalize each trip - trips.stream() - .map(TripVO::getId) - .map(this::denormalizeById) - .forEach(result -> { - operationCount.add(result.getOperationCount()); - batchCount.add(result.getBatchCount()); - invalidBatchCount.add(result.getInvalidBatchCount()); - }); - - offset += pageSize; - tripCount += trips.size(); - hasMoreData = trips.size() >= pageSize; - if (tripCount > tripTotal) { - tripTotal = tripCount; - progression.adaptTotal(tripTotal); - } - } while (hasMoreData); - - // Success log - progression.setCurrent(tripCount); - progression.setMessage(String.format("Trips denormalization finished, in %s - %s trips, %s operations, %s batches - %s invalid batch tree (skipped)", - TimeUtils.printDurationFrom(startTime), - tripCount, - operationCount, - batchCount, - invalidBatchCount)); - log.debug(progression.getMessage()); - - return DenormalizeTripResultVO.builder() - .tripCount(tripCount) - .operationCount(operationCount.intValue()) - .batchCount(batchCount.intValue()) - .invalidBatchCount(invalidBatchCount.intValue()) - .executionTime(System.currentTimeMillis() - startTime) - .build(); - } - - @Override - public DenormalizeTripResultVO denormalizeById(int tripId) { - long startTime = System.currentTimeMillis(); - long operationTotal = operationService.countByTripId(tripId); - - // Load denormalized options - int programId = tripService.getProgramIdById(tripId); - DenormalizedBatchOptions options = denormalizedBatchService.createOptionsByProgramId(programId); - - boolean hasMoreData; - int offset = 0; - int pageSize = 10; - int operationCount = 0; - MutableInt batchesCount = new MutableInt(0); - MutableInt errorCount = new MutableInt(0); - do { - // Fetch some operations - List operations = operationService.findAllByTripId(tripId, - offset, pageSize, // Page - OperationVO.Fields.ID, SortDirection.ASC, // Sort by id, to keep continuity between pages - OperationFetchOptions.builder() - .withChildrenEntities(false) - .withMeasurementValues(false) - .build()); - - operations.forEach(operation -> { - try { - List batches = denormalizedBatchService.denormalizeAndSaveByOperationId(operation.getId(), options); - batchesCount.add(CollectionUtils.size(batches)); - } catch (SumarisBusinessException be) { - log.error(be.getMessage()); - errorCount.increment(); - } - catch (Exception e) { - log.error(e.getMessage(), e); - errorCount.increment(); - } - }); - - offset += pageSize; - operationCount += operations.size(); - hasMoreData = operations.size() >= pageSize; - if (operationCount > operationTotal) { - operationTotal = operationCount; - } - } while (hasMoreData); - - return DenormalizeTripResultVO.builder() - .tripCount(1) - .operationCount(operationCount) - .batchCount(batchesCount.intValue()) - .invalidBatchCount(errorCount.intValue()) - .executionTime(System.currentTimeMillis() - startTime) - .build(); - } -} diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/DenormalizedBatchService.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchService.java similarity index 85% rename from sumaris-core/src/main/java/net/sumaris/core/service/data/DenormalizedBatchService.java rename to sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchService.java index d52f779ddc..08979f7491 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/DenormalizedBatchService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchService.java @@ -1,27 +1,26 @@ -package net.sumaris.core.service.data; - -/*- +/* * #%L - * SUMARiS:: Core + * SUMARiS * %% - * Copyright (C) 2018 SUMARiS Consortium + * Copyright (C) 2019 SUMARiS Consortium * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as * published by the Free Software Foundation, either version 3 of the * License, or (at your option) any later version. - * + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public * License along with this program. If not, see * . * #L% */ +package net.sumaris.core.service.data.denormalize; import net.sumaris.core.vo.data.batch.BatchVO; import net.sumaris.core.vo.data.batch.DenormalizedBatchOptions; @@ -46,5 +45,9 @@ public interface DenormalizedBatchService { List denormalizeAndSaveBySaleId(int saleId, @Nullable DenormalizedBatchOptions options); + @Transactional(readOnly = true) DenormalizedBatchOptions createOptionsByProgramId(int programId); + + @Transactional(readOnly = true) + DenormalizedBatchOptions createOptionsByProgramLabel(String programLabel); } diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchServiceImpl.java new file mode 100644 index 0000000000..90ac8d729a --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchServiceImpl.java @@ -0,0 +1,1348 @@ +/* + * #%L + * SUMARiS + * %% + * Copyright (C) 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package net.sumaris.core.service.data.denormalize; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.sumaris.core.dao.data.batch.BatchRepository; +import net.sumaris.core.dao.data.batch.DenormalizedBatchRepository; +import net.sumaris.core.dao.data.batch.InvalidSamplingBatchException; +import net.sumaris.core.event.config.ConfigurationEvent; +import net.sumaris.core.event.config.ConfigurationReadyEvent; +import net.sumaris.core.event.config.ConfigurationUpdatedEvent; +import net.sumaris.core.exception.SumarisTechnicalException; +import net.sumaris.core.model.IEntity; +import net.sumaris.core.model.TreeNodeEntities; +import net.sumaris.core.model.administration.programStrategy.ProgramPropertyEnum; +import net.sumaris.core.model.annotation.EntityEnums; +import net.sumaris.core.model.referential.QualityFlags; +import net.sumaris.core.model.referential.StatusEnum; +import net.sumaris.core.model.referential.location.LocationLevels; +import net.sumaris.core.model.referential.pmfm.MethodEnum; +import net.sumaris.core.model.referential.pmfm.ParameterEnum; +import net.sumaris.core.model.referential.pmfm.QualitativeValueEnum; +import net.sumaris.core.model.referential.pmfm.UnitEnum; +import net.sumaris.core.model.referential.taxon.TaxonGroupTypeEnum; +import net.sumaris.core.service.administration.programStrategy.ProgramService; +import net.sumaris.core.service.data.OperationService; +import net.sumaris.core.service.data.SaleService; +import net.sumaris.core.service.referential.conversion.RoundWeightConversionService; +import net.sumaris.core.service.referential.conversion.WeightLengthConversionService; +import net.sumaris.core.service.referential.taxon.TaxonGroupService; +import net.sumaris.core.util.Beans; +import net.sumaris.core.util.Numbers; +import net.sumaris.core.util.StringUtils; +import net.sumaris.core.util.TimeUtils; +import net.sumaris.core.vo.administration.programStrategy.ProgramFetchOptions; +import net.sumaris.core.vo.administration.programStrategy.ProgramVO; +import net.sumaris.core.vo.administration.programStrategy.Programs; +import net.sumaris.core.vo.data.batch.*; +import net.sumaris.core.vo.filter.ReferentialFilterVO; +import net.sumaris.core.vo.referential.TaxonGroupVO; +import net.sumaris.core.vo.referential.conversion.RoundWeightConversionFilterVO; +import net.sumaris.core.vo.referential.conversion.RoundWeightConversionVO; +import net.sumaris.core.vo.referential.conversion.WeightLengthConversionFilterVO; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.mutable.MutableDouble; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.mutable.MutableShort; +import org.nuiton.i18n.I18n; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; + +import javax.annotation.Nullable; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.*; +import java.util.function.Function; + +@Service("denormalizedBatchService") +@RequiredArgsConstructor +@Slf4j +public class DenormalizedBatchServiceImpl implements DenormalizedBatchService { + + public static final int INTERMEDIATE_DECIMAL_SCALE = 24; // intermediate scale can be maximized + public static final int WEIGHT_DECIMAL_SCALE = 6; // grams precision + + protected final DenormalizedBatchRepository denormalizedBatchRepository; + + protected final BatchRepository batchRepository; + + protected final ProgramService programService; + + protected final OperationService operationService; + + protected final SaleService saleService; + + protected final TaxonGroupService taxonGroupService; + + protected final WeightLengthConversionService weightLengthConversionService; + + protected final RoundWeightConversionService roundWeightConversionService; + + private boolean canEnableRtpWeight = true; + + @EventListener({ConfigurationReadyEvent.class, ConfigurationUpdatedEvent.class}) + public void onConfigurationReady(ConfigurationEvent event) { + // Check useful enumerations + try { + checkBaseEnumerations(); + } catch (Exception e) { + log.warn(e.getMessage()); + } + + // Check is can enable RTP + { + boolean enableRtp = true; + List errorMessages = Lists.newArrayList(); + + // Check enumerations used for RTP + try { + checkRtpEnumerations(); + } catch (Exception e) { + enableRtp = false; + errorMessages.add(e.getMessage()); + } + + // Check statistical rectangle level ids + if (Beans.getStream(LocationLevels.getStatisticalRectangleLevelIds()).anyMatch(id -> id < 0)) { + enableRtp = false; + errorMessages.add(I18n.t("sumaris.error.missingSomeRectangleLocationLevel")); + + } + if (this.canEnableRtpWeight != enableRtp) { + this.canEnableRtpWeight = enableRtp; + if (!enableRtp) log.warn(I18n.t("sumaris.error.denormalization.batch.cannotEnableRtpWeight", "\n" + Joiner.on("\n\t- ").join(errorMessages))); + } + } + } + + @Override + public List denormalize(@NonNull BatchVO catchBatch, @NonNull final DenormalizedBatchOptions options) { + if (options.isEnableRtpWeight()) { + Preconditions.checkArgument(canEnableRtpWeight, I18n.t("sumaris.error.denormalization.batch.cannotEnableRtpWeight", "See startup error")); + + // Check options + Preconditions.checkNotNull(options.getDateTime(), "Required options.dateTime when RTP weight enabled"); + Preconditions.checkNotNull(options.getRoundWeightCountryLocationId(), "Required options.roundWeightCountryLocationId when RTP weight enabled"); + Preconditions.checkNotNull(options.getDefaultLandingDressingId(), "Required options.defaultLandingDressingId when RTP weight enabled"); + Preconditions.checkNotNull(options.getDefaultDiscardDressingId(), "Required options.defaultDiscardDressingId when RTP weight enabled"); + Preconditions.checkNotNull(options.getDefaultLandingPreservationId(), "Required options.defaultLandingPreservationId when RTP weight enabled"); + Preconditions.checkNotNull(options.getDefaultDiscardDressingId(), "Required options.defaultDiscardDressingId when RTP weight enabled"); + + } + + long startTime = System.currentTimeMillis(); + final MutableShort flatRankOrder = new MutableShort(0); + + List result = TreeNodeEntities.streamAllAndMap(catchBatch, (source, p) -> { + TempDenormalizedBatchVO target = createTempVO(source); + TempDenormalizedBatchVO parent = p != null ? (TempDenormalizedBatchVO)p : null; + boolean isLeaf = source.isLeaf(); // Do not use 'target', because children are added later + + // Add to parent's children + if (parent != null) parent.addChildren(target); + + // Depth level + if (parent == null) { + target.setTreeLevel((short) 1); // First level + if (target.getIsLanding() == null) target.setIsLanding(false); + if (target.getIsDiscard() == null) target.setIsDiscard(false); + } else { + target.setTreeLevel((short) (parent.getTreeLevel() + 1)); + // Inherit taxon group + if (target.getInheritedTaxonGroup() == null && parent.getInheritedTaxonGroup() != null) { + target.setInheritedTaxonGroup(parent.getInheritedTaxonGroup()); + } + // Inherit taxon name + if (target.getInheritedTaxonName() == null && parent.getInheritedTaxonName() != null) { + target.setInheritedTaxonName(parent.getInheritedTaxonName()); + } + // Exhaustive inventory + if (target.getExhaustiveInventory() == null) { + // Always true, when: + // - taxon name is defined + // - taxon group is defined and taxon Name disable (in options) + if (target.getInheritedTaxonName() != null) { + target.setExhaustiveInventory(Boolean.TRUE); + } else if (target.getInheritedTaxonGroup() != null && !options.isEnableTaxonName()) { + target.setExhaustiveInventory(Boolean.TRUE); + } else if (parent.getExhaustiveInventory() != null) { + target.setExhaustiveInventory(parent.getExhaustiveInventory()); + } + } + // Inherit location + if (parent.getLocationId() != null) { + target.setLocationId(parent.getLocationId()); + } + // Inherit landing / discard + if (target.getIsLanding() == null) { + target.setIsLanding(parent.getIsLanding()); + } + if (target.getIsDiscard() == null) { + target.setIsDiscard(parent.getIsDiscard()); + } + + // Inherit quality flag + if (parent.getQualityFlagId() != null) { + if (target.getQualityFlagId() == null) { + target.setQualityFlagId(parent.getQualityFlagId()); + } + // Keep the worst value, if current has a value + else { + target.setQualityFlagId(QualityFlags.worst(parent.getQualityFlagId(), target.getQualityFlagId())); + } + } + + // If current quality is invalid + if (QualityFlags.isInvalid(target.getQualityFlagId())) { + // Force both parent and current parent exhaustive inventory to FALSE + // NOTE Allegro: + // mantis Allegro #12951 - remontée des poids selon le niveau de qualité + // Si un des lots fils (direct ou indirect) est invalide + // (c'est à dire si le code du niveau de qualité appartient à la liste des niveaux invalides) + // alors il faut considérer que l'inventaire exhaustif est non. + // Le but est de stopper la remontée des poids calculés + // s'il y a au moins un lot invalide parmi les fils, isExhaustive = false + parent.setExhaustiveInventory(Boolean.FALSE); + target.setExhaustiveInventory(Boolean.FALSE); + } + + // Inherit sorting values + Beans.getStream(parent.getSortingValues()) + .forEach(svSource -> { + // Make sure sorting value not already exists + Beans.getStream(target.getSortingValues()) + .filter(svTarget -> Objects.equals(svTarget.getPmfmId(), svSource.getPmfmId())) + .findFirst() + .ifPresentOrElse(svTarget -> { + Beans.copyProperties(svSource, svTarget, IEntity.Fields.ID); + svTarget.setIsInherited(true); + svTarget.setRankOrder(svSource.getRankOrder() / 10); + }, + () -> { + DenormalizedBatchSortingValueVO svTarget = new DenormalizedBatchSortingValueVO(); + Beans.copyProperties(svSource, svTarget, IEntity.Fields.ID); + svTarget.setIsInherited(true); + svTarget.setRankOrder(svSource.getRankOrder() / 10); + target.addSortingValue(svTarget); + }); + }); + + // Compute Alive weight, on leaf batch + // (to be able to compute indirect alive weight later) + if (isLeaf) { + computeAliveWeightFactor(target, options, true) + .ifPresent(target::setAliveWeightFactor); + } + + } + + return target; + }) + + // Sort + .sorted(Comparator.comparing(DenormalizedBatches::computeFlatOrder)) + + .map(target -> { + // Compute flat rank order + flatRankOrder.increment(); + target.setFlatRankOrder(flatRankOrder.getValue()); + + // Compute tree indent (run once, on the root batch) + if (target.getParent() == null) computeTreeIndent(target); + + return target; + }) + .toList(); + + // If only the catch batch + if (CollectionUtils.size(result) == 1) { + DenormalizedBatchVO target = result.get(0); + target.setElevateWeight(target.getWeight()); + } else { + + // Compute RTP weight from length (if enabled) + if (options.isEnableRtpWeight()) { + computeRtpWeights(result, options); + } + + // Compute indirect values + computeIndirectValues(result, options); + + // compute elevate factors + computeElevateFactor(result, options); + + // Elevate weight + computeElevatedValues(result, options); + + // Indirect elevated values + computeIndirectElevatedValues(result, options); + + } + + // Log + if (log.isDebugEnabled()) { + log.debug("Batches denormalization succeed, in {}:\n{}", + TimeUtils.printDurationFrom(startTime), + DenormalizedBatches.dumpAsString(result, true, true)); + //log.debug("Batches denormalization succeed, in {}", TimeUtils.printDurationFrom(startTime)); + } + + return result; + } + + @Override + public List denormalizeAndSaveByOperationId(int operationId, @Nullable DenormalizedBatchOptions options) { + BatchVO catchBatch = batchRepository.getCatchBatchByOperationId(operationId, BatchFetchOptions.builder() + .withChildrenEntities(true) + .withMeasurementValues(true) + .withRecorderDepartment(false) + .build()); + if (catchBatch == null) return null; + + long startTime = System.currentTimeMillis(); + log.debug("Batches denormalization of operation {id: {}}...", operationId); + + // Compute options, for the operation's program + if (options == null) { + int programId = operationService.getProgramIdById(operationId); + options = createOptionsByProgramId(programId); + } + + // Denormalize batches + List batches = denormalize(catchBatch, options); + + // Save denormalized batches + batches = denormalizedBatchRepository.saveAllByOperationId(operationId, batches); + + log.debug("Batches denormalization of operation {id: {}} [OK] in {}", operationId, TimeUtils.printDurationFrom(startTime)); + return batches; + } + + @Override + public List denormalizeAndSaveBySaleId(int saleId, @Nullable DenormalizedBatchOptions options) { + BatchVO catchBatch = batchRepository.getCatchBatchBySaleId(saleId, BatchFetchOptions.builder() + .withChildrenEntities(true) + .withMeasurementValues(true) + .withRecorderDepartment(false) + .build()); + if (catchBatch == null) return null; + + long startTime = System.currentTimeMillis(); + log.debug("Denormalize batches of sale {id: {}}...", saleId); + + // Compute options, for the sale's program + if (options == null) { + int programId = saleService.getProgramIdById(saleId); + options = createOptionsByProgramId(programId); + } + + // Denormalize batches + List denormalizedBatches = denormalize(catchBatch, options); + + // Save denormalized batches + List result = denormalizedBatchRepository.saveAllBySaleId(saleId, denormalizedBatches); + + log.debug("Denormalize batches of sale {id: {}} [OK] in {}", saleId, TimeUtils.printDurationFrom(startTime)); + return result; + } + + @Override + public DenormalizedBatchOptions createOptionsByProgramId(int programId) { + + ProgramVO program = programService.get(programId, ProgramFetchOptions.builder() + .withProperties(true) + .withLocations(false) + .withStrategies(false) + .build()); + + return createOptionsByProgram(program); + } + + @Override + public DenormalizedBatchOptions createOptionsByProgramLabel(String programLabel) { + + ProgramVO program = programService.getByLabel(programLabel, ProgramFetchOptions.builder() + .withProperties(true) + .withLocations(false) + .withStrategies(false) + .build()); + + return createOptionsByProgram(program); + } + + /* -- protected methods -- */ + + protected DenormalizedBatchOptions createOptionsByProgram(@NonNull ProgramVO program) { + Preconditions.checkNotNull(program.getProperties()); + + // Get ids of taxon group without weight + String taxonGroupsNoWeight = Optional.ofNullable(Programs.getProperty(program, ProgramPropertyEnum.TRIP_BATCH_TAXON_GROUPS_NO_WEIGHT)).orElse(""); + Integer[] taxonGroupIdsNoWeight = Arrays.stream(taxonGroupsNoWeight.split(",")) + .map(String::trim) + .map(label -> taxonGroupService.findAllByFilter(ReferentialFilterVO.builder() + .label(label) + .levelIds(new Integer[]{TaxonGroupTypeEnum.FAO.getId()}) + .statusIds(new Integer[]{StatusEnum.ENABLE.getId()}) + .build()).stream().findFirst()) + .filter(Optional::isPresent) + .map(Optional::get) + .map(TaxonGroupVO::getId) + .toArray(Integer[]::new); + + Integer roundWeightConversionCountryId = Programs.getPropertyAsInteger(program, ProgramPropertyEnum.TRIP_BATCH_ROUND_WEIGHT_CONVERSION_COUNTRY_ID); + + if (roundWeightConversionCountryId == null || roundWeightConversionCountryId < 0) { + log.warn("Missing or invalid value for program property '{}'. Will not be able to compute round weight, in batch denormalization!", ProgramPropertyEnum.TRIP_BATCH_ROUND_WEIGHT_CONVERSION_COUNTRY_ID.getKey()); + } + + return DenormalizedBatchOptions.builder() + .taxonGroupIdsNoWeight(taxonGroupIdsNoWeight) + .enableTaxonName(Programs.getPropertyAsBoolean(program, ProgramPropertyEnum.TRIP_BATCH_TAXON_NAME_ENABLE)) + .enableTaxonGroup(Programs.getPropertyAsBoolean(program, ProgramPropertyEnum.TRIP_BATCH_TAXON_GROUP_ENABLE)) + .enableRtpWeight(canEnableRtpWeight && Programs.getPropertyAsBoolean(program, ProgramPropertyEnum.TRIP_BATCH_LENGTH_WEIGHT_CONVERSION_ENABLE)) + .roundWeightCountryLocationId(roundWeightConversionCountryId) + .build(); + } + + protected void computeRtpWeights(List batches, DenormalizedBatchOptions options) { + // Select leafs + List leafBatches = batches.stream() + .map(target -> (TempDenormalizedBatchVO) target) + .filter(target -> !target.hasChildren()) + .toList(); + + int maxRtpWeightDiffPct = options.getMaxRtpWeightDiffPct(); + log.debug("Computing RTP weights on leafs..."); + + // For each leaf, try to compute a RTP weight + leafBatches.forEach(batch -> { + log.trace("- {}", batch.getLabel()); + + computeRtpContextWeight(batch, options) + .ifPresent(rtpContextWeight -> { + // Apply to the batch + batch.setRtpContextWeight(rtpContextWeight); + + // Check diff with existing weight + if (batch.getWeight() != null && maxRtpWeightDiffPct > 0) { + + // Compute diff between two weights + double errorPct = DenormalizedBatches.computeWeightDiffPercent(rtpContextWeight, batch.getWeight()); + + // If delta > max % => warn + if (errorPct > maxRtpWeightDiffPct) { + + // Replace weight, if it was a RTP (should be wrong computation in the App ?) + if (Objects.equals(batch.getWeightMethodId(), MethodEnum.CALCULATED_WEIGHT_LENGTH.getId())) { + log.warn("Batch {} has a invalid RTP weight (computed: {}, actual: {}, delta: {}%). Fixing the RTP weight using the computed value.", + batch.getLabel(), + batch.getRtpContextWeight(), + batch.getWeight(), + errorPct + ); + batch.setWeight(rtpContextWeight); + } else { + log.warn("Batch {} has a invalid weight (RTP: {}, actual: {} => delta: {}%)", + batch.getLabel(), + batch.getRtpContextWeight(), + batch.getWeight(), + errorPct + ); + } + } + } + }); + } + ); + } + protected void computeIndirectValues(List batches, DenormalizedBatchOptions options) { + + List revertBatches = batches.stream() + .map(target -> (TempDenormalizedBatchVO) target) + // Reverse order (start from leaf) + .sorted(Collections.reverseOrder(Comparator.comparing(DenormalizedBatchVO::getFlatRankOrder, Short::compareTo))) + .toList(); + + MutableInt changesCount = new MutableInt(0); + MutableInt loopCounter = new MutableInt(0); + do { + changesCount.setValue(0); + loopCounter.increment(); + log.debug("Computing indirect values (pass #{}) ...", loopCounter); + + // For each (leaf -> root) + revertBatches.forEach(batch -> { + boolean changed = false; + + log.trace("- {}", batch.getLabel()); + + // Indirect context weight + Double indirectContextWeight = computeIndirectContextWeight(batch, options); + changed = changed || !Objects.equals(indirectContextWeight, batch.getIndirectContextWeight()); + batch.setIndirectContextWeight(indirectContextWeight); + + // Indirect RTP weight from length (if enabled) + if (options.isEnableRtpWeight()) { + Double indirectRtpContextWeight = computeIndirectRtpContextWeight(batch, options); + changed = changed || !Objects.equals(indirectRtpContextWeight, batch.getIndirectRtpContextWeight()); + batch.setIndirectRtpContextWeight(indirectRtpContextWeight); + } + + // Indirect individual count + BigDecimal indirectIndividualCount = computeIndirectIndividualCount(batch); + changed = changed || !Objects.equals(indirectIndividualCount, batch.getIndirectIndividualCountDecimal()); + batch.setIndirectIndividualCountDecimal(indirectIndividualCount); + + // Compute alive weight factor + if (batch.isLeaf()) { + Double aliveWeightFactor = computeAliveWeightFactor(batch, options, batch.isLeaf()).orElse(null); + changed = changed || !Objects.equals(aliveWeightFactor, batch.getAliveWeightFactor()); + batch.setAliveWeightFactor(aliveWeightFactor); + } + else { + Double aliveWeightFactor = computeIndirectAliveWeightFactor(batch, options); + changed = changed || !Objects.equals(aliveWeightFactor, batch.getAliveWeightFactor()); + batch.setAliveWeightFactor(aliveWeightFactor); + } + + if (changed) changesCount.increment(); + }); + + log.trace("Computing indirect values (pass #{}) [OK] - {} changes", loopCounter, changesCount); + } + + // Continue while changes has been applied on tree + while (changesCount.intValue() > 0); + } + + /** + * Compute elevation factors + */ + protected void computeElevateFactor(List batches, DenormalizedBatchOptions options) { + log.debug("Computing elevation factors..."); + + // For each (root -> leaf) + batches.stream() + .map(target -> (TempDenormalizedBatchVO) target) + .forEach(target -> { + TempDenormalizedBatchVO parent = (TempDenormalizedBatchVO) target.getParent(); + + log.trace("{} {}", target.getTreeIndent(), target.getLabel()); + + BigDecimal samplingFactor = target.getSamplingFactor() != null ? target.getSamplingFactor() : new BigDecimal(1); + + // Elevate context factor (=samplingFactor x parent value) + BigDecimal elevateContextFactor = samplingFactor; + if (parent != null) { + elevateContextFactor = elevateContextFactor.multiply(parent.getElevateContextFactor()); + } + target.setElevateContextFactor(elevateContextFactor); + + // Taxon elevation factor (=samplingFactor x parent value - BUT not if parent has no taxonGroup/taxonName) + if (target.hasTaxonGroup() || target.hasTaxonName()) { + BigDecimal taxonElevateFactor = samplingFactor; + // Apply parent factor (only if has taxonGroup) + if (parent != null && (parent.hasTaxonName() || parent.hasTaxonName())) { + taxonElevateFactor = taxonElevateFactor.multiply(parent.getTaxonElevateFactor()); + } + target.setTaxonElevateFactor(taxonElevateFactor); + } + + // Elevation factor (alive weight) = elevateContextFactor x aliveWeightFactor + if (target.getAliveWeightFactor() != null) { + BigDecimal elevateFactor = elevateContextFactor.multiply(new BigDecimal(target.getAliveWeightFactor())); + target.setElevateFactor(elevateFactor); + } + else { + target.setElevateFactor(elevateContextFactor); + } + }); + } + + /** + * Compute elevated values + */ + protected void computeElevatedValues(List batches, DenormalizedBatchOptions options) { + MutableInt changesCount = new MutableInt(0); + MutableInt loopCounter = new MutableInt(0); + + do { + changesCount.setValue(0); + loopCounter.increment(); + log.debug("Computing elevated values (pass #{}) ...", loopCounter); + + // For each (root -> leaf) + batches.stream() + .map(batch -> (TempDenormalizedBatchVO) batch) + .forEach(batch -> { + boolean changed = false; + log.trace("{} {}", batch.getTreeIndent(), batch.getLabel()); + + // Base weight + BigDecimal contextWeight = Numbers.firstNotNullAsBigDecimal(batch.getWeight(), batch.getIndirectContextWeight()); + + if (contextWeight != null) { + // Elevate contextual weight + { + Double elevateContextWeight = contextWeight.multiply(batch.getElevateContextFactor()) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + changed = changed || !Objects.equals(elevateContextWeight, batch.getElevateContextWeight()); + batch.setElevateContextWeight(elevateContextWeight); + } + + // Taxon elevate context weight + if (batch.getTaxonElevateFactor() != null) { + Double taxonElevateContextWeight = contextWeight.multiply(batch.getTaxonElevateFactor()).doubleValue(); + changed = changed || !Objects.equals(taxonElevateContextWeight, batch.getTaxonElevateContextWeight()); + batch.setTaxonElevateContextWeight(taxonElevateContextWeight); + } + + // Elevate weight (alive weight) + { + Double elevateWeight = contextWeight.multiply(batch.getElevateFactor()) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + changed = changed || !Objects.equals(elevateWeight, batch.getElevateWeight()); + batch.setElevateWeight(elevateWeight); + } + } + + if (options.isEnableRtpWeight()) { + + BigDecimal rtpContextWeight = Numbers.firstNotNullAsBigDecimal(batch.getRtpContextWeight(), batch.getIndirectRtpContextWeight()); + if (rtpContextWeight != null) { + // Elevate RTP context weight + { + Double elevateRtpContextWeight = rtpContextWeight.multiply(batch.getElevateContextFactor()) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + changed = changed || !Objects.equals(elevateRtpContextWeight, batch.getElevateRtpContextWeight()); + batch.setElevateRtpContextWeight(elevateRtpContextWeight); + } + + // Indirect RTP weight (from indirect RTP context weight, converted to alive) + { + BigDecimal aliveWeightFactor = Numbers.firstNotNullAsBigDecimal(batch.getAliveWeightFactor(), new BigDecimal(1)); + Double indirectRtpWeight = rtpContextWeight.multiply(aliveWeightFactor) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + changed = changed || !Objects.equals(indirectRtpWeight, batch.getIndirectRtpWeight()); + batch.setIndirectRtpWeight(indirectRtpWeight); + } + + // Elevate RTP weight + { + Double elevateRtpWeight = rtpContextWeight.multiply(batch.getElevateFactor()) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + changed = changed || !Objects.equals(elevateRtpWeight, batch.getElevateRtpWeight()); + batch.setElevateRtpWeight(elevateRtpWeight); + } + } + } + + BigDecimal individualCount = batch.getIndividualCount() != null ? new BigDecimal(batch.getIndividualCount()) : batch.getIndirectIndividualCountDecimal(); + if (individualCount != null) { + // Taxon elevate individual count + if (batch.getTaxonElevateContextWeight() != null && batch.getTaxonElevateFactor() != null) { + Integer taxonElevateIndividualCount = individualCount.multiply(batch.getTaxonElevateFactor()) + .divide(new BigDecimal(1), 0, RoundingMode.HALF_UP) // Round to half up + .intValue(); + changed = changed || !Objects.equals(taxonElevateIndividualCount, batch.getTaxonElevateIndividualCount()); + batch.setTaxonElevateIndividualCount(taxonElevateIndividualCount); + } + + // Elevate individual count + Integer elevateIndividualCount = individualCount.multiply(batch.getElevateFactor()) + .divide(new BigDecimal(1), 0, RoundingMode.HALF_UP) // Round to half up + .intValue(); + changed = changed || !Objects.equals(elevateIndividualCount, batch.getElevateIndividualCount()); + batch.setElevateIndividualCount(elevateIndividualCount); + + // Set indirect individualCount + if (batch.getIndirectIndividualCountDecimal() != null) { + batch.setIndirectIndividualCount(batch.getIndirectIndividualCountDecimal() + .divide(new BigDecimal(1), 0, RoundingMode.HALF_UP) // Round to half up + .intValue()); + } + } + + if (changed) { + //log.trace("{} {} - changes!", target.getTreeIndent(), target.getLabel()); + changesCount.increment(); + } + }); + + log.trace("Computing elevated values (pass #{}) [OK] - {} changes", loopCounter, changesCount); + } while (changesCount.intValue() > 0); + } + + protected void computeIndirectElevatedValues(List batches, DenormalizedBatchOptions options) { + + List revertBatches = batches.stream() + .map(target -> (TempDenormalizedBatchVO) target) + // Reverse order (start from leaf) + .sorted(Collections.reverseOrder(Comparator.comparing(DenormalizedBatchVO::getFlatRankOrder, Short::compareTo))) + .toList(); + + MutableInt changesCount = new MutableInt(0); + MutableInt loopCounter = new MutableInt(0); + do { + changesCount.setValue(0); + loopCounter.increment(); + log.debug("Computing indirect elevated values (pass #{}) ...", loopCounter); + + // For each (leaf -> root) + revertBatches + .forEach(batch -> { + boolean changed = false; + + log.trace("- {}", batch.getLabel()); + + if (batch.getElevateWeight() == null) { + // No context weight to elevate: so use children elevate weight + Double indirectElevateWeight = computeIndirectElevateWeight(batch, options); + changed = changed || !Objects.equals(indirectElevateWeight, batch.getIndirectElevateWeight()) + || !Objects.equals(indirectElevateWeight, batch.getElevateWeight()); + batch.setIndirectElevateWeight(indirectElevateWeight); + batch.setElevateWeight(indirectElevateWeight); + } + + if (options.isEnableRtpWeight() && batch.getElevateRtpWeight() == null) { + Double indirectElevateRtpWeight = computeIndirectElevateRtpWeight(batch, options); + changed = changed || !Objects.equals(indirectElevateRtpWeight, batch.getIndirectRtpElevateWeight()) + || !Objects.equals(indirectElevateRtpWeight, batch.getElevateRtpWeight()); + batch.setIndirectRtpElevateWeight(indirectElevateRtpWeight); + batch.setElevateRtpWeight(indirectElevateRtpWeight); + } + + // Check weight = 0 AND individual + boolean zeroWeightWithIndividual = batch.getElevateWeight() != null && batch.getElevateWeight() == 0d + && batch.getElevateIndividualCount() != null && batch.getElevateIndividualCount() > 0; + if (zeroWeightWithIndividual) { + String message = String.format("Invalid batch {id: %s, label: '%s'}: elevateWeight=0 but elevateIndividualCount > 0", + batch.getId(), batch.getLabel()); + if (options.isAllowZeroWeightWithIndividual()) log.warn(message); + else throw new InvalidSamplingBatchException(message); + } + + if (changed) { + //log.trace("{} {} - changes!", target.getTreeIndent(), target.getLabel()); + changesCount.increment(); + } + }); + + log.trace("Computing indirect elevated values (pass #{}) [OK] - {} changes", loopCounter, changesCount); + } while (changesCount.intValue() > 0); + } + + protected Optional computeRtpContextWeight(TempDenormalizedBatchVO batch, DenormalizedBatchOptions options) { + // Already computed: skip + if (batch.getRtpContextWeight() != null) return Optional.of(batch.getRtpContextWeight()); + + // No individual count: skip + // TODO: should we use '1' as default individual count ? + if (batch.getIndividualCount() == null) return Optional.empty(); + + // No taxon: skip + Integer referenceTaxonId = batch.getTaxonName() != null + ? batch.getTaxonName().getReferenceTaxonId() + : (batch.getInheritedTaxonName() != null ? batch.getInheritedTaxonName().getId() : null); + if (referenceTaxonId == null) return Optional.empty(); + + return Beans.getStream(batch.getSortingValues()) + // Filter on length measure + .filter(sv -> sv.getNumericalValue() != null + && sv.getParameter() != null && sv.getParameter().getId() != null + && weightLengthConversionService.isWeightLengthParameter(sv.getParameter().getId()) + ) + .map(measure -> { + // Get the individual's sex (or not sexed as default) + Integer sexId = DenormalizedBatches.getSexId(batch).orElse(QualitativeValueEnum.SEX_UNSEXED.getId()); + // Get length precision (default = 1 - see Allegro mantis #8330) + Double lengthPrecision = measure.getPmfm().getPrecision() != null ? measure.getPmfm().getPrecision() : 1d; + + return weightLengthConversionService.loadFirstByFilter(WeightLengthConversionFilterVO.builder() + .month(options.getMonth()) + .year(options.getYear()) + .referenceTaxonIds(new Integer[]{referenceTaxonId}) + .childLocationIds(options.getFishingAreaLocationIds()) + .lengthPmfmIds(new Integer[]{measure.getPmfmId()}) + .sexIds(sexId >= 0 ? new Integer[]{sexId} : null) + .build()) + .map(conversion -> weightLengthConversionService.computedWeight(conversion, + measure.getNumericalValue(), // Length + measure.getUnit().getLabel(), + lengthPrecision, + batch.getIndividualCount(), + UnitEnum.KG.getLabel(), + 6 // = mg precision + ) + ); + }) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst() + // Convert alive RTP weight into dressing/preservation weight + .flatMap(rtpAliveWeight -> convertAliveWeightToContext(batch, options, rtpAliveWeight)) + .map(BigDecimal::doubleValue); + } + + protected Double computeIndirectContextWeight(TempDenormalizedBatchVO batch, + DenormalizedBatchOptions options) { + return computeIndirectWeight(batch, options, + DenormalizedBatchVO::getWeight, + DenormalizedBatchVO::getIndirectContextWeight, + true, + TempDenormalizedBatchVO::getAliveWeightFactor + ); + } + + protected Double computeIndirectRtpContextWeight(TempDenormalizedBatchVO batch, + DenormalizedBatchOptions options) { + return computeIndirectWeight(batch, options, + TempDenormalizedBatchVO::getRtpContextWeight, + TempDenormalizedBatchVO::getIndirectRtpContextWeight, + true, + TempDenormalizedBatchVO::getAliveWeightFactor + ); + } + + protected Double computeIndirectElevateWeight(TempDenormalizedBatchVO batch, + DenormalizedBatchOptions options) { + return computeIndirectWeight(batch, options, + TempDenormalizedBatchVO::getElevateWeight, + TempDenormalizedBatchVO::getIndirectElevateWeight, + false, + null // Skip control on same dressing/preservation + ); + } + + protected Double computeIndirectElevateRtpWeight(TempDenormalizedBatchVO batch, + DenormalizedBatchOptions options) { + return computeIndirectWeight(batch, options, + TempDenormalizedBatchVO::getElevateRtpWeight, + TempDenormalizedBatchVO::getIndirectRtpElevateWeight, + false, + null // Skip control on same dressing/preservation + ); + } + + protected Double computeIndirectWeight(TempDenormalizedBatchVO batch, + DenormalizedBatchOptions options, + Function weightGetter, + Function indirectWeightGetter, + boolean applySamplingRatio, + @Nullable Function aliveWeightFactorGetter) { + // Already computed: skip + Double indirectWeight = indirectWeightGetter.apply(batch); + if (indirectWeight != null) return indirectWeight; + + if (applySamplingRatio) { + // Sampling batch + if (DenormalizedBatches.isSamplingBatch(batch)) { + try { + Double samplingWeight = computeSamplingWeightAndRatio(batch, false, + options, weightGetter, indirectWeightGetter, aliveWeightFactorGetter); + if (samplingWeight != null) return samplingWeight; + } catch (InvalidSamplingBatchException e) { + // May be not a sampling batch ? (e.g. a species batch) + indirectWeight = computeSumChildrenWeight(batch, options, + weightGetter, indirectWeightGetter, true, aliveWeightFactorGetter); + if (indirectWeight != null) return indirectWeight; + throw e; + } + // Invalid sampling batch: Continue if not set + } + + // Child batch is a sampling batch + if (DenormalizedBatches.isParentOfSamplingBatch(batch)) { + return computeParentSamplingWeight(batch, false, weightGetter, indirectWeightGetter); + } + } + + // Has children (not a leaf batch) + if (batch.hasChildren()) { + // Compute sum from children's weight + return computeSumChildrenWeight(batch, options, weightGetter, indirectWeightGetter, applySamplingRatio, aliveWeightFactorGetter); + } + // Leaf batch: use the current weight as default + else { + return weightGetter.apply(batch); + } + } + + protected Double computeSamplingWeightAndRatio(TempDenormalizedBatchVO batch, + boolean checkArgument, + DenormalizedBatchOptions options, + Function weightGetter, + Function indirectWeightGetter, + Function aliveWeightFactorGetter) { + if (checkArgument) Preconditions.checkArgument(DenormalizedBatches.isSamplingBatch(batch)); + + TempDenormalizedBatchVO parent = (TempDenormalizedBatchVO) batch.getParent(); + boolean parentExhaustiveInventory = DenormalizedBatches.isExhaustiveInventory(parent); + Double parentWeight = weightGetter.apply(parent); + Double weight = weightGetter.apply(batch); + Double samplingWeight = null; + Double samplingRatio = batch.getSamplingRatio(); + BigDecimal samplingFactor = null; + final int scale = INTERMEDIATE_DECIMAL_SCALE; + + // Ignore invalid value + if (samplingRatio != null && (Double.isNaN(samplingRatio) || Double.isInfinite(batch.getSamplingRatio()))) { + samplingRatio = null; + } + + if (samplingRatio != null) { + samplingFactor = samplingRatio <= 0 + ? new BigDecimal(0) + : new BigDecimal(1).divide(new BigDecimal(samplingRatio), scale, RoundingMode.HALF_UP); + + // Try to restore sampling ratio from text (more accuracy) + if (samplingRatio > 0 && StringUtils.isNotBlank(batch.getSamplingRatioText()) && batch.getSamplingRatioText().contains("/")) { + String[] parts = batch.getSamplingRatioText().split("/", 2); + try { + BigDecimal shouldBeSamplingWeight = new BigDecimal(parts[0]); + BigDecimal shouldBeParentWeight = new BigDecimal(parts[1]); + samplingRatio = shouldBeParentWeight.doubleValue() <= 0 + ? 0d + : shouldBeSamplingWeight.divide(shouldBeParentWeight, scale, RoundingMode.HALF_UP).doubleValue(); + samplingFactor = shouldBeSamplingWeight.doubleValue() <= 0 + ? new BigDecimal(0) + : shouldBeParentWeight.divide(shouldBeSamplingWeight, scale, RoundingMode.HALF_UP); + } catch (Exception e) { + log.warn("Cannot parse samplingRatioText on batch {id: {}, label: '{}', saplingRatioText: '{}'} : {}", + batch.getId(), + batch.getLabel(), + batch.getSamplingRatioText(), + e.getMessage()); + } + } + } else if (parentExhaustiveInventory && parentWeight != null && weight != null) { + if (weight > parentWeight) { + throw new InvalidSamplingBatchException(String.format("Invalid batch weight {id: %s, label: '%s', weight: %s}. Should be <= %s kg (parent weight)", + batch.getId(), batch.getLabel(), weight, + parentWeight)); + } + if (parentWeight <= 0) { + samplingRatio = 0d; + samplingFactor = new BigDecimal(0); + } + else { + samplingRatio = new BigDecimal(weight) + .divide(new BigDecimal(parentWeight), scale, RoundingMode.HALF_UP) + .doubleValue(); + samplingFactor = weight == 0d + ? new BigDecimal(0) + : new BigDecimal(parentWeight).divide(new BigDecimal(weight), scale, RoundingMode.HALF_UP); + } + } else if (parentExhaustiveInventory && parentWeight != null && batch.hasChildren()) { + samplingWeight = computeSumChildrenWeight(batch, options, weightGetter, indirectWeightGetter, true, aliveWeightFactorGetter); + if (samplingWeight != null) { + if (parentWeight <= 0d || samplingWeight <= 0d) { + samplingRatio = 0d; + samplingFactor = new BigDecimal(0); + } + else { + samplingRatio = new BigDecimal(samplingWeight).divide(new BigDecimal(parentWeight), scale, RoundingMode.HALF_UP).doubleValue(); + samplingFactor = new BigDecimal(parentWeight).divide(new BigDecimal(samplingWeight), scale, RoundingMode.HALF_UP); + } + } + } else if ((!parentExhaustiveInventory || parentWeight == null) && batch.hasChildren()) { + samplingWeight = computeSumChildrenWeight(batch, options, weightGetter, indirectWeightGetter, true, aliveWeightFactorGetter); + if (samplingWeight != null) { + samplingRatio = 1d; + samplingFactor = new BigDecimal(1); + } + } + + // When taxon group without weight: compute the simpling ratio by individual count + else if (parent.getTaxonGroupId() != null + && ArrayUtils.isNotEmpty(options.getTaxonGroupIdsNoWeight()) + && ArrayUtils.contains(options.getTaxonGroupIdsNoWeight(), parent.getInheritedTaxonGroup().getId())) { + // TODO + log.warn("Batch {label: '{}'} - TODO try to compute samplingRatio using individualCount parent/child (taxon group no weight)", batch.getLabel()); + } + + if (samplingRatio == null || samplingFactor == null) { + // Use default value (samplingRatio=1) if: + // - batch has no children + // - batch is parent of a sampling batch + if (CollectionUtils.isEmpty(batch.getChildren())) { + samplingRatio = 1d; + samplingFactor = new BigDecimal(1); + } else { + throw new InvalidSamplingBatchException(String.format("Invalid sampling batch {id: %s, label: '%s'}: cannot get or compute the sampling ratio", + batch.getId(), batch.getLabel())); + } + } + + + // Remember values + batch.setSamplingRatio(samplingRatio); + batch.setSamplingFactor(samplingFactor); + + // Find the weight of current sampling batch + if (samplingWeight == null) { + if (weight != null) { + samplingWeight = weight; + } else if (parentExhaustiveInventory) { + if (parentWeight != null) { + samplingWeight = new BigDecimal(parentWeight).multiply(new BigDecimal(samplingRatio)) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + } + } + } + + return samplingWeight; + } + + + protected Double computeParentSamplingWeight(TempDenormalizedBatchVO parent, + boolean checkArgument, + Function weightGetter, + Function indirectWeightGetter + ) { + if (checkArgument) Preconditions.checkArgument(DenormalizedBatches.isParentOfSamplingBatch(parent)); + + // Use reference weight, if any + Double parentWeight = weightGetter.apply(parent); + if (parentWeight != null) return parentWeight; + + TempDenormalizedBatchVO samplingBatch = (TempDenormalizedBatchVO) CollectionUtils.extractSingleton(parent.getChildren()); + Double samplingWeight = weightGetter.apply(samplingBatch); + Double samplingIndirectWeight = indirectWeightGetter.apply(samplingBatch); + int scale = INTERMEDIATE_DECIMAL_SCALE; + + Double samplingRatio = null; + BigDecimal samplingFactor = null; + if (samplingBatch.getSamplingRatio() != null) { + samplingRatio = samplingBatch.getSamplingRatio(); + samplingFactor = samplingRatio <= 0d + ? new BigDecimal(0) + : new BigDecimal(1).divide(new BigDecimal(samplingRatio), scale, RoundingMode.HALF_UP); + + // Try to use the sampling ratio text (more accuracy) + if (samplingRatio > 0 && StringUtils.isNotBlank(samplingBatch.getSamplingRatioText()) && samplingBatch.getSamplingRatioText().contains("/")) { + String[] parts = samplingBatch.getSamplingRatioText().split("/", 2); + try { + BigDecimal shouldBeSamplingWeight = new BigDecimal(parts[0]); + BigDecimal shouldBeParentWeight = new BigDecimal(parts[1]); + // If ratio text use the sampling weight, we have the parent weight + if (Objects.equals(shouldBeSamplingWeight, samplingWeight) + || Objects.equals(shouldBeSamplingWeight, samplingIndirectWeight)) { + parentWeight = shouldBeParentWeight.doubleValue(); + } + samplingRatio = shouldBeParentWeight.doubleValue() <= 0d + ? 0 + : shouldBeSamplingWeight.divide(shouldBeParentWeight, scale, RoundingMode.HALF_UP).doubleValue(); + samplingFactor = shouldBeSamplingWeight.doubleValue() <= 0 + ? new BigDecimal(0) + : shouldBeParentWeight.divide(shouldBeSamplingWeight, scale, RoundingMode.HALF_UP); + } catch (Exception e) { + log.warn(String.format("Cannot parse samplingRatioText on batch {id: %s, label: '%s', saplingRatioText: '%s'} : %s", + samplingBatch.getId(), + samplingBatch.getLabel(), + samplingBatch.getSamplingRatioText(), + e.getMessage())); + } + } + } + + if (samplingRatio == null) + throw new SumarisTechnicalException(String.format("Invalid fraction batch {id: %s, label: '%s'}: cannot get or compute the sampling ratio", + samplingBatch.getId(), samplingBatch.getLabel())); + + if (parentWeight == null) { + if (samplingWeight != null) { + parentWeight = new BigDecimal(samplingWeight).multiply(samplingFactor) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + } else if (samplingIndirectWeight != null) { + parentWeight = new BigDecimal(samplingIndirectWeight).multiply(samplingFactor) + .divide(new BigDecimal(1), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP) + .doubleValue(); + } + } + + return parentWeight; + } + + protected Double computeSumChildrenWeight(@NonNull DenormalizedBatchVO batch, + @NonNull DenormalizedBatchOptions options, + @NonNull Function weightGetter, + @NonNull Function indirectWeightGetter, + boolean applySamplingRatio, + @Nullable Function aliveWeightFactorGetter) { + // Cannot compute children sum, when: + // - Not exhaustive inventory + // - No children + if (!DenormalizedBatches.isExhaustiveInventory(batch) + || !batch.hasChildren()) { + return null; + } + + // We track children alive weight factor. If not SAME => cannot compute sum + MutableDouble childrenAliveWeightFactor = new MutableDouble(-1d); + + try { + return Beans.getStream(batch.getChildren()) + .map(child -> (TempDenormalizedBatchVO) child) + .mapToDouble(child -> { + if (aliveWeightFactorGetter != null) { + Double aliveWeightFactor = aliveWeightFactorGetter.apply(child); + if (aliveWeightFactor == null) aliveWeightFactor = 1d; + // Not set: update and continue + if (childrenAliveWeightFactor.doubleValue() == -1d) { + childrenAliveWeightFactor.setValue(aliveWeightFactor); + } + // Control that all children have alive weight factor. If not, we cannot compute sum. + else if (!Objects.equals(childrenAliveWeightFactor.getValue(), aliveWeightFactor)) { + // Stop here, because we cannot sum all children's weight + throw new SumarisTechnicalException(String.format("No indirect weight" + + " (a child has a different dressing/preservation: {id: %s, label: '%s'})", child.getId(), child.getLabel())); + } + } + // Use child weight, if any + Double weight = weightGetter.apply(child); + if (weight != null) return weight; + + // Compute indirect weight + Double indirectWeight = computeIndirectWeight(child, options, weightGetter, indirectWeightGetter, applySamplingRatio, aliveWeightFactorGetter); + if (indirectWeight != null) return indirectWeight; + + // Stop here, because we cannot sum all children's weight + throw new SumarisTechnicalException(String.format("No indirect weight" + + " (a child has no weight {id: %s, label: '%s'})", child.getId(), child.getLabel())); + }).sum(); + } catch (SumarisTechnicalException e) { + log.trace(e.getMessage()); + return null; + } + } + + protected BigDecimal computeIndirectIndividualCount(TempDenormalizedBatchVO batch) { + // Already computed: skip + if (batch.getIndirectIndividualCountDecimal() != null) return batch.getIndirectIndividualCountDecimal(); + + // Cannot compute when: + // - Not exhaustive inventory + // - No children + if (!DenormalizedBatches.isExhaustiveInventory(batch) + || !batch.hasChildren()) { + return null; + } + + try { + return Beans.getStream(batch.getChildren()) + .map(child -> (TempDenormalizedBatchVO) child) + .map(child -> { + if (child.getIndividualCount() != null) { + BigDecimal samplingFactor = Optional.ofNullable(child.getSamplingFactor()).orElse(new BigDecimal(1)); + return samplingFactor.multiply(new BigDecimal(child.getIndividualCount())); + } + if (child.hasChildren()) { + BigDecimal indirectIndividualCount = computeIndirectIndividualCount(child); + if (indirectIndividualCount != null) { + return indirectIndividualCount; + } + } + throw new SumarisTechnicalException(String.format("No indirect individual count," + + " (some child batch has no individual count {id: %s, label: '%s'})", child.getId(), child.getLabel())); + }).reduce(new BigDecimal(0), BigDecimal::add); + } catch (SumarisTechnicalException e) { + log.trace(e.getMessage()); + return null; + } + } + + protected void computeTreeIndent(DenormalizedBatchVO target) { + computeTreeIndent(target, "", true); + } + + protected void computeTreeIndent(DenormalizedBatchVO target, String inheritedTreeIndent, boolean isLast) { + if (target.getParent() == null) { + target.setTreeIndent("-"); + } else { + target.setTreeIndent(inheritedTreeIndent + (isLast ? "|_" : "|-")); + } + + List children = target.getChildren(); + if (CollectionUtils.isNotEmpty(children)) { + String childrenTreeIndent = inheritedTreeIndent + (isLast ? " " : "| "); + for (int i = 0; i < children.size(); i++) { + computeTreeIndent(children.get(i), childrenTreeIndent, i == children.size() - 1); + } + } + } + + protected TempDenormalizedBatchVO createTempVO(BatchVO source) { + TempDenormalizedBatchVO target = new TempDenormalizedBatchVO(); + denormalizedBatchRepository.copy(source, target, true); + return target; + } + + /** + * Convert an alive weight into batch's dressing and preservation. + * If dressing/preservation are whole/fresh, then return unchanged weight + * @param batch + * @param options + * @param aliveWeight + * @return + */ + protected Optional convertAliveWeightToContext(TempDenormalizedBatchVO batch, + DenormalizedBatchOptions options, + BigDecimal aliveWeight) { + // Apply inverse conversion (alive weight / conversion factor) + return computeAliveWeightFactor(batch, options, batch.isLeaf()) + .map(conversionFactor -> { + + // Check conversion is not negative or zero (should never occur) + if (conversionFactor <= 0) throw new SumarisTechnicalException("Invalid round weight conversion. Coefficient should be > 0"); + + // No conversion: skip + if (conversionFactor == 1d) return aliveWeight; + + // Apply inverse conversion + return aliveWeight.divide(new BigDecimal(conversionFactor), WEIGHT_DECIMAL_SCALE, RoundingMode.HALF_UP); + }); + } + + protected Optional computeAliveWeightFactor(TempDenormalizedBatchVO batch, + DenormalizedBatchOptions options, + boolean applyDressingAndPreservationDefaults) { + + if (batch.getAliveWeightFactor() != null) return Optional.of(batch.getAliveWeightFactor()); + + Integer taxonGroupId = batch.getTaxonGroupId(); + if (taxonGroupId == null) return Optional.empty(); + + // No weight for this species: + if (ArrayUtils.isNotEmpty(options.getTaxonGroupIdsNoWeight()) + && ArrayUtils.contains(options.getTaxonGroupIdsNoWeight(), taxonGroupId)) { + return Optional.empty(); + } + + // Get dressing, or 'WHL - Whole' by default + Integer dressingId = DenormalizedBatches.getDressingId(batch) + .orElseGet(() -> { + // Apply defaults + if (applyDressingAndPreservationDefaults) { + return Boolean.TRUE.equals(batch.getIsLanding()) + ? options.getDefaultLandingDressingId() + : Boolean.TRUE.equals(batch.getIsDiscard()) + ? options.getDefaultDiscardDressingId() + : QualitativeValueEnum.DRESSING_WHOLE.getId(); + } + return null; + }); + + // Get preservation, or 'FRE - Fresh' by default + Integer preservationId = DenormalizedBatches.getPreservationId(batch) + .orElseGet(() -> { + // Apply default preservation + if (applyDressingAndPreservationDefaults) { + return Boolean.TRUE.equals(batch.getIsLanding()) + ? options.getDefaultLandingPreservationId() + : Boolean.TRUE.equals(batch.getIsDiscard()) + ? options.getDefaultDiscardPreservationId() + : QualitativeValueEnum.PRESERVATION_FRESH.getId(); + } + return null; + }); + + // Skip (e.g. if enableDefaultsDressingAndPreservation is 'false') + if (dressingId == null || preservationId == null) return Optional.empty(); + + // Find the best conversion coefficient + Optional conversion = roundWeightConversionService.findFirstByFilter(RoundWeightConversionFilterVO.builder() + .taxonGroupIds(new Integer[]{taxonGroupId}) + .dressingIds(new Integer[]{dressingId}) + .preservingIds(new Integer[]{preservationId}) + .locationIds(new Integer[]{options.getRoundWeightCountryLocationId()}) + .date(options.getDay()) + .build()); + + if (conversion.isEmpty()) { + log.warn("No RoundWeightConversion found for {taxonGroupId: {}, dressingId: {}, preservationId: {}, locationId: {}}", + taxonGroupId, + dressingId, + preservationId, + options.getRoundWeightCountryLocationId()); + } + + + return conversion.map(RoundWeightConversionVO::getConversionCoefficient); + } + + protected Double computeIndirectAliveWeightFactor(TempDenormalizedBatchVO batch, DenormalizedBatchOptions options) { + if (batch.getAliveWeightFactor() != null) return batch.getAliveWeightFactor(); + + // No taxon group, or no child => no indirect value + if (!batch.hasTaxonGroup() || !batch.hasChildren() || !DenormalizedBatches.isExhaustiveInventory(batch)) return null; + + // Collect all children factors + List childAliveFactors = batch.getChildren() + .stream() + .map(child -> computeIndirectAliveWeightFactor((TempDenormalizedBatchVO) child, options)) + .toList(); + + // Peek the first value (should be not null) + Double firstAliveFactor = childAliveFactors.get(0); + if (firstAliveFactor == null) return null; + + // Check same value on each child. If not: return empty (cannot compute indirect value) + boolean alwaysSameValue = childAliveFactors.size() == 1 || !childAliveFactors.stream().anyMatch(value -> !firstAliveFactor.equals(value)); + if (!alwaysSameValue) { + log.trace("No indirect alive weight. No unique value found in children"); + return null; + } + + return firstAliveFactor; + } + + private void checkBaseEnumerations() { + EntityEnums.checkResolved( + QualitativeValueEnum.LANDING, + QualitativeValueEnum.DISCARD + ); + } + private void checkRtpEnumerations() { + EntityEnums.checkResolved( + ParameterEnum.SEX, + QualitativeValueEnum.SEX_UNSEXED, + QualitativeValueEnum.DRESSING_WHOLE, + QualitativeValueEnum.DRESSING_GUTTED, + QualitativeValueEnum.PRESERVATION_FRESH); + } +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedOperationService.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedOperationService.java new file mode 100644 index 0000000000..89f6dc3fa6 --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedOperationService.java @@ -0,0 +1,57 @@ +package net.sumaris.core.service.data.denormalize; + +/*- + * #%L + * SUMARiS:: Core + * %% + * Copyright (C) 2018 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + + +import lombok.NonNull; +import net.sumaris.core.vo.data.OperationVO; +import net.sumaris.core.vo.data.batch.DenormalizedBatchOptions; +import net.sumaris.core.vo.filter.OperationFilterVO; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Nullable; + +/** + * @author BLA + * + * Service in charge of operation data + * + */ +@Transactional +public interface DenormalizedOperationService { + + @Transactional(readOnly = true) + DenormalizedBatchOptions createOptionsByProgramId(int programId); + + @Transactional(readOnly = true) + DenormalizedBatchOptions createOptionsByProgramLabel(String programLabel); + + @Transactional(readOnly = true) + DenormalizedBatchOptions createOptionsByOperation(@NonNull OperationVO operation, + @Nullable DenormalizedBatchOptions inheritedOptions); + + @Transactional(propagation = Propagation.NOT_SUPPORTED) + DenormalizedTripResultVO denormalizeByFilter(@NonNull OperationFilterVO filter, @Nullable DenormalizedBatchOptions options); + +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripResultVO.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripResultVO.java similarity index 78% rename from sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripResultVO.java rename to sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripResultVO.java index fc6bbd9037..9b263f357b 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripResultVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripResultVO.java @@ -24,16 +24,24 @@ import lombok.Builder; import lombok.Data; +import net.sumaris.core.model.technical.job.JobStatusEnum; +import net.sumaris.core.vo.technical.job.IJobResultVO; import java.io.Serializable; @Data @Builder -public class DenormalizeTripResultVO implements Serializable { +public class DenormalizedTripResultVO implements IJobResultVO, Serializable { private long tripCount; private long operationCount; private long batchCount; + + private long tripErrorCount; private long invalidBatchCount; private long executionTime; + + private String message; + + private JobStatusEnum status; } diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripService.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripService.java similarity index 82% rename from sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripService.java rename to sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripService.java index 33daab06fa..86180ddd85 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizeTripService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripService.java @@ -36,14 +36,14 @@ * */ @Transactional -public interface DenormalizeTripService { +public interface DenormalizedTripService { @Transactional(propagation = Propagation.NOT_SUPPORTED) - DenormalizeTripResultVO denormalizeByFilter(@NonNull TripFilterVO filter); + DenormalizedTripResultVO denormalizeByFilter(@NonNull TripFilterVO filter); @Transactional(propagation = Propagation.NOT_SUPPORTED) - DenormalizeTripResultVO denormalizeByFilter(TripFilterVO filter, IProgressionModel progression); + DenormalizedTripResultVO denormalizeByFilter(TripFilterVO filter, IProgressionModel progression); @Transactional - DenormalizeTripResultVO denormalizeById(int tripId); + DenormalizedTripResultVO denormalizeById(int tripId); } diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripServiceImpl.java new file mode 100644 index 0000000000..7d3cd4c9d3 --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/service/data/denormalize/DenormalizedTripServiceImpl.java @@ -0,0 +1,192 @@ +package net.sumaris.core.service.data.denormalize; + +/*- + * #%L + * SUMARiS:: Core + * %% + * Copyright (C) 2018 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.sumaris.core.dao.technical.SortDirection; +import net.sumaris.core.model.IProgressionModel; +import net.sumaris.core.model.ProgressionModel; +import net.sumaris.core.service.data.TripService; +import net.sumaris.core.util.StringUtils; +import net.sumaris.core.util.TimeUtils; +import net.sumaris.core.vo.data.TripFetchOptions; +import net.sumaris.core.vo.data.TripVO; +import net.sumaris.core.vo.data.batch.DenormalizedBatchOptions; +import net.sumaris.core.vo.filter.OperationFilterVO; +import net.sumaris.core.vo.filter.TripFilterVO; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.mutable.MutableInt; +import org.springframework.stereotype.Service; + +import java.util.Arrays; +import java.util.List; + +@Service("denormalizeTripService") +@RequiredArgsConstructor +@Slf4j +public class DenormalizedTripServiceImpl implements DenormalizedTripService { + + private final TripService tripService; + + private final DenormalizedBatchService denormalizedBatchService; + + private final DenormalizedOperationService denormalizedOperationService; + + + + @Override + public DenormalizedTripResultVO denormalizeByFilter(@NonNull TripFilterVO filter) { + ProgressionModel progress = new ProgressionModel(); + progress.addPropertyChangeListener(ProgressionModel.Fields.MESSAGE, (event) -> { + if (event.getNewValue() != null) log.debug(event.getNewValue().toString()); + }); + return denormalizeByFilter(filter, new ProgressionModel()); + } + + @Override + public DenormalizedTripResultVO denormalizeByFilter(@NonNull TripFilterVO tripFilter, @NonNull IProgressionModel progression) { + long startTime = System.currentTimeMillis(); + + progression.setCurrent(0); + progression.setMessage(String.format("Starting trips denormalization... filter: %s", tripFilter)); + + TripFetchOptions tripFetchOptions = TripFetchOptions.builder() + .withChildrenEntities(false) + .withMeasurementValues(false) + .withRecorderPerson(false) + .build(); + + long tripTotal = tripService.countByFilter(tripFilter); + progression.setTotal(tripTotal); + + boolean hasMoreData; + int offset = 0; + int pageSize = 10; + int tripCount = 0; + MutableInt tripErrorCount = new MutableInt(0); + MutableInt operationCount = new MutableInt(0); + MutableInt batchCount = new MutableInt(0); + MutableInt invalidBatchCount = new MutableInt(0); + List messages = Lists.newArrayList(); + + if (tripTotal > 0) { + progression.setCurrent(0); + progression.setMessage(String.format("Processing trips denormalization... 0/%s", tripTotal)); + + do { + // Fetch some trips + List trips = tripService.findAll(tripFilter, + offset, pageSize, // Page + TripVO.Fields.ID, SortDirection.ASC, // Sort by id, to keep continuity between pages + tripFetchOptions); + + if (offset > 0 && offset % (pageSize * 2) == 0) { + progression.setCurrent(offset); + progression.setMessage(String.format("Processing trips denormalization... %s/%s", offset, tripTotal)); + //log.trace(progression.getMessage()); + } + + // Denormalize each trip + trips.stream().parallel() + .forEach(trip -> { + // Load denormalized options + DenormalizedBatchOptions programOptions = denormalizedOperationService.createOptionsByProgramId(trip.getProgram().getId()); + + // Create operations filter, for this trip + OperationFilterVO operationFilter = OperationFilterVO.builder() + .tripId(trip.getId()) + .includedIds(tripFilter.getOperationIds()) + .hasNoChildOperation(true) + .build(); + + try { + // Denormalize trip's operation + DenormalizedTripResultVO result = denormalizedOperationService.denormalizeByFilter(operationFilter, programOptions); + + operationCount.add(result.getOperationCount()); + batchCount.add(result.getBatchCount()); + invalidBatchCount.add(result.getInvalidBatchCount()); + if (StringUtils.isNotBlank(result.getMessage())) { + messages.addAll(Splitter.on("\n").splitToList(result.getMessage())); + } + } + catch (Exception e) { + tripErrorCount.increment(); + String message = String.format("Error while during denormalization of trip #%s: %s", trip.getId(), e.getMessage()); + log.error(message, e); + messages.add(message); + } + }); + + offset += pageSize; + tripCount += trips.size(); + hasMoreData = trips.size() >= pageSize; + if (tripCount > tripTotal) { + tripTotal = tripCount; + progression.adaptTotal(tripTotal); + } + } while (hasMoreData); + } + + // Success log + progression.setCurrent(tripCount); + progression.setMessage(String.format("Trips denormalization finished, in %s - %s trips, %s operations, %s batches - %s trips in error, %s invalid batch trees (skipped)", + TimeUtils.printDurationFrom(startTime), + tripCount, + operationCount, + batchCount, + tripErrorCount, + invalidBatchCount)); + //log.debug(progression.getMessage()); + + return DenormalizedTripResultVO.builder() + .tripCount(tripCount) + .tripErrorCount(tripErrorCount.intValue()) + .operationCount(operationCount.intValue()) + .batchCount(batchCount.intValue()) + .invalidBatchCount(invalidBatchCount.intValue()) + .message(CollectionUtils.isNotEmpty(messages) ? String.join("\n", messages) : null) + .executionTime(System.currentTimeMillis() - startTime) + .build(); + } + + @Override + public DenormalizedTripResultVO denormalizeById(int tripId) { + // Load denormalized options + int programId = tripService.getProgramIdById(tripId); + DenormalizedBatchOptions programOptions = denormalizedBatchService.createOptionsByProgramId(programId); + + // Create operation filter, for this trip + OperationFilterVO operationFilter = OperationFilterVO.builder() + .tripId(tripId) + .hasNoChildOperation(true) + .build(); + + return denormalizedOperationService.denormalizeByFilter(operationFilter, programOptions); + } + +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/referential/LocationServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/referential/LocationServiceImpl.java index ed7dfd6ef8..9141a33b15 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/referential/LocationServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/referential/LocationServiceImpl.java @@ -35,12 +35,13 @@ import net.sumaris.core.dao.referential.ValidityStatusRepository; import net.sumaris.core.dao.referential.location.*; import net.sumaris.core.dao.technical.Page; +import net.sumaris.core.dao.technical.SortDirection; import net.sumaris.core.event.config.ConfigurationEvent; import net.sumaris.core.event.config.ConfigurationReadyEvent; import net.sumaris.core.event.config.ConfigurationUpdatedEvent; -import net.sumaris.core.event.entity.EntityInsertEvent; -import net.sumaris.core.event.entity.EntityUpdateEvent; +import net.sumaris.core.exception.SumarisTechnicalException; import net.sumaris.core.model.referential.Status; +import net.sumaris.core.model.referential.StatusEnum; import net.sumaris.core.model.referential.ValidityStatus; import net.sumaris.core.model.referential.ValidityStatusEnum; import net.sumaris.core.model.referential.location.*; @@ -51,11 +52,10 @@ import net.sumaris.core.vo.referential.LocationVO; import net.sumaris.core.vo.referential.ReferentialFetchOptions; import net.sumaris.core.vo.referential.ReferentialVO; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.locationtech.jts.geom.Geometry; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; -import org.springframework.context.event.EventListener; import org.springframework.core.io.ResourceLoader; import org.springframework.dao.DataAccessException; import org.springframework.scheduling.annotation.Async; @@ -450,27 +450,44 @@ public void updateLocationHierarchy() { } @Override - public String getLocationLabelByLatLong(Number latitude, Number longitude) { + public Optional getStatisticalRectangleLabelByLatLong(Number latitude, Number longitude) { if (longitude == null || latitude == null) { throw new IllegalArgumentException("Arguments 'latitude' and 'longitude' should not be null."); } // Try to find a statistical rectangle String rectangleLabel = Locations.getRectangleLabelByLatLong(latitude, longitude); - if (StringUtils.isNotBlank(rectangleLabel)) return rectangleLabel; + if (StringUtils.isNotBlank(rectangleLabel)) { + return Optional.of(rectangleLabel); + } - // TODO: find it from spatial query ? - // Otherwise, return null - return null; + // Otherwise, return empty + return Optional.empty(); } @Override - public Integer getLocationIdByLatLong(Number latitude, Number longitude) { - String locationLabel = getLocationLabelByLatLong(latitude, longitude); - if (locationLabel == null) return null; - Optional location = referentialDao.findByUniqueLabel(Location.class.getSimpleName(), locationLabel); - return location.map(ReferentialVO::getId).orElse(null); + public Optional getStatisticalRectangleIdByLatLong(Number latitude, Number longitude) { + return getStatisticalRectangleLabelByLatLong(latitude, longitude) + // Resolve the location, by its label (should be unique, for statistical rectangle - if not dao will return error) + .map(rectangleLabel -> { + List matches = referentialDao.findByFilter(Location.class.getSimpleName(), + ReferentialFilterVO.builder() + .label(rectangleLabel) + .levelIds(LocationLevels.getStatisticalRectangleLevelIds()) + .build(), 0, 2, null, null, null); + if (CollectionUtils.isEmpty(matches)) return null; + try { + // Extract singleton (= unique check, because of size=2) + return CollectionUtils.extractSingleton(matches); + } + catch (IllegalArgumentException e) { + throw new SumarisTechnicalException(String.format("More than one statistical rectangle found with label '%s'", rectangleLabel)); + } + }) + .filter(Objects::nonNull) + // Extract the id + .map(ReferentialVO::getId); } @Override diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialService.java b/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialService.java index 0e414963c1..076e084a48 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialService.java @@ -25,6 +25,7 @@ import net.sumaris.core.dao.technical.SortDirection; import net.sumaris.core.model.referential.IReferentialWithStatusEntity; import net.sumaris.core.vo.filter.IReferentialFilter; +import net.sumaris.core.vo.referential.ReferentialFetchOptions; import net.sumaris.core.vo.referential.ReferentialTypeVO; import net.sumaris.core.vo.referential.ReferentialVO; import org.springframework.transaction.annotation.Transactional; @@ -51,7 +52,9 @@ public interface ReferentialService { List findByFilter(String entityName, IReferentialFilter filter, int offset, int size); @Transactional(readOnly = true) - List findByFilter(String entityName, IReferentialFilter filter, int offset, int size, String sortAttribute, SortDirection sortDirection); + List findByFilter(String entityName, IReferentialFilter filter, int offset, int size, + String sortAttribute, SortDirection sortDirection, + ReferentialFetchOptions fetchOptions); @Transactional(readOnly = true) Long countByFilter(String entityName, IReferentialFilter filter); diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialServiceImpl.java index 1b830c9160..766c8dc560 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/referential/ReferentialServiceImpl.java @@ -28,6 +28,7 @@ import net.sumaris.core.dao.referential.ReferentialDao; import net.sumaris.core.dao.referential.ReferentialEntities; import net.sumaris.core.dao.technical.SortDirection; +import net.sumaris.core.dao.technical.jpa.IFetchOptions; import net.sumaris.core.event.config.ConfigurationEvent; import net.sumaris.core.event.config.ConfigurationReadyEvent; import net.sumaris.core.event.config.ConfigurationUpdatedEvent; @@ -39,6 +40,7 @@ import net.sumaris.core.model.referential.IReferentialWithStatusEntity; import net.sumaris.core.vo.filter.IReferentialFilter; import net.sumaris.core.vo.filter.ReferentialFilterVO; +import net.sumaris.core.vo.referential.ReferentialFetchOptions; import net.sumaris.core.vo.referential.ReferentialTypeVO; import net.sumaris.core.vo.referential.ReferentialVO; import org.nuiton.i18n.I18n; @@ -110,16 +112,20 @@ public ReferentialVO getLevelById(String entityName, int levelId) { } @Override - public List findByFilter(String entityName, IReferentialFilter filter, int offset, int size, String sortAttribute, SortDirection sortDirection) { + public List findByFilter(String entityName, + IReferentialFilter filter, int offset, int size, + String sortAttribute, SortDirection sortDirection, + ReferentialFetchOptions fetchOptions) { return referentialDao.findByFilter(entityName, filter != null ? filter : new ReferentialFilterVO(), offset, size, sortAttribute, - sortDirection); + sortDirection, + fetchOptions); } @Override public List findByFilter(String entityName, IReferentialFilter filter, int offset, int size) { return findByFilter(entityName, filter != null ? filter : new ReferentialFilterVO(), offset, size, IItemReferentialEntity.Fields.LABEL, - SortDirection.ASC); + SortDirection.ASC, null); } @Override diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionService.java b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionService.java index a959aa2033..e11078c7f5 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionService.java @@ -29,15 +29,22 @@ import org.springframework.transaction.annotation.Transactional; import java.util.List; +import java.util.Optional; @Transactional public interface RoundWeightConversionService { List findByFilter(RoundWeightConversionFilterVO filter, Page page, RoundWeightConversionFetchOptions fetchOptions); + Optional findFirstByFilter(RoundWeightConversionFilterVO filter); + + Optional findFirstByFilter(RoundWeightConversionFilterVO filter, RoundWeightConversionFetchOptions fetchOptions); + long countByFilter(RoundWeightConversionFilterVO filter); List saveAll(List source); void deleteAllById(List ids); + + } diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceImpl.java index f8f5e9f495..18674d5c08 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceImpl.java @@ -22,26 +22,34 @@ package net.sumaris.core.service.referential.conversion; -import com.google.common.collect.ListMultimap; +import com.google.common.base.Preconditions; +import lombok.NonNull; +import net.sumaris.core.config.CacheConfiguration; import net.sumaris.core.dao.referential.conversion.RoundWeightConversionRepository; import net.sumaris.core.dao.technical.Page; -import net.sumaris.core.model.referential.location.LocationLevels; -import net.sumaris.core.service.referential.LocationService; -import net.sumaris.core.util.Beans; -import net.sumaris.core.vo.filter.LocationFilterVO; -import net.sumaris.core.vo.referential.LocationVO; +import net.sumaris.core.dao.technical.SortDirection; +import net.sumaris.core.model.referential.conversion.RoundWeightConversion; import net.sumaris.core.vo.referential.conversion.RoundWeightConversionFetchOptions; import net.sumaris.core.vo.referential.conversion.RoundWeightConversionFilterVO; import net.sumaris.core.vo.referential.conversion.RoundWeightConversionVO; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; @Service("roundWeightConversionService") public class RoundWeightConversionServiceImpl implements RoundWeightConversionService { + private static final Page FIND_FIRST_PAGE = Page.builder().size(1) + .sortBy(RoundWeightConversion.Fields.START_DATE) + .sortDirection(SortDirection.DESC) + .build(); + @Resource private RoundWeightConversionRepository roundWeightConversionRepository; @@ -52,6 +60,28 @@ public List findByFilter(RoundWeightConversionFilterVO return roundWeightConversionRepository.findAll(filter, page, fetchOptions); } + @Override + public Optional findFirstByFilter(@NonNull RoundWeightConversionFilterVO filter) { + return findFirstByFilter(filter, RoundWeightConversionFetchOptions.DEFAULT); + } + + @Override + @Cacheable(cacheNames = CacheConfiguration.Names.ROUND_WEIGHT_CONVERSION_FIRST_BY_FILTER, key = "#filter.hashCode() * #fetchOptions.hashCode()") + public Optional findFirstByFilter( + @NonNull RoundWeightConversionFilterVO filter, @NonNull RoundWeightConversionFetchOptions fetchOptions) { + Preconditions.checkArgument(ArrayUtils.isNotEmpty(filter.getTaxonGroupIds()), "Require at least one taxonGroupId"); + Preconditions.checkArgument(ArrayUtils.isNotEmpty(filter.getLocationIds()), "Require at least one locationIds"); + + // Try to find a conversion factor + List matches = findByFilter(filter, + FIND_FIRST_PAGE, + fetchOptions + ); + if (CollectionUtils.isNotEmpty(matches)) return Optional.of(matches.get(0)); + + return Optional.empty(); + } + @Override public long countByFilter(RoundWeightConversionFilterVO filter) { return roundWeightConversionRepository.count(filter); diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionService.java b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionService.java index e24181ac31..76532fb39c 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionService.java @@ -28,22 +28,59 @@ import net.sumaris.core.vo.referential.conversion.WeightLengthConversionVO; import org.springframework.transaction.annotation.Transactional; +import javax.annotation.Nullable; import java.math.BigDecimal; import java.util.List; +import java.util.Optional; @Transactional public interface WeightLengthConversionService { - List findByFilter(WeightLengthConversionFilterVO filter, Page page, WeightLengthConversionFetchOptions fetchOptions); + @Transactional(readOnly = true) + List findByFilter(WeightLengthConversionFilterVO filter, Page page, + @Nullable WeightLengthConversionFetchOptions fetchOptions); + @Transactional(readOnly = true) long countByFilter(WeightLengthConversionFilterVO filter); + /** + * Get the best fit weight-length conversion. + * Required at least a reference taxon and location. + * Will try to load using this order + *
    + *
  • pmfmId + year + month
    • + *
  • pmfmId + year (without month)
    • + *
  • pmfmId + month (without year)
    • + *
  • TODO: Loop using parameterId (without pmfmId). If found, will convert unit
    • + *
+ * @param filter + * @param page + * @param fetchOptions + * @return + */ + @Transactional(readOnly = true) + Optional loadFirstByFilter(WeightLengthConversionFilterVO filter); + + @Transactional(readOnly = true) + Optional loadFirstByFilter(WeightLengthConversionFilterVO filter, @Nullable WeightLengthConversionFetchOptions fetchOptions); + + List saveAll(List source); void deleteAllById(List ids); + @Transactional(readOnly = true) BigDecimal computedWeight(WeightLengthConversionVO conversion, Number length, - int scale, - Number individualCount); + String lengthUnit, + @Nullable Number lengthPrecision, + @Nullable Number individualCount, + String weightUnit, + int weightScale); + + @Transactional(readOnly = true) + boolean isWeightLengthParameter(int parameterId); + + @Transactional(readOnly = true) + boolean isWeightLengthPmfm(int pmfmId); } diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceImpl.java index ee892d6a02..a9a258b795 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceImpl.java @@ -23,32 +23,45 @@ package net.sumaris.core.service.referential.conversion; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimap; import lombok.NonNull; +import net.sumaris.core.config.CacheConfiguration; import net.sumaris.core.dao.referential.conversion.WeightLengthConversionRepository; import net.sumaris.core.dao.technical.Page; +import net.sumaris.core.dao.technical.SortDirection; import net.sumaris.core.model.referential.StatusEnum; +import net.sumaris.core.model.referential.conversion.WeightLengthConversion; import net.sumaris.core.model.referential.location.LocationLevels; +import net.sumaris.core.model.referential.pmfm.UnitEnum; import net.sumaris.core.service.referential.LocationService; import net.sumaris.core.service.referential.pmfm.PmfmService; import net.sumaris.core.util.Beans; +import net.sumaris.core.util.conversion.UnitConversions; import net.sumaris.core.vo.filter.LocationFilterVO; import net.sumaris.core.vo.filter.PmfmPartsVO; import net.sumaris.core.vo.referential.LocationVO; import net.sumaris.core.vo.referential.conversion.WeightLengthConversionFetchOptions; import net.sumaris.core.vo.referential.conversion.WeightLengthConversionFilterVO; import net.sumaris.core.vo.referential.conversion.WeightLengthConversionVO; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.cache.annotation.Caching; import org.springframework.stereotype.Service; +import javax.annotation.Nullable; import javax.annotation.Resource; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Iterator; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; @Service("weightLengthConversionService") @@ -69,8 +82,8 @@ public List findByFilter(WeightLengthConversionFilterV WeightLengthConversionFetchOptions fetchOptions) { List result = weightLengthConversionRepository.findAll(filter, page, fetchOptions); - // Add length pmfm Ids - if (fetchOptions != null && fetchOptions.isWithRectangleLabels()) { + // Add pmfm Ids into VO + if (fetchOptions != null && fetchOptions.isWithLengthPmfmIds()) { // Group by [parameter, unit] Joiner mapKeyJoiner = Joiner.on('|'); Splitter mapKeySplitter = Splitter.on('|').limit(2); @@ -128,29 +141,113 @@ public long countByFilter(WeightLengthConversionFilterVO filter) { return weightLengthConversionRepository.count(filter); } + @Override + public Optional loadFirstByFilter(WeightLengthConversionFilterVO filter) { + return loadFirstByFilter(filter, WeightLengthConversionFetchOptions.DEFAULT); + } + + @Override + @Cacheable(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_FIRST_BY_FILTER, key = "#filter.hashCode() * #fetchOptions.hashCode()") + public Optional loadFirstByFilter(@NonNull WeightLengthConversionFilterVO filter, @NonNull WeightLengthConversionFetchOptions fetchOptions) { + Preconditions.checkArgument(ArrayUtils.isNotEmpty(filter.getReferenceTaxonIds()), "Require at least on referenceTaxonId"); + Preconditions.checkArgument(ArrayUtils.isNotEmpty(filter.getLocationIds()) + || ArrayUtils.isNotEmpty(filter.getChildLocationIds()) + || ArrayUtils.isNotEmpty(filter.getRectangleLabels()), "Require at least one of rectangleLabels, childLocationIds or locationIds"); + + final Page page = Page.builder().size(1) + .sortBy(filter.getYear() != null ? WeightLengthConversion.Fields.YEAR : WeightLengthConversion.Fields.START_MONTH) + .sortDirection(SortDirection.DESC) + .build(); + + // First, try with full filter + List matches = this.findByFilter(filter, page, fetchOptions); + if (CollectionUtils.isNotEmpty(matches)) return Optional.of(matches.get(0)); + + if (filter.getYear() != null && filter.getMonth() != null) { + // Retry on year only (without month) + WeightLengthConversionFilterVO filterWithoutMonth = filter.clone(); // Copy, to keep original filter unchanged + filterWithoutMonth.setMonth(null); + matches = this.findByFilter(filterWithoutMonth, page, fetchOptions); + if (CollectionUtils.isNotEmpty(matches)) return Optional.of(matches.get(0)); + + // Retry on month only (without year) + WeightLengthConversionFilterVO filterWithoutYear = filter.clone(); // Copy, to keep original filter unchanged + filterWithoutYear.setYear(null); + page.setSortBy(WeightLengthConversion.Fields.YEAR); + matches = this.findByFilter(filterWithoutYear, page, fetchOptions); + if (CollectionUtils.isNotEmpty(matches)) return Optional.of(matches.get(0)); + } + + // Retry on parameter Id (=skip unit match) + if (ArrayUtils.isNotEmpty(filter.getLengthPmfmIds()) && ArrayUtils.isEmpty(filter.getLengthParameterIds())) { + // TODO loop on parameterIds then if result + } + + // Not found + return Optional.empty(); + } + @Override public BigDecimal computedWeight(@NonNull WeightLengthConversionVO conversion, @NonNull Number length, - int scale, - Number individualCount) { + @NonNull String lengthUnit, + @Nullable Number lengthPrecision, + @Nullable Number individualCount, + @NonNull String weightUnit, + int weightScale) { + BigDecimal lengthDecimal = new BigDecimal(length.toString()); + + // Convert length to expected unit + String conversionLengthUnit = conversion.getLengthUnit() != null + ? conversion.getLengthUnit().getLabel() + : UnitEnum.valueOf(conversion.getLengthUnitId()).getLabel(); + if (!Objects.equals(conversionLengthUnit, lengthUnit)) { + // create a conversion factor + BigDecimal lengthUnitConversion = BigDecimal.valueOf(UnitConversions.lengthToMeterConversion(conversionLengthUnit)) + .divide(new BigDecimal(UnitConversions.lengthToMeterConversion(lengthUnit))); + + // Convert length to the expected unit + lengthDecimal = lengthDecimal.multiply(lengthUnitConversion); + + // Round to half of the precision (see Allegro mantis #5598) + if (lengthPrecision != null) { + // length += 0.5 * precision * lengthUnitConversion + lengthDecimal = lengthDecimal.add( + new BigDecimal("0.5") + .multiply(new BigDecimal(lengthPrecision.toString())) + .multiply(lengthUnitConversion)); + } + } // CoefA * length ^ CoefB - BigDecimal result = new BigDecimal(conversion.getConversionCoefficientA().toString()) - .multiply(new BigDecimal( - Math.pow(length.doubleValue(), conversion.getConversionCoefficientB().doubleValue()) + BigDecimal weightKg = BigDecimal.valueOf(conversion.getConversionCoefficientA()) + .multiply(BigDecimal.valueOf( + Math.pow(lengthDecimal.doubleValue(), conversion.getConversionCoefficientB()) )) // * individual count .multiply(new BigDecimal(individualCount != null ? individualCount.toString() : "1")); - // Compute alive weight - - // Round to scale - result = result.divide(new BigDecimal(1), scale, RoundingMode.HALF_UP); + // Convert to expected weight unit (kg by default) + if (!Objects.equals(weightUnit, "kg")) { + return weightKg.divide( + BigDecimal.valueOf(UnitConversions.weightToKgConversion(weightUnit)), + weightScale, + RoundingMode.HALF_UP + ); + } - return result; + // Round to expected weight scale + return weightKg.divide(new BigDecimal(1), weightScale, RoundingMode.HALF_UP); } @Override + @Caching( + evict = { + @CacheEvict(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_FIRST_BY_FILTER, allEntries = true), + @CacheEvict(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PARAMETER_ID, allEntries = true), + @CacheEvict(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PMFM_ID, allEntries = true) + } + ) public List saveAll(List sources) { return sources.stream() .map(weightLengthConversionRepository::save) @@ -158,7 +255,30 @@ public List saveAll(List sou } @Override + @Caching( + evict = { + @CacheEvict(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_FIRST_BY_FILTER, allEntries = true), + @CacheEvict(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PARAMETER_ID, allEntries = true), + @CacheEvict(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PMFM_ID, allEntries = true) + } + ) public void deleteAllById(List ids) { weightLengthConversionRepository.deleteAllById(ids); } + + @Override + @Cacheable(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PARAMETER_ID) + public boolean isWeightLengthParameter(int parameterId) { + return weightLengthConversionRepository.count(WeightLengthConversionFilterVO.builder() + .lengthParameterIds(new Integer[]{parameterId}) + .build()) > 0; + } + + @Override + @Cacheable(cacheNames = CacheConfiguration.Names.WEIGHT_LENGTH_CONVERSION_IS_LENGTH_PMFM_ID) + public boolean isWeightLengthPmfm(int pmfmId) { + return weightLengthConversionRepository.count(WeightLengthConversionFilterVO.builder() + .lengthPmfmIds(new Integer[]{pmfmId}) + .build()) > 0; + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionService.java b/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionService.java index c1ab07829f..c36c8de456 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionService.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionService.java @@ -25,13 +25,19 @@ import io.reactivex.rxjava3.core.Observable; import net.sumaris.core.event.job.JobProgressionVO; +import net.sumaris.core.model.IProgressionModel; import net.sumaris.core.vo.technical.job.JobVO; +import java.util.concurrent.Callable; import java.util.concurrent.Future; import java.util.function.Function; public interface JobExecutionService { - JobVO run(JobVO job, Function> asyncMethod); + JobVO run(JobVO job, Callable configurationLoader, + Function> asyncMethod); + + JobVO run(JobVO job, Function> callableFuture); Observable watchJobProgression(Integer id); + } diff --git a/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionServiceImpl.java b/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionServiceImpl.java index b560d918fa..7b2e8c09aa 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionServiceImpl.java +++ b/sumaris-core/src/main/java/net/sumaris/core/service/technical/JobExecutionServiceImpl.java @@ -27,12 +27,19 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import io.reactivex.rxjava3.core.Observable; +import io.reactivex.rxjava3.disposables.Disposable; import io.reactivex.rxjava3.schedulers.Schedulers; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import net.sumaris.core.event.job.JobEndEvent; +import net.sumaris.core.event.job.JobProgressionEvent; import net.sumaris.core.event.job.JobProgressionVO; +import net.sumaris.core.event.job.JobStartEvent; +import net.sumaris.core.exception.SumarisTechnicalException; import net.sumaris.core.jms.JmsConfiguration; import net.sumaris.core.jms.JmsJobEventProducer; +import net.sumaris.core.model.IProgressionModel; +import net.sumaris.core.model.ProgressionModel; import net.sumaris.core.model.social.EventLevelEnum; import net.sumaris.core.model.social.EventTypeEnum; import net.sumaris.core.model.social.SystemRecipientEnum; @@ -40,20 +47,24 @@ import net.sumaris.core.service.social.UserEventService; import net.sumaris.core.util.Assert; import net.sumaris.core.util.Dates; +import net.sumaris.core.util.reactive.Observables; import net.sumaris.core.vo.administration.user.PersonVO; import net.sumaris.core.vo.social.UserEventVO; import net.sumaris.core.vo.technical.job.JobFilterVO; +import net.sumaris.core.vo.technical.job.IJobResultVO; import net.sumaris.core.vo.technical.job.JobVO; import net.sumaris.server.security.ISecurityContext; import org.apache.commons.collections4.CollectionUtils; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.jms.annotation.JmsListener; +import org.springframework.scheduling.annotation.AsyncResult; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; import javax.annotation.PostConstruct; import javax.jms.Message; +import java.beans.PropertyChangeListener; import java.sql.Timestamp; import java.util.Date; import java.util.List; @@ -78,17 +89,19 @@ public class JobExecutionServiceImpl implements JobExecutionService { private final ISecurityContext securityContext; private final UserEventService userEventService; + private final ApplicationEventPublisher publisher; private final Map>> jobProgressionConsumerMap = new ConcurrentHashMap<>(); private final Map> jobFutureMap = new ConcurrentHashMap<>(); public JobExecutionServiceImpl(JobService jobService, UserEventService userEventService, ObjectMapper objectMapper, - Optional> securityContext) { + Optional> securityContext, ApplicationEventPublisher publisher) { this.jobService = jobService; this.userEventService = userEventService; this.objectMapper = objectMapper; this.securityContext = securityContext.orElse(null); + this.publisher = publisher; } @PostConstruct @@ -97,7 +110,14 @@ protected void init() { } @Override - public JobVO run(JobVO job, Function> asyncMethod) { + public JobVO run(JobVO job, Function> callableFuture) { + return run(job, null, callableFuture); + } + + @Override + public JobVO run(JobVO job, + Callable configurationLoader, + Function> callableFuture) { Assert.notBlank(job.getName()); Assert.notNull(job.getType()); @@ -111,6 +131,19 @@ public JobVO run(JobVO job, Function> asyncMethod) { job.setStatus(JobStatusEnum.PENDING); job.setStartDate(new Date()); + // Load job configuration + if (configurationLoader != null) { + try { + Object configuration = configurationLoader.call(); + // Store configuration into the job (as json) + if (configuration != null) { + job.setConfiguration(objectMapper.writeValueAsString(configuration)); + } + } catch (Exception e) { + throw new SumarisTechnicalException(e); + } + } + // Save job job = jobService.save(job); @@ -118,12 +151,100 @@ public JobVO run(JobVO job, Function> asyncMethod) { sendUserEvent(EventLevelEnum.INFO, job); // Execute async method - Future future = asyncMethod.apply(job); + Future future = start(job, callableFuture); jobFutureMap.put(job.getId(), future); return job; } + public Future start(final JobVO job, + Function> callableFuture) { + final int jobId = job.getId(); + + // Publish job start event + publisher.publishEvent(new JobStartEvent(jobId, job)); + + // Create progression model and listener to throttle events + ProgressionModel progressionModel = new ProgressionModel(); + job.setProgressionModel(progressionModel); + + // Create listeners to throttle events + io.reactivex.rxjava3.core.Observable progressionObservable = Observable.create(emitter -> { + + // Create listener on bean property and emit the value + PropertyChangeListener listener = evt -> { + ProgressionModel progression = (ProgressionModel) evt.getSource(); + JobProgressionVO jobProgression = JobProgressionVO.fromModelBuilder(progression) + .id(jobId) + .name(job.getName()) + .build(); + emitter.onNext(jobProgression); + + if (progression.isCompleted()) { + // complete observable + emitter.onComplete(); + } + }; + + // Add listener on current progression and message + progressionModel.addPropertyChangeListener(ProgressionModel.Fields.CURRENT, listener); + progressionModel.addPropertyChangeListener(ProgressionModel.Fields.MESSAGE, listener); + }); + + Disposable progressionSubscription = progressionObservable + // throttle for 500ms to filter unnecessary flow + .throttleLatest(500, TimeUnit.MILLISECONDS, true) + // Publish job progression event + .subscribe(jobProgressionVO -> publisher.publishEvent(new JobProgressionEvent(jobId, jobProgressionVO))); + + // Execute import + try { + R result = null; + + try { + + // Start job + Future resultFuture = callableFuture.apply(progressionModel); + + // Wait job result + result = resultFuture.get(); + + // Extract the job status, from result + if (result instanceof IJobResultVO) { + job.setStatus(((IJobResultVO)result).getStatus()); + } + else { + log.warn("Cannot read job's status, from result (not implements IWithJobStatusVO). Will use SUCCESS"); + job.setStatus(JobStatusEnum.SUCCESS); + } + + } catch (Exception e) { + // Set failed status + // TODO + //job.setStatus(JobStatusEnum.FAILED); + job.setStatus(JobStatusEnum.ERROR); + } + + // Store result as report + if (result != null) { + try { + // Serialize result in job report (as json) + job.setReport(objectMapper.writeValueAsString(result)); + } catch (JsonProcessingException e) { + throw new SumarisTechnicalException(e); + } + } + + return new AsyncResult<>(result); + + } finally { + + // Publish job end event + publisher.publishEvent(new JobEndEvent(jobId, job)); + Observables.dispose(progressionSubscription); + } + } + @Scheduled(fixedRate = 1, timeUnit = TimeUnit.SECONDS) public void waitJobFuture() { synchronized (jobFutureMap) { diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/administration/programStrategy/Programs.java b/sumaris-core/src/main/java/net/sumaris/core/vo/administration/programStrategy/Programs.java index d9b7fdf537..84437d3f36 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/administration/programStrategy/Programs.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/administration/programStrategy/Programs.java @@ -23,8 +23,8 @@ package net.sumaris.core.vo.administration.programStrategy; import lombok.NonNull; +import net.sumaris.core.model.administration.programStrategy.ProgramPropertyEnum; import org.apache.commons.collections4.MapUtils; -import org.nuiton.config.ConfigOptionDef; /** * Helper class for program @@ -35,11 +35,15 @@ protected Programs(){ // Helper class } - public static boolean getPropertyAsBoolean(@NonNull ProgramVO source, ConfigOptionDef option) { - return MapUtils.getBooleanValue(source.getProperties(), option.getKey(), Boolean.getBoolean(option.getDefaultValue())); + public static boolean getPropertyAsBoolean(@NonNull ProgramVO source, ProgramPropertyEnum property) { + return MapUtils.getBooleanValue(source.getProperties(), property.getKey(), Boolean.getBoolean(property.getDefaultValue())); } - public static String getProperty(@NonNull ProgramVO source, ConfigOptionDef option) { - return MapUtils.getString(source.getProperties(), option.getKey(), option.getDefaultValue()); + public static String getProperty(@NonNull ProgramVO source, ProgramPropertyEnum property) { + return MapUtils.getString(source.getProperties(), property.getKey(), property.getDefaultValue()); + } + + public static Integer getPropertyAsInteger(@NonNull ProgramVO source, ProgramPropertyEnum property) { + return MapUtils.getInteger(source.getProperties(), property.getKey(), property.getDefaultValue() != null ? Integer.parseInt(property.getDefaultValue()) : null); } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/data/OperationFetchOptions.java b/sumaris-core/src/main/java/net/sumaris/core/vo/data/OperationFetchOptions.java index 8341ae729a..ac82b395c7 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/data/OperationFetchOptions.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/data/OperationFetchOptions.java @@ -53,7 +53,7 @@ public static OperationFetchOptions clone(OperationFetchOptions options) { private boolean withObservers = true; @Builder.Default - private boolean withChildrenEntities = false; + private boolean withChildrenEntities = false; // If tru, enable other property (positions, fishing areas, batches, samples, etc.) @Builder.Default private boolean withMeasurementValues = false; @@ -67,6 +67,14 @@ public static OperationFetchOptions clone(OperationFetchOptions options) { @Builder.Default private boolean withTrip = false; + private boolean withPositions = false; + + private boolean withFishingAreas = false; + + private boolean withBatches = false; + + private boolean withSamples = false; + public OperationFetchOptions clone() { OperationFetchOptions target = new OperationFetchOptions(); Beans.copyProperties(this, target); diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/data/VesselPositionVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/data/VesselPositionVO.java index b368efba14..b6e7520c90 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/data/VesselPositionVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/data/VesselPositionVO.java @@ -25,6 +25,7 @@ import lombok.Data; import lombok.ToString; import lombok.experimental.FieldNameConstants; +import net.sumaris.core.dao.data.IPosition; import net.sumaris.core.model.IUpdateDateEntity; import net.sumaris.core.model.IValueObject; import net.sumaris.core.vo.administration.user.DepartmentVO; @@ -33,7 +34,7 @@ @Data @FieldNameConstants -public class VesselPositionVO implements IUpdateDateEntity, IValueObject { +public class VesselPositionVO implements IUpdateDateEntity, IValueObject, IPosition { private Integer id; private Date dateTime; diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchOptions.java b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchOptions.java index ea832a5b0e..e9847ad381 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchOptions.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchOptions.java @@ -22,15 +22,22 @@ package net.sumaris.core.vo.data.batch; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; -import net.sumaris.core.vo.data.IDataFetchOptions; +import lombok.NoArgsConstructor; +import net.sumaris.core.model.referential.pmfm.QualitativeValueEnum; +import net.sumaris.core.util.Beans; +import net.sumaris.core.util.Dates; import javax.annotation.Nullable; -import java.util.List; +import java.util.Date; @Data @Builder +@AllArgsConstructor +@NoArgsConstructor public class DenormalizedBatchOptions { public static final DenormalizedBatchOptions DEFAULT = DenormalizedBatchOptions.builder().build(); @@ -39,12 +46,65 @@ public static DenormalizedBatchOptions nullToDefault(@Nullable DenormalizedBatch return options != null ? options : DEFAULT; } + @Builder.Default + private boolean force = false; // Should recompute if denormalization already done ? + @Builder.Default private boolean enableTaxonGroup = true; @Builder.Default private boolean enableTaxonName = true; - private List taxonGroupIdsNoWeight; + @Builder.Default + private boolean enableRtpWeight = false; + + @Builder.Default + private boolean allowZeroWeightWithIndividual = true; + + private Integer[] taxonGroupIdsNoWeight; + + private Integer roundWeightCountryLocationId; // Country location, used to find a round weight conversion + + private Integer[] fishingAreaLocationIds; // Fishing areas used to find a weight length conversion + + private Date dateTime; + + @Builder.Default + private Integer defaultLandingDressingId = QualitativeValueEnum.DRESSING_GUTTED.getId(); // /!\ in SIH Adagio, the denormalization job use WHL as default + + @Builder.Default + private Integer defaultDiscardDressingId = QualitativeValueEnum.DRESSING_WHOLE.getId(); + + @Builder.Default + private Integer defaultLandingPreservationId = QualitativeValueEnum.PRESERVATION_FRESH.getId(); + + @Builder.Default + private Integer defaultDiscardPreservationId = QualitativeValueEnum.PRESERVATION_FRESH.getId(); + + @Builder.Default + private int maxRtpWeightDiffPct = 10; // 10% max pct between RTP weight and weight + + @JsonIgnore + public int getMonth() { + return dateTime != null ? Dates.getMonth(dateTime) + 1: null; + } + + @JsonIgnore + public int getYear() { + return dateTime != null ? Dates.getYear(dateTime) : null; + } + + /** + * Get date, wuthout time. Useful for increase stability of cache keys + * @return + */ + @JsonIgnore + public Date getDay() { + return dateTime != null ? Dates.resetTime(dateTime) : null; + } + + public DenormalizedBatchOptions clone() { + return Beans.clone(this, DenormalizedBatchOptions.class); + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchSortingValueVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchSortingValueVO.java index 5f681f988e..fa7e740f61 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchSortingValueVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchSortingValueVO.java @@ -27,13 +27,14 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import lombok.experimental.FieldNameConstants; +import net.sumaris.core.model.IValueObject; import net.sumaris.core.vo.referential.PmfmVO; import net.sumaris.core.vo.referential.ReferentialVO; @Data @FieldNameConstants @EqualsAndHashCode -public class DenormalizedBatchSortingValueVO { +public class DenormalizedBatchSortingValueVO implements IValueObject { @EqualsAndHashCode.Exclude private Integer id; @@ -54,4 +55,6 @@ public class DenormalizedBatchSortingValueVO { @ToString.Exclude private DenormalizedBatchVO batch; + private Integer batchId; + } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchVO.java index c0a97d8109..36d595f83b 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatchVO.java @@ -61,11 +61,17 @@ public class DenormalizedBatchVO private Short flatRankOrder; private Double weight; private Double indirectWeight; + + private Double indirectRtpWeight; + private Double elevateRtpWeight; private Double elevateContextWeight; private Double indirectContextWeight; private Double elevateWeight; + private Double taxonElevateContextWeight; private Integer individualCount; private Integer indirectIndividualCount; + + private Integer taxonElevateIndividualCount; private Integer elevateIndividualCount; private Double samplingRatio; private String samplingRatioText; diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatches.java b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatches.java index b8678c0d87..d541e97227 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatches.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/DenormalizedBatches.java @@ -23,16 +23,24 @@ package net.sumaris.core.vo.data.batch; import com.google.common.base.Joiner; +import lombok.NonNull; import net.sumaris.core.dao.data.batch.BatchSpecifications; -import net.sumaris.core.util.UnicodeChars; +import net.sumaris.core.model.referential.QualityFlags; +import net.sumaris.core.model.referential.pmfm.ParameterEnum; +import net.sumaris.core.model.referential.pmfm.PmfmEnum; +import net.sumaris.core.util.Beans; +import net.sumaris.core.util.Numbers; import net.sumaris.core.util.StringUtils; +import net.sumaris.core.util.UnicodeChars; import net.sumaris.core.vo.referential.IReferentialVO; +import net.sumaris.core.vo.referential.ReferentialVO; import org.apache.commons.collections4.CollectionUtils; import javax.annotation.Nullable; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.*; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -46,7 +54,13 @@ protected DenormalizedBatches() { public static boolean isExhaustiveInventory(DenormalizedBatchVO b) { return !DenormalizedBatches.isCatchBatch(b) - && (b.getInheritedTaxonName() != null || Boolean.TRUE.equals(b.getExhaustiveInventory())); + // Batch a taxon => always exhaustive (not child with another taxon) + && (b.getInheritedTaxonName() != null + || ( + // If batch is marked has exhaustive + Boolean.TRUE.equals(b.getExhaustiveInventory()) + ) + ); } public static boolean isCatchBatch(DenormalizedBatchVO b) { @@ -55,11 +69,13 @@ public static boolean isCatchBatch(DenormalizedBatchVO b) { public static boolean isSamplingBatch(DenormalizedBatchVO b) { return b.getSamplingRatio() != null || - (b.getParent() != null && CollectionUtils.size(b.getParent().getChildren()) == 1 - && ( - (b.getParent().getWeight() != null && b.getWeight() != null) - || !hasOwnedSortingValue(b) - ) + ( + // Should have a parent, and parent should have only one child + b.getParent() != null && CollectionUtils.size(b.getParent().getChildren()) == 1 + // Self or parent should have a weight + && (b.getParent().getWeight() != null && b.getWeight() != null) + // Should not have sorting values, nor taxon or reference taxon + && !hasOwnedSortingValue(b) && b.getTaxonGroup() == null && b.getTaxonName() == null ); } @@ -83,18 +99,25 @@ public static double computeFlatOrder(DenormalizedBatchVO b) { + (b.getRankOrder() != null ? b.getRankOrder().doubleValue() : 1d) * Math.pow(10, -1 * (b.getTreeLevel() - 1 ) * 3); } - public static String dumpAsString(List sources, boolean withHierarchicalLabel, boolean useUnicode) { Joiner joiner = Joiner.on(' ').skipNulls(); + String arrowDown = useUnicode ? UnicodeChars.ARROW_DOWN : "~"; + return sources.stream() .sorted(Comparator.comparing(DenormalizedBatches::computeFlatOrder)) + // Cast as temp batch (more fields can be displayed) + .map(vo -> (vo instanceof TempDenormalizedBatchVO) + ? (TempDenormalizedBatchVO) vo + : Beans.clone(vo, TempDenormalizedBatchVO.class)) .map(source -> { String treeIndent = useUnicode ? replaceTreeUnicode(source.getTreeIndent()) : source.getTreeIndent(); String hierarchicalLabel = withHierarchicalLabel ? generateHierarchicalLabel(source) : null; + double elevateContextFactor = Numbers.doubleValue(source.getElevateContextFactor(), 1d); + double aliveFactor = source.getAliveWeightFactor() != null ? source.getAliveWeightFactor() : 1d; boolean hasSpecies = source.getTaxonGroup() != null || source.getTaxonName() != null; return joiner.join( treeIndent, @@ -124,37 +147,118 @@ public static String dumpAsString(List sources, // Indirect weight ((source.getIndirectWeight() != null && !Objects.equals(source.getIndirectWeight(), source.getWeight())) - ? String.format("(%s %s kg)", useUnicode ? UnicodeChars.ARROW_DOWN : "~", source.getIndirectWeight()) : null), + ? String.format("(%s%s kg)", arrowDown, source.getIndirectWeight()) : null), + // RTP Context Weight + (source.getRtpContextWeight() != null ? String.format("(RTP=%s kg)", source.getRtpContextWeight()) : null), + + // Indirect RTP Context Weight + ((source.getIndirectRtpContextWeight() != null && !Objects.equals(source.getIndirectRtpContextWeight(), source.getRtpContextWeight()))? String.format("(%sRTP=%skg)", arrowDown, source.getIndirectRtpWeight()) : null), // Individual count (source.getIndividualCount() != null ? String.format("[%s indiv]", source.getIndividualCount()) : null), // Indirect individual count ((source.getIndirectIndividualCount() != null && !Objects.equals(source.getIndirectIndividualCount(), source.getIndividualCount())) - ? String.format("(%s %s indiv)", useUnicode ? UnicodeChars.ARROW_DOWN : "~", source.getIndirectIndividualCount()) : null), + ? String.format("(%s%s indiv)", arrowDown, source.getIndirectIndividualCount()) : null), + + // Elevate context factor + (1d != elevateContextFactor ? String.format("x%s", elevateContextFactor) : null), + + // Alive factor = elevate factor / context factor) + (1d != aliveFactor ? String.format("x%s", aliveFactor) : null), "=>", // Elevated weight - (source.getElevateWeight() != null ? String.format("%s kg", source.getElevateWeight()) : null), + ((source.getElevateWeight() != null && !Objects.equals(source.getElevateWeight(), source.getIndirectElevateWeight())) ? String.format("%s kg", source.getElevateWeight()) : null), + + // Indirect elevated weight + (source.getIndirectElevateWeight() != null ? String.format("(%s%s kg)", arrowDown, source.getIndirectElevateWeight()) : null), + + // Elevated RTP weight + ((source.getElevateRtpWeight() != null && !Objects.equals(source.getElevateRtpWeight(), source.getIndirectRtpElevateWeight()))? String.format("(RTP=%s kg)", source.getElevateRtpWeight()) : null), + + // Indirect elevated RTP weight + (source.getIndirectRtpElevateWeight() != null ? String.format("(%sRTP=%s kg)", arrowDown, source.getIndirectRtpElevateWeight()) : null), // Elevated individual count (source.getElevateIndividualCount() != null ? String.format("%s indiv", source.getElevateIndividualCount()) : null) - // Elevation factor - //, String.format("(x %s)", ((TempDenormalizedBatchVO)source).getElevateFactor()) ).replace("[ ]+", " "); }).collect(Collectors.joining("\n")); } + public static boolean hasSomeInvalidChild(DenormalizedBatchVO source) { + if (!source.hasChildren()) return false; + return source.getChildren().stream() + .anyMatch(child -> QualityFlags.isInvalid(child.getQualityFlagId()) + || hasSomeInvalidChild(child) // Loop on children + ); + } + + public static Optional getSexId(DenormalizedBatchVO batch) { + return getSortingQualitativeValueIdByParameterId(batch, ParameterEnum.SEX.getId()); + } + + public static Optional getDressingId(DenormalizedBatchVO batch) { + return getSortingQualitativeValueIdByPmfmId(batch, PmfmEnum.DRESSING.getId()); + } + + public static Optional getPreservationId(DenormalizedBatchVO batch) { + return getSortingQualitativeValueIdByPmfmId(batch, PmfmEnum.PRESERVATION.getId()); + } + + public static Optional getSortingQualitativeValueIdByPmfmId(DenormalizedBatchVO batch, int pmfmId) { + return getSortingQualitativeValueIdByFilter( + batch, + sv -> sv.getPmfmId() == pmfmId || (sv.getPmfm() != null && sv.getPmfm().getId() == pmfmId) + ); + } + + public static Optional getSortingQualitativeValueIdByParameterId(DenormalizedBatchVO batch, int parameterId) { + return getSortingQualitativeValueIdByFilter( + batch, + sv -> sv.getParameter() != null && sv.getParameter().getId() == parameterId + ); + } + + public static Optional getSortingQualitativeValueIdByFilter(DenormalizedBatchVO batch, Predicate filterFn) { + return Beans.getStream(batch.getSortingValues()) + .filter(sv -> sv.getQualitativeValue() != null) + .filter(filterFn::test) + .map(DenormalizedBatchSortingValueVO::getQualitativeValue) + .map(ReferentialVO::getId) + .findFirst(); + } + + /** + * Compute diff (%) between two weights + * @param weight1 + * @param weight2 + * @return + */ + public static double computeWeightDiffPercent(@NonNull Number weight1, @NonNull Number weight2) { + BigDecimal w1 = new BigDecimal(weight1.toString()); + BigDecimal w2 = new BigDecimal(weight2.toString()); + + // (ABS(w1 - w2) / w1) * 100 + return w1.subtract(w2).abs() + .divide(w1) + .multiply(new BigDecimal(100)) + // Round to 2 decimal + .divide(new BigDecimal(1), 2, RoundingMode.HALF_UP) + .doubleValue(); + } + + /* -- internal functions -- */ protected static String replaceTreeUnicode(String treeIndent) { return treeIndent.replace("|-", "\u02EB") .replace("|_", "\u02EA") - .replace(" ", "\t"); - //.replace("\t\t", "\t"); + .replace(" ", "\t") + .replace("\t\t", "\t"); } protected static String getExhaustiveInventoryAsString(DenormalizedBatchVO source, boolean useUnicode) { diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/TempDenormalizedBatchVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/TempDenormalizedBatchVO.java index dcaa6a6b6b..829d5e3a3d 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/TempDenormalizedBatchVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/data/batch/TempDenormalizedBatchVO.java @@ -22,6 +22,7 @@ package net.sumaris.core.vo.data.batch; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.experimental.FieldNameConstants; @@ -33,11 +34,69 @@ @EqualsAndHashCode public class TempDenormalizedBatchVO extends DenormalizedBatchVO { + // Factors + private BigDecimal samplingFactor; + private BigDecimal elevateContextFactor; private BigDecimal elevateFactor; + private BigDecimal taxonElevateFactor; - //private Double contextWeight; - //private Double sumChildContextWeight; - //private Double sumChildRoundWeight; - //private Double sumChildRTPWeight; + private Double aliveWeightFactor; + private Double indirectAliveWeightFactor; + + // Individual count + private BigDecimal indirectIndividualCountDecimal; + + // Weights + private Double rtpContextWeight; + + /** + * Indirect RTP weight (not alive weight, and not elevate) + */ + private Double indirectRtpContextWeight; + + /** + * Elevate RTP weights (keeping dressing/perservation = not alive weight) + */ + private Double elevateRtpContextWeight; + + private Double indirectRtpElevateWeight; + + private Double indirectElevateWeight; + + private Integer taxonGroupId; + private Integer referenceTaxonId; + + @JsonIgnore + public Integer getTaxonGroupId() { + if (taxonGroupId == null) { + taxonGroupId = this.getTaxonGroup() != null + ? this.getTaxonGroup().getId() + : ( + this.getInheritedTaxonGroup() != null + ? this.getInheritedTaxonGroup().getId() + // TODO: return the calculated taxon group ? + : null); + } + return taxonGroupId; + } + + public Integer getReferenceTaxonId() { + if (referenceTaxonId == null) { + referenceTaxonId = this.getTaxonName() != null + ? this.getTaxonName().getReferenceTaxonId() + : (this.getInheritedTaxonName() != null + ? this.getInheritedTaxonName().getReferenceTaxonId() + : null); + } + return referenceTaxonId; + } + + public boolean hasTaxonGroup() { + return getTaxonGroupId() != null; + } + + public boolean hasTaxonName() { + return getReferenceTaxonId() != null; + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/filter/OperationFilterVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/filter/OperationFilterVO.java index 1dc1118571..7b66c6cb72 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/filter/OperationFilterVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/filter/OperationFilterVO.java @@ -22,12 +22,14 @@ * #L% */ +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.experimental.FieldNameConstants; import net.sumaris.core.model.data.DataQualityStatusEnum; +import net.sumaris.core.util.Beans; import java.util.Date; @@ -58,4 +60,10 @@ public static OperationFilterVO nullToEmpty(OperationFilterVO f) { private Integer[] qualityFlagIds; private DataQualityStatusEnum[] dataQualityStatus; private Integer[] boundingBox; + private Boolean needBatchDenormalization; + + @JsonIgnore + public OperationFilterVO clone() { + return Beans.clone(this, OperationFilterVO.class); + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/filter/TripFilterVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/filter/TripFilterVO.java index 430a47d3e9..0697d12616 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/filter/TripFilterVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/filter/TripFilterVO.java @@ -59,6 +59,8 @@ public static TripFilterVO nullToEmpty(@Nullable TripFilterVO filter) { private Integer[] includedIds; private Integer tripId; + private Integer[] operationIds; + private Integer observedLocationId; private Integer[] qualityFlagIds; diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialFetchOptions.java b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialFetchOptions.java index 75faef00ec..66e0cf20d1 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialFetchOptions.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialFetchOptions.java @@ -22,13 +22,19 @@ * #L% */ +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import net.sumaris.core.dao.technical.jpa.IFetchOptions; @Builder +@Data +@AllArgsConstructor +@NoArgsConstructor public class ReferentialFetchOptions implements IFetchOptions { - + @Builder.Default + private boolean withProperties = false; } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialVO.java index 3e6139241a..b21ea729c4 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/ReferentialVO.java @@ -24,8 +24,10 @@ import lombok.*; import lombok.experimental.FieldNameConstants; +import net.sumaris.core.model.ITreeNodeEntity; import java.util.Date; +import java.util.Map; @Data @NoArgsConstructor @@ -33,7 +35,8 @@ @Builder @FieldNameConstants @EqualsAndHashCode(onlyExplicitlyIncluded = true) -public class ReferentialVO implements IReferentialVO, IReferentialWithLevelVO { +public class ReferentialVO implements IReferentialVO, + IReferentialWithLevelVO { @EqualsAndHashCode.Include private Integer id; @@ -49,7 +52,7 @@ public class ReferentialVO implements IReferentialVO, IReferentialWithL //@EqualsAndHashCode.Exclude private Integer levelId; - + private ReferentialVO level; private Integer parentId; private ReferentialVO parent; @@ -58,5 +61,7 @@ public class ReferentialVO implements IReferentialVO, IReferentialWithL // Metadata //@EqualsAndHashCode.Exclude private String entityName; + + private Map properties; } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/RoundWeightConversionFilterVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/RoundWeightConversionFilterVO.java index be4f4f5aaf..1fa9509b42 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/RoundWeightConversionFilterVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/RoundWeightConversionFilterVO.java @@ -26,6 +26,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import net.sumaris.core.util.Beans; import java.util.Date; @@ -45,4 +46,8 @@ public static RoundWeightConversionFilterVO nullToEmpty(RoundWeightConversionFil Integer[] dressingIds; Integer[] preservingIds; Date date; + + public RoundWeightConversionFilterVO clone() { + return Beans.clone(this, RoundWeightConversionFilterVO.class); + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/WeightLengthConversionFilterVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/WeightLengthConversionFilterVO.java index 7ac6e54573..c2293825a4 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/WeightLengthConversionFilterVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/referential/conversion/WeightLengthConversionFilterVO.java @@ -22,17 +22,15 @@ package net.sumaris.core.vo.referential.conversion; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; +import lombok.*; +import net.sumaris.core.util.Beans; import java.util.Date; @Data +@Builder @NoArgsConstructor @AllArgsConstructor -@Builder public class WeightLengthConversionFilterVO { public static WeightLengthConversionFilterVO nullToEmpty(WeightLengthConversionFilterVO filter) { @@ -43,6 +41,7 @@ public static WeightLengthConversionFilterVO nullToEmpty(WeightLengthConversionF Integer[] referenceTaxonIds; Integer[] locationIds; + Integer[] childLocationIds; String[] rectangleLabels; Integer[] sexIds; @@ -51,6 +50,10 @@ public static WeightLengthConversionFilterVO nullToEmpty(WeightLengthConversionF Integer[] lengthUnitIds; Integer[] lengthPmfmIds; - Integer month; + Integer month; // 1=January Integer year; + + public WeightLengthConversionFilterVO clone() { + return Beans.clone(this, WeightLengthConversionFilterVO.class); + } } diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/technical/job/IJobResultVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/technical/job/IJobResultVO.java new file mode 100644 index 0000000000..5eb896de5c --- /dev/null +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/technical/job/IJobResultVO.java @@ -0,0 +1,33 @@ +package net.sumaris.core.vo.technical.job; + +/*- + * #%L + * Quadrige3 Core :: Model Shared + * %% + * Copyright (C) 2017 - 2022 Ifremer + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import net.sumaris.core.model.technical.job.JobStatusEnum; + +import java.io.Serializable; + +public interface IJobResultVO extends Serializable { + + JobStatusEnum getStatus(); + +} diff --git a/sumaris-core/src/main/java/net/sumaris/core/vo/technical/job/JobVO.java b/sumaris-core/src/main/java/net/sumaris/core/vo/technical/job/JobVO.java index 419b08bd01..65edf8eb4d 100644 --- a/sumaris-core/src/main/java/net/sumaris/core/vo/technical/job/JobVO.java +++ b/sumaris-core/src/main/java/net/sumaris/core/vo/technical/job/JobVO.java @@ -27,6 +27,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import lombok.experimental.FieldNameConstants; +import net.sumaris.core.model.IProgressionModel; import net.sumaris.core.model.IUpdateDateEntity; import net.sumaris.core.model.IValueObject; import net.sumaris.core.model.technical.job.JobStatusEnum; @@ -64,6 +65,9 @@ public class JobVO implements private Date updateDate; + @JsonIgnore + private IProgressionModel progressionModel; + @Override public int compareTo(JobVO o) { return this.hashCode() - o.hashCode(); diff --git a/sumaris-core/src/main/resources/i18n/sumaris-core_en_GB.properties b/sumaris-core/src/main/resources/i18n/sumaris-core_en_GB.properties index 6e8ac4bd77..72b109e5b8 100644 --- a/sumaris-core/src/main/resources/i18n/sumaris-core_en_GB.properties +++ b/sumaris-core/src/main/resources/i18n/sumaris-core_en_GB.properties @@ -63,9 +63,11 @@ sumaris.core.job.progression.pending= sumaris.error.authenticate.failed=Error while authenticating\: %s sumaris.error.authenticate.unauthorized=Error while authenticating\: Access denied sumaris.error.connect=Unable to connect to the server +sumaris.error.denormalization.batch.cannotEnableRtpWeight=Cannot enable RTP weight in batch denormalization\: %s sumaris.error.department.logo.notFound= sumaris.error.entity.notfoundByLabel=Entity %s with label %s not found sumaris.error.io.openExternalEditor= +sumaris.error.missingSomeRectangleLocationLevel=Some location level for statistical rectangle (LocationLevelEnum) cannot be resolved. sumaris.error.notFound=Entity %s with id %s not found sumaris.error.person.avatar.notFound= sumaris.error.person.notFound= diff --git a/sumaris-core/src/main/resources/i18n/sumaris-core_fr_FR.properties b/sumaris-core/src/main/resources/i18n/sumaris-core_fr_FR.properties index 0ee4bdfa5e..444bcc8da0 100644 --- a/sumaris-core/src/main/resources/i18n/sumaris-core_fr_FR.properties +++ b/sumaris-core/src/main/resources/i18n/sumaris-core_fr_FR.properties @@ -62,8 +62,10 @@ sumaris.core.job.import.start.debug=Démarrage du traitement du fichier [%s] sumaris.core.job.pending=La tache '%s' est en attente d'exécution sumaris.core.job.progression.pending=En attente d'exécution sumaris.core.job.run=La tache '%s' est en cours d'exécution +sumaris.error.denormalization.batch.cannotEnableRtpWeight=Impossible de calculer les poids RTP, dans la dénormalisation des lots\: %s sumaris.error.department.logo.notFound=Logo non trouvé sumaris.error.entity.notfoundByLabel=Entitée %s avec le label %s non trouvé +sumaris.error.missingSomeRectangleLocationLevel=Un ou plusieurs niveaux de lieux pour les rectangles statistique (LocationLevelEnum) n'ont pas pu être résolu. sumaris.error.person.avatar.notFound=Avatar non trouvé sumaris.error.person.notFound=Utilisateur non trouvé sumaris.error.person.register.duplicatedPerson=Un utilisateur avec le même email, ou le même nom/prénom existe déjà. diff --git a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/adap/db-changelog-2.1.0.xml b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/adap/db-changelog-2.1.0.xml new file mode 100644 index 0000000000..f6ed32b1c6 --- /dev/null +++ b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/adap/db-changelog-2.1.0.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + ${sqlCheck.empty.sql} + + ${sqlCheck.adap.sql} + + + + + + + ${sqlCheck.adap.sql} + + SELECT COUNT(*) from SOFTWARE_PROPERTY where LABEL='sumaris.extraction.batch.denormalization.enable' AND SOFTWARE_FK=2 + + + + + + + + + + + + + + + + + + + ${sqlCheck.adap.sql} + + select count(distinct sm1.batch_fk) + from sorting_measurement_b sm1 + inner join sorting_measurement_b sm2 on sm1.batch_fk = sm2.batch_fk + where sm2.qualitative_value_fk = 191 and sm1.qualitative_value_fk = 339 + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog-0.16.0.xml b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog-0.16.0.xml index ad1e8e3f6d..db39099eb2 100644 --- a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog-0.16.0.xml +++ b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog-0.16.0.xml @@ -1,26 +1,3 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + LABEL='2.1.0' + + + + 2.1.0 + + - Add some columns in table DENORMALIZED_BATCH; + + + + + - Add some columns in table DENORMALIZED_BATCH; + + + + diff --git a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog.xml b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog.xml index e3877ae96d..05c1739b5d 100644 --- a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog.xml +++ b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/hsqldb/db-changelog.xml @@ -174,6 +174,9 @@ + + + diff --git a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/db-changelog-2.1.0.xml b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/db-changelog-2.1.0.xml new file mode 100644 index 0000000000..3cd43294d4 --- /dev/null +++ b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/db-changelog-2.1.0.xml @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + LABEL='2.1.0' + + + + 2.1.0 + + - Add missing sequences; + + + + + - Add missing sequences (parameter_group_seq) + + + + diff --git a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/db-changelog.xml b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/db-changelog.xml index e8b60883aa..f9051c850c 100644 --- a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/db-changelog.xml +++ b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/db-changelog.xml @@ -55,6 +55,9 @@ + + + diff --git a/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/open/db-changelog-2.1.0.xml b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/open/db-changelog-2.1.0.xml new file mode 100644 index 0000000000..52c02086c7 --- /dev/null +++ b/sumaris-core/src/main/resources/net/sumaris/core/db/changelog/postgresql/open/db-changelog-2.1.0.xml @@ -0,0 +1,75 @@ + + + + + + + + + + + ${sqlCheck.empty.sql} + + ${sqlCheck.open.sql} + + + + + + + + ${sqlCheck.open.sql} + + (select p.start_date_time from operation p where p.id=o.operation_fk) + ]]> + + + + (select p.start_date_time from operation p where p.id=o.operation_fk); + commit; + ]]> + + + + + + + ${sqlCheck.open.sql} + + (select p.fishing_start_date_time from operation p where p.id=o.operation_fk) + ]]> + + + + (select p.fishing_start_date_time from operation p where p.id=o.operation_fk); + commit; + ]]> + + + diff --git a/sumaris-core/src/test/java/net/sumaris/core/dao/DatabaseFixtures.java b/sumaris-core/src/test/java/net/sumaris/core/dao/DatabaseFixtures.java index b6b6d70b19..02caf30a6f 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/dao/DatabaseFixtures.java +++ b/sumaris-core/src/test/java/net/sumaris/core/dao/DatabaseFixtures.java @@ -256,6 +256,10 @@ public Integer getTaxonGroupMNZ() { return 1122; } + public Integer getTaxonGroupCOD() { + return 1161; + } + public Integer getTaxonGroupIdWithManyTaxonName() { return 1122; // MNZ - Baudroie } diff --git a/sumaris-core/src/test/java/net/sumaris/core/model/referential/QualityFlagsTest.java b/sumaris-core/src/test/java/net/sumaris/core/model/referential/QualityFlagsTest.java new file mode 100644 index 0000000000..952c929921 --- /dev/null +++ b/sumaris-core/src/test/java/net/sumaris/core/model/referential/QualityFlagsTest.java @@ -0,0 +1,50 @@ +package net.sumaris.core.model.referential; + +/*- + * #%L + * SUMARiS:: Core + * %% + * Copyright (C) 2018 - 2019 SUMARiS Consortium + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import net.sumaris.core.util.crypto.MD5Util; +import org.junit.Assert; +import org.junit.Test; + +public class QualityFlagsTest { + + @Test + public void worst() { + + // Test using enums + { + QualityFlagEnum worstQualityFlags = QualityFlags.worst(QualityFlagEnum.NOT_QUALIFIED, QualityFlagEnum.BAD, QualityFlagEnum.FIXED); + Assert.assertEquals(QualityFlagEnum.BAD, worstQualityFlags); + } + + // Test using ids + { + Integer worstQualityFlags = QualityFlags.worst(QualityFlagEnum.NOT_QUALIFIED.getId(), QualityFlagEnum.BAD.getId(), QualityFlagEnum.FIXED.getId()); + Assert.assertEquals(QualityFlagEnum.BAD.getId(), worstQualityFlags); + + worstQualityFlags = QualityFlags.worst(QualityFlagEnum.NOT_QUALIFIED.getId(), QualityFlagEnum.GOOD.getId(), QualityFlagEnum.FIXED.getId()); + Assert.assertEquals(QualityFlagEnum.FIXED.getId(), worstQualityFlags); + } + } + +} diff --git a/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizeBatchServiceWriteTest.java b/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizeBatchServiceWriteTest.java index a1dd894b2c..f30170530e 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizeBatchServiceWriteTest.java +++ b/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizeBatchServiceWriteTest.java @@ -28,7 +28,6 @@ import net.sumaris.core.model.TreeNodeEntities; import net.sumaris.core.service.AbstractServiceTest; import net.sumaris.core.service.data.BatchService; -import net.sumaris.core.service.data.DenormalizedBatchService; import net.sumaris.core.vo.data.batch.BatchVO; import net.sumaris.core.vo.data.batch.DenormalizedBatchVO; import org.junit.Assert; diff --git a/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchServiceReadTest.java b/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchServiceReadTest.java index e6f5c768f3..36b0a667e9 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchServiceReadTest.java +++ b/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizedBatchServiceReadTest.java @@ -27,11 +27,9 @@ import net.sumaris.core.model.administration.programStrategy.AcquisitionLevelEnum; import net.sumaris.core.service.AbstractServiceTest; import net.sumaris.core.service.data.DataTestUtils; -import net.sumaris.core.service.data.DenormalizedBatchService; import net.sumaris.core.vo.data.batch.BatchVO; import net.sumaris.core.vo.data.batch.DenormalizedBatchOptions; import net.sumaris.core.vo.data.batch.DenormalizedBatchVO; -import net.sumaris.core.vo.data.batch.DenormalizedBatches; import org.apache.commons.lang3.mutable.MutableInt; import org.junit.Assert; import org.junit.Assume; diff --git a/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizeTripServiceTest.java b/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizedTripServiceTest.java similarity index 86% rename from sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizeTripServiceTest.java rename to sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizedTripServiceTest.java index b29bd4ca1d..cf54f97fe1 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizeTripServiceTest.java +++ b/sumaris-core/src/test/java/net/sumaris/core/service/data/denormalize/DenormalizedTripServiceTest.java @@ -33,19 +33,19 @@ import org.springframework.beans.factory.annotation.Autowired; @Slf4j -public class DenormalizeTripServiceTest extends AbstractServiceTest{ +public class DenormalizedTripServiceTest extends AbstractServiceTest{ @ClassRule public static final DatabaseResource dbResource = DatabaseResource.writeDb(); @Autowired - private DenormalizeTripService service; + private DenormalizedTripService service; @Test public void denormalizeById() { long startTime = System.currentTimeMillis(); - DenormalizeTripResultVO result = service.denormalizeById(fixtures.getTripIdWithBatches()); + DenormalizedTripResultVO result = service.denormalizeById(fixtures.getTripIdWithBatches()); // Observers Assert.assertNotNull(result); @@ -57,7 +57,7 @@ public void denormalizeById() { @Test public void denormalizeByFilter() { - DenormalizeTripResultVO result = service.denormalizeByFilter(TripFilterVO.builder() + DenormalizedTripResultVO result = service.denormalizeByFilter(TripFilterVO.builder() .tripId(fixtures.getTripIdWithBatches()) .build()); diff --git a/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceReadTest.java b/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceReadTest.java index 18970e3d36..8fab37152c 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceReadTest.java +++ b/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceReadTest.java @@ -51,11 +51,11 @@ public class LocationServiceReadTest extends AbstractServiceTest { @Test public void getLocationLabelByLatLong() { // Check label with a position inside the Atlantic sea - String label = service.getLocationLabelByLatLong(47.6f, -5.05f); + String label = service.getStatisticalRectangleLabelByLatLong(47.6f, -5.05f).orElse(null); assertEquals("24E4", label); // Check label with a position inside the Mediterranean sea - label = service.getLocationLabelByLatLong(42.27f, 5.4f); + label = service.getStatisticalRectangleLabelByLatLong(42.27f, 5.4f).orElse(null); assertEquals("M24C2", label); } diff --git a/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceWriteTest.java b/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceWriteTest.java index e49bc4d4b1..d7d7f840ef 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceWriteTest.java +++ b/sumaris-core/src/test/java/net/sumaris/core/service/referential/LocationServiceWriteTest.java @@ -30,7 +30,6 @@ import org.junit.runners.MethodSorters; import org.springframework.beans.factory.annotation.Autowired; -import java.io.File; import java.io.FileNotFoundException; import java.io.PrintStream; @@ -71,14 +70,14 @@ public void b_updateLocationHierarchy() throws FileNotFoundException { @Test public void getLocationIdByLatLong() { // Check label with a position inside the Atlantic sea - Integer locationId = service.getLocationIdByLatLong(47.6f, -5.05f); + Integer locationId = service.getStatisticalRectangleIdByLatLong(47.6f, -5.05f).orElse(null); assertNotNull("Location Id could not found in DB, in the Atlantic Sea. Bad enumeration value for LocationLevelEnum.RECTANGLE_ICES ?", locationId); assertEquals(new Integer(115), locationId); // =id of location '24E4' // Check label with a position inside the Mediterranean sea - locationId = service.getLocationIdByLatLong(42.27f, 5.4f); + locationId = service.getStatisticalRectangleIdByLatLong(42.27f, 5.4f).orElse(null); assertNotNull("Location Id could not found in DB, in the Mediterranean Sea. Bad enumeration value for LocationLevelEnum.RECTANGLE_GFCM ?", locationId); - assertEquals(new Integer(140), locationId); // =id of location 'M24C2' + assertEquals(Integer.valueOf(140), locationId); // =id of location 'M24C2' } protected void printLocationPorts(PrintStream out, String indentation) { diff --git a/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceReadTest.java b/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceReadTest.java index 5def3aecad..46d72426d3 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceReadTest.java +++ b/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/RoundWeightConversionServiceReadTest.java @@ -25,19 +25,28 @@ */ import net.sumaris.core.dao.DatabaseResource; +import net.sumaris.core.dao.technical.Page; +import net.sumaris.core.dao.technical.SortDirection; import net.sumaris.core.model.referential.StatusEnum; +import net.sumaris.core.model.referential.conversion.RoundWeightConversion; import net.sumaris.core.model.referential.location.LocationLevels; +import net.sumaris.core.model.referential.pmfm.QualitativeValueEnum; import net.sumaris.core.service.AbstractServiceTest; import net.sumaris.core.service.referential.LocationService; +import net.sumaris.core.util.Dates; import net.sumaris.core.vo.filter.LocationFilterVO; import net.sumaris.core.vo.referential.conversion.*; +import org.apache.commons.collections4.CollectionUtils; import org.junit.Assert; import org.junit.Assume; import org.junit.ClassRule; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import java.text.ParseException; +import java.util.Date; import java.util.List; +import java.util.Objects; public class RoundWeightConversionServiceReadTest extends AbstractServiceTest { @@ -59,7 +68,7 @@ public void countByFilter() { } @Test - public void findByFilter() { + public void findByFilter() throws ParseException { // Count all long countAll = service.countByFilter(null); @@ -94,6 +103,37 @@ public void findByFilter() { } } + @Test + public void findByFilterOnEndDate() { + + service.findByFilter(RoundWeightConversionFilterVO.builder() + .statusIds(new Integer[]{StatusEnum.ENABLE.getId()}) + .build(), null, RoundWeightConversionFetchOptions.DEFAULT) + .stream() + .filter(conversion -> conversion.getEndDate() != null) + .forEach(conversion -> { + + // Try to find the same row again, after adding 12 hours to the end date + // /!\ in SIH-ADAGIO, start/end are filled with a day precision (almost), so '2012-12-31 00:00:00' stands for '2012-12-31 23:59:59' + RoundWeightConversionFilterVO filter = RoundWeightConversionFilterVO.builder() + .taxonGroupIds(new Integer[]{conversion.getTaxonGroupId()}) + .locationIds(new Integer[]{conversion.getLocationId()}) + .dressingIds(new Integer[]{conversion.getDressingId()}) + .preservingIds(new Integer[]{conversion.getPreservingId()}) + .statusIds(new Integer[]{conversion.getStatusId()}) + .date(Dates.addHours(conversion.getEndDate(), 12)) // +12h + .build(); + List result = service.findByFilter(filter, null, RoundWeightConversionFetchOptions.DEFAULT); + + Assert.assertNotNull(result); + Assert.assertEquals(1, result.size()); + + // Should be same item + Assert.assertEquals(conversion.getId(), result.get(0).getId()); + Assert.assertEquals(conversion.getEndDate(), result.get(0).getEndDate()); + }); + } + protected void assertAllValid(List sources, RoundWeightConversionFetchOptions fetchOptions) { sources.forEach(s -> this.assertValid(s, fetchOptions)); } diff --git a/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceReadTest.java b/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceReadTest.java index 7b58375ecd..f6c172d718 100644 --- a/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceReadTest.java +++ b/sumaris-core/src/test/java/net/sumaris/core/service/referential/conversion/WeightLengthConversionServiceReadTest.java @@ -130,7 +130,7 @@ public void convertWeight() { WeightLengthConversionVO conversion = conversions.get(0); - BigDecimal weight = service.computedWeight(conversion, 15d, 3, 1d); + BigDecimal weight = service.computedWeight(conversion, 15d, "cm", 0.5, 1d, "kg", 3); log.info("Computed weight ofr COD (15cm): {}kg", weight); } diff --git a/sumaris-core/src/test/resources/application-test-hsqldb.properties b/sumaris-core/src/test/resources/application-test-hsqldb.properties index f4c5787966..b60bb3fafa 100644 --- a/sumaris-core/src/test/resources/application-test-hsqldb.properties +++ b/sumaris-core/src/test/resources/application-test-hsqldb.properties @@ -21,9 +21,9 @@ spring.datasource.initialization-mode=always spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver spring.datasource.username=sa spring.datasource.password= -sumaris.persistence.sequence.startWith=1000\ +sumaris.persistence.sequence.startWith=1000 # For DEV ONLY: -#spring.datasource.url=jdbc:hsqldb:hsql://localhost/sumaris +spring.datasource.url=jdbc:hsqldb:hsql://localhost/sumaris # General JPA properties diff --git a/sumaris-extraction/pom.xml b/sumaris-extraction/pom.xml index 9d21594730..b871b8d5c9 100644 --- a/sumaris-extraction/pom.xml +++ b/sumaris-extraction/pom.xml @@ -5,7 +5,7 @@ net.sumaris sumaris-pod - 2.0.6 + 2.1.0 sumaris-extraction diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfiguration.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfiguration.java index 71f8eac3c3..5a556fc2e2 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfiguration.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfiguration.java @@ -110,10 +110,19 @@ public boolean enableTechnicalTablesUpdate() { public boolean enableAdagioOptimization() { return getApplicationConfig().getOptionAsBoolean(SumarisConfigurationOption.ENABLE_ADAGIO_OPTIMIZATION.getKey()); } + public String getAdagioSchema() { return getApplicationConfig().getOption(SumarisConfigurationOption.DB_ADAGIO_SCHEMA.getKey()); } + public boolean enableBatchDenormalization() { + return getApplicationConfig().getOptionAsBoolean(ExtractionConfigurationOption.EXTRACTION_BATCH_DENORMALISATION_ENABLE.getKey()); + } + + public void setEnableBatchDenormalization(boolean value) { + getApplicationConfig().setOption(ExtractionConfigurationOption.EXTRACTION_BATCH_DENORMALISATION_ENABLE.getKey(), String.valueOf(value)); + } + public File getTempDirectory() { return getApplicationConfig().getOptionAsFile(SumarisConfigurationOption.TMP_DIRECTORY.getKey()); } diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfigurationOption.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfigurationOption.java index 047d37539d..7503a5d4ca 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfigurationOption.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/config/ExtractionConfigurationOption.java @@ -102,7 +102,14 @@ public enum ExtractionConfigurationOption implements ConfigOptionDef { n("sumaris.config.option.extraction.query.timeout.description"), String.valueOf(5 * 60 * 1000), // 5min Integer.class, - false) + false), + + EXTRACTION_BATCH_DENORMALISATION_ENABLE( + "sumaris.extraction.batch.denormalization.enable", + n("sumaris.config.option.extraction.batch.denormalization.enable.description"), + Boolean.FALSE.toString(), + Boolean.class, + false), ; /** Configuration key. */ diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/administration/ExtractionStrategyDaoImpl.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/administration/ExtractionStrategyDaoImpl.java index c61a054d4b..01338b4695 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/administration/ExtractionStrategyDaoImpl.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/administration/ExtractionStrategyDaoImpl.java @@ -35,6 +35,7 @@ import net.sumaris.core.model.technical.extraction.IExtractionType; import net.sumaris.core.util.Dates; import net.sumaris.core.util.StringUtils; +import net.sumaris.extraction.core.config.ExtractionConfiguration; import net.sumaris.extraction.core.dao.ExtractionBaseDaoImpl; import net.sumaris.extraction.core.dao.technical.Daos; import net.sumaris.extraction.core.dao.technical.xml.XMLQuery; @@ -44,6 +45,7 @@ import net.sumaris.extraction.core.vo.administration.ExtractionStrategyContextVO; import net.sumaris.extraction.core.vo.administration.ExtractionStrategyFilterVO; import org.apache.commons.collections4.CollectionUtils; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Repository; @@ -61,6 +63,7 @@ */ @Repository("extractionStrategyDao") @Lazy +@ConditionalOnBean({ExtractionConfiguration.class}) @Slf4j public class ExtractionStrategyDaoImpl extends ExtractionBaseDaoImpl diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/free2/ExtractionFree2TripDaoImpl.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/free2/ExtractionFree2TripDaoImpl.java index 2ff05ee340..6803e85c9a 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/free2/ExtractionFree2TripDaoImpl.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/free2/ExtractionFree2TripDaoImpl.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableSet; import lombok.extern.slf4j.Slf4j; -import net.sumaris.core.dao.technical.schema.SumarisDatabaseMetadata; import net.sumaris.core.model.technical.extraction.IExtractionType; import net.sumaris.extraction.core.dao.technical.Daos; import net.sumaris.extraction.core.dao.technical.xml.XMLQuery; @@ -39,9 +38,7 @@ import net.sumaris.core.service.administration.programStrategy.ProgramService; import net.sumaris.core.service.administration.programStrategy.StrategyService; import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; -import org.springframework.core.io.ResourceLoader; import org.springframework.stereotype.Repository; import java.util.Set; @@ -56,11 +53,16 @@ public class ExtractionFree2TripDaoImpl implements Free2Specification { - @Autowired - protected StrategyService strategyService; + protected final StrategyService strategyService; - @Autowired - protected ProgramService programService; + protected final ProgramService programService; + + public ExtractionFree2TripDaoImpl(StrategyService strategyService, ProgramService programService) { + super(); + this.strategyService = strategyService; + this.programService = programService; + this.enableTripSamplingMethodColumn = false; // No SAMPLING_METHOD in this format + } @Override public Set getManagedTypes() { @@ -134,14 +136,14 @@ protected void fillContextTableNames(C context) { super.fillContextTableNames(context); // Set table names - context.setTripTableName(TABLE_NAME_PREFIX + TRIP_SHEET_NAME + "_" + context.getId()); - context.setStationTableName(TABLE_NAME_PREFIX + STATION_SHEET_NAME + "_" + context.getId()); - context.setGearTableName(TABLE_NAME_PREFIX + GEAR_SHEET_NAME + "_" + context.getId()); - context.setRawSpeciesListTableName(TABLE_NAME_PREFIX + "RAW_SL" + "_" + context.getId()); - context.setStrategyTableName(TABLE_NAME_PREFIX + STRATEGY_SHEET_NAME + "_" + context.getId()); - context.setDetailTableName(TABLE_NAME_PREFIX + DETAIL_SHEET_NAME + "_" + context.getId()); - context.setSpeciesListTableName(TABLE_NAME_PREFIX + SPECIES_LIST_SHEET_NAME + "_" + context.getId()); - context.setSpeciesLengthTableName(TABLE_NAME_PREFIX + SPECIES_LENGTH_SHEET_NAME + "_" + context.getId()); + context.setTripTableName(formatTableName(TABLE_NAME_PREFIX + TRIP_SHEET_NAME + "_%s", context.getId())); + context.setStationTableName(formatTableName(TABLE_NAME_PREFIX + STATION_SHEET_NAME + "_%s", context.getId())); + context.setGearTableName(formatTableName(TABLE_NAME_PREFIX + GEAR_SHEET_NAME + "_%s", context.getId())); + context.setRawSpeciesListTableName(formatTableName(TABLE_NAME_PREFIX + "RAW_SL" + "_%s", context.getId())); + context.setStrategyTableName(formatTableName(TABLE_NAME_PREFIX + STRATEGY_SHEET_NAME + "_%s", context.getId())); + context.setDetailTableName(formatTableName(TABLE_NAME_PREFIX + DETAIL_SHEET_NAME + "_%s", context.getId())); + context.setSpeciesListTableName(formatTableName(TABLE_NAME_PREFIX + SPECIES_LIST_SHEET_NAME + "_%s", context.getId())); + context.setSpeciesLengthTableName(formatTableName(TABLE_NAME_PREFIX + SPECIES_LENGTH_SHEET_NAME + "_%s", context.getId())); // Set sheet names context.setTripSheetName(TRIP_SHEET_NAME); diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/AggregationPmfmTripDaoImpl.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/AggregationPmfmTripDaoImpl.java index 3c9948ae3d..aa8b9eeec4 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/AggregationPmfmTripDaoImpl.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/AggregationPmfmTripDaoImpl.java @@ -90,13 +90,26 @@ public R aggregate(IExtractionTypeWithTablesVO source, @Nullable F if (sheetName != null && context.hasSheet(sheetName)) return context; try { - // Sample table - long rowCount = createSampleTable(source, context); - if (rowCount == 0) return context; - if (sheetName != null && context.hasSheet(sheetName)) return context; - - // Release table - createReleaseTable(source, context); + // If only CL expected: skip station/species aggregation + if (!CL_SHEET_NAME.equals(sheetName)) { + + // Restore the previous (HH) row count + long rowCount = countFrom(context.getStationTableName()); + + // Sample table + if (rowCount != 0) { + rowCount = createSampleTable(source, context); + if (sheetName != null && context.hasSheet(sheetName)) return context; + } + + // Release table (=sub sample) + if (rowCount != 0) { + rowCount = createReleaseTable(source, context); + if (sheetName != null && context.hasSheet(sheetName)) return context; + } + + return context; + } } catch (PersistenceException e) { // If error,clean created tables first, then rethrow the exception diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/ExtractionPmfmTripDaoImpl.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/ExtractionPmfmTripDaoImpl.java index d75702f827..113aec5cc0 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/ExtractionPmfmTripDaoImpl.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/pmfm/ExtractionPmfmTripDaoImpl.java @@ -295,7 +295,6 @@ protected XMLQuery createSpeciesLengthQuery(C context) { // - Hide sex columns, then replace by a new columns xmlQuery.setGroup("sex", false); xmlQuery.setGroup("lengthClass", false); - xmlQuery.setGroup("numberAtLength", false); // Add pmfm columns String pmfmsColumns = injectPmfmColumns(context, xmlQuery, @@ -315,6 +314,7 @@ protected XMLQuery createSpeciesLengthQuery(C context) { // Enable group, need by pmfms columns (if any) xmlQuery.setGroup("pmfms", hasPmfmsColumnsInjected); + xmlQuery.setGroup("numberAtLength", !hasPmfmsColumnsInjected); // Enable taxon columns, if enable by program property (inherited from SL or directly from HL) boolean enableTaxonColumns = this.enableSpeciesListTaxon(context) || this.enableSpeciesLengthTaxon(context); diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/rdb/ExtractionRdbTripDaoImpl.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/rdb/ExtractionRdbTripDaoImpl.java index 02db1bed68..ad0cf52499 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/rdb/ExtractionRdbTripDaoImpl.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/dao/trip/rdb/ExtractionRdbTripDaoImpl.java @@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import net.sumaris.core.dao.technical.schema.SumarisTableMetadata; import net.sumaris.core.exception.DataNotFoundException; @@ -41,32 +40,35 @@ import net.sumaris.core.model.technical.extraction.IExtractionType; import net.sumaris.core.service.administration.programStrategy.ProgramService; import net.sumaris.core.service.administration.programStrategy.StrategyService; +import net.sumaris.core.service.data.denormalize.DenormalizedOperationService; +import net.sumaris.core.service.data.denormalize.DenormalizedBatchService; import net.sumaris.core.util.Beans; import net.sumaris.core.util.StringUtils; import net.sumaris.core.util.TimeUtils; import net.sumaris.core.vo.administration.programStrategy.DenormalizedPmfmStrategyVO; import net.sumaris.core.vo.administration.programStrategy.PmfmStrategyFetchOptions; +import net.sumaris.core.vo.data.batch.DenormalizedBatchOptions; +import net.sumaris.core.vo.filter.OperationFilterVO; import net.sumaris.core.vo.filter.PmfmStrategyFilterVO; import net.sumaris.core.vo.referential.PmfmValueType; -import net.sumaris.extraction.core.dao.technical.Daos; +import net.sumaris.extraction.core.config.ExtractionConfiguration; import net.sumaris.extraction.core.dao.ExtractionBaseDaoImpl; +import net.sumaris.extraction.core.dao.technical.Daos; import net.sumaris.extraction.core.dao.technical.xml.XMLQuery; import net.sumaris.extraction.core.dao.trip.ExtractionTripDao; -import net.sumaris.extraction.core.type.LiveExtractionTypeEnum; import net.sumaris.extraction.core.specification.data.trip.RdbSpecification; -import net.sumaris.extraction.core.vo.*; +import net.sumaris.extraction.core.type.LiveExtractionTypeEnum; +import net.sumaris.extraction.core.vo.ExtractionFilterVO; +import net.sumaris.extraction.core.vo.ExtractionPmfmColumnVO; import net.sumaris.extraction.core.vo.trip.ExtractionTripFilterVO; import net.sumaris.extraction.core.vo.trip.rdb.ExtractionRdbTripContextVO; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.context.annotation.Lazy; -import org.springframework.core.io.Resource; -import org.springframework.core.io.ResourceLoader; import org.springframework.stereotype.Repository; import javax.persistence.PersistenceException; -import java.io.IOException; -import java.net.URL; import java.util.*; import java.util.stream.Collectors; @@ -77,6 +79,7 @@ * @author Benoit Lavenier */ @Repository("extractionRdbTripDao") +@ConditionalOnBean({ExtractionConfiguration.class}) @Lazy @Slf4j public class ExtractionRdbTripDaoImpl @@ -97,6 +100,15 @@ public class ExtractionRdbTripDaoImpl getManagedTypes() { @@ -114,6 +126,7 @@ public R execute(F filter) { context.setUpdateDate(new Date()); context.setType(LiveExtractionTypeEnum.RDB); context.setTableNamePrefix(TABLE_NAME_PREFIX); + context.setEnableBatchDenormalization(extractionConfiguration.enableBatchDenormalization()); // Start log Long startTime = null; @@ -127,6 +140,7 @@ public R execute(F filter) { else { filterInfo.append("(without filter)"); } + filterInfo.append("\n - Batch denormalization: " + context.isEnableBatchDenormalization()); log.info("Starting extraction #{} (trips)... {}", context.getId(), filterInfo); } @@ -153,6 +167,11 @@ public R execute(F filter) { if (sheetName != null && context.hasSheet(sheetName)) return context; } + // Execute batch denormalization (if enable) + if (rowCount != 0 && context.isEnableBatchDenormalization()) { + denormalizeBatches(context); + } + // Species Raw table if (rowCount != 0) { rowCount = createRawSpeciesListTable(context, tripFilter.isExcludeInvalidStation()); @@ -246,7 +265,7 @@ protected long createTripTable(C context) { if (count > 0) { // Update self sampling columns - updateTripSamplingMethod(context); + if (enableTripSamplingMethodColumn) updateTripSamplingMethod(context); // Clean row using generic filter count -= cleanRow(context.getTripTableName(), context.getFilter(), context.getTripSheetName()); @@ -360,6 +379,27 @@ protected XMLQuery createStationQuery(C context) { return xmlQuery; } + protected void denormalizeBatches(C context) { + String stationsTableName = context.getStationTableName(); + List programLabels = getTripProgramLabels(context); + programLabels.forEach(programLabel -> { + String sql = String.format("SELECT distinct %s from %s where %s='%s'", + RdbSpecification.COLUMN_STATION_NUMBER, stationsTableName, RdbSpecification.COLUMN_PROJECT, programLabel); + Integer[] operationIds = query(sql, Integer.class).toArray(Integer[]::new); + + DenormalizedBatchOptions options = denormalizedOperationService.createOptionsByProgramLabel(programLabel); + // DEBUG + //options.setEnableRtpWeight(false); + + denormalizedOperationService.denormalizeByFilter(OperationFilterVO.builder() + .programLabel(programLabel) + .includedIds(operationIds) + .hasNoChildOperation(true) + .needBatchDenormalization(true) + .build(), options); + }); + } + /** * Create raw table (with hidden columns used by sub table - e.g. SAMPLE_ID) * @param context @@ -375,18 +415,25 @@ protected long createRawSpeciesListTable(C context, boolean excludeInvalidStatio // Clean row using generic filter long count = countFrom(tableName); if (count > 0) { - cleanRow(tableName, context.getFilter(), context.getSpeciesListSheetName()); + count -= cleanRow(tableName, context.getFilter(), context.getSpeciesListSheetName()); } // Add as a raw table (to be able to clean it later) context.addRawTableName(tableName); + // DEBUG + //context.addTableName(tableName, "RAW_SL", rawXmlQuery.getHiddenColumnNames(), rawXmlQuery.hasDistinctOption()); + return count; } protected XMLQuery createRawSpeciesListQuery(C context, boolean excludeInvalidStation) { - XMLQuery xmlQuery = createXMLQuery(context, "createRawSpeciesListTable"); + String queryName = context.isEnableBatchDenormalization() + ? "createRawSpeciesListDenormalizeTable" + : "createRawSpeciesListTable"; + + XMLQuery xmlQuery = createXMLQuery(context, queryName); xmlQuery.bind("stationTableName", context.getStationTableName()); xmlQuery.bind("rawSpeciesListTableName", context.getRawSpeciesListTableName()); @@ -394,6 +441,7 @@ protected XMLQuery createRawSpeciesListQuery(C context, boolean excludeInvalidSt xmlQuery.bind("catchCategoryPmfmId", String.valueOf(PmfmEnum.DISCARD_OR_LANDING.getId())); xmlQuery.bind("landingQvId", String.valueOf(QualitativeValueEnum.LANDING.getId())); xmlQuery.bind("discardQvId", String.valueOf(QualitativeValueEnum.DISCARD.getId())); + xmlQuery.bind("lengthPmfmIds", Daos.getSqlInNumbers(getSpeciesLengthPmfmIds())); // Exclude not valid station xmlQuery.setGroup("excludeInvalidStation", excludeInvalidStation); @@ -416,7 +464,7 @@ protected long createSpeciesListTable(C context) { // Clean row using generic filter if (count > 0) { - cleanRow(tableName, context.getFilter(), context.getSpeciesListSheetName()); + count -= cleanRow(tableName, context.getFilter(), context.getSpeciesListSheetName()); } // Add result table to context diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/service/ExtractionServiceImpl.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/service/ExtractionServiceImpl.java index 78231b0149..f2f43b6971 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/service/ExtractionServiceImpl.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/service/ExtractionServiceImpl.java @@ -107,7 +107,7 @@ @Slf4j @Service("extractionManager") @RequiredArgsConstructor -@ConditionalOnBean({ExtractionAutoConfiguration.class}) +@ConditionalOnBean({ExtractionConfiguration.class}) public class ExtractionServiceImpl implements ExtractionService { private final ExtractionConfiguration configuration; diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/specification/data/trip/RdbSpecification.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/specification/data/trip/RdbSpecification.java index bb4227a9a6..29ce081843 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/specification/data/trip/RdbSpecification.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/specification/data/trip/RdbSpecification.java @@ -44,6 +44,7 @@ public interface RdbSpecification { String COLUMN_YEAR = "year"; String COLUMN_VESSEL_IDENTIFIER = "vessel_identifier"; String COLUMN_TRIP_CODE = "trip_code"; + String COLUMN_STATION_NUMBER = "station_number"; String COLUMN_SAMPLING_METHOD = "sampling_method"; @@ -61,6 +62,8 @@ public interface RdbSpecification { String[] SHEET_NAMES = { TR_SHEET_NAME, HH_SHEET_NAME, + + //"RAW_SL", // -- For DEBUG SL_SHEET_NAME, HL_SHEET_NAME, // CA_SHEET_NAME diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionContextVO.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionContextVO.java index e77e0d0239..62c31c7417 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionContextVO.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionContextVO.java @@ -57,6 +57,8 @@ public class ExtractionContextVO implements IExtractionTypeWithTablesVO { Date updateDate; + boolean enableBatchDenormalization; + @FieldNameConstants.Exclude Map sheetNameByTableNames = new LinkedHashMap<>(); diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionFilterCriterionVO.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionFilterCriterionVO.java index a3634bfb3e..ad7fe12d54 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionFilterCriterionVO.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/ExtractionFilterCriterionVO.java @@ -44,13 +44,16 @@ public String toString() { StringBuilder sb = new StringBuilder(); if (this.getSheetName() != null) sb.append("Sheet: ").append(this.getSheetName()); if (this.getName() != null && this.getOperator() != null){ - sb.append(", ").append(this.getName()).append(this.getOperator()); + sb.append(", ").append(this.getName()) + .append(' ') + .append(this.getOperator()) + .append(' '); } else { sb.append(", ").append("Value: "); } if (this.getValue() != null) sb.append(this.getValue()); - if (this.getValues() != null) sb.append('[').append(Joiner.on(',').join(this.getValues())).append(']'); + if (this.getValues() != null) sb.append('(').append(Joiner.on(',').join(this.getValues())).append(')'); return sb.toString(); } } diff --git a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/trip/ExtractionTripFilterVO.java b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/trip/ExtractionTripFilterVO.java index 835c8ab667..9c6383a91f 100644 --- a/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/trip/ExtractionTripFilterVO.java +++ b/sumaris-extraction/src/main/java/net/sumaris/extraction/core/vo/trip/ExtractionTripFilterVO.java @@ -42,7 +42,7 @@ public class ExtractionTripFilterVO extends TripFilterVO { private Page page; - private boolean excludeInvalidStation = true; + private boolean excludeInvalidStation = false; public String toString(String separator) { separator = (separator == null) ? ", " : separator; diff --git a/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_en_GB.properties b/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_en_GB.properties index b5cc6632c7..44a46b7db5 100644 --- a/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_en_GB.properties +++ b/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_en_GB.properties @@ -1,5 +1,6 @@ sumaris.config.option.auth.allExtractionTypeAccess.role.description= sumaris.config.option.extraction.accessNotSelfExtraction.role.description= +sumaris.config.option.extraction.batch.denormalization.enable.description= sumaris.config.option.extraction.cli.frequency.description= sumaris.config.option.extraction.cli.output.format.description= sumaris.config.option.extraction.enabled.description= diff --git a/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_fr_FR.properties b/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_fr_FR.properties index 565890c0fd..fe7ef32ae9 100644 --- a/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_fr_FR.properties +++ b/sumaris-extraction/src/main/resources/i18n/sumaris-extraction_fr_FR.properties @@ -1,5 +1,6 @@ sumaris.config.option.auth.allExtractionTypeAccess.role.description= sumaris.config.option.extraction.accessNotSelfExtraction.role.description= +sumaris.config.option.extraction.batch.denormalization.enable.description= sumaris.config.option.extraction.cli.frequency.description= sumaris.config.option.extraction.cli.output.format.description= sumaris.config.option.extraction.enabled.description= diff --git a/sumaris-extraction/src/main/resources/static/api/extraction/md/apase-v1.0.md b/sumaris-extraction/src/main/resources/static/api/extraction/md/apase-v1.0.md new file mode 100644 index 0000000000..2e1e8a6508 --- /dev/null +++ b/sumaris-extraction/src/main/resources/static/api/extraction/md/apase-v1.0.md @@ -0,0 +1,201 @@ +# Spéficiation du format d'extraction APASE + +APASE extraction format is based on the [ICES RDB data exchange format](https://www.ices.dk/data/Documents/RDB/RDB%20Exchange%20Format.pdf). +EU/ICES have defined common format and processing tools, for fisheries statistics (.e.G R scripts - see COST project). + + +## Trip (TR) + +A commercial fishing trip that has been sampled on board or a sample from a fish market. + +| Field name | Type | Req. | Basic checks | Comments | +|-------------------------------|---------|---------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type * | String | M | | Fixed value ”TR”. | +| Sampling type * | String | M | Code list | “S” = sea sampling | +| Landing country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year * | Integer | M | 1 900 − 3 000 | | +| Project * | String | M | Code list | National project name. Code list is editable. | +| Trip code * | String | M | String 50 | National coding system. | +| Vessel length | Integer | O | 3 − 160 | Over-all length in metres. | +| Vessel power | Integer | O | 4 − 8 500 | Vessel power (kW). | +| Vessel size | Integer | O | 1 − 2 500 | Gross registered tonnes (GRT). | +| Vessel type | Integer | M | Code list | 1 = stern trawler, 2 = side trawler, 3 = gillnetter, 4 = other boats. | +| Harbour | String | O | Code list | Landing harbour. | +| Number of sets/hauls on trip | Integer | O | 1-300 | Total number of hauls/sets taken during the trip. Both the stations where biological measures were taken and the stations that were not worked up should be counted here. | +| Days at sea | Integer | O | 1-60 | In days. | +| Vessel identifier (encrypted) | Integer | O | 1 − 999 999 | Encrypted vessel identifier. Id encrypted so that no-one can map the Id to the real vessel. | +| Sampling country | String | M | Code list | ISO 3166 – 1 alpha-3 codes. The country that did the sampling. | +| Sampling method | String | M | Code list | “Observer” or “SelfSampling”. | + +Custom columns, only for APASE: + +| Field name | Type | Req. | Basic checks | Comments | +|---------------------|---------|------|--------------|----------------------------------| +| Departure date time | String | M | | “YYYY-MM-DD HH:MM:SS” (ISO 8601) | +| Return date time | String | M | | “YYYY-MM-DD HH:MM:SS” (ISO 8601) | +| Trip comments | String | O | | Comments | + + +> Req. stand for required. In the Req. column the “M” stands for mandatory and “O” stands for optional. +> +> `*` = key field + +## Fishing Gear (FG) + +Fishing gear, used on trips + +| Field name | Type | Req. | Basic checks | Comments | +|-------------------------------|---------|---------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type * | String | M | | Fixed value ”FG | +| Sampling type * | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | +| Landing country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year * | Integer | M | 1 900 − 3 000 | | +| Project * | String | M | Code list | National project name. Code list is editable. | +| Trip code * | String | M | String 50 | National coding system. | +| Gear identifier * | Integer | M | 1 − 999 999 | Gear identifier. Unique for the trip. | +| Gear type * | String | M | Code list | “OTB” or “OTT” | +| Sub gear identifier | Integer | O | 1 - 99 | Sub gear identifier. Unique for the trip. | +| Gear label | String | M | | Free text. Not unique | +| Buoy weight kg | Integer | | | | +| Door type | String | | | | +| Entremise length | Double | | | | +| Groundrope type | String | | | | +| Headline cumulative length | Double | | | | +| Mesh gauge ass mm | Integer | | | | +| Mesh gauge back mm | Integer | | | | +| Mesh gauge belly mm | Integer | | | | +| Mesh gauge ext mm | Integer | | | | +| Mesh gauge gor mm | Integer | | | | +| Mesh gauge lower wing mm | Integer | | | | +| Mesh gauge upper wing mm | Integer | | | | +| Nb buoy | double | | | | +| Rig type | String | | | | +| Select device apase | String | | | | +| Stone grid | String | | | | +| Sweep length | double | | | | +| Tickler chain | String | | | | +| Vertical opening estimated | double | | | | +| Vertical opening instrument | double | | | | + +> TODO: Fill all columns + +## Fishing station (HH) + +Detailed information about a fishing operation, e.g a haul (for OTB gear) or a sub-haul (for OTT gear). + +| Field name | Type | Req. | Basic checks | Comments | +|------------------------------------------|---------|------|------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type * | String | M | | Fixed value ”HH”. | +| Sampling type * | String | M | Code list | “S” = sea sampling | +| Landing country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year * | Integer | M | 1 900 − 3 000 | | +| Project * | String | M | Code list | National project name. Code list is editable. | +| Trip code * | String | M | String 50 | National coding system. | +| Station number * | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | +| Fishing validity | String | M/O | Code list | I = Invalid, V = Valid. Mandatory for sampling type “S” and “M”. When a haul is invalid, then no SL and HL records are allowed. | +| Aggregation level | String | M/O | Code list | H = haul, T = trip. Mandatory for sampling type “S” and “M”. If more than one station exist for the same trip, then all should be “H” (= haul). | +| Catch registration | String | M | Code list | The parts (landings/discards) of the catch, registered as: "All", "Lan", "Dis", "None". | +| Species registration | String | M | Code list | The species in the catch, registered as "All", "Par", "None". | +| Date | String | M | “1900–01-01” to “2025–12–31” | “YYYY-MM-DD” (ISO 8601). Fishing starting date. If aggregation level is “T”, the day = day of first station no. Fishing starting date. | +| Time | String | O | 00:00-23:59 | Starting time. “HH:MM” in UTC/GTM (No daylight saving/summer time). “Meaning the time in London”. If aggregation level is “T”, the time shoot = time shot of first station no. | +| Fishing duration | Integer | M | 5 − 99 999 | In minutes. | +| Pos.Start.Lat.dec. | Dec | M | 20.00000 − 80.00000 | Shooting (start) position in decimal degrees of latitude. | +| Pos.Start.Lon.dec. | Dec | M | −31.00000 − 31.00000 | Shooting (start) position in decimal degrees of longitude. | +| Pos.Stop.Lat.dec. | Dec | M | 20.00000 − 80.00000 | Hauling (stop) position in decimal degrees of latitude. | +| Pos.Stop.Lon.dec. | Dec | M | −31.00000 − 31.00000 | Hauling (stop) position in decimal degrees of longitude. | +| Area | String | O | Code list | Area level 3 (level 4 for Baltic, Mediterranean, and Black Seas) in the Data Collection Regulation (EC, 2008a, 2008b). | +| Statistical rectangle | String | M | Code list | Area level 5 in the Data Collection Regulation (EC, 2008a, 2008b). This is the ICES statistical rectangles (e.g. 41G9) except for the Mediterranean and Black Seas, where GFCM geographical subareas (GSAs) are used. | +| Subpolygon | String | O | Code list | National level as defined by each country as child nodes (substratification) of the ICES rectangles. It is recommended that this is coordinated internationally, e.g. through the Regional Coordination Meetings (EC RCMs). | +| Main fishing depth | Integer | O | 1−999 | Depth from surface to groundrope in metres. | +| Main water depth | Integer | O | 1−999 | Depth from surface in metres. | +| Fishing activity category National | String | O | Code list | | +| Fishing activity category European lvl 5 | String | O | Code list | | +| Fishing activity category European lvl 6 | String | O | Code list | | + +Custom columns, only for APASE: + +| Field name | Type | Req. | Basic checks | Comments | +|-------------------------|---------|------|--------------|-------------------------------------------------------------------| +| Station comments | String | O | | Comments | +| Gear identifier * | Integer | M | | Gear identifier. | +| Tag operation | String | O | | Free tag, used to associate stations. Only on ”OTB” gear type | +| Sorting end date time | String | O | | “YYYY-MM-DD HH:MM:SS” (ISO 8601) | +| Sorting start date time | String | O | | “YYYY-MM-DD HH:MM:SS” (ISO 8601) | +| Diurnal operation | String | O | Y or N | | +| Gear speed | Double | O | | Gear speed, in nautical miles | +| Gear depth | Double | O | | Gear depth, in meters | +| Sea state | String | O | Code list | From ”0” = calm, to ”9” = phenomenal | +| Wind force beaufort | String | O | 0 - 12 | Beaufort scale | +| Wind cardinal direction | String | O | Code list | E, N, NE, NO, I, S, SE, SO | +| Rectilinear operation | String | O | Y or N | | +| Seabed features | String | O | Code list | | + + +## Species list (SL) + +The sorting strata defined by species, catch category, etc. + +| Field name | Type | Req. | Basic checks | Comments | +|----------------------------------|---------|------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type | String | M | | Fixed value ”SL”. | +| Sampling type | String | M | Code list | “S” = sea sampling | +| Landing country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year | Integer | M | 1 900 − 3 000 | | +| Project | String | M | Code list | National project name. Code list is editable. | +| Trip code | String | M | String 50 | National coding system. | +| Station number | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | +| Species | Integer | M | Code list | The AphiaID, which is a 6 digit code, is used for the species in the species field. The AphiaIDs are maintained by WoRMS. Only species AphiaIDs with status “Accepted” or “Alternate Representation” is allowed. | +| Catch category | String | M | Code list | The fate of the catch: “DIS” = discard, “LAN” = landing. | +| Landing category * | String | M | Code list | The intended usage at the time of landing. This should match the same field in CL record (whether or not the fish was actually used for this or another purpose): “IND” = industry or “HUC” = human consumption. | +| Commercial size category scale * | String | O | Code list | Commercial sorting scale code (optional for “Unsorted”) | +| Commercial size category * | String | O | Code list | Commercial sorting category in the given scale (optional for “Unsorted”). (EC, 2006) and later amendments when scale is “EU”. | +| Subsampling category * | String | O | Code list | “VRAC“ = Vrac or “H-VRAC“ = Hors Vrac. Mandatory for catch category = “DIS“ | +| Sex * | String | O | Code list | M = Male, F = Female, T = Transitional 2 (optional for “Unsexed”) | +| Weight | Integer | M | 1 − 9 999 999 999 | Whole weight in grammes. Decimals not allowed. Weight of the corresponding stratum (Species – Catch category – size category – Sex). | +| Subsample weight | Integer | O | 1 − 9 999 999 999 | Whole weight in grammes. Decimals not allowed. For sea sampling: the live weight of the subsample of the corresponding stratum. For market sampling: the sample weight is the whole weight of the fish measured (e.g. the summed weight of the fish in one or more boxes). | +| Length code | | | | Class: 1 mm = “mm”, 0.5 cm = “scm”; 1 cm = “cm”; 2.5 cm = “25 mm”, 5 cm = “5 cm”. | + +Custom columns, only for APASE: + +| Field name | Type | Req. | Basic checks | Comments | +|-------------------|---------|------|--------------|--------------------------------------------------------------------------------| +| Sub gear position | String | O | Code list | “B” = Bâbord, “T” = Tribord. Mandatory for “OTT“ gears. Empty for “OTB“ gears. | +| Sub gear number | Integer | O | 1 - 99 | Mandatory for “OTT“ gears. Empty for “OTB“ gears. | +| Catch weight * | Integer | M | | The total catch weight for “OTB“ gears, or sub catch weight for “OTT“. | + +## Haul length (HL) + +Length frequency in the subsample of the stratum. +One record represents one length class. + + +| Field name | Type | Req. | Basic checks | Comments | +|----------------------------------|---------|------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type | String | M | | Fixed value ”SL”. | +| Sampling type | String | M | Code list | “S” = sea sampling | +| Landing country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year | Integer | M | 1 900 − 3 000 | | +| Project | String | M | Code list | National project name. Code list is editable. | +| Trip code | String | M | String 50 | National coding system. | +| Station number | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | +| Species | Integer | M | Code list | The AphiaID, which is a 6 digit code, is used for the species in the species field. The AphiaIDs are maintained by WoRMS. Only species AphiaIDs with status “Accepted” or “Alternate Representation” is allowed. | +| Catch category | String | M | Code list | The fate of the catch: “LAN” = Landing, “BMS” = Below Minimum Size landing, “DIS” = Discard or “REGDIS” = Logbook Registered Discard. | +| Landing category * | String | M | Code list | The intended usage at the time of landing. This should match the same field in CL record (whether or not the fish was actually used for this or another purpose): “IND” = industry or “HUC” = human consumption. | +| Commercial size category scale * | String | O | Code list | Commercial sorting scale code (optional for “Unsorted”) | +| Commercial size category * | String | O | Code list | Commercial sorting category in the given scale (optional for “Unsorted”). (EC, 2006) and later amendments when scale is “EU”. | +| Subsampling category * | String | O | Code list | “VRAC“ = Vrac or “H-VRAC“ = Hors Vrac. Mandatory for catch category = “DIS“ | +| Sex * | String | O | Code list | M = Male, F = Female, T = Transitional 2 (optional for “Unsexed”) | +| Individual sex | String | O | Code list | If M = Male, = , F = Female, T = Transitional = (optional for “Unsexed”). Only different from “Sex” if individual length distribution is obtained on HL-level (and not on SL-level). | +| Length class * | Integer | M | 1−3 999 | In mm. Identifier: lower bound of size class, e.g. 650 for 65 – 66 cm. | +| Number at length * | Integer | M | 1−999 | (not raised to whole catch) Length classes with zero should be excluded from the record. | + +Custom columns, only for APASE: + +| Field name | Type | Req. | Basic checks | Comments | +|-----------------------|--------|------|--------------|-------------------------------------------------------------------------------------------------------------------------| +| Measure type | String | O | Code list | Measure type. “LT“ = length total, “LC“ = length carapace, etc. | diff --git a/sumaris-extraction/src/main/resources/static/api/extraction/md/rdb-v1.3.md b/sumaris-extraction/src/main/resources/static/api/extraction/md/rdb-v1.3.md index c135e0d74d..349425d181 100644 --- a/sumaris-extraction/src/main/resources/static/api/extraction/md/rdb-v1.3.md +++ b/sumaris-extraction/src/main/resources/static/api/extraction/md/rdb-v1.3.md @@ -3,54 +3,168 @@ Download the full documentation at : https://www.ices.dk/data/Documents/RDB/RDB%20Exchange%20Format.pdf (PDF) + +## Data types and record types +The following data types are defined: +- CS = Commercial fisheries sampling +- CL = Commercial fisheries landings statistics +- CE = Commercial fisheries effort statistics. + +Each of these three data types consists of data of specific record types (see below). + +| Data type | Record types | +|-----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------| +| CS (S (Commercial fisheries sampling) | TR (trip), HH (haul header), SL (species list), HL (haul length), CA (catch aged) = SMAWL (Sex-Maturity-Age-Weight-Length) | +| CL (Commercial fisheries landings statistics) | CL (commercial fisheries landings statistics) | +| CE (Commercial fisheries effort statistics) | CE (Commercial fisheries effort statistics) | + + +The record types are given in a specific hierarchy (Figure 1) and order within the data +file. Each data record consists of a range of data fields. The required order is given +below. + +> Figure 1. TODO + + Req. stand for required. In the Req. column the “M” stands for mandatory and “O” stands for optional. -## Trip record (TR) - -| Field name | Type | Req. | Basic checks | Comments | -|---------------------|----------|--------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| Record type | String | M | | Fixed value TR. | -| Sampling type | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | -| Landing country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | -| Vessel flag country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | -| Year | Integer | M | 1 900 − 3 000 | | -| Project | String | M | Code list | National project name. Code list is editable. | -| Trip code | String | M | String 50 | National coding system. | -| (...) | | | | | - -## Station record (HH) - -| Field name | Type | Req. | Basic checks | Comments | -|----------------------|---------|------|------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Record type | String | M | | Fixed value HH. | -| Sampling type | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | -| Landing country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | -| Vessel flag country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | -| Year | Integer | M | 1 900 − 3 000 | | -| Project | String | M | Code list | National project name. Code list is editable. | -| Trip code | String | M | String 50 | National coding system. | -| Station number | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | -| Fishing validity | String | M/O | Code list | I = Invalid, V = Valid. Mandatory for sampling type “S” and “M”. When a haul is invalid, then no SL and HL records are allowed. | -| Aggregation level | String | M/O | Code list | H = haul, T = trip. Mandatory for sampling type “S” and “M”. If more than one station exist for the same trip, then all should be “H” (= haul). | -| Catch registration | String | M | Code list | The parts (landings/discards) of the catch, registered as:"All", "Lan", "Dis", "None". | -| Species registration | String | M | Code list | The species in the catch, registered as "All", "Par", "None". | -| Date | String | M | “1900–01-01” to “2025–12–31” | “YYYY-MM-DD” (ISO 8601). If aggregation level is “T”, the day = day of first station no. Fishing starting date. | -| Time | String | O | 00:00-23:59 | Starting time. “HH:MM” in UTC/GTM (No daylight saving/summer time). “Meaning the time in London”. If aggregation level is “T”, the time shoot = time shot of first station no. | -| (...) | | | | | - -## Species list record (SL) - - -| Field name | Type | Req. | Basic checks | Comments | -|---------------------|----------|------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Record type | String | M | | Fixed value SL. | -| Sampling type | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | -| Landing country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | -| Vessel flag country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | -| Year | Integer | M | 1 900 − 3 000 | | -| Project | String | M | Code list | National project name. Code list is editable. | -| Trip code | String | M | String 50 | National coding system. | -| Station number | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | -| Species | Integer | M | Code list | The AphiaID, which is a 6 digit code, is used for the species in the species field. The AphiaIDs are maintained by WoRMS. Only species AphiaIDs with status “Accepted” or “Alternate Representation” is allowed. | -| Catch category | String | M | Code list | The fate of the catch: “LAN” = Landing, “BMS” = Below Minimum Size landing, “DIS” = Discard or “REGDIS” = Logbook Registered Discard. | -| (...) | | | | | +## Trip (TR) + +A commercial fishing trip that has been sampled on board or a sample from a fish market. + +| Field name | Type | Req. | Basic checks | Comments | +|-------------------------------|---------|---------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type * | String | M | | Fixed value TR. | +| Sampling type * | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | +| Landing country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year * | Integer | M | 1 900 − 3 000 | | +| Project * | String | M | Code list | National project name. Code list is editable. | +| Trip code * | String | M | String 50 | National coding system. | +| Vessel length | Integer | O | 3 − 160 | Over-all length in metres. | +| Vessel power | Integer | O | 4 − 8 500 | Vessel power (kW). | +| Vessel size | Integer | O | 1 − 2 500 | Gross registered tonnes (GRT). | +| Vessel type | Integer | M | Code list | 1 = stern trawler, 2 = side trawler, 3 = gillnetter, 4 = other boats. | +| Harbour | String | O | Code list | Landing harbour. | +| Number of sets/hauls on trip | Integer | O | 1-300 | Total number of hauls/sets taken during the trip. Both the stations where biological measures were taken and the stations that were not worked up should be counted here. | +| Days at sea | Integer | O | 1-60 | In days. | +| Vessel identifier (encrypted) | Integer | O | 1 − 999 999 | Encrypted vessel identifier. Id encrypted so that no-one can map the Id to the real vessel. | +| Sampling country | String | M | Code list | ISO 3166 – 1 alpha-3 codes. The country that did the sampling. | +| Sampling method | String | M | Code list | “Observer” or “SelfSampling”. | + +> `*` = key field + +## Fishing station (HH) + +Detailed information about a fishing operation, e.g a haul. + +> HH = Haul head + + +| Field name | Type | Req. | Basic checks | Comments | +|------------------------------------------|---------|------|------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type * | String | M | | Fixed value HH. | +| Sampling type * | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | +| Landing country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country * | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year * | Integer | M | 1 900 − 3 000 | | +| Project * | String | M | Code list | National project name. Code list is editable. | +| Trip code * | String | M | String 50 | National coding system. | +| Station number * | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | +| Fishing validity | String | M/O | Code list | I = Invalid, V = Valid. Mandatory for sampling type “S” and “M”. When a haul is invalid, then no SL and HL records are allowed. | +| Aggregation level | String | M/O | Code list | H = haul, T = trip. Mandatory for sampling type “S” and “M”. If more than one station exist for the same trip, then all should be “H” (= haul). | +| Catch registration | String | M | Code list | The parts (landings/discards) of the catch, registered as:"All", "Lan", "Dis", "None". | +| Species registration | String | M | Code list | The species in the catch, registered as "All", "Par", "None". | +| Date | String | M | “1900–01-01” to “2025–12–31” | “YYYY-MM-DD” (ISO 8601). Fishing starting date. If aggregation level is “T”, the day = day of first station no. Fishing starting date. | +| Time | String | O | 00:00-23:59 | Starting time. “HH:MM” in UTC/GTM (No daylight saving/summer time). “Meaning the time in London”. If aggregation level is “T”, the time shoot = time shot of first station no. | +| Fishing duration | | | 5 − 99 999 | In minutes. | +| Pos.Start.Lat.dec. | Dec | O | 20.00000 − 80.00000 | Shooting (start) position in decimal degrees of latitude. | +| Pos.Start.Lon.dec. | Dec | O | −31.00000 − 31.00000 | Shooting (start) position in decimal degrees of longitude. | +| Pos.Stop.Lat.dec. | Dec | O | 20.00000 − 80.00000 | Hauling (stop) position in decimal degrees of latitude. | +| Pos.Stop.Lon.dec. | Dec | O | −31.00000 − 31.00000 | Hauling (stop) position in decimal degrees of longitude. | +| Area | String | O | Code list | Area level 3 (level 4 for Baltic, Mediterranean, and Black Seas) in the Data Collection Regulation (EC, 2008a, 2008b). | +| Statistical rectangle | String | O | Code list | Area level 5 in the Data Collection Regulation (EC, 2008a, 2008b). This is the ICES statistical rectangles (e.g. 41G9) except for the Mediterranean and Black Seas, where GFCM geographical subareas (GSAs) are used. | +| Subpolygon | String | O | Code list | National level as defined by each country as child nodes (substratification) of the ICES rectangles. It is recommended that this is coordinated internationally, e.g. through the Regional Coordination Meetings (EC RCMs). | +| Main fishing depth | Integer | O | 1−999 | Depth from surface to groundrope in metres. | +| Main water depth | Integer | O | 1−999 | Depth from surface in metres. | +| Fishing activity category National | String | O | Code list | | +| Fishing activity category European lvl 5 | String | O | Code list | | +| Fishing activity category European lvl 6 | String | O | Code list | | +| Gear type | String | M | Code list | | +| Mesh size | Integer | O | 1−999 | Stretch measure. | +| Selection device | Integer | O | | Not mounted = 0, Exit window / selection panel = 1, grid = 2. A selection device is defined as a square-meshed panel or window that is inserted into a towed net. | +| Mesh size in selection device | Integer | O | | In mm. The mesh size of a square-meshed panel or window shall mean the largest determinable mesh size of such a panel or window. | + +## Species list (SL) + +The sorting strata defined by species, catch category, etc. + +| Field name | Type | Req. | Basic checks | Comments | +|----------------------------------|---------|------|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type | String | M | | Fixed value SL. | +| Sampling type | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | +| Landing country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year | Integer | M | 1 900 − 3 000 | | +| Project | String | M | Code list | National project name. Code list is editable. | +| Trip code | String | M | String 50 | National coding system. | +| Station number | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | +| Species | Integer | M | Code list | The AphiaID, which is a 6 digit code, is used for the species in the species field. The AphiaIDs are maintained by WoRMS. Only species AphiaIDs with status “Accepted” or “Alternate Representation” is allowed. | +| Catch category | String | M | Code list | The fate of the catch: “LAN” = Landing, “BMS” = Below Minimum Size landing, “DIS” = Discard or “REGDIS” = Logbook Registered Discard. | +| Landing category * | String | M | Code list | The intended usage at the time of landing. This should match the same field in CL record (whether or not the fish was actually used for this or another purpose): “IND” = industry or “HUC” = human consumption. | +| Commercial size category scale * | String | O | Code list | Commercial sorting scale code (optional for “Unsorted”) | +| Commercial size category * | String | O | Code list | Commercial sorting category in the given scale (optional for “Unsorted”). (EC, 2006) and later amendments when scale is “EU”. | +| Subsampling category * | String | O | Code list | Used when different fractions of the same species are subsampled at different levels. Typically used when few large specimens are taken out from the total catch before the many small fish are subsampled. | +| Sex * | String | O | Code list | M = Male, F = Female, T = Transitional 2 (optional for “Unsexed”) | +| Weight | Integer | M | 1 − 9 999 999 999 | Whole weight in grammes. Decimals not allowed. Weight of the corresponding stratum (Species – Catch category – size category – Sex). | +| Subsample weight | Integer | O | 1 − 9 999 999 999 | Whole weight in grammes. Decimals not allowed. For sea sampling: the live weight of the subsample of the corresponding stratum. For market sampling: the sample weight is the whole weight of the fish measured (e.g. the summed weight of the fish in one or more boxes). | +| Length code | | | | Class: 1 mm = “mm”, 0.5 cm = “scm”; 1 cm = “cm”; 2.5 cm = “25 mm”, 5 cm = “5 cm”. | + +## Haul length (HL) + +Length frequency in the subsample of the stratum. +One record represents one length class. + + +| Field name | Type | Req. | Basic checks | Comments | +|----------------------------------|---------|------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Record type | String | M | | Fixed value SL. | +| Sampling type | String | M | Code list | “S” = sea sampling, “M” = market sampling of known fishing trips, “D” = market sampling of mixed trips, “V” = vendor. | +| Landing country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the country where the vessel is landing and selling the catch. | +| Vessel flag country | String | M | Code list | ISO 3166 – 1 alpha-3 codes: the flag country of the vessel. This can be different from the landing country (see description of Landing country). | +| Year | Integer | M | 1 900 − 3 000 | | +| Project | String | M | Code list | National project name. Code list is editable. | +| Trip code | String | M | String 50 | National coding system. | +| Station number | Integer | M | 1-99999 | Sequential numbering of hauls. Starting by 1 for each new trip. If the “Aggregation level” is T then this “Station number” should be 999. | +| Species | Integer | M | Code list | The AphiaID, which is a 6 digit code, is used for the species in the species field. The AphiaIDs are maintained by WoRMS. Only species AphiaIDs with status “Accepted” or “Alternate Representation” is allowed. | +| Catch category | String | M | Code list | The fate of the catch: “LAN” = Landing, “BMS” = Below Minimum Size landing, “DIS” = Discard or “REGDIS” = Logbook Registered Discard. | +| Landing category * | String | M | Code list | The intended usage at the time of landing. This should match the same field in CL record (whether or not the fish was actually used for this or another purpose): “IND” = industry or “HUC” = human consumption. | +| Commercial size category scale * | String | O | Code list | Commercial sorting scale code (optional for “Unsorted”) | +| Commercial size category * | String | O | Code list | Commercial sorting category in the given scale (optional for “Unsorted”). (EC, 2006) and later amendments when scale is “EU”. | +| Subsampling category * | String | O | Code list | Used when different fractions of the same species are subsampled at different levels. Typically used when few large specimens are taken out from the total catch before the many small fish are subsampled. | +| Sex * | String | O | Code list | M = Male, F = Female, T = Transitional 2 (optional for “Unsexed”) | +| Individual sex | String | O | Code list | If M = Male, = , F = Female, T = Transitional = (optional for “Unsexed”). Only different from “Sex” if individual length distribution is obtained on HL-level (and not on SL-level). | +| Length class * | Integer | M | 1−3 999 | In mm. Identifier: lower bound of size class, e.g. 650 for 65 – 66 cm. | +| Number at length * | Integer | M | 1−999 | (not raised to whole catch) Length classes with zero should be excluded from the record. | + + +## Catch aged (CA) + +> CA = SMAWL (Sex-Maturity-Age-Weight-Length) + +Sex-Maturity-Age-Weight distribution sampled representatively from the +length groups. + +One record represents one fish. + +## Commercial fisheries landings statistics (CL) + +Official landings statistics with some modifiers for misreporting. + +> Not supported yet + +## Commercial fisheries effort statistics (CE) + +Effort statistics from logbooks. + +> Not supported yet \ No newline at end of file diff --git a/sumaris-extraction/src/main/resources/xmlQuery/aggRdb/v1_3/createStationTable.xml b/sumaris-extraction/src/main/resources/xmlQuery/aggRdb/v1_3/createStationTable.xml index 0cc9a38d4a..d6d178b6f9 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/aggRdb/v1_3/createStationTable.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/aggRdb/v1_3/createStationTable.xml @@ -46,8 +46,12 @@ 1=1 - T.VESSEL_IDENTIFIER in (&vesselIds) - T.TRIP_CODE in (&tripCodes) + + &vesselIds + + + &tripCodes + @@ -92,6 +96,8 @@ + + @@ -109,12 +115,7 @@ = &startDate]]> - - &vesselIds - - - &tripCodes - + S.SAMPLING_TYPE, S.LANDING_COUNTRY, S.VESSEL_FLAG_COUNTRY, S.YEAR, S.PROJECT, diff --git a/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createGearTable.xml b/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createGearTable.xml index 5f117cbfcc..b96c64ba8e 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createGearTable.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createGearTable.xml @@ -14,11 +14,12 @@ diff --git a/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createTripTable.xml b/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createTripTable.xml index 831f3cd388..5912a4ffa6 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createTripTable.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/free2/v1_9/createTripTable.xml @@ -4,9 +4,9 @@ - + - + diff --git a/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTable.xml b/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTable.xml index 30a3b7a8cf..fae5bbd998 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTable.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTable.xml @@ -27,7 +27,7 @@ - + INNER JOIN SORTING_MEASUREMENT_B SM ON SM.BATCH_FK = B.ID LEFT OUTER JOIN QUALITATIVE_VALUE QV ON QV.ID = SM.QUALITATIVE_VALUE_FK diff --git a/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTaxon.xml b/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTaxon.xml index f461d984ca..3ca106c24b 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTaxon.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/pmfmTrip/v1_0/injectionSpeciesLengthTaxon.xml @@ -22,7 +22,6 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + &stationTableName + + INNER JOIN DENORMALIZED_BATCH SPECIE_B ON SPECIE_B.OPERATION_FK = S.STATION_NUMBER AND SPECIE_B.IS_LANDING <> SPECIE_B.IS_DISCARD + INNER JOIN TAXON_GROUP TG ON TG.ID = SPECIE_B.INHERITED_TAXON_GROUP_FK + INNER JOIN DENORMALIZED_BATCH_SORT_VAL CATCH_CATEGORY_SV ON CATCH_CATEGORY_SV.BATCH_FK=SPECIE_B.ID AND CATCH_CATEGORY_SV.PMFM_FK=&catchCategoryPmfmId AND CATCH_CATEGORY_SV.IS_INHERITED=0 + LEFT OUTER JOIN DENORMALIZED_BATCH SAMPLING_B ON SAMPLING_B.PARENT_BATCH_FK = SPECIE_B.ID AND SAMPLING_B.SAMPLING_RATIO > 0 + + + LEFT OUTER JOIN DENORMALIZED_BATCH LENGTH_B ON LENGTH_B.PARENT_BATCH_FK=COALESCE(SAMPLING_B.ID, SPECIE_B.ID) + LEFT OUTER JOIN DENORMALIZED_BATCH_SORT_VAL LENGTH_SV ON LENGTH_SV.BATCH_FK=LENGTH_B.ID AND LENGTH_SV.PMFM_FK in (&lengthPmfmIds) AND IS_INHERITED=0 + LEFT OUTER JOIN UNIT ON UNIT.ID = LENGTH_SV.UNIT_FK + LEFT OUTER JOIN TAXON_NAME TN ON TN.REFERENCE_TAXON_FK = LENGTH_B.REFERENCE_TAXON_FK + + 1=1 + + + + + + SPECIE_B.WEIGHT IS NOT NULL + + + + + + + SAMPLING_TYPE, LANDING_COUNTRY, VESSEL_FLAG_COUNTRY, YEAR, PROJECT, TRIP_CODE, STATION_NUMBER, SPECIES, + CATCH_CATEGORY, LANDING_CATEGORY, COMMERCIAL_SIZE_CATEGORY_SCALE, COMMERCIAL_SIZE_CATEGORY, SUBSAMPLING_CATEGORY, + SEX, LENGTH_CODE, SAMPLE_ID, SAMPLE_RANK_ORDER, + SPECIE_B.ELEVATE_WEIGHT, SAMPLING_B.SAMPLING_RATIO, SAMPLING_B.ELEVATE_RTP_WEIGHT, SPECIE_B.ELEVATE_RTP_WEIGHT + + + S.STATION_NUMBER, SAMPLE_RANK_ORDER + + + + diff --git a/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createRawSpeciesListTable.xml b/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createRawSpeciesListTable.xml index 4a86b2f204..def2981862 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createRawSpeciesListTable.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createRawSpeciesListTable.xml @@ -34,7 +34,7 @@ - + - - + + @@ -63,6 +63,9 @@ + + + diff --git a/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesLengthTable.xml b/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesLengthTable.xml index 7763aba54e..29ff8bf9d0 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesLengthTable.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesLengthTable.xml @@ -28,7 +28,7 @@ SL.SAMPLE_ID B.ID - (SELECT QV.LABEL FROM SORTING_MEASUREMENT_B SM INNER JOIN QUALITATIVE_VALUE QV ON QV.ID=SM.QUALITATIVE_VALUE_FK WHERE SM.BATCH_FK = B.ID and SM.PMFM_FK=&sexPmfmId) + (SELECT QV.LABEL FROM SORTING_MEASUREMENT_B SM_SEX INNER JOIN QUALITATIVE_VALUE QV ON QV.ID=SM_SEX.QUALITATIVE_VALUE_FK WHERE SM_SEX.BATCH_FK = B.ID and SM_SEX.PMFM_FK=&sexPmfmId) CAST(CASE PMFM_LENGTH.UNIT_FK WHEN &centimeterUnitId THEN SM_LENGTH.NUMERICAL_VALUE*10 WHEN &millimeterUnitId THEN SM_LENGTH.NUMERICAL_VALUE ELSE null END AS INTEGER) COALESCE(B.INDIVIDUAL_COUNT,1) B.COMMENTS diff --git a/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesListTable.xml b/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesListTable.xml index d7fd847dd9..76e297dba8 100644 --- a/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesListTable.xml +++ b/sumaris-extraction/src/main/resources/xmlQuery/rdb/v1_3/createSpeciesListTable.xml @@ -44,8 +44,13 @@ - - + + @@ -60,9 +65,8 @@ SAMPLING_TYPE, LANDING_COUNTRY, VESSEL_FLAG_COUNTRY, YEAR, PROJECT, TRIP_CODE, STATION_NUMBER, SPECIES, CATCH_CATEGORY, LANDING_CATEGORY, COMMERCIAL_SIZE_CATEGORY_SCALE, COMMERCIAL_SIZE_CATEGORY, SUBSAMPLING_CATEGORY, - SEX, LENGTH_CODE + SEX, LENGTH_CODE, SUBSAMPLING_WEIGHT, SUBSAMPLING_RATIO - diff --git a/sumaris-extraction/src/test/java/net/sumaris/extraction/core/DatabaseFixtures.java b/sumaris-extraction/src/test/java/net/sumaris/extraction/core/DatabaseFixtures.java index 65b2474629..7986a0d96a 100644 --- a/sumaris-extraction/src/test/java/net/sumaris/extraction/core/DatabaseFixtures.java +++ b/sumaris-extraction/src/test/java/net/sumaris/extraction/core/DatabaseFixtures.java @@ -88,4 +88,15 @@ public int getYearRdbProduct() { return 2012; } + public int getTripIdByProgramLabel(String programLabel) { + switch (programLabel) { + case "APASE": + return 70; + case "ADAP-MER": + return 100; + default: + throw new IllegalArgumentException("Add trip id for program " + programLabel); + } + } + } diff --git a/sumaris-extraction/src/test/java/net/sumaris/extraction/core/InitTests.java b/sumaris-extraction/src/test/java/net/sumaris/extraction/core/InitTests.java index 10ed78ba31..08426d2c80 100644 --- a/sumaris-extraction/src/test/java/net/sumaris/extraction/core/InitTests.java +++ b/sumaris-extraction/src/test/java/net/sumaris/extraction/core/InitTests.java @@ -40,7 +40,7 @@ public static void main(String[] args) { try { // Force replacement - initTests.setReplaceDbIfExists(true); + //initTests.setReplaceDbIfExists(true); initTests.before(); } catch (Throwable ex) { diff --git a/sumaris-extraction/src/test/java/net/sumaris/extraction/core/service/hsqldb/ExtractionServiceHsqlDbTest.java b/sumaris-extraction/src/test/java/net/sumaris/extraction/core/service/hsqldb/ExtractionServiceHsqlDbTest.java index 9e4a402176..9389dd8e92 100644 --- a/sumaris-extraction/src/test/java/net/sumaris/extraction/core/service/hsqldb/ExtractionServiceHsqlDbTest.java +++ b/sumaris-extraction/src/test/java/net/sumaris/extraction/core/service/hsqldb/ExtractionServiceHsqlDbTest.java @@ -22,9 +22,22 @@ package net.sumaris.extraction.core.service.hsqldb; +import net.sumaris.core.service.data.TripService; +import net.sumaris.core.vo.data.TripVO; import net.sumaris.extraction.core.DatabaseResource; +import net.sumaris.extraction.core.config.ExtractionConfiguration; import net.sumaris.extraction.core.service.ExtractionServiceTest; +import net.sumaris.extraction.core.specification.data.trip.RdbSpecification; +import net.sumaris.extraction.core.type.LiveExtractionTypeEnum; +import net.sumaris.extraction.core.vo.trip.ExtractionTripFilterVO; +import org.junit.Assert; +import org.junit.Assume; import org.junit.ClassRule; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.File; +import java.io.IOException; /** * @author Benoit LAVENIER @@ -34,4 +47,86 @@ public class ExtractionServiceHsqlDbTest extends ExtractionServiceTest { @ClassRule public static final DatabaseResource dbResource = DatabaseResource.writeDb(); -} + @Autowired + protected ExtractionConfiguration extractionConfiguration; + + @Autowired + protected TripService tripService; + + @Test + public void executeRdbWithDenormalisation() throws IOException { + + // Enable batch optimization, in extraction + extractionConfiguration.setEnableBatchDenormalization(true); + Assert.assertTrue(extractionConfiguration.enableBatchDenormalization()); + + // Create filter for a trip (APASE) + ExtractionTripFilterVO filter = createFilterForTrip(fixtures.getTripIdByProgramLabel("APASE")); + + // TODO remove this + filter.setSheetName(RdbSpecification.SL_SHEET_NAME); + filter.setPreview(true); + + // Test the RDB format + File outputFile = service.executeAndDumpTrips(LiveExtractionTypeEnum.RDB, filter); + Assert.assertTrue(outputFile.exists()); + File root = unpack(outputFile, LiveExtractionTypeEnum.RDB.getLabel()); + + // TR.csv + { + File tripFile = new File(root, RdbSpecification.TR_SHEET_NAME + ".csv"); + Assert.assertTrue(countLineInCsvFile(tripFile) > 1); + } + + // HH.csv + { + File stationFile = new File(root, RdbSpecification.HH_SHEET_NAME + ".csv"); + Assert.assertTrue(countLineInCsvFile(stationFile) > 1); + + // Make sure this column exists (column with a 'dbms' attribute) + assertHasColumn(stationFile, RdbSpecification.COLUMN_FISHING_TIME); + } + + // SL.csv + { + File speciesListFile = new File(root, RdbSpecification.SL_SHEET_NAME + ".csv"); + Assert.assertTrue(countLineInCsvFile(speciesListFile) > 1); + + // Make sure this column exists (column with a 'dbms' attribute) + assertHasColumn(speciesListFile, RdbSpecification.COLUMN_WEIGHT); + } + } + + protected ExtractionTripFilterVO createFilterForTrip(int tripId) { + TripVO trip = loadAndValidateTripById(70 /*APASE trip*/); + + // Create extraction filter + ExtractionTripFilterVO filter = new ExtractionTripFilterVO(); + filter.setProgramLabel(trip.getProgram().getLabel()); + filter.setTripId(trip.getId()); + + return filter; + } + + protected TripVO loadAndValidateTripById(int tripId) { + // Load + TripVO trip = tripService.get(tripId); + Assume.assumeNotNull(trip); + + // Control + if (trip.getControlDate() == null) { + trip = tripService.control(trip); + Assume.assumeNotNull(trip); + Assume.assumeNotNull(trip.getControlDate()); + } + + // Validate + if (trip.getValidationDate() == null) { + trip = tripService.validate(trip); + Assume.assumeNotNull(trip); + Assume.assumeNotNull(trip.getValidationDate()); + } + + return trip; + } +} \ No newline at end of file diff --git a/sumaris-extraction/src/test/resources/application-hsqldb.properties b/sumaris-extraction/src/test/resources/application-hsqldb.properties index b4445f092c..7ff509d8d8 100644 --- a/sumaris-extraction/src/test/resources/application-hsqldb.properties +++ b/sumaris-extraction/src/test/resources/application-hsqldb.properties @@ -1,13 +1,12 @@ # SUMARiS options sumaris.name=SUMARiS sumaris.test.data.common=data-hsqldb-01-common.xml -sumaris.test.data.additional=data-hsqldb-02-program.xml,data-hsqldb-02-program-OBSDEB.xml,data-hsqldb-02-program-OBSBIO.xml,data-hsqldb-02-program-ACOST.xml,data-hsqldb-02-program-PIFIL.xml,data-hsqldb-02-program-APASE.xml,data-hsqldb-03-data.xml,data-hsqldb-04-pendings.xml,data-hsqldb-05-extracts.xml,data-hsqldb-06-configs.xml,data-hsqldb-07-backgrounds.xml - +sumaris.test.data.additional=data-hsqldb-02-program.xml,data-hsqldb-02-program-ADAP.xml,data-hsqldb-02-program-OBSDEB.xml,data-hsqldb-02-program-OBSBIO.xml,data-hsqldb-02-program-ACOST.xml,data-hsqldb-02-program-PIFIL.xml,data-hsqldb-02-program-APASE.xml,data-hsqldb-03-data.xml,data-hsqldb-04-pendings.xml,data-hsqldb-05-extracts.xml,data-hsqldb-06-configs.xml,data-hsqldb-07-backgrounds.xml # Disable JMS, and cache -spring.jms.enabled=false spring.cache.enabled=false spring.cache.type=NONE +spring.jms.enabled=false spring.activemq.enabled=false # Spring: Common properties diff --git a/sumaris-importation/pom.xml b/sumaris-importation/pom.xml index de18503005..4138606d22 100644 --- a/sumaris-importation/pom.xml +++ b/sumaris-importation/pom.xml @@ -3,7 +3,7 @@ sumaris-pod net.sumaris - 2.0.6 + 2.1.0 4.0.0 diff --git a/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportService.java b/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportService.java index 9d4a44ecfc..45924393c2 100644 --- a/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportService.java +++ b/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportService.java @@ -28,11 +28,10 @@ import net.sumaris.importation.core.service.vessel.vo.SiopVesselImportResultVO; import org.springframework.scheduling.annotation.Async; +import javax.annotation.Nullable; import javax.transaction.Transactional; -import java.io.File; import java.io.IOException; import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; public interface SiopVesselImportService { @@ -47,9 +46,10 @@ public interface SiopVesselImportService { * @throws IOException */ @Transactional - SiopVesselImportResultVO importFromFile(SiopVesselImportContextVO context, IProgressionModel progressionModel) throws IOException; + SiopVesselImportResultVO importFromFile(SiopVesselImportContextVO context, @Nullable IProgressionModel progressionModel) throws IOException; @Async("jobTaskExecutor") - Future asyncImportFromFile(SiopVesselImportContextVO context, JobVO job); + Future asyncImportFromFile(SiopVesselImportContextVO context, + @Nullable IProgressionModel progressionModel); } diff --git a/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportServiceImpl.java b/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportServiceImpl.java index 2a53e6a4c2..609b08644f 100644 --- a/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportServiceImpl.java +++ b/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/SiopVesselImportServiceImpl.java @@ -91,6 +91,7 @@ import org.springframework.scheduling.annotation.AsyncResult; import org.springframework.stereotype.Service; +import javax.annotation.Nullable; import java.beans.PropertyChangeListener; import java.io.File; import java.io.IOException; @@ -224,15 +225,11 @@ public class SiopVesselImportServiceImpl implements SiopVesselImportService { protected final ApplicationContext applicationContext; - private final ObjectMapper objectMapper; - - private final ApplicationEventPublisher publisher; - private boolean running = false; @Override public SiopVesselImportResultVO importFromFile(@NonNull SiopVesselImportContextVO context, - IProgressionModel progressionModel) throws IOException { + @Nullable IProgressionModel progressionModel) throws IOException { Files.checkExists(context.getProcessingFile()); Preconditions.checkNotNull(context.getRecorderPersonId()); @@ -442,87 +439,27 @@ else if (processedKeys.contains(uniqueKey)) { } @Override - public Future asyncImportFromFile(SiopVesselImportContextVO context, JobVO job) { - int jobId = job.getId(); - - final SiopVesselImportService self = applicationContext.getBean(SiopVesselImportService.class); + public Future asyncImportFromFile(@NonNull SiopVesselImportContextVO context, + @Nullable IProgressionModel progressionModel) { + SiopVesselImportResultVO result; try { - // Affect context to job (as json) - job.setConfiguration(objectMapper.writeValueAsString(context)); - } catch (JsonProcessingException e) { - throw new SumarisTechnicalException(e); - } + result = applicationContext.getBean(SiopVesselImportService.class) + .importFromFile(context, progressionModel); - // Publish job start event - publisher.publishEvent(new JobStartEvent(jobId, job)); - - // Create progression model and listener to throttle events - ProgressionModel progressionModel = new ProgressionModel(); - io.reactivex.rxjava3.core.Observable progressionObservable = Observable.create(emitter -> { - - // Create listener on bean property and emit the value - PropertyChangeListener listener = evt -> { - ProgressionModel progression = (ProgressionModel) evt.getSource(); - JobProgressionVO jobProgression = JobProgressionVO.fromModelBuilder(progression) - .id(jobId) - .name(job.getName()) - .build(); - emitter.onNext(jobProgression); - - if (progression.isCompleted()) { - // complete observable - emitter.onComplete(); - } - }; + // Set result status + result.setStatus(result.hasError() ? JobStatusEnum.ERROR : JobStatusEnum.SUCCESS); - // Add listener on current progression and message - progressionModel.addPropertyChangeListener(ProgressionModel.Fields.CURRENT, listener); - progressionModel.addPropertyChangeListener(ProgressionModel.Fields.MESSAGE, listener); - }); - - Disposable progressionSubscription = progressionObservable - // throttle for 500ms to filter unnecessary flow - .throttleLatest(500, TimeUnit.MILLISECONDS, true) - // Publish job progression event - .subscribe(jobProgressionVO -> publisher.publishEvent(new JobProgressionEvent(jobId, jobProgressionVO))); - - // Execute import - try { - SiopVesselImportResultVO result; - - try { - result = self.importFromFile(context, progressionModel); + } catch (Exception e) { + // Result is kept in context + result = context.getResult(); + result.setMessage(t("sumaris.import.vessel.error.detail", ExceptionUtils.getStackTrace(e))); - // Set result status - job.setStatus(result.hasError() ? JobStatusEnum.ERROR : JobStatusEnum.SUCCESS); - - } catch (Exception e) { - // Result is kept in context - result = context.getResult(); - result.setMessage(t("sumaris.import.vessel.error.detail", ExceptionUtils.getStackTrace(e))); - - // Set failed status - // TODO - //job.setStatus(JobStatusEnum.FAILED); - job.setStatus(JobStatusEnum.ERROR); - } - - try { - // Serialize result in job report (as json) - job.setReport(objectMapper.writeValueAsString(result)); - } catch (JsonProcessingException e) { - throw new SumarisTechnicalException(e); - } - - return new AsyncResult<>(result); - - } finally { - - // Publish job end event - publisher.publishEvent(new JobEndEvent(jobId, job)); - Observables.dispose(progressionSubscription); + // Set failed status + result.setStatus(JobStatusEnum.ERROR); } + + return new AsyncResult<>(result); } /* -- protected methods -- */ diff --git a/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/vo/SiopVesselImportResultVO.java b/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/vo/SiopVesselImportResultVO.java index 01450286b3..ff667e3be6 100644 --- a/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/vo/SiopVesselImportResultVO.java +++ b/sumaris-importation/src/main/java/net/sumaris/importation/core/service/vessel/vo/SiopVesselImportResultVO.java @@ -26,12 +26,14 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import net.sumaris.core.model.technical.job.JobStatusEnum; +import net.sumaris.core.vo.technical.job.IJobResultVO; @Data @Builder @NoArgsConstructor @AllArgsConstructor -public class SiopVesselImportResultVO { +public class SiopVesselImportResultVO implements IJobResultVO { private Integer inserts; private Integer updates; @@ -42,6 +44,8 @@ public class SiopVesselImportResultVO { private String message; + private JobStatusEnum status; + public boolean hasError() { return this.errors != null && this.errors > 0; } diff --git a/sumaris-importation/src/main/java/net/sumaris/importation/server/graphql/VesselImportGraphQLService.java b/sumaris-importation/src/main/java/net/sumaris/importation/server/graphql/VesselImportGraphQLService.java index e5b19af242..091cc3886b 100644 --- a/sumaris-importation/src/main/java/net/sumaris/importation/server/graphql/VesselImportGraphQLService.java +++ b/sumaris-importation/src/main/java/net/sumaris/importation/server/graphql/VesselImportGraphQLService.java @@ -22,6 +22,7 @@ package net.sumaris.importation.server.graphql; +import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Preconditions; import io.leangen.graphql.annotations.GraphQLArgument; import io.leangen.graphql.annotations.GraphQLQuery; @@ -35,6 +36,7 @@ import net.sumaris.core.vo.technical.job.JobVO; import net.sumaris.importation.core.service.vessel.SiopVesselImportService; import net.sumaris.importation.core.service.vessel.vo.SiopVesselImportContextVO; +import net.sumaris.importation.core.service.vessel.vo.SiopVesselImportResultVO; import net.sumaris.server.http.graphql.GraphQLApi; import net.sumaris.server.security.IFileController; import net.sumaris.server.security.ISecurityContext; @@ -94,7 +96,8 @@ public JobVO importSiopVessels(@GraphQLArgument(name = "fileName") String fileNa .build(); // Execute importJob by JobService (async) - return jobExecutionService.run(importJob, (job) -> siopVesselImportService.asyncImportFromFile(context, job)); + return jobExecutionService.run(importJob, () -> context, + (progression) -> siopVesselImportService.asyncImportFromFile(context, progression)); } } diff --git a/sumaris-rdf/pom.xml b/sumaris-rdf/pom.xml index bac1988b08..3d42306abe 100644 --- a/sumaris-rdf/pom.xml +++ b/sumaris-rdf/pom.xml @@ -3,7 +3,7 @@ sumaris-pod net.sumaris - 2.0.6 + 2.1.0 4.0.0 diff --git a/sumaris-server/pom.xml b/sumaris-server/pom.xml index e65e1cfeee..2216398963 100644 --- a/sumaris-server/pom.xml +++ b/sumaris-server/pom.xml @@ -4,7 +4,7 @@ net.sumaris sumaris-pod - 2.0.6 + 2.1.0 sumaris-server diff --git a/sumaris-server/src/main/java/net/sumaris/server/config/SumarisServerConfiguration.java b/sumaris-server/src/main/java/net/sumaris/server/config/SumarisServerConfiguration.java index 58c61d94df..2a0c6a2222 100644 --- a/sumaris-server/src/main/java/net/sumaris/server/config/SumarisServerConfiguration.java +++ b/sumaris-server/src/main/java/net/sumaris/server/config/SumarisServerConfiguration.java @@ -22,14 +22,10 @@ package net.sumaris.server.config; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; import lombok.extern.slf4j.Slf4j; import net.sumaris.core.config.SumarisConfiguration; import net.sumaris.core.config.SumarisConfigurationOption; import net.sumaris.server.http.security.AuthTokenTypeEnum; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.compress.utils.Lists; import org.apache.commons.lang3.StringUtils; import org.nuiton.config.ApplicationConfig; import org.nuiton.version.Version; @@ -38,9 +34,7 @@ import java.io.File; import java.util.List; -import java.util.Objects; import java.util.TimeZone; -import java.util.stream.Collectors; /** *

SumarisServerConfiguration class.

@@ -97,41 +91,6 @@ public SumarisServerConfiguration(ConfigurableEnvironment env, String... args) { super(env, args); } - public List getConfigurationOptionAsNumbers(String optionKey) { - List result = (List) complexOptionsCache.getIfPresent(optionKey); - - // Not exists in cache - if (result == null) { - String ids = applicationConfig.getOption(optionKey); - if (StringUtils.isBlank(ids)) { - result = ImmutableList.of(); - } else { - final List invalidIds = Lists.newArrayList(); - result = Splitter.on(",").omitEmptyStrings().trimResults() - .splitToList(ids) - .stream() - .map(id -> { - try { - return Integer.parseInt(id); - } catch (Exception e) { - invalidIds.add(id); - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(invalidIds)) { - log.error("Skipping invalid values found in configuration option '{}': {}", optionKey, invalidIds); - } - } - - // Add to cache - complexOptionsCache.put(optionKey, result); - } - return result; - } - public List getAccessNotSelfDataDepartmentIds() { return getConfigurationOptionAsNumbers(SumarisServerConfigurationOption.ACCESS_NOT_SELF_DATA_DEPARTMENT_IDS.getKey()); } diff --git a/sumaris-server/src/main/java/net/sumaris/server/http/graphql/referential/ReferentialGraphQLService.java b/sumaris-server/src/main/java/net/sumaris/server/http/graphql/referential/ReferentialGraphQLService.java index 776fb815d1..033a0985ad 100644 --- a/sumaris-server/src/main/java/net/sumaris/server/http/graphql/referential/ReferentialGraphQLService.java +++ b/sumaris-server/src/main/java/net/sumaris/server/http/graphql/referential/ReferentialGraphQLService.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import io.leangen.graphql.annotations.*; +import io.leangen.graphql.execution.ResolutionEnvironment; import io.reactivex.rxjava3.core.BackpressureStrategy; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -40,6 +41,7 @@ import net.sumaris.core.vo.referential.*; import net.sumaris.server.http.graphql.GraphQLApi; import net.sumaris.server.http.graphql.GraphQLHelper; +import net.sumaris.server.http.graphql.GraphQLUtils; import net.sumaris.server.http.security.AuthService; import net.sumaris.server.http.security.IsAdmin; import net.sumaris.server.http.security.IsUser; @@ -52,6 +54,7 @@ import java.util.Date; import java.util.List; +import java.util.Set; @Service @GraphQLApi @@ -94,7 +97,10 @@ public List findReferentialsByFilter( @GraphQLArgument(name = "offset", defaultValue = "0") Integer offset, @GraphQLArgument(name = "size", defaultValue = "1000") Integer size, @GraphQLArgument(name = "sortBy", defaultValue = ReferentialVO.Fields.LABEL) String sort, - @GraphQLArgument(name = "sortDirection", defaultValue = "asc") String direction) { + @GraphQLArgument(name = "sortDirection", defaultValue = "asc") String direction, + @GraphQLEnvironment() ResolutionEnvironment env) { + + // Metier: special case to be able to sort on join attribute (e.g. taxonGroup) if (Metier.class.getSimpleName().equalsIgnoreCase(entityName)) { @@ -109,12 +115,18 @@ public List findReferentialsByFilter( restrictProgramFilter(entityName, filter); } + Set fields = GraphQLUtils.fields(env); + return referentialService.findByFilter(entityName, ReferentialFilterVO.nullToEmpty(filter), offset == null ? 0 : offset, size == null ? 1000 : size, sort == null ? ReferentialVO.Fields.LABEL : sort, - SortDirection.fromString(direction, SortDirection.ASC)); + SortDirection.fromString(direction, SortDirection.ASC), + ReferentialFetchOptions.builder() + .withProperties(fields.contains(ReferentialVO.Fields.PROPERTIES)) + .build() + ); } @GraphQLQuery(name = "referentialsCount", description = "Get referentials count") diff --git a/sumaris-server/src/main/java/net/sumaris/server/service/administration/AccountServiceImpl.java b/sumaris-server/src/main/java/net/sumaris/server/service/administration/AccountServiceImpl.java index ec2764bb31..f1108696ea 100644 --- a/sumaris-server/src/main/java/net/sumaris/server/service/administration/AccountServiceImpl.java +++ b/sumaris-server/src/main/java/net/sumaris/server/service/administration/AccountServiceImpl.java @@ -54,7 +54,6 @@ import org.apache.commons.collections.CollectionUtils; import org.nuiton.i18n.I18n; import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.event.EventListener; import org.springframework.core.convert.support.GenericConversionService; import org.springframework.stereotype.Service; @@ -73,19 +72,14 @@ public class AccountServiceImpl implements AccountService { private final SumarisServerConfiguration configuration; private final PersonRepository personRepository; - private final UserSettingsService userSettingsService; - private final UserTokenRepository userTokenRepository; private final PersonService personService; private final UserMessageService userMessageService; private final ServerCryptoService serverCryptoService; - private AccountService self; // loop back to force transactional handling - private String serverUrl; - @Autowired public AccountServiceImpl(SumarisServerConfiguration serverConfiguration, PersonService personService, PersonRepository personRepository, @@ -96,7 +90,7 @@ public AccountServiceImpl(SumarisServerConfiguration serverConfiguration, UserMessageService userMessageService) { this.personService = personService; this.personRepository = personRepository; - this.userSettingsService =userSettingsService; + this.userSettingsService = userSettingsService; this.userTokenRepository = userTokenRepository; this.configuration = serverConfiguration; this.serverCryptoService = serverCryptoService; diff --git a/sumaris-test-shared/pom.xml b/sumaris-test-shared/pom.xml index 26272dd8a1..618ae4cb3d 100644 --- a/sumaris-test-shared/pom.xml +++ b/sumaris-test-shared/pom.xml @@ -4,7 +4,7 @@ net.sumaris sumaris-pod - 2.0.6 + 2.1.0 sumaris-test-shared diff --git a/sumaris-test-shared/src/main/java/net/sumaris/core/test/InitTests.java b/sumaris-test-shared/src/main/java/net/sumaris/core/test/InitTests.java index 7c46319b1d..09747dd01d 100644 --- a/sumaris-test-shared/src/main/java/net/sumaris/core/test/InitTests.java +++ b/sumaris-test-shared/src/main/java/net/sumaris/core/test/InitTests.java @@ -182,7 +182,7 @@ protected void before() throws Throwable { initServiceLocator(); boolean isFileDatabase = Daos.isFileDatabase(config.getJdbcURL()); - boolean needSchemaUpdate = true; + boolean needSchemaUpdate = config.isLiquibaseEnabled(); if (isFileDatabase) { log.info("Init test data in database... [" + config.getJdbcURL() + "]"); diff --git a/sumaris-test-shared/src/main/resources/data-hsqldb-01-common.xml b/sumaris-test-shared/src/main/resources/data-hsqldb-01-common.xml index 0633965bfe..2087ef1ee7 100644 --- a/sumaris-test-shared/src/main/resources/data-hsqldb-01-common.xml +++ b/sumaris-test-shared/src/main/resources/data-hsqldb-01-common.xml @@ -118,6 +118,7 @@ + @@ -138,6 +139,7 @@ + @@ -173,6 +175,8 @@ + + @@ -183,6 +187,7 @@ + @@ -247,6 +252,11 @@ + + + + + @@ -275,6 +285,8 @@ + + @@ -1053,7 +1065,7 @@ - + @@ -1307,7 +1319,7 @@ - + @@ -1765,7 +1777,7 @@ - + @@ -1834,7 +1846,8 @@ - + + @@ -1957,44 +1970,88 @@ CONVERSION_COEFFICIENT_A="0.00000081" CONVERSION_COEFFICIENT_B="2.97" DESCRIPTION="OBSMER Conan 1978"/> + + + + - + + - - - - - + + - - + + + + + + + + @@ -2060,6 +2117,8 @@ + + diff --git a/sumaris-test-shared/src/main/resources/data-hsqldb-02-program-ADAP.xml b/sumaris-test-shared/src/main/resources/data-hsqldb-02-program-ADAP.xml index f818b8a849..ef293acdc3 100644 --- a/sumaris-test-shared/src/main/resources/data-hsqldb-02-program-ADAP.xml +++ b/sumaris-test-shared/src/main/resources/data-hsqldb-02-program-ADAP.xml @@ -58,10 +58,10 @@ - + - + diff --git a/sumaris-test-shared/src/main/resources/data-hsqldb-03-data.xml b/sumaris-test-shared/src/main/resources/data-hsqldb-03-data.xml index 0f0e085d4c..e5826e41a2 100644 --- a/sumaris-test-shared/src/main/resources/data-hsqldb-03-data.xml +++ b/sumaris-test-shared/src/main/resources/data-hsqldb-03-data.xml @@ -183,7 +183,7 @@ - + @@ -499,25 +499,74 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -556,18 +605,19 @@ + + + + - - @@ -576,8 +626,68 @@ + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +