Add an option to remove suffix from keys in JSON format

box · May 31, 2024 · 54b7b72 · 54b7b72
1 parent 688b652
commit 54b7b72
Show file tree

Hide file tree

Showing 11 changed files with 218 additions and 112 deletions.
diff --git a/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java b/cli/src/main/java/com/box/l10n/mojito/cli/command/ImportLocalizedAssetCommand.java
@@ -22,8 +22,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.stream.Collectors;
-
 import org.fusesource.jansi.Ansi;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -71,7 +69,7 @@ public class ImportLocalizedAssetCommand extends Command {
       names = {Param.REPOSITORY_LOCALES_MAPPING_LONG, Param.REPOSITORY_LOCALES_MAPPING_SHORT},
       arity = 1,
       required = false,
-      description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION) // TODO(ja) this is a wrong description, keys and avlues are swapped, the documentation is wrong and can't be shared with pull for exmaple
+      description = Param.REPOSITORY_LOCALES_MAPPING_DESCRIPTION)
   String localeMappingParam;
 
   @Parameter(
@@ -133,12 +131,11 @@ public class ImportLocalizedAssetCommand extends Command {
   boolean continueOnError = false;
 
   @Parameter(
-          names = {"--apply-integrity-checks"},
-          arity = 0,
-          description = "Apply integrity checks")
+      names = {"--apply-integrity-checks"},
+      arity = 0,
+      description = "Apply integrity checks")
   boolean applyIntegrityChecks = false;
 
-
   @Autowired AssetClient assetClient;
 
   @Autowired LocaleClient localeClient;
@@ -179,29 +176,36 @@ public void execute() throws CommandException {
             directoriesIncludePatterns,
             directoriesExcludePatterns)) {
 
-      List<ImportLocalizedAssetBody> list = getLocalesForImport().stream().map(locale -> doImportFileMatch(sourceFileMatch, locale)).filter(Objects::nonNull).toList();
-
-      list.forEach(importLocalizedAssetForContent -> {
-        try {
-          commandHelper.waitForPollableTask(importLocalizedAssetForContent.getPollableTask().getId());
-        } catch (CommandException e) {
-          if (continueOnError) {
-            consoleWriter
+      List<ImportLocalizedAssetBody> list =
+          getLocalesForImport().stream()
+              .map(locale -> doImportFileMatch(sourceFileMatch, locale))
+              .filter(Objects::nonNull)
+              .toList();
+
+      list.forEach(
+          importLocalizedAssetForContent -> {
+            try {
+              commandHelper.waitForPollableTask(
+                  importLocalizedAssetForContent.getPollableTask().getId());
+            } catch (CommandException e) {
+              if (continueOnError) {
+                consoleWriter
                     .a("   Error while importing: ")
                     .fg(Ansi.Color.RED)
                     .a(sourceFileMatch.getPath().toString())
                     .println();
-          } else {
-            throw e;
-          }
-        }
-      });
+              } else {
+                throw e;
+              }
+            }
+          });
     }
 
     consoleWriter.fg(Ansi.Color.GREEN).newLine().a("Finished").println(2);
   }
 
-  protected ImportLocalizedAssetBody doImportFileMatch(FileMatch fileMatch, Locale locale) throws CommandException {
+  protected ImportLocalizedAssetBody doImportFileMatch(FileMatch fileMatch, Locale locale)
+      throws CommandException {
     try {
       logger.info("Importing for locale: {}", locale.getBcp47Tag());
       Path targetPath = getTargetPath(fileMatch, locale);
@@ -222,33 +226,30 @@ protected ImportLocalizedAssetBody doImportFileMatch(FileMatch fileMatch, Locale
         if (continueOnError) {
           if (!commandHelper.getFileContent(fileMatch.getPath()).isBlank()) {
             consoleWriter
-                    .a("   Missing file, skipping: ")
-                    .fg(Ansi.Color.YELLOW)
-                    .a(targetPath.toString())
-                    .println();
+                .a("   Missing file, skipping: ")
+                .fg(Ansi.Color.YELLOW)
+                .a(targetPath.toString())
+                .println();
           }
           return null;
         } else {
           throw e;
         }
       }
-      return
-
-          assetClient.importLocalizedAssetForContent(
-              assetByPathAndRepositoryId.getId(),
-              locale.getId(),
-              fileContent,
-              statusForEqualTarget,
-              fileMatch.getFileType().getFilterConfigIdOverride(),
-              commandHelper.getFilterOptionsOrDefaults(
-                  fileMatch.getFileType(), filterOptionsParam));
+      return assetClient.importLocalizedAssetForContent(
+          assetByPathAndRepositoryId.getId(),
+          locale.getId(),
+          fileContent,
+          statusForEqualTarget,
+          fileMatch.getFileType().getFilterConfigIdOverride(),
+          commandHelper.getFilterOptionsOrDefaults(fileMatch.getFileType(), filterOptionsParam));
     } catch (AssetNotFoundException ex) {
       if (continueOnError) {
         consoleWriter
-                .a("   Asset not found: ")
-                .fg(Ansi.Color.YELLOW)
-                .a(fileMatch.getPath().toString())
-                .println();
+            .a("   Asset not found: ")
+            .fg(Ansi.Color.YELLOW)
+            .a(fileMatch.getPath().toString())
+            .println();
       } else {
         throw new CommandException(
             "No asset for file [" + fileMatch.getPath() + "] into repo [" + repositoryParam + "]",

diff --git a/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java b/common/src/main/java/com/box/l10n/mojito/LocaleMappingHelper.java
@@ -32,6 +32,8 @@ public Map<String, String> getLocaleMapping(String localeMapppingParam) {
    */
   public Map<String, String> getInverseLocaleMapping(String localeMapppingParam) {
 
+    // TODO(ja) that needs to be a multimap because multiple localized files can be imported for the
+    // same locale
     Map<String, String> inverseLocaleMapping = null;
 
     if (localeMapppingParam != null) {

diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/JSONFilter.java
@@ -48,6 +48,18 @@ public class JSONFilter extends net.sf.okapi.filters.json.JSONFilter {
    */
   boolean noteKeepOrReplace = false;
 
+  /**
+   * Remove the suffix from the key.
+   *
+   * <p>Typically useful for FormatJS:
+   *
+   * <p>"text-unit-name": { "defaultMessage": "example", "description": "example description" }
+   *
+   * <p>text unit name would be text-unit-name/defaultMessage. With removeKeySuffix set, the
+   * "/defaultMessage" can be removed.
+   */
+  String removeKeySuffix = null;
+
   NoteAnnotation noteAnnotation;
   UsagesAnnotation usagesAnnotation;
   String currentKeyName;
@@ -110,6 +122,7 @@ void applyFilterOptions(RawDocument input) {
       filterOptions.getString("usagesKeyPattern", s -> usagesKeyPattern = Pattern.compile(s));
       filterOptions.getBoolean("noteKeepOrReplace", b -> noteKeepOrReplace = b);
       filterOptions.getBoolean("usagesKeepOrReplace", b -> usagesKeepOrReplace = b);
+      filterOptions.getString("removeKeySuffix", s -> removeKeySuffix = s);
       filterOptions.getBoolean(
           "convertToHtmlCodes",
           b -> {
@@ -135,6 +148,7 @@ public void handleKey(String key, JsonValueTypes valueType, JsonKeyTypes keyType
 
   @Override
   public void handleValue(String value, JsonValueTypes valueType) {
+    removeKeySuffixIfMatch(value);
     extractNoteIfMatch(value);
     extractUsageIfMatch(value);
     super.handleValue(value, valueType);
@@ -170,6 +184,15 @@ void addNote(String value) {
     noteAnnotation.add(note);
   }
 
+  void removeKeySuffixIfMatch(String key) {
+    if (removeKeySuffix != null) {
+      if (key.endsWith(removeKeySuffix)) {
+        key = key.substring(0, key.length() - removeKeySuffix.length());
+        logger.debug("Remove suffix from key: {}", key);
+      }
+    }
+  }
+
   void extractNoteIfMatch(String value) {
     if (noteKeyPattern != null) {
       Matcher m = noteKeyPattern.matcher(currentKeyName);

diff --git a/...va/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java b/...va/com/box/l10n/mojito/rest/resttemplate/FormLoginAuthenticationCsrfTokenInterceptor.java
@@ -44,7 +44,9 @@ public class FormLoginAuthenticationCsrfTokenInterceptor implements ClientHttpRe
 
   public static final String CSRF_PARAM_NAME = "_csrf";
   public static final String CSRF_HEADER_NAME = "X-CSRF-TOKEN";
-  public static final String COOKIE_SESSION_NAME = "SESSION"; // ""JSESSIONID"; // TODO(ja) must be a change in the server we're now setting SESSION - I sort of remember toying server confi with that at some point
+  public static final String COOKIE_SESSION_NAME =
+      "SESSION"; // ""JSESSIONID"; // TODO(ja) must be a change in the server we're now setting
+  // SESSION - I sort of remember toying server confi with that at some point
 
   @Autowired FormLoginConfig formLoginConfig;
 
@@ -123,9 +125,9 @@ public ClientHttpResponse intercept(
    * @return
    * @throws IOException
    */
-static  AtomicInteger countFound = new AtomicInteger(0);
-static AtomicInteger countNotFound = new AtomicInteger(0);
+  static AtomicInteger countFound = new AtomicInteger(0);
 
+  static AtomicInteger countNotFound = new AtomicInteger(0);
 
   @Override
   public ClientHttpResponse intercept(
@@ -138,8 +140,9 @@ public ClientHttpResponse intercept(
       startAuthenticationAndInjectCsrfToken(request);
     }
 
-    if (countNotFound.get() > 5 && countFound.get() == 0){
-      logger.error("Session is never found, check the server / client configuration SESSION/JSESSIONID");
+    if (countNotFound.get() > 5 && countFound.get() == 0) {
+      logger.error(
+          "Session is never found, check the server / client configuration SESSION/JSESSIONID");
     }
 
     ClientHttpResponse clientHttpResponse = execution.execute(request, body);

diff --git a/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java b/webapp/src/main/java/com/box/l10n/mojito/entity/Drop.java
@@ -35,8 +35,8 @@
     attributeNodes = {
       @NamedAttributeNode(value = "repository", subgraph = "Drop.legacy.repository"),
       @NamedAttributeNode(value = "importPollableTask", subgraph = "Drop.legacy.subTask"),
-            @NamedAttributeNode(value = "exportPollableTask", subgraph = "Drop.legacy.subTask"),
-            @NamedAttributeNode(value = "translationKits", subgraph = "Drop.legacy.translationKits")
+      @NamedAttributeNode(value = "exportPollableTask", subgraph = "Drop.legacy.subTask"),
+      @NamedAttributeNode(value = "translationKits", subgraph = "Drop.legacy.translationKits")
     },
     subgraphs = {
       @NamedSubgraph(
@@ -45,9 +45,9 @@
       @NamedSubgraph(
           name = "Drop.legacy.repository",
           attributeNodes = {@NamedAttributeNode("createdByUser")}),
-        @NamedSubgraph(
-            name = "Drop.legacy.translationKits",
-            attributeNodes = {@NamedAttributeNode("locale")})
+      @NamedSubgraph(
+          name = "Drop.legacy.translationKits",
+          attributeNodes = {@NamedAttributeNode("locale")})
     })
 public class Drop extends AuditableEntity {
 

diff --git a/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java b/webapp/src/main/java/com/box/l10n/mojito/okapi/ImportTranslationsFromLocalizedAssetStep.java
@@ -159,8 +159,8 @@ protected TMTextUnitVariant importTextUnit(
 
         tmTextUnitVariantCommentAnnotation.setMessage(integrityCheckException.getMessage());
 
-        tmTextUnitVariantCommentAnnotation.setSeverity(TMTextUnitVariantComment.Severity.ERROR); // dial it down for plural strings?
-
+        tmTextUnitVariantCommentAnnotation.setSeverity(
+            TMTextUnitVariantComment.Severity.ERROR); // dial it down for plural strings?
 
         new TMTextUnitVariantCommentAnnotations(target)
             .addAnnotation(tmTextUnitVariantCommentAnnotation);

diff --git a/...n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java b/...n/mojito/service/assetintegritychecker/integritychecker/MarkdownLinkIntegrityChecker.java
@@ -1,34 +1,31 @@
 package com.box.l10n.mojito.service.assetintegritychecker.integritychecker;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.util.LinkedHashSet;
 import java.util.Set;
 import java.util.regex.Matcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class MarkdownLinkIntegrityChecker extends RegexIntegrityChecker {
 
-    /**
-     * logger
-     */
-    static Logger logger = LoggerFactory.getLogger(MarkdownLinkIntegrityChecker.class);
+  /** logger */
+  static Logger logger = LoggerFactory.getLogger(MarkdownLinkIntegrityChecker.class);
 
-    @Override
-    public String getRegex() {
-        return "\\[(?<text>.+?)]\\((?<url>.+?)\\)";
-    }
+  @Override
+  public String getRegex() {
+    return "\\[(?<text>.+?)]\\((?<url>.+?)\\)";
+  }
 
-    @Override
-    Set<String> getPlaceholders(String string) {
-        Set<String> placeholders = new LinkedHashSet<>();
+  @Override
+  Set<String> getPlaceholders(String string) {
+    Set<String> placeholders = new LinkedHashSet<>();
 
-        if (string != null) {
-            Matcher matcher = getPattern().matcher(string);
-            while (matcher.find()) {
-                placeholders.add("[%s](%s)".formatted("--translatable--", matcher.group("url")));
-            }
-        }
-        return placeholders;
+    if (string != null) {
+      Matcher matcher = getPattern().matcher(string);
+      while (matcher.find()) {
+        placeholders.add("[%s](%s)".formatted("--translatable--", matcher.group("url")));
+      }
     }
+    return placeholders;
+  }
 }
diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java b/webapp/src/main/java/com/box/l10n/mojito/service/drop/DropRepository.java
@@ -1,13 +1,9 @@
 package com.box.l10n.mojito.service.drop;
 
 import com.box.l10n.mojito.entity.Drop;
-
-import java.util.List;
 import java.util.Optional;
-
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
-import org.springframework.data.domain.Sort;
 import org.springframework.data.jpa.domain.Specification;
 import org.springframework.data.jpa.repository.EntityGraph;
 import org.springframework.data.jpa.repository.EntityGraph.EntityGraphType;

diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMService.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/TMService.java
@@ -1244,8 +1244,8 @@ public void importLocalizedAsset(
     driver.addStep(new CheckForDoNotTranslateStep());
 
     //    driver.addStep(getConfiguredQualityStep());
-//    IntegrityCheckStep integrityCheckStep = new IntegrityCheckStep();
-//    driver.addStep(integrityCheckStep);
+    //    IntegrityCheckStep integrityCheckStep = new IntegrityCheckStep();
+    //    driver.addStep(integrityCheckStep);
 
     driver.addStep(
         new ImportTranslationsFromLocalizedAssetStep(