Refactor AndroidString classes to be able to apply proper and consist…

…ent escaping. This also greatly simplify the logic
box · Sep 30, 2024 · c621606 · c621606
1 parent bb5a999
commit c621606
Show file tree

Hide file tree

Showing 21 changed files with 203 additions and 185 deletions.
diff --git a/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml b/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rCA/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals name="plural_recipe_cook_hours">
     <item formatted="false" quantity="one">%1$,d heure à cuisiner</item>

diff --git a/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml b/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-fr-rFR/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals name="plural_recipe_cook_hours">
     <item formatted="false" quantity="one">%1$,d heure à cuisiner</item>

diff --git a/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml b/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-ja-rJP/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals name="plural_recipe_cook_hours">
     <item formatted="false" quantity="other">所要時間：%1$,d 時間</item>

diff --git a/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml b/...portAndroidStringsPostProcessing/expected/removeDescription/res/values-ru-rRU/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals name="plural_recipe_cook_hours">
     <item formatted="false" quantity="one">Приготовление: %1$,d час</item>

diff --git a/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml b/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rCA/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <!-- Header for the recipe ingredients with the ingredients countsection -->
   <!-- Delete a piece of content -->

diff --git a/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml b/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-fr-rFR/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals description="The singular and plural form for Recipe Cook Time in hours" name="plural_recipe_cook_hours">
     <item formatted="false" quantity="one">%1$,d heure à cuisiner</item>

diff --git a/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml b/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ja-rJP/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals description="The singular and plural form for Recipe Cook Time in hours" name="plural_recipe_cook_hours">
     <item formatted="false" quantity="other">所要時間：%1$,d 時間</item>

diff --git a/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml b/...ortAndroidStringsPostProcessing/expected/removeUntranslated/res/values-ru-rRU/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals description="The singular and plural form for Recipe Cook Time in hours" name="plural_recipe_cook_hours">
     <item formatted="false" quantity="one">Приготовление: %1$,d час</item>

diff --git a/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml b/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rCA/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <!-- Header for the recipe ingredients with the ingredients countsection -->
   <!-- Delete a piece of content -->

diff --git a/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml b/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-fr-rFR/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals name="plural_recipe_cook_hours">
     <item formatted="false" quantity="one">%1$,d heure à cuisiner</item>

diff --git a/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml b/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ja-rJP/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals name="plural_recipe_cook_hours">
     <item formatted="false" quantity="other">所要時間：%1$,d 時間</item>

diff --git a/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml b/...ngsPostProcessing/expected/removeUntranslatedAndDescription/res/values-ru-rRU/strings.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <resources>
   <plurals name="plural_recipe_cook_hours">
     <item formatted="false" quantity="one">Приготовление: %1$,d час</item>

diff --git a/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java b/common/src/main/java/com/box/l10n/mojito/okapi/filters/AndroidFilter.java
@@ -466,7 +466,7 @@ public String execute(String xmlContent) {
 
       if (xmlContent == null
           || xmlContent.isBlank()
-          || (!shouldApplyPostProcessingRemoveUntranslatedExcluded && !removeTranslatableFalse)) {
+          || (!shouldApplyPostProcessingRemoveUntranslatedExcluded && !hasRemoveUntranslated())) {
         return xmlContent;
       }
 
@@ -553,6 +553,11 @@ public String execute(String xmlContent) {
         StreamResult streamResult = new StreamResult(new StringWriter());
         transformer.transform(domSource, streamResult);
         String processedXmlContent = streamResult.getWriter().toString();
+
+        if (!hasStandalone) {
+          processedXmlContent = processedXmlContent.replace(" standalone=\"no\"", "");
+        }
+
         return processedXmlContent;
 
       } catch (ParserConfigurationException | SAXException | IOException | TransformerException e) {

diff --git a/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java b/common/src/test/java/com/box/l10n/mojito/okapi/filters/AndroidFilterTest.java
@@ -138,7 +138,7 @@ public void testPostProcessingKeepDescription() {
     String output = androidFilePostProcessor.execute(input);
     String expected =
         """
-            <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+            <?xml version="1.0" encoding="UTF-8"?>
             <resources>
               <string description="a description" name="pinterest2">somestring to keep</string>
               <!--testing plural-->
@@ -182,7 +182,7 @@ public void testPostProcessingRemoveDescription() {
     String output = androidFilePostProcessor.execute(input);
     String expected =
         """
-            <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+            <?xml version="1.0" encoding="UTF-8"?>
             <resources>
               <string name="pinterest2">somestring to keep</string>
               <!--testing plural-->
@@ -257,7 +257,7 @@ public void testPostProcessingRemoveTranslatableFalse() {
     String output = androidFilePostProcessor.execute(input);
     String expected =
         """
-             <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+             <?xml version="1.0" encoding="UTF-8"?>
              <resources>
                <!--testing plural-->
                <plurals name="pins2">
@@ -267,4 +267,82 @@ public void testPostProcessingRemoveTranslatableFalse() {
              """;
     assertEquals(expected, output);
   }
+
+  @Test
+  public void testPostProcessingStandaloneNo() {
+    AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor =
+        new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false);
+    String input =
+        """
+                <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+                <resources>
+                    <string name="pinterest2" description="a description" translatable="false">somestring to keep</string>
+                    <string name="pinterest">@#$untranslated$#@</string>
+                    <!--testing plural-->
+                    <plurals name="pins">
+                        <item quantity="one">@#$untranslated$#@</item>
+                        <item quantity="other">@#$untranslated$#@</item>
+                    </plurals>
+                    <plurals description="testing plural attr" name="pins2">
+                        <item quantity="one">translated</item>
+                        <item quantity="other">@#$untranslated$#@</item>
+                    </plurals>
+                    <plurals name="pins3" translatable="false">
+                        <item quantity="one">pin fr</item>
+                        <item quantity="other">pins fr</item>
+                    </plurals>
+                </resources>
+                """;
+    String output = androidFilePostProcessor.execute(input);
+    String expected =
+        """
+                 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+                 <resources>
+                   <!--testing plural-->
+                   <plurals name="pins2">
+                     <item quantity="one">translated</item>
+                   </plurals>
+                 </resources>
+                 """;
+    assertEquals(expected, output);
+  }
+
+  @Test
+  public void testPostProcessingStandaloneYes() {
+    AndroidFilter.AndroidFilePostProcessor androidFilePostProcessor =
+        new AndroidFilter.AndroidFilePostProcessor(true, true, 2, true, false);
+    String input =
+        """
+                <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+                <resources>
+                    <string name="pinterest2" description="a description" translatable="false">somestring to keep</string>
+                    <string name="pinterest">@#$untranslated$#@</string>
+                    <!--testing plural-->
+                    <plurals name="pins">
+                        <item quantity="one">@#$untranslated$#@</item>
+                        <item quantity="other">@#$untranslated$#@</item>
+                    </plurals>
+                    <plurals description="testing plural attr" name="pins2">
+                        <item quantity="one">translated</item>
+                        <item quantity="other">@#$untranslated$#@</item>
+                    </plurals>
+                    <plurals name="pins3" translatable="false">
+                        <item quantity="one">pin fr</item>
+                        <item quantity="other">pins fr</item>
+                    </plurals>
+                </resources>
+                """;
+    String output = androidFilePostProcessor.execute(input);
+    String expected =
+        """
+                 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+                 <resources>
+                   <!--testing plural-->
+                   <plurals name="pins2">
+                     <item quantity="one">translated</item>
+                   </plurals>
+                 </resources>
+                 """;
+    assertEquals(expected, output);
+  }
 }
diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocument.java
@@ -1,10 +1,7 @@
 package com.box.l10n.mojito.android.strings;
 
-import static com.google.common.base.Preconditions.checkArgument;
-
 import java.util.ArrayList;
 import java.util.List;
-import org.w3c.dom.Node;
 
 public class AndroidStringDocument {
 
@@ -29,46 +26,8 @@ public void addSingular(AndroidSingular string) {
     strings.add(string);
   }
 
-  public void addSingular(AndroidStringElement element, Node comment) {
-
-    checkArgument(element.isSingular(), "element should be singular");
-
-    addSingular(
-        new AndroidSingular(
-            element.getIdAttribute(),
-            element.getNameAttribute(),
-            element.getUnescapedContent(),
-            comment != null ? comment.getTextContent() : null));
-  }
-
   public void addPlural(AndroidPlural plural) {
     plurals.add(plural);
     strings.add(plural);
   }
-
-  public void addPlural(AndroidStringElement element, Node comment) {
-
-    checkArgument(element.isPlural(), "element should be plural");
-
-    AndroidPlural.AndroidPluralBuilder builder = AndroidPlural.builder();
-    builder.setName(element.getNameAttribute());
-    if (comment != null) {
-      builder.setComment(comment.getTextContent());
-    }
-
-    element.forEachPluralItem(builder::addItem);
-
-    addPlural(builder.build());
-  }
-
-  public void addElement(Node node, Node comment) {
-
-    AndroidStringElement element = new AndroidStringElement(node);
-
-    if (element.isSingular()) {
-      addSingular(element, comment);
-    } else if (element.isPlural()) {
-      addPlural(element, comment);
-    }
-  }
 }
diff --git a/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java b/webapp/src/main/java/com/box/l10n/mojito/android/strings/AndroidStringDocumentMapper.java
@@ -15,7 +15,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
-import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -202,7 +201,7 @@ Stream<TextUnitDTO> singularToTextUnit(AndroidSingular singular) {
     textUnit.setName(singular.getName());
     textUnit.setComment(singular.getComment());
     textUnit.setTmTextUnitId(singular.getId());
-    textUnit.setTarget(unescape(singular.getContent()));
+    textUnit.setTarget(singular.getContent());
     addTextUnitDTOAttributes(textUnit);
 
     return Stream.of(textUnit);
@@ -226,7 +225,7 @@ Stream<TextUnitDTO> pluralToTextUnits(AndroidPlural plural) {
               textUnit.setTmTextUnitId(item.getId());
               textUnit.setPluralForm(quantity);
               textUnit.setPluralFormOther(pluralFormOther);
-              textUnit.setTarget(unescape(item.getContent()));
+              textUnit.setTarget(item.getContent());
               addTextUnitDTOAttributes(textUnit);
 
               return textUnit;
@@ -286,23 +285,4 @@ static String removeInvalidControlCharacter(String str) {
 
     return withoutControlCharacters;
   }
-
-  /** should use {@link com.box.l10n.mojito.okapi.filters.AndroidFilter#unescape(String)} */
-  static String unescape(String str) {
-
-    String unescape = str;
-
-    if (StringUtils.startsWith(unescape, "\"") && StringUtils.endsWith(unescape, "\"")) {
-      unescape = unescape.substring(1, unescape.length() - 1);
-    }
-
-    unescape =
-        Strings.nullToEmpty(unescape)
-            .replaceAll("\\\\'", "'")
-            .replaceAll("\\\\\"", "\"")
-            .replaceAll("\\\\@", "@")
-            .replaceAll("\\\\n", "\n");
-
-    return unescape;
-  }
 }