diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java index 9379d2bef..598e3e658 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSPhrase.java @@ -154,13 +154,32 @@ public void push( String text = getFileContent(pluralSeparator, search, true, null); String tagForUpload = getTagForUpload(repository.getName()); - phraseClient.uploadAndWait( - projectId, - repository.getSourceLocale().getBcp47Tag(), - "xml", - repository.getName() + "-strings.xml", - text, - ImmutableList.of(tagForUpload)); + + OptionsParser optionsParser = new OptionsParser(options); + Boolean tmpUpload = optionsParser.getBoolean("nativeUpload", false); + Boolean escape = optionsParser.getBoolean("escape", true); + + // dfsdfaf=dfdsfds,dsafadsfs=dfasdff to map + + if (tmpUpload) { + phraseClient.nativeUploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload), + null); + } else { + phraseClient.uploadAndWait( + projectId, + repository.getSourceLocale().getBcp47Tag(), + "xml", + repository.getName() + "-strings.xml", + text, + ImmutableList.of(tagForUpload), + null); + } removeUnusedKeysAndTags(projectId, repository.getName(), tagForUpload); } @@ -472,6 +491,7 @@ public void pushTranslations( "xml", repository.getName() + "-strings.xml", fileContent, + null, null); } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java index ddf6181be..4bb365cdf 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClient.java @@ -5,6 +5,7 @@ import static com.box.l10n.mojito.io.Files.write; import com.box.l10n.mojito.json.ObjectMapper; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableSet; import com.phrase.client.ApiClient; import com.phrase.client.ApiException; @@ -12,10 +13,17 @@ import com.phrase.client.api.LocalesApi; import com.phrase.client.api.TagsApi; import com.phrase.client.api.UploadsApi; +import com.phrase.client.auth.ApiKeyAuth; import com.phrase.client.model.Tag; import com.phrase.client.model.TranslationKey; import com.phrase.client.model.Upload; import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -23,6 +31,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; @@ -48,16 +57,33 @@ public PhraseClient(ApiClient apiClient) { Retry.backoff(5, Duration.ofMillis(500)).maxBackoff(Duration.ofSeconds(5)); } + public Upload nativeUploadAndWait( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + String uploadId = + nativeUploadCreateFileWithRetry( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions); + return waitForUploadToFinish(projectId, uploadId); + } + public Upload uploadAndWait( String projectId, String localeId, String fileFormat, String fileName, String fileContent, - List tags) { + List tags, + String formatOptions) { String uploadId = - uploadCreateFile(projectId, localeId, fileFormat, fileName, fileContent, tags); + uploadCreateFile( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions); return waitForUploadToFinish(projectId, uploadId); } @@ -101,7 +127,8 @@ String uploadCreateFile( String fileFormat, String fileName, String fileContent, - List tags) { + List tags, + String formatOptions) { Path tmpWorkingDirectory = null; @@ -126,7 +153,8 @@ String uploadCreateFile( write(fileToUpload, fileContent); Upload upload = - uploadsApiUploadCreateWithRetry(projectId, localeId, fileFormat, tags, fileToUpload); + uploadsApiUploadCreateWithRetry( + projectId, localeId, fileFormat, tags, fileToUpload, formatOptions); return upload.getId(); } finally { @@ -136,8 +164,147 @@ String uploadCreateFile( } } + public String nativeUploadCreateFileWithRetry( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + logger.info( + "nativeUploadCreateFile: projectId: {}, localeId: {}, fileName: {}, tags: {}", + projectId, + localeId, + fileName, + tags); + + return Mono.fromCallable( + () -> + nativeUploadCreateFile( + projectId, localeId, fileFormat, fileName, fileContent, tags, formatOptions)) + .retryWhen( + retryBackoffSpec.doBeforeRetry( + doBeforeRetry -> + logAttempt( + doBeforeRetry.failure(), + "Retrying failed attempt to uploadCreate to Phrase, file: %s, project id: %s" + .formatted(fileName, projectId)))) + .doOnError( + throwable -> + rethrowExceptionWithLog( + throwable, + "Final error in UploadCreate from Phrase, file: %s, project id: %s" + .formatted(fileName, projectId))) + .block(); + } + + /** + * The official SDK does not support format_options properly, so adding a replacement method base + * on pure Java client + */ + public String nativeUploadCreateFile( + String projectId, + String localeId, + String fileFormat, + String fileName, + String fileContent, + List tags, + Map formatOptions) { + + String urlString = String.format("%s/projects/%s/uploads", apiClient.getBasePath(), projectId); + String boundary = UUID.randomUUID().toString(); + final String LINE_FEED = "\r\n"; + + StringBuilder multipartBody = new StringBuilder(); + + multipartBody.append("--").append(boundary).append(LINE_FEED); + multipartBody + .append("Content-Disposition: form-data; name=\"file\"; filename=\"") + .append(fileName) + .append("\"") + .append(LINE_FEED); + multipartBody.append("Content-Type: application/xml").append(LINE_FEED); + multipartBody.append(LINE_FEED); + multipartBody.append(fileContent).append(LINE_FEED); + + addFormField(multipartBody, boundary, "locale_id", localeId); + addFormField(multipartBody, boundary, "file_format", fileFormat); + addFormField(multipartBody, boundary, "update_translations", "true"); + addFormField(multipartBody, boundary, "update_descriptions", "true"); + + if (tags != null) { + String tagsString = String.join(",", tags); + addFormField(multipartBody, boundary, "tags", tagsString); + } + + if (formatOptions != null) { + for (Map.Entry e : formatOptions.entrySet()) { + addFormField( + multipartBody, boundary, "format_options[%s]".formatted(e.getKey()), e.getValue()); + } + } + multipartBody.append("--").append(boundary).append("--").append(LINE_FEED); + + String token = ((ApiKeyAuth) apiClient.getAuthentication("Token")).getApiKey(); + + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(urlString)) + .header("Authorization", "token " + token) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST( + HttpRequest.BodyPublishers.ofString( + multipartBody.toString(), StandardCharsets.UTF_8)) + .build(); + + HttpResponse response; + try (HttpClient client = HttpClient.newHttpClient()) { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + + int statusCode = response.statusCode(); + String responseBody = response.body(); + + if (statusCode == 201) { + JsonNode rootNode = new ObjectMapper().readTreeUnchecked(responseBody); + return rootNode.path("id").asText(); + } else { + throw new RuntimeException("Server returned status code " + statusCode + ": " + responseBody); + } + } + + /** + * Helper method to add a form field to the multipart body. + * + * @param builder The StringBuilder for the multipart body. + * @param boundary The boundary string. + * @param name The name of the form field. + * @param value The value of the form field. + */ + private static void addFormField( + StringBuilder builder, String boundary, String name, String value) { + String LINE_FEED = "\r\n"; + builder.append("--").append(boundary).append(LINE_FEED); + builder + .append("Content-Disposition: form-data; name=\"") + .append(name) + .append("\"") + .append(LINE_FEED); + builder.append(LINE_FEED); + builder.append(value).append(LINE_FEED); + } + Upload uploadsApiUploadCreateWithRetry( - String projectId, String localeId, String fileFormat, List tags, Path fileToUpload) { + String projectId, + String localeId, + String fileFormat, + List tags, + Path fileToUpload, + String formatOptions) { return Mono.fromCallable( () -> @@ -157,7 +324,7 @@ Upload uploadsApiUploadCreateWithRetry( null, null, null, - null, + formatOptions, null, null, null)) diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java index 2c70bdda0..5ed24ba05 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/phrase/PhraseClientTest.java @@ -6,7 +6,10 @@ import com.box.l10n.mojito.android.strings.AndroidStringDocumentWriter; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.phrase.client.model.Tag; +import com.phrase.client.model.Upload; +import java.io.IOException; import java.time.ZonedDateTime; import java.util.List; import java.util.Objects; @@ -41,6 +44,8 @@ public class PhraseClientTest { @Value("${test.phrase-client.projectId:}") String testProjectId; + @Autowired private PhraseClientConfig phraseClientConfig; + @Test public void testRemoveTag() { String tagForUpload = "push_2024_06_10_07_17_00_089_122"; @@ -88,57 +93,84 @@ public void testParallelDownload() { } @Test - public void test() { + public void test() throws IOException, InterruptedException { Assume.assumeNotNull(testProjectId); - for (int i = 0; i < 2; i++) { + for (int i = 1; i < 2; i++) { AndroidStringDocument source = new AndroidStringDocument(); source.addSingular( - new AndroidSingular(11L, i + "string1", "some link to a page", "test comment1")); + new AndroidSingular(11L, i + "-string1", "some link to a page", "test comment1")); source.addSingular( new AndroidSingular( 12L, - i + "string2", + i + "-string2", "some link to a page", "test comment2")); source.addSingular( new AndroidSingular( 13L, - i + "string3", + i + "-string3", "If that is your IP address click here to unblock it.", "test comment2")); source.addSingular( new AndroidSingular( 14L, - i + "string4", + i + "-string4", "If that is your IP address click here to unblock it.", "test comment2")); source.addSingular( new AndroidSingular( 15L, - i + "string5", + i + "-string5", "If that is your IP address click here to unblock it.", "test comment2")); source.addSingular( new AndroidSingular( 16L, - i + "string6", + i + "-string6", "visit your settings", "test comment2")); + source.addSingular( + new AndroidSingular(17L, i + "-string7", "a string\nwith return line", "test comment2")); + + source.addSingular( + new AndroidSingular( + 18L, i + "-string9", "a string\n\nwith two return line", "test comment2")); + + source.addSingular( + new AndroidSingular( + 19L, i + "-string10", "a string & and the & escape", "test comment2")); + + source.addSingular( + new AndroidSingular(20L, i + "-string11", "simple tag", "test comment2")); + String androidFile = new AndroidStringDocumentWriter(source, i % 2 == 0).toText(); System.out.println(androidFile); + Upload upload = + phraseClient.nativeUploadAndWait( + testProjectId, + "en", + "xml", + "strings.xml", + androidFile, + ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping"), + ImmutableMap.of("unescape_tags", "true")); + + // Upload upload = + // phraseClient.uploadAndWait( + // testProjectId, + // "en", + // "xml", + // "strings.xml", + // androidFile, + // ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping"), + // null); - phraseClient.uploadAndWait( - testProjectId, - "en", - "xml", - "strings.xml", - androidFile, - ImmutableList.of(i % 2 == 0 ? "test-escaping" : "test-no-escaping")); + System.out.println(upload); String s = phraseClient.localeDownload(