Add precommit hook to validate i18n (opensearch-project#8423)

* Add precommit hook to validate i18n

Signed-off-by: Miki <[email protected]>

* Changeset file for PR opensearch-project#8423 created/updated

---------

Signed-off-by: Miki <[email protected]>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>

(cherry picked from commit 5203139)
Signed-off-by: Miki <[email protected]>

changelogs/fragments/8423.yml

@@ -0,0 +1,2 @@
+infra:
+- Add precommit hook to validate i18n ([#8423](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/8423))

src/dev/i18n/extract_default_translations.d.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { I18nConfig } from './config';
+
+export declare function validateMessageNamespace(
+  id: string,
+  filePath: string,
+  allowedPaths: Record<string, string[]>,
+  reporter: unknown
+): void;
+
+export declare function matchEntriesWithExctractors(
+  inputPath: string,
+  options: Record<string, unknown>
+): Promise<[[string[], unknown]]>;
+
+export declare function extractMessagesFromPathToMap(
+  inputPath: string,
+  targetMap: Map<string, { message: string }>,
+  config: I18nConfig,
+  reporter: any
+): Promise<void>;

src/dev/i18n/extractors/index.d.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export declare function extractCodeMessages(
+  buffer: Buffer,
+  reporter: unknown
+): Generator<[string, { message: string; description?: string }], void>;

src/dev/i18n/index.ts

@@ -28,17 +28,18 @@
  * under the License.
  */
 
-// @ts-ignore
-export { extractMessagesFromPathToMap } from './extract_default_translations';
-// @ts-ignore
-export { matchEntriesWithExctractors } from './extract_default_translations';
+export {
+  extractMessagesFromPathToMap,
+  matchEntriesWithExctractors,
+} from './extract_default_translations';
 export {
   arrayify,
   writeFileAsync,
   readFileAsync,
   accessAsync,
   normalizePath,
   ErrorReporter,
+  FailReporter,
 } from './utils';
 export { serializeToJson, serializeToJson5 } from './serializers';
 export {

src/dev/i18n/tasks/check_default_messages_in_files.ts

@@ -0,0 +1,147 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { glob } from 'glob';
+import { createFailError, isFailError, REPO_ROOT } from '@osd/dev-utils';
+import { ErrorReporter, filterConfigPaths, I18nConfig, normalizePath, readFileAsync } from '..';
+import { extractCodeMessages } from '../extractors';
+import { validateMessageNamespace } from '../extract_default_translations';
+
+function filterEntries(entries: string[], exclude: string[]) {
+  return entries.filter((entry: string) =>
+    exclude.every((excludedPath: string) => !normalizePath(entry).startsWith(excludedPath))
+  );
+}
+
+function addMessageToMap(
+  targetMap: Map<string, { message: string }>,
+  key: string,
+  value: { message: string },
+  reporter: ErrorReporter
+) {
+  const existingValue = targetMap.get(key);
+
+  if (targetMap.has(key) && existingValue!.message !== value.message) {
+    reporter.report(
+      createFailError(`There is more than one default message for the same id "${key}":
+"${existingValue!.message}" and "${value.message}"`)
+    );
+  } else {
+    targetMap.set(key, value);
+  }
+}
+
+async function extractMessagesFromFilesToMap(
+  files: string[],
+  targetMap: Map<string, { message: string }>,
+  config: I18nConfig,
+  reporter: any
+) {
+  const availablePaths = Object.values(config.paths).flat();
+  const ignoredPatterns = [
+    '**/node_modules/**',
+    '**/__tests__/**',
+    '**/dist/**',
+    '**/target/**',
+    '**/vendor/**',
+    '**/*.test.{js,jsx,ts,tsx}',
+    '**/*.d.ts',
+  ];
+
+  const filesToCheck = files.filter(
+    (file) => glob.sync(file, { ignore: ignoredPatterns, root: REPO_ROOT }).length > 0
+  );
+
+  const fileContents = await Promise.all(
+    filterEntries(filesToCheck, config.exclude)
+      .filter((entry) => {
+        const normalizedEntry = normalizePath(entry);
+        return availablePaths.some(
+          (availablePath) =>
+            normalizedEntry.startsWith(`${normalizePath(availablePath)}/`) ||
+            normalizePath(availablePath) === normalizedEntry
+        );
+      })
+      .map(async (entry: any) => ({
+        name: entry,
+        content: await readFileAsync(entry),
+      }))
+  );
+
+  for (const { name, content } of fileContents) {
+    const reporterWithContext = reporter.withContext({ name });
+
+    try {
+      for (const [id, value] of extractCodeMessages(content, reporterWithContext)) {
+        validateMessageNamespace(id, name, config.paths, reporterWithContext);
+        addMessageToMap(targetMap, id, value, reporterWithContext);
+      }
+    } catch (error) {
+      if (!isFailError(error)) {
+        throw error;
+      }
+
+      reporterWithContext.report(error);
+    }
+  }
+}
+
+export function checkDefaultMessagesInFiles(config: I18nConfig | undefined, inputPaths: string[]) {
+  if (!config) {
+    throw new Error('Config is missing');
+  }
+  const filteredPaths = filterConfigPaths(inputPaths, config) as string[];
+  if (filteredPaths.length === 0) return;
+  return [
+    {
+      title: 'Checking default messages in files',
+      task: async (context: {
+        messages: Map<string, { message: string }>;
+        reporter: ErrorReporter;
+      }) => {
+        const { messages, reporter } = context;
+        const initialErrorsNumber = reporter.errors.length;
+
+        // Return result if no new errors were reported for this path.
+        const result = await extractMessagesFromFilesToMap(
+          filteredPaths,
+          messages,
+          config,
+          reporter
+        );
+        if (reporter.errors.length === initialErrorsNumber) {
+          return result;
+        }
+
+        throw reporter;
+      },
+    },
+  ];
+}

src/dev/i18n/tasks/check_files_for_untracked_translations.ts

@@ -0,0 +1,119 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { glob } from 'glob';
+import { REPO_ROOT } from '@osd/dev-utils';
+import { ListrContext } from '.';
+import { I18nConfig, normalizePath, readFileAsync } from '..';
+// @ts-ignore
+import { extractCodeMessages } from '../extractors';
+
+function filterEntries(entries: string[], exclude: string[]) {
+  return entries.filter((entry: string) =>
+    exclude.every((excludedPath: string) => !normalizePath(entry).startsWith(excludedPath))
+  );
+}
+
+async function checkFilesForUntrackedMessagesTask({
+  files,
+  config,
+  reporter,
+}: {
+  files: string[];
+  config: I18nConfig;
+  reporter: any;
+}) {
+  const availablePaths = Object.values(config.paths).flat();
+  const ignoredPatterns = availablePaths.concat([
+    '**/*.d.ts',
+    '**/*.test.{js,jsx,ts,tsx}',
+    '**/__fixtures__/**',
+    '**/__tests__/**',
+    '**/build/**',
+    '**/dist/**',
+    '**/node_modules/**',
+    '**/packages/osd-i18n/**',
+    '**/packages/osd-plugin-generator/template/**',
+    '**/scripts/**',
+    '**/src/dev/**',
+    '**/target/**',
+    '**/test/**',
+    '**/vendor/**',
+  ]);
+
+  const filesToCheck = files.filter(
+    (file) => glob.sync(file, { ignore: ignoredPatterns, root: REPO_ROOT }).length > 0
+  );
+
+  const fileContents = await Promise.all(
+    filterEntries(filesToCheck, config.exclude)
+      .filter((entry) => {
+        const normalizedEntry = normalizePath(entry);
+        return !availablePaths.some(
+          (availablePath) =>
+            normalizedEntry.startsWith(`${normalizePath(availablePath)}/`) ||
+            normalizePath(availablePath) === normalizedEntry
+        );
+      })
+      .map(async (entry: any) => ({
+        name: entry,
+        content: await readFileAsync(entry),
+      }))
+  );
+
+  for (const { name, content } of fileContents) {
+    const reporterWithContext = reporter.withContext({ name });
+    for (const [id] of extractCodeMessages(content, reporterWithContext)) {
+      const errorMessage = `Untracked file contains i18n label (${id}).`;
+      reporterWithContext.report(errorMessage);
+    }
+  }
+}
+
+export function checkFilesForUntrackedMessages(files: string[]) {
+  return [
+    {
+      title: `Checking untracked messages in files`,
+      task: async (context: ListrContext) => {
+        const { reporter, config } = context;
+        if (!config) {
+          throw new Error('Config is not defined');
+        }
+        const initialErrorsNumber = reporter.errors.length;
+        const result = await checkFilesForUntrackedMessagesTask({ files, config, reporter });
+        if (reporter.errors.length === initialErrorsNumber) {
+          return result;
+        }
+
+        throw reporter;
+      },
+    },
+  ];
+}

src/dev/i18n/tasks/index.ts

@@ -35,6 +35,8 @@ export { extractUntrackedMessages } from './extract_untracked_translations';
 export { checkCompatibility } from './check_compatibility';
 export { mergeConfigs } from './merge_configs';
 export { checkConfigs } from './check_configs';
+export { checkFilesForUntrackedMessages } from './check_files_for_untracked_translations';
+export { checkDefaultMessagesInFiles } from './check_default_messages_in_files';
 
 export interface ListrContext {
   config?: I18nConfig;

src/dev/i18n/utils/index.ts

@@ -54,6 +54,7 @@ export {
   arrayify,
   // classes
   ErrorReporter, // @ts-ignore
+  FailReporter, // @ts-ignore
 } from './utils';
 
 export { verifyICUMessage } from './verify_icu_message';

src/dev/i18n/utils/utils.js

@@ -342,6 +342,18 @@ export class ErrorReporter {
   }
 }
 
+export class FailReporter {
+  errors = [];
+
+  withContext(context) {
+    return { report: (error) => this.report(error, context) };
+  }
+
+  report(error, context) {
+    this.errors.push(createFailError(`Error in ${normalizePath(context.name)}\n${error}`));
+  }
+}
+
 // export function arrayify<Subj = any>(subj: Subj | Subj[]): Subj[] {
 export function arrayify(subj) {
   return Array.isArray(subj) ? subj : [subj];