diff --git a/.travis.yml b/.travis.yml index b3ec1b8..93488c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: node_js dist: bionic +node_js: 16 + # enable c++11/14 builds addons: apt: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9733d31..5e636a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 2.2.0 + +- Add language code "all" [#136](https://github.com/mapbox/vtcomposite/pull/136) + # 2.1.0 - Add language code "local" [#133](https://github.com/mapbox/vtcomposite/pull/133) diff --git a/README.md b/README.md index 3ab278a..974dd4e 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ A filtering function for modifying a tile's features and properties to support l - The script of `{language_property}`, if available, must be stored in the `{language_property}_script` property. - If `{language_property}_script` not in the `params.omit_scripts` list, use `{language_property}` when searching for matching translation. - If `{language_property}_script` is in the `params.omit_scripts` list, skip `{language_property}` when searching for matching translation. + - `all` language code returns `{language_property}`, `{language_property}_local` and all possible language properties that have different values than `{language_property}` - `params.omit_scripts` **Array>** array of scripts to skip `local` language code. - `params.language_property` **String** the primary property in features that identifies the feature in a language. - Default value: `name`. diff --git a/package-lock.json b/package-lock.json index 5182b7e..15ec944 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mapbox/vtcomposite", - "version": "2.1.0", + "version": "2.2.0-dev10", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@mapbox/vtcomposite", - "version": "2.1.0", + "version": "2.2.0-dev10", "hasInstallScript": true, "license": "ISC", "dependencies": { @@ -3143,7 +3143,8 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true + "dev": true, + "requires": {} }, "agent-base": { "version": "6.0.2", @@ -3631,7 +3632,8 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-6.0.0.tgz", "integrity": "sha512-7GPezalm5Bfi/E22PnQxDWH2iW9GTvAlUNTztemeHb6c1BniSyoeTrM87JkC0wYdi6aQrZX9p2qEiAno8aTcbw==", - "dev": true + "dev": true, + "requires": {} }, "eslint-scope": { "version": "7.1.1", @@ -4642,6 +4644,14 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "requires": { + "safe-buffer": "~5.1.0" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -4685,14 +4695,6 @@ "es-abstract": "^1.19.5" } }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", diff --git a/package.json b/package.json index 3ffcae2..3ba6ef1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@mapbox/vtcomposite", - "version": "2.1.0", + "version": "2.2.0-dev10", "description": "Compositing operations on Vector Tiles (c++ bindings using N-API)", "url": "http://github.com/mapbox/vtcomposite", "main": "./lib/index.js", diff --git a/src/vtcomposite.cpp b/src/vtcomposite.cpp index 9b2ceb0..228feea 100644 --- a/src/vtcomposite.cpp +++ b/src/vtcomposite.cpp @@ -19,6 +19,7 @@ // stl #include #include +#include #include #include @@ -652,6 +653,18 @@ struct LocalizeWorker : Napi::AsyncWorker return matching_worldviews; } + static std::string remove_hidden_prefix(std::string property_key, std::string& hidden_prefix) + { + bool has_hidden_prefix = utils::startswith(property_key, hidden_prefix); + + if (has_hidden_prefix) + { + return property_key.substr(hidden_prefix.length()); + } + + return property_key; + } + void Execute() override { try @@ -660,7 +673,8 @@ struct LocalizeWorker : Napi::AsyncWorker std::string incompatible_worldview_key; std::string compatible_worldview_key; std::vector class_key_precedence; - bool keep_every_language = true; + bool keep_all_non_hidden_languages = true; + bool is_international_tile_with_all_languages = false; std::vector language_key_precedence; if (baton_data_->return_localized_tile) @@ -672,13 +686,20 @@ struct LocalizeWorker : Napi::AsyncWorker class_key_precedence.push_back(baton_data_->hidden_prefix + baton_data_->class_property); class_key_precedence.push_back(baton_data_->class_property); - keep_every_language = false; - for (auto const& lang : baton_data_->languages) + keep_all_non_hidden_languages = false; + if (baton_data_->languages.size() == 1 && baton_data_->languages[0] == "all") { - language_key_precedence.push_back(baton_data_->language_property + "_" + lang); - language_key_precedence.push_back(baton_data_->hidden_prefix + baton_data_->language_property + "_" + lang); + is_international_tile_with_all_languages = true; + } + else + { + for (auto const& lang : baton_data_->languages) + { + language_key_precedence.push_back(baton_data_->language_property + "_" + lang); + language_key_precedence.push_back(baton_data_->hidden_prefix + baton_data_->language_property + "_" + lang); + } + language_key_precedence.push_back(baton_data_->language_property); } - language_key_precedence.push_back(baton_data_->language_property); } else { @@ -688,7 +709,7 @@ struct LocalizeWorker : Napi::AsyncWorker class_key_precedence.push_back(baton_data_->class_property); - keep_every_language = true; // reassign to the same value as default for clarity + keep_all_non_hidden_languages = true; // reassign to the same value as default for clarity language_key_precedence.push_back(baton_data_->language_property); } @@ -730,10 +751,14 @@ struct LocalizeWorker : Napi::AsyncWorker auto language_key_idx = static_cast(language_key_precedence.size()); vtzero::property_value language_value; vtzero::property_value original_language_value; - bool omit_local_langauge = false; + bool omit_local_language = false; // collect final properties std::vector> final_properties; + + // collect the languages + std::unordered_map language_properties_to_be_added_to_final_properties; + while (auto property = feature.next_property()) { // if true, we've already encounterd a property that indicates @@ -816,12 +841,40 @@ struct LocalizeWorker : Napi::AsyncWorker // wait till we are done looping through all properties before we add class value to final_properties } + // property_key starts with "name" + // or property_key starts with "_mbx_" + "name" else if ( utils::startswith(property_key, baton_data_->language_property) || utils::startswith(property_key, baton_data_->hidden_prefix + baton_data_->language_property)) { + + if (is_international_tile_with_all_languages) + { + std::string cleaned_property_key = remove_hidden_prefix(property_key, baton_data_->hidden_prefix); + + if (property_key == baton_data_->language_property) + { + // add local language name to final properties + final_properties.emplace_back( + cleaned_property_key, + property.value()); + original_language_value = property.value(); + } + else if (property_key != baton_data_->language_property + "_script") + { + // add other languages (name_xx, except name_script) to a temporary hashmap + // later encounter of the same language in the loop overwrites the former + language_properties_to_be_added_to_final_properties[cleaned_property_key] = property.value(); + } + + continue; + } + // check if the property is of higher precedence that language key encountered so far - std::uint32_t idx = static_cast(std::distance(language_key_precedence.begin(), std::find(language_key_precedence.begin(), language_key_precedence.end(), property_key))); + std::uint32_t idx = static_cast( + std::distance( + language_key_precedence.begin(), + std::find(language_key_precedence.begin(), language_key_precedence.end(), property_key))); if (idx < language_key_idx) { language_key_idx = idx; @@ -836,21 +889,21 @@ struct LocalizeWorker : Napi::AsyncWorker else if (property_key == baton_data_->language_property + "_script") { // true if script is in the omitted list - omit_local_langauge = std::any_of( + omit_local_language = std::any_of( baton_data_->omit_scripts.begin(), baton_data_->omit_scripts.end(), [&](const std::string& script) { return (script == property.value().string_value()); }); - if (keep_every_language) + if (keep_all_non_hidden_languages) { final_properties.emplace_back(property_key, property.value()); } } else { - if (keep_every_language) + if (keep_all_non_hidden_languages) { if (!utils::startswith(property_key, baton_data_->hidden_prefix)) { @@ -888,7 +941,7 @@ struct LocalizeWorker : Napi::AsyncWorker if (language_value.valid()) { // `local` language is "the original language in an acceptable script". - if (omit_local_langauge) + if (omit_local_language) { // don't need to check if `local` is in the desired list of languages // because the script of the original language is not acceptable. @@ -919,6 +972,22 @@ struct LocalizeWorker : Napi::AsyncWorker final_properties.emplace_back(baton_data_->language_property + "_local", original_language_value); } + // Check the list of languages to be added + // Only add the ones that are different from original local language to the final properties + if (is_international_tile_with_all_languages) + { + for (const auto& language_property : language_properties_to_be_added_to_final_properties) + { + std::string language_property_key = language_property.first; + vtzero::property_value language_property_value = language_property.second; + + if (language_property_value.string_value() != original_language_value.string_value()) + { + final_properties.emplace_back(language_property_key, language_property_value); + } + } + } + // build new feature(s) if (has_worldview_key) { diff --git a/test/vtcomposite-localize-language.test.js b/test/vtcomposite-localize-language.test.js index bb886c3..f7094c9 100644 --- a/test/vtcomposite-localize-language.test.js +++ b/test/vtcomposite-localize-language.test.js @@ -8,7 +8,6 @@ const mvtFixtures = require('@mapbox/mvt-fixtures'); const test = require('tape'); const zlib = require('zlib'); - test('[localize] success: buffer size stays the same when no changes needed', (assert) => { const singlePointBuffer = mvtFixtures.get('002').buffer; const params = { @@ -861,3 +860,157 @@ test('[localize language] _mbx_name_local exists in the input tile', (assert) => assert.end(); }); }); + +test('[localize language] languages=all returns name, name_local and all name_xx(s) with _mbx_ removed from property name', (assert) => { + const rawDataLanguageProperties = { + name: '你好', + name_script: 'Han', + 'name_zh-Hant': 'Nǐ hǎo', + name_en: 'hello', + name_de: 'hallo', + _mbx_name_it: 'ciao', + _mbx_name_fr: 'bonjour' + }; + const expectedLocalizedProperties = { + name: '你好', + name_local: '你好', + 'name_zh-Hant': 'Nǐ hǎo', + name_en: 'hello', + name_de: 'hallo', + name_it: 'ciao', + name_fr: 'bonjour' + }; + const params = { + buffer: mvtFixtures.create({ + layers: [ + { + version: 2, + name: 'places', + features: [ + { + id: 10, + tags: Object.keys(rawDataLanguageProperties).flatMap((_, index) => [index, index]), + type: 1, // point + geometry: [9, 54, 38] + } + ], + keys: Object.keys(rawDataLanguageProperties), + values: Object.values(rawDataLanguageProperties).map((value) => ({ string_value: value })), + extent: 4096 + } + ] + }).buffer, + languages: ['all'] + }; + + localize(params, (err, buffer) => { + assert.notOk(err); + const info = vtinfo(buffer); + const properties = info.layers.places.feature(0).properties; + assert.equal(info.layers.places.length, 1, 'expected number of features'); + assert.deepEqual(properties, expectedLocalizedProperties, 'expected properties'); + assert.end(); + }); +}); + +test('[localize language] languages=all returns only all name_xx(s) that are different from name', (assert) => { + const rawDataLanguageProperties = { + name: 'place', + name_script: 'Latin', + name_en: 'place', // same as `name`, expected to be dropped + name_de: 'place DE', + _mbx_name_it: 'place', // same as `name`, expected to be dropped + _mbx_name_fr: 'place FR', + }; + const expectedLocalizedProperties = { + name: 'place', + name_local: 'place', + name_de: 'place DE', + name_fr: 'place FR' + }; + const params = { + buffer: mvtFixtures.create({ + layers: [ + { + version: 2, + name: 'places', + features: [ + { + id: 10, + tags: Object.keys(rawDataLanguageProperties).flatMap((_, index) => [index, index]), + type: 1, // point + geometry: [9, 54, 38] + } + ], + keys: Object.keys(rawDataLanguageProperties), + values: Object.values(rawDataLanguageProperties).map((value) => ({ string_value: value })), + extent: 4096 + } + ] + }).buffer, + languages: ['all'] + }; + + localize(params, (err, buffer) => { + assert.notOk(err); + const info = vtinfo(buffer); + const properties = info.layers.places.feature(0).properties; + assert.equal(info.layers.places.length, 1, 'expected number of features'); + assert.deepEqual(properties, expectedLocalizedProperties, 'expected properties'); + assert.end(); + }); +}); + +test('[localize language] languages=all returns the last value of the same language properties', (assert) => { + const rawDataLanguageProperties = { + name: '你好', + name_script: 'Han', + 'name_zh-Hant': 'Nǐ hǎo', + name_en: 'EN first value - to be overwritten', + _mbx_name_de: 'DE first value - to be overwritten', + name_de: 'hallo', + _mbx_name_it: 'ciao', + _mbx_name_fr: 'bonjour', + _mbx_name_en: 'hello' + }; + const expectedLocalizedProperties = { + name: '你好', + name_local: '你好', + 'name_zh-Hant': 'Nǐ hǎo', + name_en: 'hello', + name_de: 'hallo', + name_it: 'ciao', + name_fr: 'bonjour' + }; + const params = { + buffer: mvtFixtures.create({ + layers: [ + { + version: 2, + name: 'places', + features: [ + { + id: 10, + tags: Object.keys(rawDataLanguageProperties).flatMap((_, index) => [index, index]), + type: 1, // point + geometry: [9, 54, 38] + } + ], + keys: Object.keys(rawDataLanguageProperties), + values: Object.values(rawDataLanguageProperties).map((value) => ({ string_value: value })), + extent: 4096 + } + ] + }).buffer, + languages: ['all'] + }; + + localize(params, (err, buffer) => { + assert.notOk(err); + const info = vtinfo(buffer); + const properties = info.layers.places.feature(0).properties; + assert.equal(info.layers.places.length, 1, 'expected number of features'); + assert.deepEqual(properties, expectedLocalizedProperties, 'expected properties'); + assert.end(); + }); +});