From 3f0095b57aed860a651081e5784027c3a6bcde52 Mon Sep 17 00:00:00 2001
From: Timothy Flynn <trflynn89@pm.me>
Date: Sun, 10 Oct 2021 11:31:49 -0400
Subject: [PATCH] LibUnicode: Skip unknown languages and territories

Some CLDR languages.json / territories.json files contain localizations
for some lanuages/territories that are otherwise not present in the CLDR
database. We already don't generate anything in UnicodeLocale.cpp for
these anomalies, but this will stop us from even storing that data in
the generator's memory.

This doesn't affect the output of the generator, but will have an effect
after an upcoming commit to unique-ify all of the strings in the CLDR.
---
 .../LibUnicode/GenerateUnicodeLocale.cpp      | 28 +++++++++++++++----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/Meta/Lagom/Tools/CodeGenerators/LibUnicode/GenerateUnicodeLocale.cpp b/Meta/Lagom/Tools/CodeGenerators/LibUnicode/GenerateUnicodeLocale.cpp
index 957bdb229d..ee4bbb8d86 100644
--- a/Meta/Lagom/Tools/CodeGenerators/LibUnicode/GenerateUnicodeLocale.cpp
+++ b/Meta/Lagom/Tools/CodeGenerators/LibUnicode/GenerateUnicodeLocale.cpp
@@ -227,7 +227,7 @@ static void parse_identity(String locale_path, UnicodeLocaleData& locale_data, L
     }
 }
 
-static void parse_locale_languages(String locale_path, Locale& locale)
+static void parse_locale_languages(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
 {
     LexicalPath languages_path(move(locale_path));
     languages_path = languages_path.append("languages.json"sv);
@@ -245,11 +245,14 @@ static void parse_locale_languages(String locale_path, Locale& locale)
     auto const& languages_object = locale_display_names_object.as_object().get("languages"sv);
 
     languages_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
+        if (!locale_data.languages.contains_slow(key))
+            return;
+
         locale.languages.set(key, value.as_string());
     });
 }
 
-static void parse_locale_territories(String locale_path, Locale& locale)
+static void parse_locale_territories(String locale_path, UnicodeLocaleData& locale_data, Locale& locale)
 {
     LexicalPath territories_path(move(locale_path));
     territories_path = territories_path.append("territories.json"sv);
@@ -267,6 +270,9 @@ static void parse_locale_territories(String locale_path, Locale& locale)
     auto const& territories_object = locale_display_names_object.as_object().get("territories"sv);
 
     territories_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
+        if (!locale_data.territories.contains_slow(key))
+            return;
+
         locale.territories.set(key, value.as_string());
     });
 }
@@ -426,6 +432,7 @@ static Core::DirIterator path_to_dir_iterator(String path)
 
 static void parse_all_locales(String core_path, String locale_names_path, String misc_path, String numbers_path, UnicodeLocaleData& locale_data)
 {
+    auto identity_iterator = path_to_dir_iterator(locale_names_path);
     auto locale_names_iterator = path_to_dir_iterator(move(locale_names_path));
     auto misc_iterator = path_to_dir_iterator(move(misc_path));
     auto numbers_iterator = path_to_dir_iterator(move(numbers_path));
@@ -452,6 +459,18 @@ static void parse_all_locales(String core_path, String locale_names_path, String
         return builder.build();
     };
 
+    while (identity_iterator.has_next()) {
+        auto locale_path = identity_iterator.next_full_path();
+        VERIFY(Core::File::is_directory(locale_path));
+
+        auto language = remove_variants_from_path(locale_path);
+        if (!language.has_value())
+            continue;
+
+        auto& locale = locale_data.locales.ensure(*language);
+        parse_identity(locale_path, locale_data, locale);
+    }
+
     while (locale_names_iterator.has_next()) {
         auto locale_path = locale_names_iterator.next_full_path();
         VERIFY(Core::File::is_directory(locale_path));
@@ -461,9 +480,8 @@ static void parse_all_locales(String core_path, String locale_names_path, String
             continue;
 
         auto& locale = locale_data.locales.ensure(*language);
-        parse_identity(locale_path, locale_data, locale);
-        parse_locale_languages(locale_path, locale);
-        parse_locale_territories(locale_path, locale);
+        parse_locale_languages(locale_path, locale_data, locale);
+        parse_locale_territories(locale_path, locale_data, locale);
         parse_locale_scripts(locale_path, locale_data, locale);
     }