Language fixes (#5384)

* Add some comments

* Decouple language settings

* Normalize on read/write

* Refactor

* Support device locale on app startup

* Cleanup, port to web

* Clean up comments

* Comment

* Try not to mutate

* Protect util handling, update test

* Dedupe array values

package.json

···
     "await-lock": "^2.2.2",
     "babel-plugin-transform-remove-console": "^6.9.4",
     "base64-js": "^1.5.1",
+    "bcp-47": "^2.1.0",
     "bcp-47-match": "^2.0.3",
     "date-fns": "^2.30.0",
     "deprecated-react-native-prop-types": "^5.0.0",

src/components/AppLanguageDropdown.tsx

···
       if (sanitizedLang !== value) {
         setLangPrefs.setAppLanguage(sanitizeAppLanguageSetting(value))
       }
-      setLangPrefs.setPrimaryLanguage(value)
-      setLangPrefs.setContentLanguage(value)
 
       // reset feeds to refetch content
       resetPostsFeedQueries(queryClient)

src/components/AppLanguageDropdown.web.tsx

···
       if (sanitizedLang !== value) {
         setLangPrefs.setAppLanguage(sanitizeAppLanguageSetting(value))
       }
-      setLangPrefs.setPrimaryLanguage(value)
-      setLangPrefs.setContentLanguage(value)
 
       // reset feeds to refetch content
       resetPostsFeedQueries(queryClient)

src/locale/deviceLocales.ts

···
+import {getLocales as defaultGetLocales, Locale} from 'expo-localization'
+
+import {dedupArray} from '#/lib/functions'
+
+type LocalWithLanguageCode = Locale & {
+  languageCode: string
+}
+
+/**
+ * Normalized locales
+ *
+ * Handles legacy migration for Java devices.
+ *
+ * {@link https://github.com/bluesky-social/social-app/pull/4461}
+ * {@link https://xml.coverpages.org/iso639a.html}
+ */
+export function getLocales() {
+  const locales = defaultGetLocales?.() ?? []
+  const output: LocalWithLanguageCode[] = []
+
+  for (const locale of locales) {
+    if (typeof locale.languageCode === 'string') {
+      if (locale.languageCode === 'in') {
+        // indonesian
+        locale.languageCode = 'id'
+      }
+      if (locale.languageCode === 'iw') {
+        // hebrew
+        locale.languageCode = 'he'
+      }
+      if (locale.languageCode === 'ji') {
+        // yiddish
+        locale.languageCode = 'yi'
+      }
+
+      // @ts-ignore checked above
+      output.push(locale)
+    }
+  }
+
+  return output
+}
+
+export const deviceLocales = getLocales()
+
+/**
+ * BCP-47 language tag without region e.g. array of 2-char lang codes
+ *
+ * {@link https://docs.expo.dev/versions/latest/sdk/localization/#locale}
+ */
+export const deviceLanguageCodes = dedupArray(
+  deviceLocales.map(l => l.languageCode),
+)

src/locale/helpers.ts

···
   return AppLanguage.en
 }
 
+/**
+ * Handles legacy migration for Java devices.
+ *
+ * {@link https://github.com/bluesky-social/social-app/pull/4461}
+ * {@link https://xml.coverpages.org/iso639a.html}
+ */
 export function fixLegacyLanguageCode(code: string | null): string | null {
-  // handle some legacy code conversions, see https://xml.coverpages.org/iso639a.html
   if (code === 'in') {
     // indonesian
     return 'id'
···
   }
   return code
 }
+
+/**
+ * Find the first language supported by our translation infra. Values should be
+ * in order of preference, and match the values of {@link AppLanguage}.
+ *
+ * If no match, returns `en`.
+ */
+export function findSupportedAppLanguage(languageTags: (string | undefined)[]) {
+  const supported = new Set(Object.values(AppLanguage))
+  for (const tag of languageTags) {
+    if (!tag) continue
+    if (supported.has(tag as AppLanguage)) {
+      return tag
+    }
+  }
+  return AppLanguage.en
+}

src/platform/detection.ts

···
 import {Platform} from 'react-native'
-import {getLocales} from 'expo-localization'
-
-import {fixLegacyLanguageCode} from '#/locale/helpers'
-import {dedupArray} from 'lib/functions'
 
 export const isIOS = Platform.OS === 'ios'
 export const isAndroid = Platform.OS === 'android'
···
   // @ts-ignore we know window exists -prf
   global.window.matchMedia(isMobileWebMediaQuery)?.matches
 export const isIPhoneWeb = isWeb && /iPhone/.test(navigator.userAgent)
-
-export const deviceLocales = dedupArray(
-  getLocales?.()
-    .map?.(locale => fixLegacyLanguageCode(locale.languageCode))
-    .filter(code => typeof code === 'string'),
-) as string[]

src/state/persisted/index.ts

···
   tryStringify,
 } from '#/state/persisted/schema'
 import {PersistedApi} from './types'
+import {normalizeData} from './util'
 
 export type {PersistedAccount, Schema} from '#/state/persisted/schema'
 export {defaults} from '#/state/persisted/schema'
···
   key: K,
   value: Schema[K],
 ): Promise<void> {
-  _state = {
+  _state = normalizeData({
     ..._state,
     [key]: value,
-  }
+  })
   await writeToStorage(_state)
 }
 write satisfies PersistedApi['write']
···
     })
   }
   if (rawData) {
-    return tryParse(rawData)
+    const parsed = tryParse(rawData)
+    if (parsed) {
+      return normalizeData(parsed)
+    }
   }
 }

src/state/persisted/index.web.ts

···
   tryStringify,
 } from '#/state/persisted/schema'
 import {PersistedApi} from './types'
+import {normalizeData} from './util'
 
 export type {PersistedAccount, Schema} from '#/state/persisted/schema'
 export {defaults} from '#/state/persisted/schema'
···
   } catch (e) {
     // Ignore and go through the normal path.
   }
-  _state = {
+  _state = normalizeData({
     ..._state,
     [key]: value,
-  }
+  })
   writeToStorage(_state)
   broadcast.postMessage({event: {type: UPDATE_EVENT, key}})
   broadcast.postMessage({event: UPDATE_EVENT}) // Backcompat while upgrading
···
       return lastResult
     } else {
       const result = tryParse(rawData)
-      lastRawData = rawData
-      lastResult = result
-      return result
+      if (result) {
+        lastRawData = rawData
+        lastResult = normalizeData(result)
+        return lastResult
+      }
     }
   }
 }

src/state/persisted/schema.ts

···
 import {z} from 'zod'
 
+import {deviceLanguageCodes, deviceLocales} from '#/locale/deviceLocales'
+import {findSupportedAppLanguage} from '#/locale/helpers'
 import {logger} from '#/logger'
-import {deviceLocales} from '#/platform/detection'
 import {PlatformInfo} from '../../../modules/expo-bluesky-swiss-army'
 
 const externalEmbedOptions = ['show', 'hide'] as const
···
     lastEmailConfirm: z.string().optional(),
   }),
   languagePrefs: z.object({
-    primaryLanguage: z.string(), // should move to server
-    contentLanguages: z.array(z.string()), // should move to server
-    postLanguage: z.string(), // should move to server
+    /**
+     * The target language for translating posts.
+     *
+     * BCP-47 2-letter language code without region.
+     */
+    primaryLanguage: z.string(),
+    /**
+     * The languages the user can read, passed to feeds.
+     *
+     * BCP-47 2-letter language codes without region.
+     */
+    contentLanguages: z.array(z.string()),
+    /**
+     * The language(s) the user is currently posting in, configured within the
+     * composer. Multiple languages are psearate by commas.
+     *
+     * BCP-47 2-letter language code without region.
+     */
+    postLanguage: z.string(),
+    /**
+     * The user's post language history, used to pre-populate the post language
+     * selector in the composer. Within each value, multiple languages are
+     * separated by values.
+     *
+     * BCP-47 2-letter language codes without region.
+     */
     postLanguageHistory: z.array(z.string()),
+    /**
+     * The language for UI translations in the app.
+     *
+     * BCP-47 2-letter language code with or without region,
+     * to match with {@link AppLanguage}.
+     */
     appLanguage: z.string(),
   }),
   requireAltTextEnabled: z.boolean(), // should move to server
···
     lastEmailConfirm: undefined,
   },
   languagePrefs: {
-    primaryLanguage: deviceLocales[0] || 'en',
-    contentLanguages: deviceLocales || [],
-    postLanguage: deviceLocales[0] || 'en',
-    postLanguageHistory: (deviceLocales || [])
+    primaryLanguage: deviceLanguageCodes[0] || 'en',
+    contentLanguages: deviceLanguageCodes || [],
+    postLanguage: deviceLanguageCodes[0] || 'en',
+    postLanguageHistory: (deviceLanguageCodes || [])
       .concat(['en', 'ja', 'pt', 'de'])
       .slice(0, 6),
-    appLanguage: deviceLocales[0] || 'en',
+    // try full language tag first, then fallback to language code
+    appLanguage: findSupportedAppLanguage([
+      deviceLocales.at(0)?.languageTag,
+      deviceLanguageCodes[0],
+    ]),
   },
   requireAltTextEnabled: false,
   largeAltBadgeEnabled: false,

src/state/persisted/util.ts

···
+import {parse} from 'bcp-47'
+
+import {dedupArray} from '#/lib/functions'
+import {logger} from '#/logger'
+import {Schema} from '#/state/persisted/schema'
+
+export function normalizeData(data: Schema) {
+  const next = {...data}
+
+  /**
+   * Normalize language prefs to ensure that these values only contain 2-letter
+   * country codes without region.
+   */
+  try {
+    const langPrefs = {...next.languagePrefs}
+    langPrefs.primaryLanguage = normalizeLanguageTagToTwoLetterCode(
+      langPrefs.primaryLanguage,
+    )
+    langPrefs.contentLanguages = dedupArray(
+      langPrefs.contentLanguages.map(lang =>
+        normalizeLanguageTagToTwoLetterCode(lang),
+      ),
+    )
+    langPrefs.postLanguage = langPrefs.postLanguage
+      .split(',')
+      .map(lang => normalizeLanguageTagToTwoLetterCode(lang))
+      .filter(Boolean)
+      .join(',')
+    langPrefs.postLanguageHistory = dedupArray(
+      langPrefs.postLanguageHistory.map(postLanguage => {
+        return postLanguage
+          .split(',')
+          .map(lang => normalizeLanguageTagToTwoLetterCode(lang))
+          .filter(Boolean)
+          .join(',')
+      }),
+    )
+    next.languagePrefs = langPrefs
+  } catch (e: any) {
+    logger.error(`persisted state: failed to normalize language prefs`, {
+      safeMessage: e.message,
+    })
+  }
+
+  return next
+}
+
+export function normalizeLanguageTagToTwoLetterCode(lang: string) {
+  const result = parse(lang).language
+  return result ?? lang
+}

src/state/session/__tests__/session-test.ts

···
   },
 }))
 
+jest.mock('expo-localization', () => ({
+  getLocales: () => [],
+}))
+
 describe('session', () => {
   it('can log in and out', () => {
     let state = getInitialState([])

src/view/com/modals/lang-settings/ContentLanguagesSettings.tsx

···
 import React from 'react'
 import {StyleSheet, View} from 'react-native'
-import {ScrollView} from '../util'
-import {Text} from '../../util/text/Text'
-import {usePalette} from 'lib/hooks/usePalette'
-import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
-import {deviceLocales} from 'platform/detection'
-import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
-import {LanguageToggle} from './LanguageToggle'
-import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
 import {Trans} from '@lingui/macro'
+
+import {deviceLanguageCodes} from '#/locale/deviceLocales'
 import {useModalControls} from '#/state/modals'
 import {
   useLanguagePrefs,
   useLanguagePrefsApi,
 } from '#/state/preferences/languages'
+import {usePalette} from 'lib/hooks/usePalette'
+import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
+import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
+import {Text} from '../../util/text/Text'
+import {ScrollView} from '../util'
+import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
+import {LanguageToggle} from './LanguageToggle'
 
 export const snapPoints = ['100%']
 
···
     langs.sort((a, b) => {
       const hasA =
         langPrefs.contentLanguages.includes(a.code2) ||
-        deviceLocales.includes(a.code2)
+        deviceLanguageCodes.includes(a.code2)
       const hasB =
         langPrefs.contentLanguages.includes(b.code2) ||
-        deviceLocales.includes(b.code2)
+        deviceLanguageCodes.includes(b.code2)
       if (hasA === hasB) return a.name.localeCompare(b.name)
       if (hasA) return -1
       return 1

src/view/com/modals/lang-settings/PostLanguagesSettings.tsx

···
 import React from 'react'
 import {StyleSheet, View} from 'react-native'
-import {ScrollView} from '../util'
-import {Text} from '../../util/text/Text'
-import {usePalette} from 'lib/hooks/usePalette'
-import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
-import {deviceLocales} from 'platform/detection'
-import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
-import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
-import {ToggleButton} from 'view/com/util/forms/ToggleButton'
 import {Trans} from '@lingui/macro'
+
+import {deviceLanguageCodes} from '#/locale/deviceLocales'
 import {useModalControls} from '#/state/modals'
 import {
+  hasPostLanguage,
   useLanguagePrefs,
   useLanguagePrefsApi,
-  hasPostLanguage,
 } from '#/state/preferences/languages'
+import {usePalette} from 'lib/hooks/usePalette'
+import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
+import {ToggleButton} from 'view/com/util/forms/ToggleButton'
+import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
+import {Text} from '../../util/text/Text'
+import {ScrollView} from '../util'
+import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
 
 export const snapPoints = ['100%']
 
···
     langs.sort((a, b) => {
       const hasA =
         hasPostLanguage(langPrefs.postLanguage, a.code2) ||
-        deviceLocales.includes(a.code2)
+        deviceLanguageCodes.includes(a.code2)
       const hasB =
         hasPostLanguage(langPrefs.postLanguage, b.code2) ||
-        deviceLocales.includes(b.code2)
+        deviceLanguageCodes.includes(b.code2)
       if (hasA === hasB) return a.name.localeCompare(b.name)
       if (hasA) return -1
       return 1

yarn.lock

···
   resolved "https://registry.yarnpkg.com/bcp-47-match/-/bcp-47-match-2.0.3.tgz#603226f6e5d3914a581408be33b28a53144b09d0"
   integrity sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==
 
+bcp-47@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/bcp-47/-/bcp-47-2.1.0.tgz#7e80734c3338fe8320894981dccf4968c3092df6"
+  integrity sha512-9IIS3UPrvIa1Ej+lVDdDwO7zLehjqsaByECw0bu2RRGP73jALm6FYbzI5gWbgHLvNdkvfXB5YrSbocZdOS0c0w==
+  dependencies:
+    is-alphabetical "^2.0.0"
+    is-alphanumerical "^2.0.0"
+    is-decimal "^2.0.0"
+
 better-opn@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-3.0.2.tgz#f96f35deaaf8f34144a4102651babcf00d1d8817"
···
   resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.2.0.tgz#d33fa7bac284f4de7af949638c9d68157c6b92e8"
   integrity sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==
 
+is-alphabetical@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-2.0.1.tgz#01072053ea7c1036df3c7d19a6daaec7f19e789b"
+  integrity sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==
+
+is-alphanumerical@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz#7c03fbe96e3e931113e57f964b0a368cc2dfd875"
+  integrity sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==
+  dependencies:
+    is-alphabetical "^2.0.0"
+    is-decimal "^2.0.0"
+
 is-arguments@^1.0.4:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b"
···
   integrity sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==
   dependencies:
     has-tostringtag "^1.0.0"
+
+is-decimal@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-2.0.1.tgz#9469d2dc190d0214fd87d78b78caecc0cc14eef7"
+  integrity sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==
 
 is-directory@^0.3.1:
   version "0.3.1"