public void ConvertToPalasoConformPrivateUseRfc5646Tag(string flexConformPrivateUseRfc5646Tag)
{
string language = String.Empty;
string script = String.Empty;
string region = String.Empty;
string variant = String.Empty;
string privateUse = String.Empty;
var tokens = flexConformPrivateUseRfc5646Tag.Split(new[] { '-' });
for (int position = 0; position < tokens.Length; ++position)
{
string currentToken = tokens[position];
if (position == 0)
{
if (!currentToken.Equals("x", StringComparison.OrdinalIgnoreCase))
{
throw new ValidationException(String.Format("The rfctag {0} does not start with 'x-' or 'X-'.",
flexConformPrivateUseRfc5646Tag));
}
language = currentToken;
}
else if (position == 1 && !StandardSubtags.IsValidIso15924ScriptCode(currentToken))
{
language = language + '-' + currentToken;
}
else if (StandardSubtags.IsValidIso15924ScriptCode(currentToken))
{
if (!String.IsNullOrEmpty(region) || !String.IsNullOrEmpty(variant))
{
throw new ValidationException(
String.Format(
"The rfctag '{0}' contains a misplaced Script subtag (i.e. it was preceded by a region or variant subtag.",
flexConformPrivateUseRfc5646Tag));
}
script = currentToken;
}
else if (StandardSubtags.IsValidIso3166RegionCode(currentToken))
{
if (!String.IsNullOrEmpty(variant))
{
throw new ValidationException(
String.Format(
"The rfctag '{0}' contains a misplaced Region subtag (i.e. it was preceded by a variant subtag.",
flexConformPrivateUseRfc5646Tag));
}
region = currentToken;
}
else if (StandardSubtags.IsValidRegisteredVariantCode(currentToken))
{
variant = variant + currentToken;
}
else
{
privateUse = String.IsNullOrEmpty(privateUse) ? currentToken : privateUse + '-' + currentToken;
}
}
variant = IetfLanguageTag.ConcatenateVariantAndPrivateUse(variant, privateUse);
ConvertToPalasoConformPrivateUseRfc5646Tag(language, script, region, variant);
}
/// <summary>
/// Filter out tags that contain a region marker unless the caller has already specified that region
/// markers are allowed in language tags. Note that li.LanguageTag can be just a search string the
/// user has typed, which might be a (partial) language tag or might be (part of) a language name.
/// If the tag doesn't actually parse as a language tag, we assume the user is typing something other
/// than a language tag and consider it not to be something we'd filter out as specifying a region.
/// </summary>
private bool RegionalDialectsFilter(LanguageInfo li)
{
if (IncludeRegionalDialects)
{
return(true);
}
// always include Chinese languages with region codes
if (li.LanguageTag.IsOneOf("zh-CN", "zh-TW"))
{
return(true);
}
// written this way to avoid having to catch predictable exceptions as the user is typing
string language;
string script;
string region;
string variant;
if (IetfLanguageTag.TryGetParts(li.LanguageTag, out language, out script, out region, out variant))
{
return(string.IsNullOrEmpty(region)); // OK only if no region.
}
return(true); // Not a tag? Don't filter it out.
}
/// <summary>
/// Creates a new writing system.
/// </summary>
/// <param name="languageSubtag">The language subtag.</param>
/// <param name="scriptSubtag">The script subtag.</param>
/// <param name="regionSubtag">The region subtag.</param>
/// <param name="variantSubtags">The variant subtags.</param>
/// <returns></returns>
public CoreWritingSystemDefinition Create(LanguageSubtag languageSubtag, ScriptSubtag scriptSubtag, RegionSubtag regionSubtag, IEnumerable <VariantSubtag> variantSubtags)
{
lock (m_syncRoot)
{
VariantSubtag[] variantSubtagsArray = variantSubtags.ToArray();
string langTag = IetfLanguageTag.Create(languageSubtag, scriptSubtag, regionSubtag, variantSubtagsArray);
CoreWritingSystemDefinition ws;
m_repo.WritingSystemFactory.Create(langTag, out ws);
if (ws.Language != null && languageSubtag != null && ws.Language.Name != languageSubtag.Name)
{
ws.Language = new LanguageSubtag(ws.Language, languageSubtag.Name);
}
if (ws.Script != null && scriptSubtag != null && ws.Script.Name != scriptSubtag.Name)
{
ws.Script = new ScriptSubtag(ws.Script, scriptSubtag.Name);
}
if (ws.Region != null && regionSubtag != null && ws.Region.Name != regionSubtag.Name)
{
ws.Region = new RegionSubtag(ws.Region, regionSubtag.Name);
}
for (int i = 0; i < Math.Min(ws.Variants.Count, variantSubtagsArray.Length); i++)
{
if (ws.Variants[i].Code == variantSubtagsArray[i].Code && ws.Variants[i].Name != variantSubtagsArray[i].Name)
{
ws.Variants[i] = new VariantSubtag(ws.Variants[i], variantSubtagsArray[i].Name);
}
}
if (ws.Language != null && !string.IsNullOrEmpty(ws.Language.Name))
{
ws.Abbreviation = ws.Language.Name.Length > 3 ? ws.Language.Name.Substring(0, 3) : ws.Language.Name;
}
else
{
ws.Abbreviation = ws.LanguageTag;
}
if (ws.DefaultCollation == null)
{
string message;
if (SystemCollator.ValidateLanguageTag(ws.LanguageTag, out message))
{
ws.DefaultCollation = new SystemCollationDefinition {
LanguageTag = ws.LanguageTag
}
}
;
else
{
ws.DefaultCollation = new IcuRulesCollationDefinition("standard");
}
}
if (ws.DefaultFont == null)
{
ws.DefaultFont = new FontDefinition("Charis SIL");
}
ws.AcceptChanges();
return(ws);
}
}
public LanguageSuggestion(IWritingSystemFactory wsFactory, string languageTag, string keyboardLayout)
: base(wsFactory)
{
_languageTag = languageTag;
_keyboardLayout = keyboardLayout;
LanguageSubtag languageSubtag;
ScriptSubtag scriptSubtag;
RegionSubtag regionSubtag;
IEnumerable <VariantSubtag> variantSubtags;
IetfLanguageTag.TryGetSubtags(languageTag, out languageSubtag, out scriptSubtag, out regionSubtag,
out variantSubtags);
var s = new StringBuilder();
if (!string.IsNullOrEmpty(languageSubtag.Name))
{
s.Append(languageSubtag.Name);
}
if (scriptSubtag != null && !string.IsNullOrEmpty(scriptSubtag.Name) && !IetfLanguageTag.IsScriptImplied(languageTag))
{
s.AppendFormat("-{0}", scriptSubtag.Name);
}
if (regionSubtag != null && !string.IsNullOrEmpty(regionSubtag.Name))
{
s.AppendFormat("-{0}", regionSubtag.Name);
}
Label = s.ToString();
}
public static string CurrentVersion(string languageTag)
{
string language, script, region, variant;
IetfLanguageTag.TryGetParts(languageTag, out language, out script, out region, out variant);
return(CurrentVersion(language, script, region, variant));
}
public void IsValidRegionCode_InvlidRegionCode_ReturnsFalse()
{
Assert.That(IetfLanguageTag.IsValidRegionCode("a"), Is.False);
Assert.That(IetfLanguageTag.IsValidRegionCode("abc"), Is.False);
Assert.That(IetfLanguageTag.IsValidRegionCode("12"), Is.False);
Assert.That(IetfLanguageTag.IsValidRegionCode("1234"), Is.False);
}
public override WritingSystemDefinition ShowDialogIfNeededAndGetDefinition()
{
WaitCursor.Show();
try
{
var variants = new List <VariantSubtag> {
WellKnownSubtags.IpaVariant
};
variants.AddRange(_templateWritingSystemDefinition.Variants);
WritingSystemDefinition ws;
WritingSystemFactory.Create(IetfLanguageTag.Create(_templateWritingSystemDefinition.Language, null,
_templateWritingSystemDefinition.Region, variants), out ws);
string ipaFontName = _fontsForIpa.FirstOrDefault(FontExists);
if (!string.IsNullOrEmpty(ipaFontName))
{
ws.DefaultFont = new FontDefinition(ipaFontName);
}
ws.Abbreviation = "ipa";
ws.DefaultFontSize = _templateWritingSystemDefinition.DefaultFontSize;
IKeyboardDefinition ipaKeyboard = Keyboard.Controller.AvailableKeyboards.FirstOrDefault(k => k.Id.ToLower().Contains("ipa"));
if (ipaKeyboard != null)
{
ws.Keyboard = ipaKeyboard.Id;
}
return(ws);
}
finally
{
WaitCursor.Hide();
}
}
private void WriteLanguageTagElements(XElement identityElem, string languageTag)
{
string language, script, region, variant;
IetfLanguageTag.TryGetParts(languageTag, out language, out script, out region, out variant);
// language element is required
identityElem.SetAttributeValue("language", "type", language);
// write the rest if they have contents
if (!string.IsNullOrEmpty(script))
{
identityElem.SetAttributeValue("script", "type", script);
}
else
{
identityElem.Elements("script").Remove();
}
if (!string.IsNullOrEmpty(region))
{
identityElem.SetAttributeValue("territory", "type", region);
}
else
{
identityElem.Elements("territory").Remove();
}
if (!string.IsNullOrEmpty(variant))
{
identityElem.SetAttributeValue("variant", "type", variant);
}
else
{
identityElem.Elements("variant").Remove();
}
}
internal void EnsureIeftLanguageTagsUnique(IEnumerable <LdmlMigrationInfo> migrationInfo)
{
var uniqueLanguageTags = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
foreach (LdmlMigrationInfo info in migrationInfo)
{
LdmlMigrationInfo currentInfo = info;
if (uniqueLanguageTags.Contains(currentInfo.LanguageTagAfterMigration))
{
if (currentInfo.LanguageTagBeforeMigration.Equals(currentInfo.LanguageTagAfterMigration, StringComparison.InvariantCultureIgnoreCase))
{
// We want to change the other, because we are the same. Even if the other is the same, we'll change it anyway.
LdmlMigrationInfo otherInfo = _migrationInfo.First(
i => i.LanguageTagAfterMigration.Equals(currentInfo.LanguageTagAfterMigration, StringComparison.InvariantCultureIgnoreCase)
);
otherInfo.LanguageTagAfterMigration = IetfLanguageTag.ToUniqueLanguageTag(
otherInfo.LanguageTagAfterMigration, uniqueLanguageTags);
uniqueLanguageTags.Add(otherInfo.LanguageTagAfterMigration);
}
else
{
currentInfo.LanguageTagAfterMigration = IetfLanguageTag.ToUniqueLanguageTag(
currentInfo.LanguageTagAfterMigration, uniqueLanguageTags);
uniqueLanguageTags.Add(currentInfo.LanguageTagAfterMigration);
}
}
else
{
uniqueLanguageTags.Add(currentInfo.LanguageTagAfterMigration);
}
}
}
public void IsValidLanguageCode_InvalidLanguageCode_ReturnsFalse()
{
Assert.That(IetfLanguageTag.IsValidLanguageCode("a"), Is.False);
Assert.That(IetfLanguageTag.IsValidLanguageCode("0"), Is.False);
Assert.That(IetfLanguageTag.IsValidLanguageCode("abcdefghi"), Is.False);
Assert.That(IetfLanguageTag.IsValidLanguageCode("123456789"), Is.False);
Assert.That(IetfLanguageTag.IsValidLanguageCode("abcdefgh-abc-def-ghi-jkl"), Is.False);
}
public void ToUniqueLanguageTag_LanguageTagAlreadyContainsADuplicateMarker_DuplicateNumberIsMaintainedAndNewOneIsIntroduced()
{
var existingTags = new[] { "en-Zxxx-x-dupl0-audio", "en-Zxxx-x-audio-dupl1" };
var ws = new WritingSystemDefinition("en-Zxxx-x-dupl0-audio");
ws.LanguageTag = IetfLanguageTag.ToUniqueLanguageTag(ws.LanguageTag, existingTags);
Assert.That(ws.LanguageTag, Is.EqualTo("en-Zxxx-x-dupl0-audio-dupl1"));
}
public void ToUniqueLanguageTag_IdIsNull()
{
var existingTags = new[] { "en-Zxxx-x-audio" };
var ws = new WritingSystemDefinition("de");
ws.LanguageTag = IetfLanguageTag.ToUniqueLanguageTag(ws.LanguageTag, existingTags);
Assert.That(ws.Id, Is.Null);
}
public void ToUniqueLanguageTag_ADuplicatewithHigherNumberAlreadyExists_DuplicateMarkerWithLowNumberIsAppended()
{
var existingTags = new[] { "en-Zxxx-x-audio", "en-Zxxx-x-audio-dupl1" };
var ws = new WritingSystemDefinition("en-Zxxx-x-audio");
ws.LanguageTag = IetfLanguageTag.ToUniqueLanguageTag(ws.LanguageTag, existingTags);
Assert.That(ws.LanguageTag, Is.EqualTo("en-Zxxx-x-audio-dupl0"));
}
[TestCase("x-thisiswaytoolong")] // > 15 characters (should be 8 but langtags.json has up to 15)
public void TryGetVariantSubtags_PrivateUseRulesHonored(string privateUseVariantTag)
{
IEnumerable <VariantSubtag> variantSubtags;
// Test both with and without custom name
Assert.That(IetfLanguageTag.TryGetVariantSubtags(privateUseVariantTag, out variantSubtags, "Bad Variant"), Is.False);
Assert.That(IetfLanguageTag.TryGetVariantSubtags(privateUseVariantTag, out variantSubtags), Is.False);
}
public void ToUniqueLanguageTag_IsAlreadyUnique_NothingChanges()
{
var existingTags = new[] { "en-Zxxx-x-audio" };
var ws = new WritingSystemDefinition("de");
ws.LanguageTag = IetfLanguageTag.ToUniqueLanguageTag(ws.LanguageTag, existingTags);
Assert.That(ws.LanguageTag, Is.EqualTo("de"));
}
public void IsValidPrivateUseCode_ValidPrivateUseCode_Returns_True()
{
Assert.That(IetfLanguageTag.IsValidPrivateUseCode("a"), Is.True);
Assert.That(IetfLanguageTag.IsValidPrivateUseCode("A"), Is.True);
Assert.That(IetfLanguageTag.IsValidPrivateUseCode("1"), Is.True);
Assert.That(IetfLanguageTag.IsValidPrivateUseCode("abcdefgh"), Is.True);
Assert.That(IetfLanguageTag.IsValidPrivateUseCode("12345678"), Is.True);
}
public void IsValidLanguageCode_ValidLanguageCode_ReturnsTrue()
{
Assert.That(IetfLanguageTag.IsValidLanguageCode("ab"), Is.True);
Assert.That(IetfLanguageTag.IsValidLanguageCode("abcdefgh"), Is.True);
Assert.That(IetfLanguageTag.IsValidLanguageCode("abcdefgh-abc"), Is.True);
Assert.That(IetfLanguageTag.IsValidLanguageCode("abcdefgh-abc-def"), Is.True);
Assert.That(IetfLanguageTag.IsValidLanguageCode("abcdefgh-abc-def-ghi"), Is.True);
}
public void Validate_ValidTag_ReturnsTrue()
{
string message;
Assert.That(
IetfLanguageTag.Validate(new LanguageSubtag("en"), new ScriptSubtag("Latn"), new RegionSubtag("US"),
new List <VariantSubtag>(), out message), Is.True);
}
public void Validate_InvalidTag_ReturnsFalse()
{
string message;
Assert.That(
IetfLanguageTag.Validate(new LanguageSubtag("a"), new ScriptSubtag(""), new RegionSubtag(""),
new List <VariantSubtag>(), out message), Is.False);
}
public void ToLanguageTag_FWIcuLocales_ReturnsLanguageTags()
{
// convert older FW language tags
Assert.AreEqual("slu", IetfLanguageTag.ToLanguageTag("eslu"));
// other possibilities from FW6.0.6
Assert.AreEqual("qaa-x-bcd", IetfLanguageTag.ToLanguageTag("x123"));
Assert.AreEqual("qaa-x-kac", IetfLanguageTag.ToLanguageTag("xka2"));
}
public void ToUniqueLanguageTag_IsNotUnique_DuplicateMarkerIsAppended()
{
var existingTags = new[] { "en-Zxxx-x-audio" };
var ws = new WritingSystemDefinition("en-Zxxx-x-audio");
ws.LanguageTag = IetfLanguageTag.ToUniqueLanguageTag(ws.LanguageTag, existingTags);
Assert.That(ws.LanguageTag, Is.EqualTo("en-Zxxx-x-audio-dupl0"));
}
public override WritingSystemDefinition ShowDialogIfNeededAndGetDefinition()
{
WritingSystemDefinition ws = WritingSystemFactory.Create(_templateWritingSystemDefinition);
ws.LanguageTag = IetfLanguageTag.ToUniqueLanguageTag(
ws.LanguageTag, _existingWritingSystemsForLanguage.Select(w => w.LanguageTag));
return(ws);
}
/// <summary>
/// Some languages in langtags.json have not been normalized to have a default tag without a script marker
/// in one of its entries. For some uses of the data, we really want to see only the default tags but we
/// also don't want to not see any languages. So scan through the data for cases where every tag associated
/// with a language contains a script marker and choose one as the default to receive a minimal tag that is
/// equal to the language code alone. (The one found in the most countries is chosen by default.)
/// </summary>
private void EnsureDefaultTags()
{
HashSet<string> tagSet = new HashSet<string>();
foreach (var langInfo in _codeToLanguageIndex.Values)
tagSet.Add(langInfo.LanguageTag);
var tagList = tagSet.ToList();
tagList.Sort((a,b) => string.Compare(a, b, StringComparison.Ordinal));
var prevLang = string.Empty;
var countChanged = 0;
for (var i = 0; i < tagList.Count; ++i)
{
var tag = tagList[i];
string language;
string script;
string region;
string variant;
if (!IetfLanguageTag.TryGetParts(tag, out language, out script, out region, out variant))
{
prevLang = tag; // shouldn't happen, but if it does...
continue;
}
// Check for a language without a simple tag that has a tag with a script.
// (not quite foolproof in theory since a tag with region or variant might sort
// in front of a tag with a script, but good enough in practice)
if (language == prevLang || string.IsNullOrEmpty(script))
{
prevLang = language;
continue;
}
// Go through all the entries for this language so we can attempt to choose
// the "best" for being the default;
var langInfo = _codeToLanguageIndex[tag];
while (i + 1 < tagList.Count)
{
var tagNext = tagList[i + 1];
if (tagNext.StartsWith(language + "-"))
{
++i;
var langInfoNext = _codeToLanguageIndex[tagNext];
// choose the one that's more widespread unless the name information
// indicates a possibly less widespread region of origin.
if (langInfoNext.Names.Count >= langInfo.Names.Count &&
langInfoNext.Countries.Count > langInfo.Countries.Count)
{
langInfo = langInfoNext;
}
}
else
{
break;
}
}
langInfo.LanguageTag = language; // force tag to default form arbitrarily for now.
++countChanged;
prevLang = language;
}
Debug.WriteLine($"LanguageLookup.EnsureDefaultTags() changed {countChanged} language tags");
}
public void SplitVariantAndPrivateUse_PrivateUseWithCapitalXDashOnly_ReturnsPrivateUseWithxDash()
{
string variant;
string privateUse;
IetfLanguageTag.SplitVariantAndPrivateUse("X-audio", out variant, out privateUse);
Assert.That(variant, Is.EqualTo(String.Empty));
Assert.That(privateUse, Is.EqualTo("audio"));
}
public void SplitVariantAndPrivateUse_VariantAndPrivateUseWithCapitalXDash_ReturnsConcatenatedVariantAndPrivateUse()
{
string variant;
string privateUse;
IetfLanguageTag.SplitVariantAndPrivateUse("1901-X-audio", out variant, out privateUse);
Assert.That(variant, Is.EqualTo("1901"));
Assert.That(privateUse, Is.EqualTo("audio"));
}
public void SplitVariantAndPrivateUse_BogusVariantBadPrivateUse_HappilysplitsOnxDash()
{
string variant;
string privateUse;
IetfLanguageTag.SplitVariantAndPrivateUse("bogusVariant-X-audio-emic-etic", out variant, out privateUse);
Assert.That(variant, Is.EqualTo("bogusVariant"));
Assert.That(privateUse, Is.EqualTo("audio-emic-etic"));
}
public void Canonicalize_ImplicitScript_SuppressesScript()
{
Assert.That(IetfLanguageTag.Canonicalize("en-Latn-US"), Is.EqualTo("en-US"));
Assert.That(IetfLanguageTag.Canonicalize("zh-hans-Cn-x-stuff"), Is.EqualTo("zh-CN-x-stuff"));
Assert.That(IetfLanguageTag.Canonicalize("zH-hans-Cn"), Is.EqualTo("zh-CN"));
Assert.That(IetfLanguageTag.Canonicalize("zH-Hant-tW-x-stuff"), Is.EqualTo("zh-TW-x-stuff"));
Assert.That(IetfLanguageTag.Canonicalize("Zh-hant-Tw"), Is.EqualTo("zh-TW"));
Assert.That(IetfLanguageTag.Canonicalize("oro-Latn"), Is.EqualTo("oro"));
}
public void ToIcuLocale_UnknownLanguageTags_ReturnsIcuLocales()
{
// language, private use script
Assert.AreEqual("en_Fake", IetfLanguageTag.ToIcuLocale("en-Qaaa-x-Fake"));
// language, private use script, private use region
Assert.AreEqual("en_Fake_QD", IetfLanguageTag.ToIcuLocale("en-Qaaa-QM-x-Fake-QD"));
// private use language, script
Assert.AreEqual("xzzz_Latn", IetfLanguageTag.ToIcuLocale("qaa-Latn-x-zzz"));
}
public void SplitVariantAndPrivateUse_VariantOnly_ReturnsVariant()
{
string variant;
string privateUse;
IetfLanguageTag.SplitVariantAndPrivateUse("1901", out variant, out privateUse);
Assert.That(variant, Is.EqualTo("1901"));
Assert.That(privateUse, Is.EqualTo(String.Empty));
}
public void SplitVariantAndPrivateUse_VariantAndPrivateUse_ReturnsVariantAndPrivateUse()
{
string variant;
string privateUse;
IetfLanguageTag.SplitVariantAndPrivateUse("1901-x-audio", out variant, out privateUse);
Assert.That(variant, Is.EqualTo("1901"));
Assert.That(privateUse, Is.EqualTo("audio"));
}
