// There's some confusion between the Palaso migrator and our version 19 migrator about whether an old language
// tag should have multiple X's if it has more than one private-use component. Since such X's are not
// significant, ignore them.
private bool TryGetNewTag(string oldTag, out string newTag)
{
string key = RemoveMultipleX(oldTag.ToLowerInvariant());
if (m_tagMap.TryGetValue(key, out newTag))
{
return(!newTag.Equals(oldTag, StringComparison.OrdinalIgnoreCase));
}
var cleaner = new IetfLanguageTagCleaner(oldTag);
cleaner.Clean();
// FieldWorks needs to handle this special case.
if (cleaner.Language.ToLowerInvariant() == "cmn")
{
var region = cleaner.Region;
if (string.IsNullOrEmpty(region))
{
region = "CN";
}
cleaner = new IetfLanguageTagCleaner("zh", cleaner.Script, region, cleaner.Variant, cleaner.PrivateUse);
}
newTag = cleaner.GetCompleteTag();
while (m_tagMap.Values.Contains(newTag, StringComparer.OrdinalIgnoreCase))
{
// We can't use this tag because it would conflict with what we are mapping something else to.
cleaner = new IetfLanguageTagCleaner(cleaner.Language, cleaner.Script, cleaner.Region, cleaner.Variant,
WritingSystemIdMigrator.GetNextDuplPart(cleaner.PrivateUse));
newTag = cleaner.GetCompleteTag();
}
m_tagMap[key] = newTag;
return(!newTag.Equals(oldTag, StringComparison.OrdinalIgnoreCase));
}
public void CompleteTagConstructor_HasInvalidLanguageName_MovedToPrivateUse()
{
var cleaner = new IetfLanguageTagCleaner("234");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-x-234"));
}
public void CompleteTagConstructor_XDashBeforeValidLanguageNameInVariant_NoChange()
{
var cleaner = new IetfLanguageTagCleaner("", "", "", "x-de", "");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("x-de"));
}
public void Language_XDashBeforeString_AddsQaa()
{
var cleaner = new IetfLanguageTagCleaner("x-blah");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-x-blah"));
}
public void CompleteTagConstructor_LanguageNameWithAudio_GetZxxxAdded()
{
var cleaner = new IetfLanguageTagCleaner("aaa-x-audio");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("aaa-Zxxx-x-audio"));
}
private IEnumerable <Tuple <string, string> > GetLanguageAndKeyboardCombinations()
{
foreach (InputLanguage language in InputLanguage.InstalledInputLanguages)
{
CultureInfo culture;
try
{
//http://www.wesay.org/issues/browse/WS-34598
//Oddly enough, this can throw. It seems like it might have to do with a badly applied .Net patch
//http://www.ironspeed.com/Designer/3.2.4/WebHelp/Part_VI/Culture_ID__XXX__is_not_a_supported_culture.htm and others
culture = language.Culture;
}
catch (CultureNotFoundException)
{
continue;
}
if (culture.EnglishName.StartsWith("Invariant"))
{
continue;
}
string region = string.Empty;
if (Environment.OSVersion.Platform != PlatformID.Unix)
{
region = GetRegion(language);
}
var cleaner = new IetfLanguageTagCleaner(culture.TwoLetterISOLanguageName, "", region, "", "");
cleaner.Clean();
yield return(Tuple.Create(cleaner.GetCompleteTag(), language.LayoutName));
}
}
public void CompleteTagConstructor_HasLanguageNameAndOtherName_OtherNameMovedToPrivateUse()
{
var cleaner = new IetfLanguageTagCleaner("abc-123");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("abc-x-123"));
}
public void CompleteTagConstructor_ValidRfctagWithLegacyIso3CodeAndPrivateUse_MigratesToRfc2LetterCodeAndPrivateUse()
{
var cleaner = new IetfLanguageTagCleaner("eng-bogus");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("en-x-bogus"));
}
public void CompleteTagConstructor_TagContainsOnlyPrivateUseWithAdditionalXDash_RedundantXDashRemoved()
{
var cleaner = new IetfLanguageTagCleaner("x-some-x-whatever");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-x-some-whatever"));
}
public void CompleteTagConstructor_PrivateUseWithAudioAndDuplicateX_MakesAudioTag()
{
var cleaner = new IetfLanguageTagCleaner("x-en-Zxxx-x-audio");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-Zxxx-x-en-Zxxx-audio"));
}
void VerifyRfcCleaner(IetfLanguageTagCleaner cleaner, string language, string script, string region, string variant, string completeTag)
{
Assert.That(cleaner.Language, Is.EqualTo(language));
Assert.That(cleaner.Script, Is.EqualTo(script));
Assert.That(cleaner.Region, Is.EqualTo(region));
Assert.That(cleaner.Variant, Is.EqualTo(variant));
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo(completeTag));
}
public void CompleteTagConstructor_QaaWithXDashBeforeValidLanguageName_NoChange()
{
var cleaner = new IetfLanguageTagCleaner("qaa-x-th");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-x-th"));
Assert.That(cleaner.Language, Is.EqualTo("qaa"));
Assert.That(cleaner.PrivateUse, Is.EqualTo("th"));
}
public void CompleteTagConstructor_ValidRfctagWithPrivateUseElements_NoChange()
{
var cleaner = new IetfLanguageTagCleaner("qaa-Zxxx-x-Zxxx-AUDIO");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-Zxxx-x-Zxxx-AUDIO"));
// Except, it should have moved things from Language, where the constructor puts them, to the appropriate places.
Assert.That(cleaner.Language, Is.EqualTo("qaa"));
Assert.That(cleaner.Script, Is.EqualTo("Zxxx"));
Assert.That(cleaner.PrivateUse, Is.EqualTo("Zxxx-AUDIO"));
}
public void CompleteTagConstructor_InvalidLanguageNameWithScript_QaaAdded()
{
var cleaner = new IetfLanguageTagCleaner("wee-Latn");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-Latn-x-wee"));
// Also when initially "Latn" is properly in the Script field.
cleaner = new IetfLanguageTagCleaner("wee", "Latn", "", "", "");
cleaner.Clean();
Assert.That(cleaner.GetCompleteTag(), Is.EqualTo("qaa-Latn-x-wee"));
}
///<summary>
/// Constructor.
///</summary>
///<param name="idsInFile"></param>
///<param name="replaceIdsInFile"></param>
///<param name="writingSystemRepository"></param>
public static void FindOrphans(
IEnumerable <string> idsInFile,
IdReplacementStrategy replaceIdsInFile,
IWritingSystemRepository writingSystemRepository)
{
List <string> originalIds = idsInFile.ToList();
List <string> updatedIds = originalIds.ToList();
foreach (string wsId in originalIds)
{
// Check if it's in the repo
if (writingSystemRepository.Contains(wsId))
{
continue;
}
string newId;
if (writingSystemRepository.WritingSystemIdHasChanged(wsId))
{
newId = writingSystemRepository.WritingSystemIdHasChangedTo(wsId);
}
else
{
// It's an orphan
// Clean it
var rfcTagCleaner = new IetfLanguageTagCleaner(wsId);
rfcTagCleaner.Clean();
newId = rfcTagCleaner.GetCompleteTag();
}
WritingSystemDefinition conformantWritingSystem;
writingSystemRepository.WritingSystemFactory.Create(newId, out conformantWritingSystem);
// If it changed, then change
if (conformantWritingSystem.LanguageTag != wsId)
{
conformantWritingSystem.LanguageTag = IetfLanguageTag.ToUniqueLanguageTag(
conformantWritingSystem.LanguageTag, updatedIds);
replaceIdsInFile(wsId, conformantWritingSystem.LanguageTag);
updatedIds.Remove(wsId);
updatedIds.Add(conformantWritingSystem.LanguageTag);
}
// Check if it's in the repo
if (writingSystemRepository.Contains(conformantWritingSystem.LanguageTag))
{
continue;
}
// It's not in the repo so set it
writingSystemRepository.Set(conformantWritingSystem);
}
writingSystemRepository.Save();
}
private bool TryGetNewLangTag(string oldTag, out string newTag)
{
if (m_tagMap.TryGetValue(oldTag, out newTag))
{
return(!newTag.Equals(oldTag));
}
var cleaner = new IetfLanguageTagCleaner(oldTag);
cleaner.Clean();
newTag = cleaner.GetCompleteTag();
while (m_tagMap.Values.Contains(newTag))
{
// We can't use this tag because it would conflict with what we are mapping something else to.
cleaner = new IetfLanguageTagCleaner(cleaner.Language, cleaner.Script, cleaner.Region, cleaner.Variant,
WritingSystemIdMigrator.GetNextDuplPart(cleaner.PrivateUse));
newTag = cleaner.GetCompleteTag();
}
m_tagMap[oldTag] = newTag;
return(!newTag.Equals(oldTag));
}
public override void Migrate(string sourceFilePath, string destinationFilePath)
{
string sourceFileName = Path.GetFileName(sourceFilePath);
var writingSystemDefinitionV1 = new WritingSystemDefinitionV1();
new LdmlAdaptorV1().Read(sourceFilePath, writingSystemDefinitionV1);
string abbreviation = writingSystemDefinitionV1.Abbreviation;
float defaultFontSize = writingSystemDefinitionV1.DefaultFontSize;
string keyboard = writingSystemDefinitionV1.Keyboard;
string spellCheckingId = writingSystemDefinitionV1.SpellCheckingId;
string defaultFontName = writingSystemDefinitionV1.DefaultFontName;
string languageName = writingSystemDefinitionV1.LanguageName.IsOneOf("Unknown Language", "Language Not Listed") ? string.Empty : writingSystemDefinitionV1.LanguageName;
string variant, privateUse;
IetfLanguageTag.SplitVariantAndPrivateUse(writingSystemDefinitionV1.Variant, out variant, out privateUse);
var langTagCleaner = new IetfLanguageTagCleaner(writingSystemDefinitionV1.Language, writingSystemDefinitionV1.Script, writingSystemDefinitionV1.Region,
variant, privateUse);
langTagCleaner.Clean();
string langTag = IetfLanguageTag.Canonicalize(langTagCleaner.GetCompleteTag());
List <string> knownKeyboards = writingSystemDefinitionV1.KnownKeyboards.Select(k => string.IsNullOrEmpty(k.Locale) ? k.Layout : string.Format("{0}_{1}", k.Locale, k.Layout)).ToList();
bool isGraphiteEnabled = false;
string legacyMapping = string.Empty;
string scriptName = string.Empty;
string regionName = string.Empty;
string variantName = string.Empty;
SystemCollationDefinition scd = null;
// Create system collation definition if applicable
if ((writingSystemDefinitionV1.SortUsing == WritingSystemDefinitionV1.SortRulesType.OtherLanguage) && (!string.IsNullOrEmpty(writingSystemDefinitionV1.SortRules)))
{
scd = new SystemCollationDefinition {
LanguageTag = writingSystemDefinitionV1.SortRules
}
}
;
// Migrate fields from legacy fw namespace, and then remove fw namespace
XElement ldmlElem = XElement.Load(sourceFilePath);
XElement fwElem = ldmlElem.Elements("special").FirstOrDefault(e => !string.IsNullOrEmpty((string)e.Attribute(XNamespace.Xmlns + "fw")));
if (fwElem != null)
{
XElement graphiteEnabledElem = fwElem.Element(FW + "graphiteEnabled");
if (graphiteEnabledElem != null)
{
if (!bool.TryParse((string)graphiteEnabledElem.Attribute("value"), out isGraphiteEnabled))
{
isGraphiteEnabled = false;
}
}
// LegacyMapping
XElement legacyMappingElem = fwElem.Element(FW + "legacyMapping");
if (legacyMappingElem != null)
{
legacyMapping = (string)legacyMappingElem.Attribute("value");
}
// ScriptName
XElement scriptNameElem = fwElem.Element(FW + "scriptName");
if (scriptNameElem != null)
{
scriptName = (string)scriptNameElem.Attribute("value");
}
// RegionName
XElement regionNameElem = fwElem.Element(FW + "regionName");
if (regionNameElem != null)
{
regionName = (string)regionNameElem.Attribute("value");
}
// VariantName
XElement variantNameElem = fwElem.Element(FW + "variantName");
if (variantNameElem != null)
{
variantName = (string)variantNameElem.Attribute("value");
}
}
// Record the details for use in PostMigrate where we change the file name to match the ieft language tag where we can.
var migrationInfo = new LdmlMigrationInfo(sourceFileName)
{
LanguageTagBeforeMigration = writingSystemDefinitionV1.Bcp47Tag,
LanguageTagAfterMigration = langTag,
RemovedPropertiesSetter = ws =>
{
if (!string.IsNullOrEmpty(abbreviation))
{
ws.Abbreviation = abbreviation;
}
if (defaultFontSize != 0)
{
ws.DefaultFontSize = defaultFontSize;
}
if (!string.IsNullOrEmpty(keyboard))
{
ws.Keyboard = keyboard;
}
if (!string.IsNullOrEmpty(spellCheckingId))
{
ws.SpellCheckingId = spellCheckingId;
}
if (!string.IsNullOrEmpty(defaultFontName))
{
ws.DefaultFont = ws.Fonts[defaultFontName];
}
if (!string.IsNullOrEmpty(languageName))
{
ws.Language = new LanguageSubtag(ws.Language, languageName);
}
ws.IsGraphiteEnabled = isGraphiteEnabled;
if (!string.IsNullOrEmpty(legacyMapping))
{
ws.LegacyMapping = legacyMapping;
}
if (!string.IsNullOrEmpty(scriptName) && ws.Script != null && ws.Script.IsPrivateUse)
{
ws.Script = new ScriptSubtag(ws.Script, scriptName);
}
if (!string.IsNullOrEmpty(regionName) && ws.Region != null && ws.Region.IsPrivateUse)
{
ws.Region = new RegionSubtag(ws.Region, regionName);
}
if (scd != null)
{
ws.DefaultCollation = scd;
}
foreach (string keyboardId in knownKeyboards)
{
IKeyboardDefinition kd;
if (!Keyboard.Controller.TryGetKeyboard(keyboardId, out kd))
{
kd = Keyboard.Controller.CreateKeyboard(keyboardId, KeyboardFormat.Unknown, Enumerable.Empty <string>());
}
ws.KnownKeyboards.Add(kd);
}
}
};
_migrationInfo.Add(migrationInfo);
// Store things that stay in ldml but are being moved: WindowsLcid, variantName, font, known keyboards, collations, font features, character sets
// misc properties
var staging = new Staging
{
WindowsLcid = writingSystemDefinitionV1.WindowsLcid,
DefaultFontName = writingSystemDefinitionV1.DefaultFontName,
SortUsing = writingSystemDefinitionV1.SortUsing,
SortRules = writingSystemDefinitionV1.SortRules,
};
// Determine if variantName is non-common private use before preserving it
if (!string.IsNullOrEmpty(variantName))
{
int index = IetfLanguageTag.GetIndexOfFirstNonCommonPrivateUseVariant(IetfLanguageTag.GetVariantSubtags(migrationInfo.LanguageTagAfterMigration));
if (index > -1)
{
staging.VariantName = variantName;
}
}
if (fwElem != null)
{
// DefaultFontFeatures
XElement fontFeatsElem = fwElem.Element(FW + "defaultFontFeatures");
if (fontFeatsElem != null && !string.IsNullOrEmpty(staging.DefaultFontName))
{
staging.DefaultFontFeatures = (string)fontFeatsElem.Attribute("value");
}
//MatchedPairs, PunctuationPatterns, QuotationMarks deprecated
// Valid Chars
XElement validCharsElem = fwElem.Element(FW + "validChars");
if (validCharsElem != null)
{
try
{
var fwValidCharsElem = XElement.Parse((string)validCharsElem.Attribute("value"));
AddCharacterSet(fwValidCharsElem, staging, "WordForming", "main");
AddCharacterSet(fwValidCharsElem, staging, "Numeric", "numeric");
AddCharacterSet(fwValidCharsElem, staging, "Other", "punctuation");
}
catch (XmlException)
{
ParseLegacyWordformingCharOverridesFile(staging);
}
}
}
_staging[sourceFileName] = staging;
}
public override void Migrate(string sourceFilePath, string destinationFilePath)
{
string sourceFileName = Path.GetFileName(sourceFilePath);
var writingSystemDefinitionV1 = new WritingSystemDefinitionV1();
new LdmlAdaptorV1().Read(sourceFilePath, writingSystemDefinitionV1);
string abbreviation = writingSystemDefinitionV1.Abbreviation;
float defaultFontSize = writingSystemDefinitionV1.DefaultFontSize;
string keyboard = writingSystemDefinitionV1.Keyboard;
string spellCheckingId = writingSystemDefinitionV1.SpellCheckingId;
string defaultFontName = writingSystemDefinitionV1.DefaultFontName;
string languageName = writingSystemDefinitionV1.LanguageName.IsOneOf("Unknown Language", "Language Not Listed") ? string.Empty : writingSystemDefinitionV1.LanguageName;
string variant, privateUse;
IetfLanguageTag.SplitVariantAndPrivateUse(writingSystemDefinitionV1.Variant, out variant, out privateUse);
var langTagCleaner = new IetfLanguageTagCleaner(writingSystemDefinitionV1.Language, writingSystemDefinitionV1.Script, writingSystemDefinitionV1.Region,
variant, privateUse);
langTagCleaner.Clean();
string langTag = IetfLanguageTag.Canonicalize(langTagCleaner.GetCompleteTag());
List<string> knownKeyboards = writingSystemDefinitionV1.KnownKeyboards.Select(k => string.IsNullOrEmpty(k.Locale) ? k.Layout : string.Format("{0}_{1}", k.Locale, k.Layout)).ToList();
bool isGraphiteEnabled = false;
string legacyMapping = string.Empty;
string scriptName = string.Empty;
string regionName = string.Empty;
string variantName = string.Empty;
SystemCollationDefinition scd = null;
// Create system collation definition if applicable
if ((writingSystemDefinitionV1.SortUsing == WritingSystemDefinitionV1.SortRulesType.OtherLanguage) && (!string.IsNullOrEmpty(writingSystemDefinitionV1.SortRules)))
scd = new SystemCollationDefinition { LanguageTag = writingSystemDefinitionV1.SortRules };
// Migrate fields from legacy fw namespace, and then remove fw namespace
XElement ldmlElem = XElement.Load(sourceFilePath);
XElement fwElem = ldmlElem.Elements("special").FirstOrDefault(e => !string.IsNullOrEmpty((string) e.Attribute(XNamespace.Xmlns + "fw")));
if (fwElem != null)
{
XElement graphiteEnabledElem = fwElem.Element(FW + "graphiteEnabled");
if (graphiteEnabledElem != null)
{
if (!bool.TryParse((string) graphiteEnabledElem.Attribute("value"), out isGraphiteEnabled))
isGraphiteEnabled = false;
}
// LegacyMapping
XElement legacyMappingElem = fwElem.Element(FW + "legacyMapping");
if (legacyMappingElem != null)
legacyMapping = (string) legacyMappingElem.Attribute("value");
// ScriptName
XElement scriptNameElem = fwElem.Element(FW + "scriptName");
if (scriptNameElem != null)
scriptName = (string) scriptNameElem.Attribute("value");
// RegionName
XElement regionNameElem = fwElem.Element(FW + "regionName");
if (regionNameElem != null)
regionName = (string) regionNameElem.Attribute("value");
// VariantName
XElement variantNameElem = fwElem.Element(FW + "variantName");
if (variantNameElem != null)
variantName = (string) variantNameElem.Attribute("value");
}
// Record the details for use in PostMigrate where we change the file name to match the ieft language tag where we can.
var migrationInfo = new LdmlMigrationInfo(sourceFileName)
{
LanguageTagBeforeMigration = writingSystemDefinitionV1.Bcp47Tag,
LanguageTagAfterMigration = langTag,
RemovedPropertiesSetter = ws =>
{
if (!string.IsNullOrEmpty(abbreviation))
ws.Abbreviation = abbreviation;
if (defaultFontSize != 0)
ws.DefaultFontSize = defaultFontSize;
if (!string.IsNullOrEmpty(keyboard))
ws.Keyboard = keyboard;
if (!string.IsNullOrEmpty(spellCheckingId))
ws.SpellCheckingId = spellCheckingId;
if (!string.IsNullOrEmpty(defaultFontName))
ws.DefaultFont = ws.Fonts[defaultFontName];
if (!string.IsNullOrEmpty(languageName))
ws.Language = new LanguageSubtag(ws.Language, languageName);
ws.IsGraphiteEnabled = isGraphiteEnabled;
if (!string.IsNullOrEmpty(legacyMapping))
ws.LegacyMapping = legacyMapping;
if (!string.IsNullOrEmpty(scriptName) && ws.Script != null && ws.Script.IsPrivateUse)
ws.Script = new ScriptSubtag(ws.Script, scriptName);
if (!string.IsNullOrEmpty(regionName) && ws.Region != null && ws.Region.IsPrivateUse)
ws.Region = new RegionSubtag(ws.Region, regionName);
if (scd != null)
ws.DefaultCollation = scd;
foreach (string keyboardId in knownKeyboards)
{
IKeyboardDefinition kd;
if (!Keyboard.Controller.TryGetKeyboard(keyboardId, out kd))
kd = Keyboard.Controller.CreateKeyboard(keyboardId, KeyboardFormat.Unknown, Enumerable.Empty<string>());
ws.KnownKeyboards.Add(kd);
}
}
};
_migrationInfo.Add(migrationInfo);
// Store things that stay in ldml but are being moved: WindowsLcid, variantName, font, known keyboards, collations, font features, character sets
// misc properties
var staging = new Staging
{
WindowsLcid = writingSystemDefinitionV1.WindowsLcid,
DefaultFontName = writingSystemDefinitionV1.DefaultFontName,
SortUsing = writingSystemDefinitionV1.SortUsing,
SortRules = writingSystemDefinitionV1.SortRules,
};
// Determine if variantName is non-common private use before preserving it
if (!string.IsNullOrEmpty(variantName))
{
int index = IetfLanguageTag.GetIndexOfFirstNonCommonPrivateUseVariant(IetfLanguageTag.GetVariantSubtags(migrationInfo.LanguageTagAfterMigration));
if (index > -1)
staging.VariantName = variantName;
}
if (fwElem != null)
{
// DefaultFontFeatures
XElement fontFeatsElem = fwElem.Element(FW + "defaultFontFeatures");
if (fontFeatsElem != null && !string.IsNullOrEmpty(staging.DefaultFontName))
staging.DefaultFontFeatures = (string) fontFeatsElem.Attribute("value");
//MatchedPairs, PunctuationPatterns, QuotationMarks deprecated
// Valid Chars
XElement validCharsElem = fwElem.Element(FW + "validChars");
if (validCharsElem != null)
{
try
{
var fwValidCharsElem = XElement.Parse((string) validCharsElem.Attribute("value"));
AddCharacterSet(fwValidCharsElem, staging, "WordForming", "main");
AddCharacterSet(fwValidCharsElem, staging, "Numeric", "numeric");
AddCharacterSet(fwValidCharsElem, staging, "Other", "punctuation");
}
catch (XmlException)
{
ParseLegacyWordformingCharOverridesFile(staging);
}
}
}
_staging[sourceFileName] = staging;
}