/// <summary/>
public bool ValidateIetfCode(string text)
{
string language;
string script;
string region;
string variant;
if (IetfLanguageTag.TryGetParts(_model.CurrentWsSetupModel?.CurrentLanguageTag, out language, out script, out region,
out variant))
{
return(text.StartsWith(language) && IetfLanguageTag.IsValid(text));
}
throw new ApplicationException("Invalid code stored in the model");
}
/// <summary>
/// Whether a configuration represents a Reversal.
/// </summary>
public static bool IsConfigurationAnOriginalReversal(DictionaryConfigurationModel configuration, LcmCache cache)
{
if (configuration.FilePath == null)
{
return(false);
}
// No configuration.WritingSystem means it is not a reversal, or that it is the AllReversalIndexes which doesn't act any different from a default config
if (!string.IsNullOrWhiteSpace(configuration.WritingSystem) && IetfLanguageTag.IsValid(configuration.WritingSystem))
{
var writingSystem = (CoreWritingSystemDefinition)cache.WritingSystemFactory.get_Engine(configuration.WritingSystem);
// The reversals start out with the filename matching the ws Id, copies will have a different file name
return(writingSystem.Id == Path.GetFileNameWithoutExtension(configuration.FilePath));
}
return(false);
}
/// <summary>
/// API to query the SLDR for an LDML file and save it locally in the SLDR cache and specified directories
/// </summary>
/// <param name="destinationPath">Destination path to save the requested LDML file</param>
/// <param name="languageTag">Current IETF language tag</param>
/// <param name="topLevelElements">List of top level element names to request. SLDR will always publish identity, so it doesn't need to be requested.
/// If null, the entire LDML file will be requested.</param>
/// <param name="filename">Saved filename</param>
/// <returns>Enum status SldrStatus if file could be retrieved and the source</returns>
public static SldrStatus GetLdmlFile(string destinationPath, string languageTag, IEnumerable <string> topLevelElements, out string filename)
{
CheckInitialized();
if (String.IsNullOrEmpty(destinationPath))
{
throw new ArgumentException("destinationPath");
}
if (!Directory.Exists(destinationPath))
{
throw new DirectoryNotFoundException("destinationPath");
}
if (String.IsNullOrEmpty(languageTag) || (!IetfLanguageTag.IsValid(languageTag)))
{
throw new ArgumentException("ietfLanguageTag");
}
if (topLevelElements == null)
{
throw new ArgumentNullException("topLevelElements");
}
string sldrLanguageTag = IetfLanguageTag.Canonicalize(languageTag);
SldrLanguageTagInfo langTagInfo;
if (LanguageTags.TryGet(sldrLanguageTag, out langTagInfo))
{
sldrLanguageTag = langTagInfo.SldrLanguageTag;
}
string[] topLevelElementsArray = topLevelElements.ToArray();
using (_sldrCacheMutex.Lock())
{
var status = SldrStatus.NotFound;
CreateSldrCacheDirectory();
string sldrCacheFilePath;
bool redirected;
do
{
string revid, uid = "", tempString;
if (destinationPath == SldrCachePath)
{
filename = string.Format("{0}.{1}", sldrLanguageTag, LdmlExtension);
}
else
{
filename = string.Format("{0}.{1}", languageTag, LdmlExtension);
// Check if LDML file already exists in destination and read revid and uid
if (!ReadSilIdentity(Path.Combine(destinationPath, filename), out tempString, out uid))
{
uid = DefaultUserId;
}
}
// If languageTag contains fonipa, don't bother trying to access the SLDR
if (sldrLanguageTag.Contains(WellKnownSubtags.IpaVariant) || sldrLanguageTag.Contains(WellKnownSubtags.AudioScript))
{
return(SldrStatus.NotFound);
}
sldrCacheFilePath = Path.Combine(SldrCachePath, !string.IsNullOrEmpty(uid) && uid != DefaultUserId ? string.Format("{0}-{1}.{2}", sldrLanguageTag, uid, LdmlExtension)
: string.Format("{0}.{1}", sldrLanguageTag, LdmlExtension));
// Read revid from cache file
ReadSilIdentity(sldrCacheFilePath, out revid, out tempString);
// Concatenate parameters for url string
string requestedElements = string.Empty;
if (topLevelElementsArray.Length > 0)
{
requestedElements = string.Format("&inc[]={0}", string.Join("&inc[]=", topLevelElementsArray));
}
string requestedUserId = !string.IsNullOrEmpty(uid) ? string.Format("&uid={0}", uid) : string.Empty;
string requestedRevid = !string.IsNullOrEmpty(revid) ? string.Format("&revid={0}", revid) : string.Empty;
string url = string.Format("{0}{1}?ext={2}&flatten=1{3}{4}{5}",
SldrRepository, sldrLanguageTag, LdmlExtension,
requestedElements, requestedUserId, requestedRevid);
string tempFilePath = sldrCacheFilePath + "." + TmpExtension;
// Using WebRequest instead of WebClient so we have access to disable AllowAutoRedirect
var webRequest = (HttpWebRequest)WebRequest.Create(Uri.EscapeUriString(url));
webRequest.AllowAutoRedirect = false;
webRequest.UserAgent = UserAgent;
webRequest.Timeout = 10000;
try
{
if (_offlineMode)
{
throw new WebException("Test mode: SLDR offline so accessing cache", WebExceptionStatus.ConnectFailure);
}
// Check the response header to see if the requested LDML file got redirected
using (var webResponse = (HttpWebResponse)webRequest.GetResponse())
{
if (webResponse.StatusCode == HttpStatusCode.NotModified)
{
// Report status that file is the most current from SLDR
status = SldrStatus.FromSldr;
redirected = false;
}
else if (webResponse.StatusCode == HttpStatusCode.MovedPermanently)
{
// Extract ietfLanguageTag from the response header
var parsedresponse = HttpUtilityFromMono.ParseQueryString(webResponse.Headers["Location"]);
sldrLanguageTag = parsedresponse.Get("ws_id").Split('?')[0];
redirected = true;
}
else
{
// Download the LDML file to a temp file in case the transfer gets interrupted
using (Stream responseStream = webResponse.GetResponseStream())
using (var fs = new FileStream(tempFilePath, FileMode.OpenOrCreate, FileAccess.Write))
{
var buff = new byte[102400];
int c;
while ((c = responseStream.Read(buff, 0, buff.Length)) > 0)
{
fs.Write(buff, 0, c);
fs.Flush();
}
}
status = SldrStatus.FromSldr;
sldrCacheFilePath = MoveTmpToCache(tempFilePath, uid);
redirected = false;
}
}
}
catch (WebException we)
{
// Return from 404 error
var errorResponse = (HttpWebResponse)we.Response;
if ((we.Status == WebExceptionStatus.ProtocolError) && (errorResponse.StatusCode == HttpStatusCode.NotFound))
{
return(SldrStatus.NotFound);
}
string sldrCacheFilename;
// Download failed so check SLDR cache
if (!string.IsNullOrEmpty(uid) && (uid != DefaultUserId))
{
sldrCacheFilename = string.Format("{0}-{1}.{2}", sldrLanguageTag, uid, LdmlExtension);
}
else
{
sldrCacheFilename = string.Format("{0}.{1}", sldrLanguageTag, LdmlExtension);
}
sldrCacheFilePath = Path.Combine(SldrCachePath, sldrCacheFilename);
if (File.Exists(sldrCacheFilePath))
{
status = SldrStatus.FromCache;
}
else
{
return(SldrStatus.UnableToConnectToSldr);
}
redirected = false;
}
finally
{
if (File.Exists(tempFilePath))
{
File.Delete(tempFilePath);
}
}
} while (redirected);
if (destinationPath != SldrCachePath)
{
// Copy from Cache to destination (w/o uid in filename), overwriting whatever used to be there
File.Copy(sldrCacheFilePath, Path.Combine(destinationPath, filename), true);
}
return(status);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="LanguageLookup"/> class.
/// </summary>
public LanguageLookup()
{
var threeToTwoLetter = new Dictionary <string, string>();
foreach (string line in LanguageRegistryResources.TwoToThreeCodes.Replace("\r\n", "\n").Split(new[] { "\n" }, StringSplitOptions.RemoveEmptyEntries))
{
string[] items = line.Split('\t');
threeToTwoLetter.Add(items[1].Trim(), items[0].Trim());
}
//LanguageIndex.txt Format: LangID CountryID NameType Name
//a language appears on one row for each of its alternative langauges
var entries = new List <string>(LanguageRegistryResources.LanguageIndex.Split(new[] { "\n" }, StringSplitOptions.RemoveEmptyEntries));
entries.Add("qaa\t?\tL\tUnlisted Language");
foreach (string entry in entries.Skip(1)) //skip the header
{
string[] items = entry.Split('\t');
if (items.Length != 4)
{
continue;
}
if (items[2].Contains('!')) //temporary suppression of entries while waiting for Ethnologue changes
{
continue;
}
string code = items[0].Trim();
string twoLetterCode;
if (threeToTwoLetter.TryGetValue(code, out twoLetterCode))
{
code = twoLetterCode;
}
string regionCode = items[1].Trim();
LanguageInfo language = GetOrCreateLanguageFromCode(code, regionCode == "?" ? "?" : StandardSubtags.RegisteredRegions[regionCode].Name);
string name = items[3].Trim();
if (items[2] == "L")
{
while (language.Names.Contains(name))
{
language.Names.Remove(name);
}
language.Names.Insert(0, name);
}
else
{
if (items[2].Contains("P"))
{
//Skip pejorative
}
else if (items[1] == ("ET"))
{
//Skip alternatives for Ethiopia, as per request
}
else if (items[0] == "gax" || items[0] == "om")
{
//For these two "Oromo" languages, skip all related languages as per request
}
else if (!language.Names.Contains(name))
{
language.Names.Add(name); //intentionally not lower-casing
}
}
}
IEnumerable <IGrouping <string, string> > languageGroups = Sldr.LanguageTags.Where(info => info.IsAvailable && IetfLanguageTag.IsValid(info.LanguageTag))
.Select(info => IetfLanguageTag.Canonicalize(info.LanguageTag))
.GroupBy(IetfLanguageTag.GetLanguagePart);
foreach (IGrouping <string, string> languageGroup in languageGroups)
{
string[] langTags = languageGroup.ToArray();
if (langTags.Length == 1)
{
string langTag = langTags[0];
LanguageInfo language;
if (langTag != languageGroup.Key && _codeToLanguageIndex.TryGetValue(languageGroup.Key, out language))
{
_codeToLanguageIndex.Remove(languageGroup.Key);
language.LanguageTag = langTag;
_codeToLanguageIndex[langTag] = language;
}
}
else
{
foreach (string langTag in langTags)
{
LanguageSubtag languageSubtag;
ScriptSubtag scriptSubtag;
RegionSubtag regionSubtag;
IEnumerable <VariantSubtag> variantSubtags;
if (IetfLanguageTag.TryGetSubtags(langTag, out languageSubtag, out scriptSubtag, out regionSubtag, out variantSubtags))
{
if (langTag == languageSubtag)
{
continue;
}
LanguageInfo language = GetOrCreateLanguageFromCode(langTag, regionSubtag == null ? "?" : regionSubtag.Name);
bool displayScript = scriptSubtag != null && !IetfLanguageTag.IsScriptImplied(langTag);
LanguageInfo otherLanguage;
if (langTag != languageSubtag && !displayScript && _codeToLanguageIndex.TryGetValue(languageSubtag, out otherLanguage) && language.Countries.SetEquals(otherLanguage.Countries))
{
language.Names.AddRange(otherLanguage.Names);
}
else
{
string name = displayScript ? string.Format("{0} ({1})", languageSubtag.Name, scriptSubtag.Name) : languageSubtag.Name;
if (!language.Names.Contains(name))
{
language.Names.Add(name); //intentionally not lower-casing
}
}
}
}
}
}
foreach (LanguageInfo languageInfo in _codeToLanguageIndex.Values)
{
foreach (string name in languageInfo.Names)
{
GetOrCreateListFromName(name).Add(languageInfo);
}
if (languageInfo.Names.Count == 0)
{
continue; // this language is suppressed
}
//Why just this small set? Only out of convenience. Ideally we'd have a db of all languages as they write it in their literature.
string localName = null;
switch (languageInfo.Names[0])
{
case "French":
localName = "français";
break;
case "Spanish":
localName = "español";
break;
case "Chinese":
localName = "中文";
break;
case "Hindi":
localName = "हिन्दी";
break;
case "Bengali":
localName = "বাংলা";
break;
case "Telugu":
localName = "తెలుగు";
break;
case "Tamil":
localName = "தமிழ்";
break;
case "Urdu":
localName = "اُردُو";
break;
case "Arabic":
localName = "العربية/عربي";
break;
case "Thai":
localName = "ภาษาไทย";
break;
case "Indonesian":
localName = "Bahasa Indonesia";
break;
}
if (!string.IsNullOrEmpty(localName))
{
if (!languageInfo.Names.Remove(localName))
{
GetOrCreateListFromName(localName).Add(languageInfo);
}
languageInfo.Names.Insert(0, localName);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="LanguageDataIndex"/> class.
/// </summary>
public LanguageDataIndex(IDictionary <string, string> sourcefiles)
{
string twotothreecodes = sourcefiles["TwoToThreeCodes.txt"];
string subtagregistry = sourcefiles["ianaSubtagRegistry.txt"];
StandardSubtags.InitialiseIanaSubtags(twotothreecodes, subtagregistry);
// First read in Ethnologue data file into temporary dictionary
var threeToTwoLetter = StandardSubtags.TwoAndThreeMap(twotothreecodes, true);
//LanguageIndex.txt Format: LangID CountryID NameType Name
//a language appears on one row for each of its alternative langauges
string languageindex = sourcefiles["LanguageIndex.txt"];
var entries = new List <string>(languageindex.Split(new[] { "\n" }, StringSplitOptions.RemoveEmptyEntries));
entries.Add("qaa\t?\tL\tUnlisted Language");
foreach (string entry in entries.Skip(1)) //skip the header
{
string[] items = entry.Split('\t');
if (items.Length != 4)
{
continue;
}
if (items[2].StartsWith("!")) //temporary suppression of entries while waiting for Ethnologue changes
{
continue;
}
// excluded by !
// all gax (ET,KE,SO) including L
// all gaz (ET) including L
// all hae (ET) including L
string code = items[0].Trim();
string twoLetterCode;
string threelettercode = code;
if (threeToTwoLetter.TryGetValue(code, out twoLetterCode))
{
code = twoLetterCode;
}
//temporary suppression of entries while waiting for Ethnologue changes (those excluded by !)
if (ExcludedCodes.Contains(code))
{
continue;
}
string regionCode = items[1].Trim();
LanguageInfo language = GetOrCreateLanguageFromCode(code, threelettercode, regionCode == "?" ? "" : StandardSubtags.RegisteredRegions[regionCode].Name);
string name = items[3].Trim();
if (items[2].Trim() == "L")
{
while (language.Names.Contains(name))
{
language.Names.Remove(name);
}
language.Names.Insert(0, name);
}
else
{
if (items[2].Contains("P"))
{
//Skip pejorative
}
else if (ExcludedRegions.Contains(StandardSubtags.RegisteredRegions[regionCode].Name))
{
//Skip alternatives for Ethiopia, as per request
}
else if (code == "gax" || code == "om")
{
//For these two "Oromo" languages, skip all related languages as per request
}
else if (!language.Names.Contains(name))
{
language.Names.Add(name); //intentionally not lower-casing
}
}
}
// Then for each registered ietf language tag create a real entry and add the ethnologue data to it
IOrderedEnumerable <LanguageSubtag> languages = StandardSubtags.RegisteredLanguages.OrderBy(lang => lang.Iso3Code);
foreach (LanguageSubtag language in languages)
{
bool singlename = false;
if (language.IsDeprecated || ExcludedCodes.Contains(language.Code))
{
continue;
}
LanguageInfo langinfo = GetOrCreateLanguageFromCode(language.Code, language.Iso3Code, null);
langinfo.DesiredName = language.Name.Replace("'", "’");
langinfo.IsMacroLanguage = language.IsMacroLanguage;
foreach (string country in langinfo.Countries)
{
if (ExcludedRegions.Contains(country))
{
singlename = true;
}
}
foreach (string name in language.Names)
{
string langname = name.Replace("'", "’");
if (!langinfo.Names.Contains(langname))
{
if (singlename && langinfo.Names.Count == 1)
{
// leave single ethnologue names
break;
}
else
{
langinfo.Names.Add(langname);
}
}
if (singlename)
{
break;
}
}
_codeToLanguageIndex.Add(language.Code, langinfo);
}
IEnumerable <IGrouping <string, string> > languageGroups = Sldr.LanguageTags.Where(info => info.IsAvailable && IetfLanguageTag.IsValid(info.LanguageTag))
.Select(info => IetfLanguageTag.Canonicalize(info.LanguageTag))
.GroupBy(IetfLanguageTag.GetLanguagePart);
foreach (IGrouping <string, string> languageGroup in languageGroups)
{
string[] langTags = languageGroup.ToArray();
if (langTags.Length == 1)
{
string langTag = langTags[0];
LanguageInfo language;
if (langTag != languageGroup.Key && _codeToLanguageIndex.TryGetValue(languageGroup.Key, out language))
{
_codeToLanguageIndex.Remove(languageGroup.Key);
language.LanguageTag = langTag;
_codeToLanguageIndex[langTag] = language;
}
}
else
{
foreach (string langTag in langTags)
{
LanguageSubtag languageSubtag;
ScriptSubtag scriptSubtag;
RegionSubtag regionSubtag;
IEnumerable <VariantSubtag> variantSubtags;
if (IetfLanguageTag.TryGetSubtags(langTag, out languageSubtag, out scriptSubtag, out regionSubtag, out variantSubtags))
{
if (langTag == languageSubtag)
{
continue;
}
LanguageInfo language = GetOrCreateLanguageFromCode(langTag, langTag, regionSubtag == null ? "" : regionSubtag.Name); // changed to default to "" 2017-04-24
bool displayScript = scriptSubtag != null && !IetfLanguageTag.IsScriptImplied(langTag);
LanguageInfo otherLanguage;
if (langTag != languageSubtag && !displayScript && _codeToLanguageIndex.TryGetValue(languageSubtag, out otherLanguage) && language.Countries.SetEquals(otherLanguage.Countries))
{
language.Names.AddRange(otherLanguage.Names);
}
else
{
string name = displayScript ? string.Format("{0} ({1})", languageSubtag.Name, scriptSubtag.Name) : languageSubtag.Name;
if (!language.Names.Contains(name))
{
language.Names.Add(name); //intentionally not lower-casing
}
}
LanguageInfo keylanguage;
if (_codeToLanguageIndex.TryGetValue(languageGroup.Key, out keylanguage))
{
language.IsMacroLanguage = keylanguage.IsMacroLanguage;
}
_codeToLanguageIndex.Add(langTag, language);
}
}
}
}
string languagecodes = sourcefiles["LanguageCodes.txt"];
var codeentries = new List <string>(languagecodes.Split(new[] { "\n" }, StringSplitOptions.RemoveEmptyEntries));
foreach (var languageCode in codeentries)
{
var data = languageCode.Split(new[] { '\t' }, StringSplitOptions.RemoveEmptyEntries);
if (data.Length < 2)
{
continue;
}
var langCode = data[0];
string twoLetterCode;
if (threeToTwoLetter.TryGetValue(langCode, out twoLetterCode))
{
langCode = twoLetterCode;
}
if (langCode == "fuv")
{
langCode = "fuv-Arab"; // special case because the script has been added to this language code
}
// which is probably something to do with the SLDR
var countryCode = data[1];
LanguageInfo lang;
if (_codeToLanguageIndex.TryGetValue(langCode, out lang))
{
lang.PrimaryCountry = StandardSubtags.RegisteredRegions[countryCode].Name;
}
}
// localise some language names
foreach (LanguageInfo languageInfo in _codeToLanguageIndex.Values)
{
if (languageInfo.Names.Count == 0)
{
continue; // this language is suppressed
}
//Why just this small set? Only out of convenience. Ideally we'd have a db of all languages as they write it in their literature.
string localName = null;
switch (languageInfo.Names[0])
{
case "French":
localName = "français";
break;
case "Spanish":
localName = "español";
break;
case "Chinese":
localName = "中文";
break;
case "Hindi":
localName = "हिन्दी";
break;
case "Bengali":
localName = "বাংলা";
break;
case "Telugu":
localName = "తెలుగు";
break;
case "Tamil":
localName = "தமிழ்";
break;
case "Urdu":
localName = "اُردُو";
break;
case "Arabic":
localName = "العربية/عربي";
break;
case "Thai":
localName = "ภาษาไทย";
break;
case "Indonesian":
localName = "Bahasa Indonesia";
break;
}
if (!string.IsNullOrEmpty(localName))
{
if (languageInfo.Names.Contains(localName))
{
languageInfo.Names.Remove(localName);
}
languageInfo.Names.Insert(0, localName);
languageInfo.DesiredName = localName;
}
switch (languageInfo.ThreeLetterTag)
{
case "itd": // 2 temporary special cases because the LanguageCodes.txt files needs to be updated with LanguageIndex.txt
languageInfo.PrimaryCountry = "Indonesia";
break;
case "xak":
languageInfo.PrimaryCountry = "Venezuela";
break;
default:
// Also set the PrimaryCountry if there is only one country
if (String.IsNullOrEmpty(languageInfo.PrimaryCountry) && languageInfo.Countries.Count == 1)
{
languageInfo.PrimaryCountry = languageInfo.Countries.First();
}
break;
}
}
// check if any languages are found in multiple countries but do not have a primary country
// there is a test for this in LanguageLookupTests.llExpectedLanguagesHaveUniquePrimaryCountries
var languagesWithoutRegions = new List <LanguageInfo>();
foreach (var lang in _codeToLanguageIndex.Values)
{
if (String.IsNullOrEmpty(lang.PrimaryCountry))
{
languagesWithoutRegions.Add(lang);
}
}
var languagesWithAmbiguousPrimaryCountry = languagesWithoutRegions.Where(l => l.Countries.Count() > 1);
foreach (var lang in languagesWithAmbiguousPrimaryCountry)
{
Console.WriteLine("Language {0}({1}) has no primary country but is found in multiple countries", lang.DesiredName, lang.LanguageTag);
}
}
