public static void RefreshOldVersionInfo()
{
var Items = new Dictionary <string, LangMatrixRow>();
//SqlServerReg.Parse(Items, LangsDesignDirs.other + "sqlserver.reg", LangsDesignDirs.other + "sqlserver-clsids.reg");
//SqlServerQuery.Parse(Items, LangsDesignDirs.other + "sqlserver.query");
//ByHand.Parse(Items, LangsDesignDirs.other + "by-hand.xml");
GoogleTrans.Parse(Items);
//SpellCheck.Parse(Items);
foreach (var kv in Items)
{
var ci = CultureInfo.GetCultureInfo(kv.Value.lang);
var lcid = ci.LCID;
if (lcid != 4096)
{
kv.Value.row[6] = lcid.ToString();
}
kv.Value.row[9] = ci.EnglishName;
}
var fullText = new LangMatrix(
Items.Select(kv => kv.Value)
);
fullText.save(LangsDesignDirs.otherappdata + "oldVersionInfo.csv");
var wrongs = LangMatrix.readLangs(LangsDesignDirs.otherappdata + "oldVersionInfo.csv").Where(l => !Langs.nameToMeta.ContainsKey(l)).ToArray();
if (wrongs.Length > 1) // "", LCID 127
{
throw new Exception();
}
}
static Rw.Common.Matrix matrixToDart(LangMatrix matrix)
{
var res = new Rw.Common.Matrix();
res.Cols.AddRange(matrix.colNames);
res.Rows.AddRange(matrix.langs.Select((val, idx) => {
var row = new Rw.Common.Row();
row.Langs.AddRange(val.Split(','));
row.Values.AddRange(matrix.data[idx].Select(cell => cell ?? ""));
return(row);
}));
return(res);
}
public static void Build()
{
var cldr = Json.Deserialize <Langs.CldrLang[]>(LangsDirs.dirCldrTexts);
var roots = cldr.SelectMany(c => c.Regions.Select(r => LocaleIdentifier.Parse(string.Format("{0}-{1}-{2}", c.Lang, c.ScriptId, r)))).ToArray();
var locs = roots.Select(root => new Locale(root)).ToArray();
// ALPHABETS
var rx = new Regex("[{} ]");
var alphas = locs.Select(loc => {
var localePattern = loc.Find("//characters");
var data = localePattern.SelectChildren(XPathNodeType.Element).OfType <XPathNavigator>().Where(al => al.Name == "exemplarCharacters").Select(al => {
var value = CldrUtils.decodeUnicodeLiteral(al.Value.Normalize().Trim('[', ']'));
value = rx.Replace(value, "");
var keyNode = al.SelectSingleNode("./@type");
var key = keyNode == null ? "root" : keyNode.Value;
//if (key == "numbers" || key == "punctuation" || string.IsNullOrEmpty(value)) return null;
return(new { key, value });
}).Where(n => n != null).ToArray();
return(new {
lang = loc.Id.ToString(),
data
});
}).Where(d => d.data.Length > 0).OrderBy(l => l.lang).ToArray();
Func <string, IEnumerable <LangMatrixRow> > alphasRes = (string key) => alphas.SelectMany(a => a.data.Where(aa => aa.key == key).Select(aa =>
new LangMatrixRow {
lang = a.lang, columnNames = Linq.Items(aa.key).ToArray(), row = Linq.Items(aa.value).ToArray()
}));
var alphaRoot = new LangMatrix(alphasRes("root"), null, true);
var alphaAuxlity = new LangMatrix(alphasRes("auxiliary"), null, true);
var alphaIndex = new LangMatrix(alphasRes("index"), null, true);
var alphaNumbers = new LangMatrix(alphasRes("numbers"), null, true);
var alphaPunctuation = new LangMatrix(alphasRes("punctuation"), null, true);
var patterns = new LangMatrix(locs.Select(loc => {
var localePattern = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localePattern").ToString();
var localeSeparator = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localeSeparator").ToString();
return(new LangMatrixRow {
lang = loc.Id.ToString(),
row = new string[] { localePattern, localeSeparator },
columnNames = new string[] { "pattern", "separator" },
});
}), null, true);
Dictionary <string, Dictionary <string, string> > langsProtocol = new Dictionary <string, Dictionary <string, string> >();
var langs = new LangMatrix(
locs.Select(loc => fromCldr(loc, "//localeDisplayNames/languages")),
langsProtocol, true
);
Dictionary <string, Dictionary <string, string> > scriptsProtocol = new Dictionary <string, Dictionary <string, string> >();
var scripts = new LangMatrix(
locs.Select(loc => fromCldr(loc, "//localeDisplayNames/scripts")),
scriptsProtocol, true
);
Dictionary <string, Dictionary <string, string> > regionsProtocol = new Dictionary <string, Dictionary <string, string> >();
var regions = new LangMatrix(
locs.Select(loc => fromCldr(loc, "//localeDisplayNames/territories")),
regionsProtocol, true
);
langs.save(LangsDesignDirs.cldr + "cldrNameLangs.csv", true);
scripts.save(LangsDesignDirs.cldr + "cldrNameScripts.csv", true);
regions.save(LangsDesignDirs.cldr + "cldrNameRegions.csv", true);
patterns.save(LangsDesignDirs.cldr + "cldrNamePatterns.csv", true);
alphaRoot.save(LangsDesignDirs.cldr + "alphaRoot.csv", true);
alphaAuxlity.save(LangsDesignDirs.cldr + "alphaAuxlity.csv", true);
alphaIndex.save(LangsDesignDirs.cldr + "alphaIndex.csv", true);
alphaNumbers.save(LangsDesignDirs.cldr + "alphaNumbers.csv", true);
alphaPunctuation.save(LangsDesignDirs.cldr + "alphaPunctuation.csv", true);
// save to DART messages
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNameLangs.msg", Protobuf.ToBytes(matrixToDart(langs)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNameScripts.msg", Protobuf.ToBytes(matrixToDart(scripts)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNameRegions.msg", Protobuf.ToBytes(matrixToDart(regions)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNamePatterns.msg", Protobuf.ToBytes(matrixToDart(patterns)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaRoot.msg", Protobuf.ToBytes(matrixToDart(alphaRoot)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaAuxlity.msg", Protobuf.ToBytes(matrixToDart(alphaAuxlity)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaIndex.msg", Protobuf.ToBytes(matrixToDart(alphaIndex)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaNumbers.msg", Protobuf.ToBytes(matrixToDart(alphaNumbers)));
File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaPunctuation.msg", Protobuf.ToBytes(matrixToDart(alphaPunctuation)));
// var localePattern = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localePattern").ToString();
// var localeSeparator = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localeSeparator").ToString();
// fromCldr(loc, "//localeDisplayNames/languages");
// fromCldr(loc, "//localeDisplayNames/scripts");
// fromCldr(loc, "//localeDisplayNames/territories");
//});
}
public static void MergeOldToCldr()
{
var olds = new LangMatrix(LangsDesignDirs.otherappdata + "oldVersionInfo.csv");
olds.langs.ForEach((l, idx) => {
var cldr = Langs.nameToMeta[Langs.oldToNew(l)];
var old = olds.data[idx];
cldr.BreakerClass = old[0];
cldr.StemmerClass = old[1];
//cldr.IsEuroTalk = old[2] != null;
//cldr.IsLingea = old[3] != null;
//cldr.IsGoethe = old[4] != null;
int.TryParse(old[6], out cldr.LCID);
cldr.GoogleTransId = old[7];
cldr.WordSpellCheckLCID = old[8] != null ? int.Parse(old[8]) : 0;
cldr.Name = old[9];
});
// prepare langGuids
var withGuid = Langs.meta.
Where(m => (m.BreakerClass != null) && m.Lang != "zh" && m.Id != "en-US" && m.Lang != "pt" && m.Lang != "sr");
var dupls2 = withGuid.GroupBy(m => m.Lang).Where(g => g.Count() > 1).ToArray();
var dupls = withGuid.GroupBy(m => m.Lang).Where(g => g.Count() > 1).Select(g => g.Key).ToArray();
var langGuids = withGuid.Where(m => !dupls.Contains(m.Lang)).ToDictionary(m => m.Lang, m => new { m.StemmerClass, m.BreakerClass });
// prepare scriptGuids
withGuid = Langs.meta.
Where(m => (m.BreakerClass != null) && m.Lang != "zh" && m.Id != "en-US" && m.Lang != "pt" && m.Lang != "sr" && m.Id != "ur-PK");
dupls2 = withGuid.GroupBy(m => m.ScriptId).Where(g => g.Count() > 1).ToArray();
dupls = withGuid.GroupBy(m => m.ScriptId).Where(g => g.Count() > 1).Select(g => g.Key).ToArray();
var scriptGuids = withGuid.Where(m => !dupls.Contains(m.ScriptId)).ToDictionary(m => m.ScriptId, m => m.BreakerClass);
//expand breaking and stemming GUID to other langs
foreach (var m in Langs.meta.Where(m => m.StemmerClass == null && m.BreakerClass == null && langGuids.ContainsKey(m.Lang)))
{
var lg = langGuids[m.Lang];
m.StemmerClass = lg.StemmerClass;
m.BreakerClass = lg.BreakerClass;
}
foreach (var m in Langs.meta.Where(m => m.BreakerClass == null && scriptGuids.ContainsKey(m.ScriptId)))
{
m.BreakerClass = scriptGuids[m.ScriptId];
}
// alphabets
var alphs = new LangMatrix(LangsDesignDirs.cldr + "alphaRoot.csv");
var ignAlphas = new HashSet <string>()
{
"Hant", "Hans", "Jpan", "Kore"
};
Langs.meta.ForEach(m => {
m.Alphabet = "";
m.AlphabetUpper = null;
if (!ignAlphas.Contains(m.ScriptId))
{
Langs.getFullNames(m).ForEach(n => m.Alphabet += alphs[n.ToString(), 0]);
}
finishAlphabet(m);
if (m.Alphabet.Length == 0)
{
m.Alphabet = null;
}
else
{
m.Alphabet = new String(m.Alphabet.Distinct().OrderBy(ch => ch).ToArray());
m.AlphabetUpper = m.Alphabet.ToUpper();
}
});
Langs.save();
}