public static void dumpObjectCount() { // dump count var lowByteToCount = database.Select((list, low) => new { list, low }).Where(li => li.list.Count() > 1).ToDictionary(li => li.low, li => li.list.Length); var lines = lowByteToCount.Select(kv => { decodeLowByte(kv.Key, out string lang, out string classUri); return(new { lang, classUri, kv.Value }); }). GroupBy(lcv => lcv.classUri). SelectMany(g => { return(g.Select(lcv => $"{lcv.lang}={lcv.classUri}={lcv.Value}"). Concat(Linq.Items($"**={g.Key}={g.Sum(lcv => lcv.Value)}"))); }). OrderBy(s => s); File.WriteAllLines(LowUtilsDirs.logs + "dump-objects-count.txt", lines); }
public static void dumpPageParts() { // dump page tree counts IEnumerable <string[]> pageDump(Page p) { yield return(new[] { "p" }); if (p.entries == null) { yield return(new[] { "p", "a noentry" }); yield break; } var ens = p.entries.Length == 1 ? "b entry" : "c entries"; yield return(new[] { "p", ens }); foreach (var en in p.entries) { if (en.otherForm == null) { yield return(new[] { "p", ens, "a noform" }); yield break; } var fms = en.otherForm.Length == 1 ? "b form" : "c forms"; yield return(new[] { "p", ens, fms }); } } var pageParts = new Dictionary <string, int>(); foreach (var s in AllLangs.SelectMany(lang => getObjsStr <Page>(lang). SelectMany(p => pageDump(p)). Select(arr => string.Join("=", arr)). SelectMany(l => Linq.Items("**=" + l, lang + "=" + l)))) { pageParts[s] = pageParts.TryGetValue(s, out int c) ? c + 1 : 1; } File.WriteAllLines(LowUtilsDirs.logs + "dump-page-parts.txt", pageParts.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key} {kv.Value}")); }
public LangMatrix(StreamReader rdr) : this() { try { var lines = rdr.ReadAllLines().ToArray(); var cell00 = lines[0].Split(new char[] { ';' }, 2)[0].Split('/'); LangMatrixRow[] rawLines = null; var data = new List <string[]>(); if (cell00.Length != 2) // => RJ import format: change COLS and ROWS { var matxOld = lines.Select(l => l.Split(';')).ToArray(); var len = matxOld[0].Length; if (!matxOld.All(r => r.Length == len)) { throw new Exception(); } var matxNew = new List <string[]>(); for (var i = 0; i < matxOld[0].Length; i++) { matxNew.Add(new string[matxOld.Length]); } for (var i = 0; i < matxOld.Length; i++) { for (var j = 0; j < matxOld[0].Length; j++) { matxNew[j][i] = matxOld[i][j].Replace('\t', ' ').Replace("@@s", ";"); } } rawLines = readRaw(matxNew); for (var i = 0; i < rawLines.Length; i++) { rawLines[i].lang = Langs.oldToNew(rawLines[i].lang); } rawLines = new LangMatrixRow[] { null }.Concat(rawLines).ToArray(); cell00 = new string[] { "lang", "" }; //var colsNum = rawLines.Length - 1; //var rowsNum = rawLines[0].row.Length + 1; // adds 1 for rawLines[].lang //langs = new string[rawLines[0].row.Length + 1]; //for (var rowIdx = 0; rowIdx < rowsNum; rowIdx++) { // if (rowIdx == 0) langs[0] = rawLines[0].lang; // else langs[rowIdx] = rawLines[0].row[rowIdx - 1]; //} //for (var i = 0; i < langs.Length; i++) langs[i] = Langs.oldToNew(langs[i]); //for (var rowIdx = 0; rowIdx < rowsNum; rowIdx++) { // var row = new string[colsNum]; // for (var colIdx = 0; colIdx < colsNum; colIdx++) { // if (rowIdx == 0) row[colIdx] = rawLines[colIdx + 1].lang; // else row[colIdx] = rawLines[colIdx + 1].row[rowIdx - 1]; // } // data.Add(row); //} } else { rawLines = readRaw(lines); } var langs = new List <string>(); var groupTheSameRows = cell00[0] == "langs"; // group by rows if (cell00[1] == "colNames") { colNames = rawLines[0].row; // save column names } rawLines.Skip(1).ForEach(l => { foreach (var lang in groupTheSameRows ? l.lang.Split(',') : Linq.Items(l.lang)) { langs.Add(lang); data.Add(l.row); } }); this.langs = langs.ToArray(); this.data = data.ToArray(); } finally { rdr.Close(); } }
public static void Build() { var cldr = Json.Deserialize <Langs.CldrLang[]>(LangsDirs.dirCldrTexts); var roots = cldr.SelectMany(c => c.Regions.Select(r => LocaleIdentifier.Parse(string.Format("{0}-{1}-{2}", c.Lang, c.ScriptId, r)))).ToArray(); var locs = roots.Select(root => new Locale(root)).ToArray(); // ALPHABETS var rx = new Regex("[{} ]"); var alphas = locs.Select(loc => { var localePattern = loc.Find("//characters"); var data = localePattern.SelectChildren(XPathNodeType.Element).OfType <XPathNavigator>().Where(al => al.Name == "exemplarCharacters").Select(al => { var value = CldrUtils.decodeUnicodeLiteral(al.Value.Normalize().Trim('[', ']')); value = rx.Replace(value, ""); var keyNode = al.SelectSingleNode("./@type"); var key = keyNode == null ? "root" : keyNode.Value; //if (key == "numbers" || key == "punctuation" || string.IsNullOrEmpty(value)) return null; return(new { key, value }); }).Where(n => n != null).ToArray(); return(new { lang = loc.Id.ToString(), data }); }).Where(d => d.data.Length > 0).OrderBy(l => l.lang).ToArray(); Func <string, IEnumerable <LangMatrixRow> > alphasRes = (string key) => alphas.SelectMany(a => a.data.Where(aa => aa.key == key).Select(aa => new LangMatrixRow { lang = a.lang, columnNames = Linq.Items(aa.key).ToArray(), row = Linq.Items(aa.value).ToArray() })); var alphaRoot = new LangMatrix(alphasRes("root"), null, true); var alphaAuxlity = new LangMatrix(alphasRes("auxiliary"), null, true); var alphaIndex = new LangMatrix(alphasRes("index"), null, true); var alphaNumbers = new LangMatrix(alphasRes("numbers"), null, true); var alphaPunctuation = new LangMatrix(alphasRes("punctuation"), null, true); var patterns = new LangMatrix(locs.Select(loc => { var localePattern = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localePattern").ToString(); var localeSeparator = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localeSeparator").ToString(); return(new LangMatrixRow { lang = loc.Id.ToString(), row = new string[] { localePattern, localeSeparator }, columnNames = new string[] { "pattern", "separator" }, }); }), null, true); Dictionary <string, Dictionary <string, string> > langsProtocol = new Dictionary <string, Dictionary <string, string> >(); var langs = new LangMatrix( locs.Select(loc => fromCldr(loc, "//localeDisplayNames/languages")), langsProtocol, true ); Dictionary <string, Dictionary <string, string> > scriptsProtocol = new Dictionary <string, Dictionary <string, string> >(); var scripts = new LangMatrix( locs.Select(loc => fromCldr(loc, "//localeDisplayNames/scripts")), scriptsProtocol, true ); Dictionary <string, Dictionary <string, string> > regionsProtocol = new Dictionary <string, Dictionary <string, string> >(); var regions = new LangMatrix( locs.Select(loc => fromCldr(loc, "//localeDisplayNames/territories")), regionsProtocol, true ); langs.save(LangsDesignDirs.cldr + "cldrNameLangs.csv", true); scripts.save(LangsDesignDirs.cldr + "cldrNameScripts.csv", true); regions.save(LangsDesignDirs.cldr + "cldrNameRegions.csv", true); patterns.save(LangsDesignDirs.cldr + "cldrNamePatterns.csv", true); alphaRoot.save(LangsDesignDirs.cldr + "alphaRoot.csv", true); alphaAuxlity.save(LangsDesignDirs.cldr + "alphaAuxlity.csv", true); alphaIndex.save(LangsDesignDirs.cldr + "alphaIndex.csv", true); alphaNumbers.save(LangsDesignDirs.cldr + "alphaNumbers.csv", true); alphaPunctuation.save(LangsDesignDirs.cldr + "alphaPunctuation.csv", true); // save to DART messages File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNameLangs.msg", Protobuf.ToBytes(matrixToDart(langs))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNameScripts.msg", Protobuf.ToBytes(matrixToDart(scripts))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNameRegions.msg", Protobuf.ToBytes(matrixToDart(regions))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\cldrNamePatterns.msg", Protobuf.ToBytes(matrixToDart(patterns))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaRoot.msg", Protobuf.ToBytes(matrixToDart(alphaRoot))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaAuxlity.msg", Protobuf.ToBytes(matrixToDart(alphaAuxlity))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaIndex.msg", Protobuf.ToBytes(matrixToDart(alphaIndex))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaNumbers.msg", Protobuf.ToBytes(matrixToDart(alphaNumbers))); File.WriteAllBytes(LangsDesignDirs.data + @"langsDesign\alphaPunctuation.msg", Protobuf.ToBytes(matrixToDart(alphaPunctuation))); // var localePattern = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localePattern").ToString(); // var localeSeparator = loc.FindOrDefault("//localeDisplayNames/localeDisplayPattern/localeSeparator").ToString(); // fromCldr(loc, "//localeDisplayNames/languages"); // fromCldr(loc, "//localeDisplayNames/scripts"); // fromCldr(loc, "//localeDisplayNames/territories"); //}); }
public static Dictionary <string, HashSet <char> > getBlockNames(string str) { return(getBlockNames(Linq.Items(str))); }
public static Dictionary <string, string> checkBlockNames(string str, string script) { return(str == null ? null : checkBlockNames(Linq.Items(str), script)); }