static public bool BatchExportLocalization(SingleLanguageDB db,
string refPath, string refFileFormatString, string overrideRefPattern,
string outPath, string outFileFormatString, int splitLine = 0)
{
return(BatchExportLocalization(db, refPath, LocalizationFileFormat.GetFileFormat(refFileFormatString),
overrideRefPattern, outPath, LocalizationFileFormat.GetFileFormat(outFileFormatString), splitLine));
}
/// <summary>
///
/// </summary>
/// <param name="original"></param>
/// <param name="translation"></param>
/// <param name="removeRedundant"></param>
/// <returns></returns>
static public List <DiffResultEntry> CheckTranslation(SingleLanguageDB original, SingleLanguageDB translation, bool ignoreSame)
{
List <DiffResultEntry> check = new List <DiffResultEntry>();
foreach (var entry in translation)
{
foreach (var kv in entry.Value)
{
string engText = original.LookupText(entry.Key, kv.Key);
if (engText == null || engText == kv.Value)
{
if (ignoreSame && engText == kv.Value)
{
continue;
}
check.Add(new DiffResultEntry(entry.Key, kv.Key, engText, kv.Value));
}
else if (!SameSpecialCharacters(engText, kv.Value))
{
check.Add(new DiffResultEntry(entry.Key, kv.Key, engText, kv.Value));
}
}
}
return(check);
}
/// <summary>
///
/// </summary>
/// <param name="db"></param>
/// <param name="input"></param>
/// <param name="mode"></param>
/// <returns></returns>
static public int ImportToSingleLanguageDBFromFile(SingleLanguageDB db, ILocalizationFile input,
LocalizationDB.ImportMode mode = LocalizationDB.ImportMode.kReplace)
{
// for now, all input file will be marked as english...
if (!input.VerifyHeader("english"))
{
return(0);
}
int importCount = 0;
int count = input.CountLines();
if (count > 0)
{
for (int i = 0; i < count; ++i)
{
if (input.IsLineLocalizationData(i))
{
string tag = "";
int ver = 0;
string data = "";
string rest = "";
input.GetData(i, ref tag, ref ver, ref data, ref rest);
if (db.Import(tag, ver, data, mode))
{
++importCount;
}
}
}
}
return(importCount);
}
public bool ImportSingleLanguageDB(SingleLanguageDB sldb, bool overwrite = false)
{
if (data.ContainsKey(sldb.Language) && !overwrite)
{
return(false);
}
data[sldb.Language] = sldb;
return(true);
}
public bool Import(string language, string tag, int ver, string text, ImportMode mode)
{
if (!data.ContainsKey(language))
{
data[language] = new SingleLanguageDB(language);
}
return(data[language].Import(tag, ver, text, mode));
}
public static bool DoDiff(string newOriginalPath, string newOriginalFormat, string oldOriginalPath, string oldOriginalFormat,
string oldTranslationPath, string oldTranslationFormat, string outputPath, string outputFormat, string diffFilePath, string language)
{
SingleLanguageDB oldOriginal = new SingleLanguageDB(language);
SingleLanguageDB newOriginal = new SingleLanguageDB(language);
SingleLanguageDB oldTranslation = new SingleLanguageDB(language);
Localization.BatchImportToSingleLanguageDB(newOriginal, newOriginalPath, newOriginalFormat);
Localization.BatchImportToSingleLanguageDB(oldOriginal, oldOriginalPath, oldOriginalFormat);
Localization.BatchImportToSingleLanguageDB(oldTranslation, oldTranslationPath, oldTranslationFormat);
var diff = Localization.Compare(oldOriginal, newOriginal, false);
YMLFile diffyml = new YMLFile();
diffyml.AppendLine(null, -1, GetLanguageTag(language) + ":", null);
foreach (var entry in diff)
{
string chitext = oldTranslation.LookupText(entry.Tag, entry.Version);
diffyml.AppendLine(null, -1, "# new: " + entry.NewText, null);
diffyml.AppendLine(null, -1, "# old: " + entry.OldText, null);
if (chitext != null)
{
diffyml.AppendLine(entry.Tag, entry.Version, chitext, null);
}
else
{
var lastestText = oldTranslation.LookupLatestText(entry.Tag);
if (lastestText != null)
{
diffyml.AppendLine(entry.Tag, entry.Version, lastestText.Item2, null);
}
else
{
diffyml.AppendLine(entry.Tag, entry.Version, entry.NewText, null);
}
}
// Remove trnaslation, prepare for export
if (chitext != null)
{
oldTranslation.Remove(entry.Tag, entry.Version);
}
}
diffyml.Write(diffFilePath);
if (outputPath != null && outputFormat != null)
{
return(Localization.BatchExportLocalization(oldTranslation, newOriginalPath, newOriginalFormat, null, outputPath, outputFormat));
}
return(true);
}
/// <summary>
/// Merge input into target.
/// </summary>
/// <param name="target"></param>
/// <param name="input"></param>
/// <param name="mode"></param>
/// <returns></returns>
static public int MergeIn(SingleLanguageDB target, SingleLanguageDB input, LocalizationDB.ImportMode mode = LocalizationDB.ImportMode.kReplace)
{
int count = 0;
foreach (var entry in input)
{
foreach (var kv in entry.Value)
{
if (target.Import(entry.Key, kv.Key, kv.Value, mode))
{
++count;
}
}
}
return(count);
}
/// <summary>
/// Find missing entries in target that are in src.
/// Can be used to detect missing translation.
/// </summary>
/// <param name="src"></param>
/// <param name="target"></param>
/// <returns>tag, ver, missing text, potential text</returns>
static public List <DiffResultEntry> GetMissingEntries(SingleLanguageDB src, SingleLanguageDB target, bool onlyLatest)
{
List <DiffResultEntry> result = new List <DiffResultEntry>();
foreach (var entry in src)
{
string tag = entry.Key;
if (onlyLatest)
{
int ver = entry.Value.Last().Key;
string text = target.LookupText(tag, ver);
if (text == null)
{
string refText = entry.Value.Last().Value;
var potential = target.LookupLatestText(tag);
if (potential != null)
{
text = potential.Item2;
}
result.Add(new DiffResultEntry(tag, ver, refText, text));
}
}
else
{
foreach (var kv in entry.Value)
{
int ver = kv.Key;
string text = target.LookupText(tag, ver);
if (text == null)
{
string refText = entry.Value.Last().Value;
var potential = target.LookupLatestText(tag);
if (potential != null)
{
text = potential.Item2;
}
result.Add(new DiffResultEntry(tag, ver, refText, text));
}
}
}
}
return(result);
}
/// <summary>
///
/// </summary>
/// <param name="db"></param>
/// <param name="path"></param>
/// <param name="fileFormat"></param>
/// <param name="overridePattern"></param>
/// <param name="mode"></param>
/// <returns></returns>
static public int BatchImportToSingleLanguageDB(SingleLanguageDB db, string path, LocalizationFileFormat fileFormat,
string overridePattern = null, LocalizationDB.ImportMode mode = LocalizationDB.ImportMode.kReplace)
{
if (!Directory.Exists(path))
{
return(0);
}
string pattern = overridePattern != null ? overridePattern : fileFormat.GetLocalizationFileInstance().DefaultNamePattern();
int totalCount = 0;
foreach (var filename in Directory.EnumerateFiles(path, pattern))
{
ILocalizationFile file = fileFormat.GetLocalizationFileInstance();
file.Read(filename);
totalCount += ImportToSingleLanguageDBFromFile(db, file, mode);
}
return(totalCount);
}
/// <summary>
/// Check if newdb has anything new, will ignore extra entries in old db.
/// Can be used to detect difference between versions.
/// </summary>
/// <param name="oldDb"></param>
/// <param name="newDb"></param>
/// <returns>tag, ver, new string, old text</returns>
static public List <DiffResultEntry> Compare(SingleLanguageDB oldDb, SingleLanguageDB newDb, bool lowercase = false)
{
List <DiffResultEntry> result = new List <DiffResultEntry>();
foreach (var entry in newDb)
{
string tag = entry.Key;
foreach (var kv in entry.Value)
{
int ver = kv.Key;
string refText = kv.Value;
string text = oldDb.LookupText(tag, ver);
if (text == null)
{
var potential = oldDb.LookupLatestText(tag);
if (potential != null)
{
text = potential.Item2;
}
result.Add(new DiffResultEntry(tag, ver, refText, text));
}
else
{
// check if string are same
if (lowercase)
{
if (refText.ToLower() != text.ToLower())
{
result.Add(new DiffResultEntry(tag, ver, refText, text));
}
}
else
{
if (refText != text)
{
result.Add(new DiffResultEntry(tag, ver, refText, text));
}
}
}
}
}
return(result);
}
/// <summary>
///
/// </summary>
/// <param name="db"></param>
/// <param name="reference"></param>
/// <param name="output"></param>
/// <returns></returns>
static public bool ExportLocalization(SingleLanguageDB db, ILocalizationFile reference, ILocalizationFile output)
{
// for now, all input file will be marked as english...
if (!reference.VerifyHeader("english"))
{
return(false);
}
output.GenHeader("english");
int count = reference.CountLines();
if (count > 0)
{
for (int i = reference.HeaderLines(); i < count; ++i)
{
if (reference.IsLineLocalizationData(i))
{
string tag = "";
int ver = 0;
string data = "";
string rest = "";
reference.GetData(i, ref tag, ref ver, ref data, ref rest);
if (db != null)
{
string localizeddata = db.LookupText(tag, ver);
if (localizeddata != null)
{
output.AppendLine(tag, ver, localizeddata, rest);
continue;
}
}
output.AppendLine(tag, ver, data, rest);
}
else
{
string data = "";
reference.GetNonLocalizationData(i, ref data);
output.AppendLine(null, 0, data, null);
}
}
}
return(true);
}
public bool Load()
{
string dbPath = rootPath;
dbPath += "\\" + name;
if (!Directory.Exists(dbPath))
{
return(false);
}
// Load;
foreach (var fname in Directory.EnumerateFiles(dbPath))
{
string filename = Path.GetFileName(fname);
SingleLanguageDB sldb = new SingleLanguageDB(filename);
if (sldb.Load(filename, dbPath))
{
data[filename] = sldb;
}
}
return(true);
}
/// <summary>
///
/// </summary>
/// <param name="translation"></param>
static public void ScanExtraSpace(SingleLanguageDB translation)
{
List <Tuple <string, int, string> > toModify = new List <Tuple <string, int, string> >();
foreach (var entry in translation)
{
foreach (var key in entry.Value.Keys)
{
string newstr = ScanExtraSpace(entry.Value[key]);
if (newstr != entry.Value[key])
{
toModify.Add(new Tuple <string, int, string>(entry.Key, key, newstr));
}
}
}
foreach (var it in toModify)
{
translation.Import(it.Item1, it.Item2, it.Item3, LocalizationDB.ImportMode.kReplace);
}
}
static public int BatchImportToSingleLanguageDB(SingleLanguageDB db, string path, string fileFormatString,
string overridePattern = null, LocalizationDB.ImportMode mode = LocalizationDB.ImportMode.kReplace)
{
return(BatchImportToSingleLanguageDB(db, path, LocalizationFileFormat.GetFileFormat(fileFormatString), overridePattern, mode));
}
public static bool DoMerge(string newOriginalPath, string newOriginalFormat, string oldOriginalPath, string oldOriginalFormat,
string oldTranslationPath, string oldTranslationFormat, string inputPath, string inputFormat, string outputPath, string outputFormat,
string diffFilePath, string language, bool checkSpecialChar, bool ignoreSame, string checkFilePath)
{
SingleLanguageDB input = new SingleLanguageDB(language);
SingleLanguageDB newOriginal = new SingleLanguageDB(language);
Localization.BatchImportToSingleLanguageDB(input, inputPath, inputFormat);
Localization.BatchImportToSingleLanguageDB(newOriginal, newOriginalPath, newOriginalFormat);
// optional
SingleLanguageDB oldOriginal = null;
SingleLanguageDB oldTranslation = null;
if (oldOriginalPath != null && oldOriginalFormat != null)
{
oldOriginal = new SingleLanguageDB(language);
Localization.BatchImportToSingleLanguageDB(oldOriginal, oldOriginalPath, oldOriginalFormat);
}
if (oldTranslationPath != null && oldTranslationFormat != null)
{
oldTranslation = new SingleLanguageDB(language);
Localization.BatchImportToSingleLanguageDB(oldTranslation, oldTranslationPath, oldTranslationFormat);
}
Dictionary <string, string> RemovedDiffChi = new Dictionary <string, string>();
Dictionary <string, string> RemovedDiffEng = new Dictionary <string, string>();
// 1. Remove diff from old translation (but allow them if they are in new original)
if (oldOriginal != null && oldTranslation != null)
{
var diff = Localization.Compare(oldOriginal, newOriginal, false);
foreach (var entry in diff)
{
string chitext = oldTranslation.LookupText(entry.Tag, entry.Version);
string newtext = newOriginal.LookupText(entry.Tag, entry.Version);
if (chitext != null && newtext == null)
{
RemovedDiffChi[entry.Tag] = chitext;
RemovedDiffEng[entry.Tag] = entry.OldText;
oldTranslation.Remove(entry.Tag, entry.Version);
}
}
}
// 2. Merge in old translation to input
if (oldTranslation != null)
{
Localization.MergeIn(input, oldTranslation, LocalizationDB.ImportMode.kIgnore);
}
// 3. Find diff entires
YMLSafeFile diffyml = new YMLSafeFile();
diffyml.AppendLine(null, -1, GetLanguageTag(language) + ":", null);
var missing = Localization.GetMissingEntries(newOriginal, input, false);
foreach (var entry in missing)
{
string chi = null;
if (RemovedDiffChi.ContainsKey(entry.Tag))
{
chi = RemovedDiffChi[entry.Tag];
}
string oldeng = null;
if (RemovedDiffChi.ContainsKey(entry.Tag))
{
oldeng = RemovedDiffEng[entry.Tag];
}
diffyml.AppendLine(null, -1, "# Missing. Original: " + entry.NewText, null);
if (oldeng != null)
{
diffyml.AppendLine(null, -1, "# old Original: " + oldeng, null);
}
diffyml.AppendLine(entry.Tag, entry.Version, chi != null ? chi : entry.OldText, null);
}
diffyml.Write(diffFilePath);
// 3. Checks
if (checkSpecialChar)
{
// check missing entries first as we will remove entries failed the check as well.
var check = Localization.CheckTranslation(newOriginal, input, ignoreSame);
YMLSafeFile checkyml = new YMLSafeFile();
checkyml.AppendLine(null, -1, GetLanguageTag(language) + ":", null);
foreach (var entry in check)
{
if (entry.NewText != null)
{
checkyml.AppendLine(null, -1, "# Check Fail, Original: " + entry.NewText, null);
checkyml.AppendLine(entry.Tag, entry.Version, entry.OldText, null);
}
// Remove trnaslation, prepare for export
input.Remove(entry.Tag, entry.Version);
}
checkyml.Write(checkFilePath);
}
// 4. Export the merged translation
return(Localization.BatchExportLocalization(input, newOriginalPath, newOriginalFormat, null, outputPath, outputFormat));
}
/// <summary>
///
/// </summary>
/// <param name="db"></param>
/// <param name="refPath"></param>
/// <param name="refFileFormat"></param>
/// <param name="overrideRefPattern"></param>
/// <param name="outPath"></param>
/// <param name="outFileFormat"></param>
/// <param name="splitLine"></param>
/// <returns></returns>
static public bool BatchExportLocalization(SingleLanguageDB db,
string refPath, LocalizationFileFormat refFileFormat, string overrideRefPattern,
string outPath, LocalizationFileFormat outFileFormat, int splitLine = 0)
{
if (!Directory.Exists(refPath))
{
return(false);
}
string refpattern = overrideRefPattern != null ? overrideRefPattern : refFileFormat.GetLocalizationFileInstance().DefaultNamePattern();
foreach (var filename in Directory.EnumerateFiles(refPath, refpattern))
{
ILocalizationFile reffile = refFileFormat.GetLocalizationFileInstance();
reffile.Read(filename);
ILocalizationFile outfile = outFileFormat.GetLocalizationFileInstance();
if (!outfile.CanWrite())
{
continue;
}
if (!ExportLocalization(db, reffile, outfile))
{
continue;
}
string newName = Path.GetFileNameWithoutExtension(filename);
Directory.CreateDirectory(outPath);
if (splitLine <= 0)
{
outfile.Write(outPath + "\\" + newName + "." + outfile.DefaultExtension());
}
else
{
int count = outfile.CountLines();
if (count > 0)
{
ILocalizationFile splitFile = outFileFormat.GetLocalizationFileInstance();
splitFile.GenHeader("english");
int contentCount = 0;
int fileCount = 1;
for (int i = outfile.HeaderLines(); i < count; ++i)
{
if (contentCount >= splitLine)
{
splitFile.Write(outPath + "\\" + newName + "-" + fileCount.ToString() + "." + splitFile.DefaultExtension());
fileCount++;
contentCount = 0;
if (i != count - 1)
{
splitFile = outFileFormat.GetLocalizationFileInstance();
splitFile.GenHeader("english");
}
}
if (outfile.IsLineLocalizationData(i))
{
string tag = "";
int ver = 0;
string data = "";
string rest = "";
outfile.GetData(i, ref tag, ref ver, ref data, ref rest);
splitFile.AppendLine(tag, ver, data, rest);
contentCount++;
}
else
{
string data = "";
outfile.GetNonLocalizationData(i, ref data);
splitFile.AppendLine(null, 0, data, null);
}
}
splitFile.Write(outPath + "\\" + newName + "-" + fileCount.ToString() + "." + splitFile.DefaultExtension());
}
}
}
return(true);
}
static bool DoRefine(Dictionary <string, string> options)
{
string outputPath = null;
string outputFormat = null;
string oldOriginPath = null;
string oldTranslationPath = null;
string oldOriginFormat = null;
string oldTranslationFormat = null;
if (!options.TryGetValue("output-path", out outputPath))
{
throw new ArgumentException("Missing output-path.");
}
if (!options.TryGetValue("output-format", out outputFormat))
{
throw new ArgumentException("Missing output-format.");
}
if (!options.TryGetValue("old-original-path", out oldOriginPath))
{
throw new ArgumentException("Missing old-original-path.");
}
if (!options.TryGetValue("old-original-format", out oldOriginFormat))
{
throw new ArgumentException("Missing old-original-format.");
}
if (!options.TryGetValue("old-translation-path", out oldTranslationPath))
{
throw new ArgumentException("Missing old-translation-path.");
}
if (!options.TryGetValue("old-translation-format", out oldTranslationFormat))
{
throw new ArgumentException("Missing old-translation-format.");
}
// all dummy english for now..
SingleLanguageDB oldEng = new SingleLanguageDB("english");
SingleLanguageDB oldTranslate = new SingleLanguageDB("english");
Localization.BatchImportToSingleLanguageDB(oldEng, oldOriginPath, oldOriginFormat);
Localization.BatchImportToSingleLanguageDB(oldTranslate, oldTranslationPath, oldTranslationFormat);
Dictionary <string, string> dict1 = new Dictionary <string, string>();
Dictionary <string, string> dict2 = new Dictionary <string, string>();
foreach (var kv in oldEng)
{
string tag = kv.Key;
if ((tag.Length > 2 && tag[1] == '_') || tag.StartsWith("PROV"))
{
continue;
}
string value = kv.Value.Last().Value;
if (tag.ToLowerInvariant().Replace("_", "").Contains(value.ToLowerInvariant().Replace(" ", "")))
{
dict1[tag] = value;
}
else
{
TextInfo ti = CultureInfo.CurrentCulture.TextInfo;
if (value == ti.ToTitleCase(value))
{
dict2[tag] = value;
}
}
}
YMLSafeFile dict1file = new YMLSafeFile();
YMLSafeFile dict2file = new YMLSafeFile();
dict1file.AppendLine(null, -1, "l_english:", null);
dict2file.AppendLine(null, -1, "l_english:", null);
foreach (var entry in dict1)
{
dict1file.AppendLine(null, -1, "# Potential Term, origin: " + entry.Value, null);
string trans = oldTranslate.LookupLatestText(entry.Key)?.Item2;
dict1file.AppendLine(entry.Key, 0, trans != null? trans : entry.Value, null);
}
foreach (var entry in dict2)
{
dict2file.AppendLine(null, -1, "# Potential Term, origin: " + entry.Value, null);
string trans = oldTranslate.LookupLatestText(entry.Key)?.Item2;
dict2file.AppendLine(entry.Key, 0, trans != null ? trans : entry.Value, null);
}
dict1file.Write(outputPath + "\\dict1." + dict1file.DefaultExtension());
dict2file.Write(outputPath + "\\dict2." + dict1file.DefaultExtension());
return(true);
}
