public async void Download()
{
StatusMessage = "ダウンロードを開始します";
ProgressMaxValue = TranslationFiles.Count();
ProgressCurrentValue = 0;
IsProgress = true;
try
{
if (!Directory.Exists(DownloadDirectoryName))
{
Directory.CreateDirectory(DownloadDirectoryName);
}
foreach (var translationFile in TranslationFiles)
{
StatusMessage = $"{translationFile.Name}をダウンロード中...";
var uri = new Uri(translationFile.Url);
await Downloader.DownloadAsync(uri,
$@"{DownloadDirectoryName}\{translationFile.Name}.csv");
translationFile.UpdateLastModifyTime();
ProgressCurrentValue += 1;
}
StatusMessage = "全てのダウンロードが完了しました";
}
finally
{
IsProgress = false;
}
}
public void NotAllLanguageTranslatedTest()
{
var groupedByLng = TranslationFiles
.GroupBy(t => t.Language)
.Select(grp => new { Lng = grp.Key, Count = grp.Count(), Files = grp.ToList() })
.ToList();
// Uncomment if new language is needed
//var newLng = "sk";
//if (!groupedByLng.Exists(t => t.Lng == newLng))
// groupedByLng.Add(new { Lng = newLng, Count = 0, Files = new List<TranslationFile>() });
var enGroup = groupedByLng.Find(f => f.Lng == "en");
var expectedCount = enGroup.Count;
var otherLngs = groupedByLng.Where(g => g.Lng != "en");
var incompleteList = otherLngs
.Where(lng => lng.Count != expectedCount)
.Select(lng => new { Issue = $"Language '{lng.Lng}' (Count={lng.Count}). Not found files:\r\n", lng.Lng, lng.Files })
.ToList();
var message = $"Next languages are not equal 'en' (Count= {expectedCount}) by translated files count:\r\n\r\n";
if (incompleteList.Count > 0)
{
var enFilePaths = enGroup.Files.Select(f => f.FilePath);
for (int i = 0; i < incompleteList.Count; i++)
{
var lng = incompleteList[i];
message += $"\r\n\r\n{i}. {lng.Issue}\r\n";
var lngFilePaths = lng.Files.Select(f => f.FilePath).ToList();
var notFoundFilePaths = enFilePaths
.Select(p => p.Replace("\\en\\", $"\\{lng.Lng}\\"))
.Where(p => !lngFilePaths.Contains(p));
message += string.Join("\r\n", notFoundFilePaths);
/* Save empty 'EN' keys to not found files */
/*foreach (var path in notFoundFilePaths)
* {
* SaveNotFoundLanguage(path.Replace($"\\{lng.Lng}\\", "\\en\\"), path);
* }*/
}
}
Assert.AreEqual(0, incompleteList.Count, message);
}
public void EnDublicatesByContentTest()
{
var allRuTranslations = TranslationFiles
.Where(file => file.Language == "ru")
.SelectMany(item => item.Translations)
.ToList();
var allEnDuplicates = TranslationFiles
.Where(file => file.Language == "en")
.SelectMany(item => item.Translations)
.GroupBy(t => t.Value)
.Where(grp => grp.Count() > 1)
.Select(grp => new { ContentKey = grp.Key, Count = grp.Count(), List = grp.ToList() })
.OrderByDescending(itm => itm.Count)
.ToList();
var duplicatesKeys = new List <TranslationItem>();
foreach (var item in allEnDuplicates)
{
var ruEquivalents = allRuTranslations
.Where(t => item.List.Select(k => k.Key).Contains(t.Key))
.GroupBy(t => t.Value)
.Select(grp => new
{
ContentKey = grp.Key,
Count = grp.Count(),
Keys = grp.Select(k => k.Key).ToList()
})
.Where(t => t.Count > 1)
.ToList();
if (!ruEquivalents.Any())
{
continue;
}
duplicatesKeys.AddRange(
item.List.Where(item => ruEquivalents
.SelectMany(k => k.Keys)
.Any(k => k == item.Key)
)
);
}
var duplicates = duplicatesKeys
.GroupBy(k => k.Value)
.Select(g => new { ContentKey = g.Key, Count = g.Count(), Keys = g.ToList() })
.ToList();
Assert.AreEqual(0, duplicates.Count, string.Join(", ", duplicates.Select(d => JObject.FromObject(d).ToString())));
}
public void FullEnDublicatesTest()
{
var fullEnDuplicates = TranslationFiles
.Where(file => file.Language == "en")
.SelectMany(item => item.Translations)
.GroupBy(t => new { t.Key, t.Value })
.Where(grp => grp.Count() > 1)
.Select(grp => new { Key = grp.Key, Count = grp.Count(), Keys = grp.ToList() })
.OrderByDescending(itm => itm.Count)
.Select(grp => new { Key = grp.Key.Key, Value = grp.Key.Value, grp.Count })
.ToList();
Assert.AreEqual(0, fullEnDuplicates.Count, string.Join("\r\n", fullEnDuplicates.Select(d => JObject.FromObject(d).ToString())));
}
public void LanguageTranslatedPercentTest()
{
var message = $"Next languages translated less then 100%:\r\n\r\n";
var groupedByLng = TranslationFiles
.GroupBy(t => t.Language)
.Select(g => new
{
Language = g.Key,
AllTranslated = g.ToList()
.SelectMany(t => t.Translations)
.ToList()
})
.Select(t => new
{
t.Language,
TotalKeysCount = t.AllTranslated.LongCount(),
EmptyKeysCount = t.AllTranslated
.Where(t => string.IsNullOrEmpty(t.Value))
.LongCount()
})
.ToList();
var i = 0;
var exists = false;
var expectedTotalKeysCount = groupedByLng.Where(t => t.Language == "en").Single().TotalKeysCount;
foreach (var lng in groupedByLng)
{
if (lng.EmptyKeysCount == 0 && lng.TotalKeysCount == expectedTotalKeysCount)
{
continue;
}
exists = true;
var translated = lng.TotalKeysCount == expectedTotalKeysCount
? Math.Round(100f - (lng.EmptyKeysCount * 100f / expectedTotalKeysCount), 1)
: Math.Round(lng.TotalKeysCount * 100f / expectedTotalKeysCount, 1);
message += $"{++i}. Language '{lng.Language}' translated by '{translated}%'\r\n";
}
Assert.AreEqual(false, exists, message);
}
public void NotFoundKeysTest()
{
var allEnKeys = TranslationFiles
.Where(file => file.Language == "en")
.SelectMany(item => item.Translations)
.Select(item => item.Key);
var allJsTranslationKeys = JavaScriptFiles
.Where(f => !f.Path.Contains("Banner.js")) // skip Banner.js (translations from firebase)
.SelectMany(j => j.TranslationKeys)
.Select(k => k.Substring(k.IndexOf(":") + 1))
.Distinct();
var notFoundJsKeys = allJsTranslationKeys.Except(allEnKeys);
Assert.AreEqual(0, notFoundJsKeys.Count(),
"Some i18n-keys are not exist in translations in 'en' language: Keys: '{0}'",
string.Join("\r\n", notFoundJsKeys));
}
public void DublicatesFilesByMD5HashTest()
{
var skipHashes = new List <string>()
{
"bcba174a8dadc0ff97f37f9a2d816d88",
"2a506ed97d0fbd0858192b755ae122d0",
"ec73989085d4e1b984c1c9dca10524da"
};
var duplicatesByMD5 = TranslationFiles
.Where(t => !skipHashes.Contains(t.Md5Hash))
.GroupBy(t => t.Md5Hash)
.Where(grp => grp.Count() > 1)
.Select(grp => new { Key = grp.Key, Count = grp.Count(), Paths = grp.ToList().Select(f => f.FilePath) })
.OrderByDescending(itm => itm.Count)
.ToList();
Assert.AreEqual(0, duplicatesByMD5.Count, "Dublicates by MD5 hash:\r\n" + string.Join("\r\n", duplicatesByMD5.Select(d => $"\r\nMD5='{d.Key}':\r\n{string.Join("\r\n", d.Paths.Select(p => p))}'")));
}
public void UselessTranslationKeysTest()
{
var allEnKeys = TranslationFiles
.Where(file => file.Language == "en")
.SelectMany(item => item.Translations)
.Select(item => item.Key)
.Where(k => !k.StartsWith("Culture_"));
var allJsTranslationKeys = JavaScriptFiles
.SelectMany(j => j.TranslationKeys)
.Select(k => k.Substring(k.IndexOf(":") + 1))
.Where(k => !k.StartsWith("Culture_"))
.Distinct();
var notFoundi18nKeys = allEnKeys.Except(allJsTranslationKeys);
Assert.AreEqual(0, notFoundi18nKeys.Count(),
"Some i18n-keys are not found in js: \r\nKeys: '\r\n{0}'",
string.Join("\r\n", notFoundi18nKeys));
}
public void RefreshTranslationFiles(bool showDirNotFoundError = false)
{
TranslationFiles.Clear();
if (Directory.Exists(SelectedFolder))
{
DirectoryInfo folder = new DirectoryInfo(SelectedFolder);
FileInfo[] fileInfos = folder.GetFiles("*.json", SearchOption.TopDirectoryOnly);
foreach (FileInfo translationFile in fileInfos)
{
StreamReader reader = new StreamReader(new FileStream(translationFile.FullName, FileMode.Open));
string jsonString = reader.ReadToEnd();
Dictionary <string, string> jsonPairs = JsonConvert.DeserializeObject <Dictionary <string, string> >(jsonString);
ObservableCollection <TranslationPair> translationPairs = new ObservableCollection <TranslationPair>(
jsonPairs.Select(pair => new TranslationPair {
Key = pair.Key, Value = pair.Value
}));
string fileName = translationFile.Name.Substring(0, translationFile.Name.Length - translationFile.Extension.Length);
TranslationFiles.Add(new TranslationFile {
Name = fileName, Path = translationFile.FullName, TranslationPairs = translationPairs
});
}
RefreshScores();
_mainWindow.UpdateTabs();
}
else
{
if (showDirNotFoundError)
{
MessageBox.Show("The selected folder wasn't found.");
}
}
}
public void RefreshScores()
{
//get unique translation keys of all files
Dictionary <string, Tuple <string, string> > uniqueTranslationKeys = new Dictionary <string, Tuple <string, string> >();
foreach (TranslationFile translationFile in TranslationFiles)
{
foreach (TranslationPair translationPair in translationFile.TranslationPairs)
{
if (!uniqueTranslationKeys.ContainsKey(translationPair.Key))
{
uniqueTranslationKeys.Add(translationPair.Key, new Tuple <string, string>(translationFile.Name, translationPair.Value));
}
}
}
//determine score for every key and add missing keys to respective files
foreach (KeyValuePair <string, Tuple <string, string> > uniqueTranslationPair in uniqueTranslationKeys)
{
double score = (double)TranslationFiles.Count(tf => tf.TranslationPairs.Any(tp => tp.Key == uniqueTranslationPair.Key && !string.IsNullOrWhiteSpace(tp.Value))) / TranslationFiles.Count();
foreach (TranslationFile translationFile in TranslationFiles)
{
TranslationPair translationPair = translationFile.TranslationPairs.FirstOrDefault(tp => tp.Key == uniqueTranslationPair.Key);
if (translationPair == null)
{
//_languageServiceClient.GetAppIdToken(APP_ID, );
//string autoTranslateValue = _languageServiceClient.Translate(
// "Bearer" + " " + APP_ID,
// uniqueTranslationPair.Value.Item2,
// uniqueTranslationPair.Value.Item1,
// translationFile.Name,
// "text/plain",
// "general"
// );
translationFile.TranslationPairs.Add(new TranslationPair {
Key = uniqueTranslationPair.Key, Value = "", Score = 0
});
}
else if (string.IsNullOrWhiteSpace(translationPair.Value))
{
translationPair.Score = 0;
//translationPair.Value = _languageServiceClient.Translate(
// "Bearer" + " " + APP_ID,
// uniqueTranslationPair.Value.Item2,
// uniqueTranslationPair.Value.Item1,
// translationFile.Name,
// "text/plain",
// "general"
// );
}
else
{
translationPair.Score = score;
}
}
}
foreach (TranslationFile translationFile in TranslationFiles)
{
translationFile.TranslationPairs = new ObservableCollection <TranslationPair>(translationFile.TranslationPairs.OrderBy(tp => tp.Score).ThenBy(tp => tp.Key));
}
}
public void SpellCheckTest()
{
const string dictionariesPath = @"..\..\..\dictionaries";
var i = 0;
var errorsCount = 0;
var message = $"Next keys have spell check issues:\r\n\r\n";
//var list = new List<SpellCheckExclude>();
var groupByLng = TranslationFiles
.GroupBy(t => t.Language)
.Select(g => new
{
Language = g.Key,
Files = g.ToList()
})
.ToList();
foreach (var group in groupByLng)
{
try
{
var language = SpellCheck.GetDictionaryLanguage(group.Language);
//var spellCheckExclude = new SpellCheckExclude(group.Language);
using (var dictionaryStream = File.OpenRead(Path.Combine(dictionariesPath, language, $"{language}.dic")))
using (var affixStream = File.OpenRead(Path.Combine(dictionariesPath, language, $"{language}.aff")))
{
var dictionary = WordList.CreateFromStreams(dictionaryStream, affixStream);
foreach (var g in group.Files)
{
foreach (var item in g.Translations)
{
var result = SpellCheck.HasSpellIssues(item.Value, group.Language, dictionary);
if (result.HasProblems)
{
message += $"{++i}. lng='{group.Language}' file='{g.FilePath}'\r\nkey='{item.Key}' value='{item.Value}'\r\nIncorrect words:\r\n{string.Join("\r\n", result.SpellIssues.Select(issue => $"'{issue.Word}' Suggestion: '{issue.Suggestions.FirstOrDefault()}'"))}\r\n\r\n";
errorsCount++;
/*foreach (var word in result.SpellIssues
* .Where(issue => issue.Suggestions.Any())
* .Select(issue => issue.Word))
* {
* if (!spellCheckExclude.Excludes.Contains(word))
* {
* spellCheckExclude.Excludes.Add(word);
* }
* }*/
}
}
}
}
//spellCheckExclude.Excludes.Sort();
//list.Add(spellCheckExclude);
}
catch (NotSupportedException)
{
// Skip not supported
continue;
}
}
//string json = JsonConvert.SerializeObject(list, Formatting.Indented);
//File.WriteAllText("../../../spellcheck-excludes.json", json);
Assert.AreEqual(0, errorsCount, message);
}
public void WrongTranslationVariablesTest()
{
var message = $"Next keys have wrong variables:\r\n\r\n";
var regVariables = new Regex("\\{\\{([^\\{].?[^\\}]+)\\}\\}", RegexOptions.Compiled | RegexOptions.Multiline);
var groupedByLng = TranslationFiles
.GroupBy(t => t.Language)
.Select(g => new
{
Language = g.Key,
TranslationsWithVariables = g.ToList()
.SelectMany(t => t.Translations)
.Where(k => k.Value.IndexOf("{{") != -1)
.Select(t => new
{
t.Key,
t.Value,
Variables = regVariables.Matches(t.Value)
.Select(m => m.Groups[1]?.Value?.Trim().Replace(", lowercase", ""))
.ToList()
})
.ToList()
})
.ToList();
var enWithVariables = groupedByLng
.Where(t => t.Language == "en")
.SelectMany(t => t.TranslationsWithVariables)
.ToList();
var otherLanguagesWithVariables = groupedByLng
.Where(t => t.Language != "en")
.ToList();
var i = 0;
var errorsCount = 0;
foreach (var lng in otherLanguagesWithVariables)
{
foreach (var t in lng.TranslationsWithVariables)
{
var enKey = enWithVariables
.Where(en => en.Key == t.Key)
.FirstOrDefault();
if (enKey == null)
{
// wrong
message += $"{++i}. lng='{lng.Language}' key='{t.Key}' has no 'en' language variant (!!!useless key!!!)\r\n\r\n";
errorsCount++;
continue;
}
if (enKey.Variables.Count != t.Variables.Count)
{
// wrong
message += $"{++i}. lng='{lng.Language}' key='{t.Key}' has less variables then 'en' language have " +
$"(en={enKey.Variables.Count}|{lng.Language}={t.Variables.Count})\r\n" +
$"'en': '{enKey.Value}'\r\n'{lng.Language}': '{t.Value}'\r\n\r\n";
errorsCount++;
}
if (!t.Variables.All(v => enKey.Variables.Contains(v)))
{
// wrong
errorsCount++;
message += $"{++i}. lng='{lng.Language}' key='{t.Key}' has not equals variables of 'en' language have\r\n\r\n" +
$"Have to be:\r\n'{enKey.Value}'\r\n\r\n{string.Join("\r\n", enKey.Variables)}\r\n\r\n" +
$"But in real:\r\n'{t.Value}'\r\n\r\n{string.Join("\r\n", t.Variables)} \r\n\r\n";
}
}
}
Assert.AreEqual(0, errorsCount, message);
}
