private void MergeDataset(TrainingDataset ds, Dictionary <string, TrainingEntry> entries)
{
var dsEntries = ds.GetEntries();
double lastProgress = -1;
for (int i = 0; i < dsEntries.Count; i++)
{
dsEntries[i].Word = dsEntries[i].Word.ToLower().Trim();
double progress = Math.Round(i * 100.0 / dsEntries.Count, 2);
if (progress != lastProgress)
{
lastProgress = progress;
Console.Write($"\r -> %{progress}");
}
if (IsValidEntry(dsEntries[i]))
{
TrainingEntry entry = GetEntry(dsEntries[i], entries);
entry.Phonetics = PreparePhonetics(dsEntries[i].Phonetics);
ConvertAudios(dsEntries[i], entry);
}
}
SaveTrainingIndex(entries);
Console.WriteLine($"\r -> Done! ");
}
public async void CSVReader_ShouldReadInputData()
{
string path = @"C:\data\input.txt";
var mockFileSystem = new MockFileSystem();
var mockInputFile = new MockFileData(
"1.1,2.1,1.0,1.0\n" +
"0.3,3.2,3.2,0.2\n" +
"5.3,1.1,2.1,0.1\n");
mockFileSystem.AddFile(@"C:\data\input.txt", mockInputFile);
IFileReader fr = new CSVReader(mockFileSystem);
TrainingDataset data = await fr.ReadInputData(path, new int[] { 2, 3 });
Assert.Equal(1.1, data.TrainingExamples.ElementAt(0).InputValues[0]);
Assert.Equal(2.1, data.TrainingExamples.ElementAt(0).InputValues[1]);
Assert.Equal(1.0, data.TrainingExamples.ElementAt(0).ExpectedOutputs[0]);
Assert.Equal(1.0, data.TrainingExamples.ElementAt(0).ExpectedOutputs[1]);
Assert.Equal(0.3, data.TrainingExamples.ElementAt(1).InputValues[0]);
Assert.Equal(3.2, data.TrainingExamples.ElementAt(1).InputValues[1]);
Assert.Equal(3.2, data.TrainingExamples.ElementAt(1).ExpectedOutputs[0]);
Assert.Equal(0.2, data.TrainingExamples.ElementAt(1).ExpectedOutputs[1]);
Assert.Equal(5.3, data.TrainingExamples.ElementAt(2).InputValues[0]);
Assert.Equal(1.1, data.TrainingExamples.ElementAt(2).InputValues[1]);
Assert.Equal(2.1, data.TrainingExamples.ElementAt(2).ExpectedOutputs[0]);
Assert.Equal(0.1, data.TrainingExamples.ElementAt(2).ExpectedOutputs[1]);
}
public void Merge()
{
Dictionary <String, TrainingEntry> entries = new Dictionary <string, TrainingEntry>();
foreach (string folder in TrainingSetFolders)
{
Console.WriteLine("Merging Dataset: " + folder);
TrainingDataset ds = new TrainingDataset(folder);
MergeDataset(ds, entries);
}
}
public async Task WriteInputData(TrainingDataset dataset, string path)
{
try
{
using StreamWriter sw = _fileSystem.File.CreateText(path);
foreach (var example in dataset.TrainingExamples)
{
await sw.WriteLineAsync(ConvertExampleToString(example));
}
foreach (var example in dataset.TestExamples)
{
await sw.WriteLineAsync(ConvertExampleToString(example));
}
}
catch
{
throw;
}
}
public HowToPronounceProvider()
{
Title = "'How To Pronounce' Training Set Provider";
ds = new TrainingDataset(@"D:\My Projects\Speech Morphing Data Gatherer\Data\");
}
public LongmanProvider()
{
Title = "Longman Training Set Provider";
ds = new TrainingDataset(@"D:\My Projects\Speech Morphing Data Gatherer\Data\");
}
public MerriamWebsterProvider()
{
Title = "Merriam-Webster Training Set Provider";
ds = new TrainingDataset(@"D:\My Projects\Speech Morphing Data Gatherer\Data\");
}
public CollinsProvider()
{
Title = "Collins Training Set Provider";
ds = new TrainingDataset(@"D:\My Projects\Speech Morphing Data Gatherer\Data\");
}
