public static ISerializer CreateUntaggedSerializer()
{
var builder = new SerializerBuilder();
return(builder.ConfigureDefaultValuesHandling(DefaultValuesHandling.Preserve)
.WithTypeInspector(x => new SortedTypeInspector(x))
.DisableAliases()
.Build());
}
public static ISerializer CreateSerializer()
{
var builder = new SerializerBuilder();
builder.WithNamingConvention(namingConvention);
builder.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults);
return(builder.Build());
}
public static void DataToYaml(TextWriter writer, IEnumerable <Dictionary <string, object> > datatable, YamlColumnNamesType yamlColumnNames = null)
{
var builder = new SerializerBuilder();
builder.ConfigureDefaultValuesHandling(DefaultValuesHandling.Preserve);
var serializer = builder.Build();
var tree = yamlColumnNames == null ? datatable : ConvertDataTableWithYamlColumns(datatable, yamlColumnNames);
serializer.Serialize(writer, tree);
}
public static ISerializer CreateDefaultSerializer()
{
var builder = new SerializerBuilder();
foreach ((string tag, var type) in nodeTypeResolver.Value.tagMappings)
{
builder.WithTagMapping(tag, type);
}
return(builder.ConfigureDefaultValuesHandling(DefaultValuesHandling.Preserve)
.WithTypeInspector(x => new SortedTypeInspector(x))
.DisableAliases()
.Build());
}
public static void WriteFile <T>(string filePath, T value, Format format)
{
var directory = Path.GetDirectoryName(filePath);
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
using (var file = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.ReadWrite))
{
using (var writer = new StreamWriter(file, new UTF8Encoding(false)))
{
switch (format)
{
case Format.Json:
{
using (var jsonTextWriter = new JsonTextWriter(writer))
{
jsonTextWriter.Formatting = Formatting.Indented;
var jsonSerializer = new JsonSerializer()
{
Formatting = Formatting.Indented,
NullValueHandling = NullValueHandling.Ignore,
};
jsonSerializer.Serialize(jsonTextWriter, value);
}
}
break;
case Format.Yaml:
{
var builder = new SerializerBuilder();
builder.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull);
var yamlSerializer = builder.Build();
yamlSerializer.Serialize(writer, value);
}
break;
}
}
}
}
public void SaveFile(string path, GraphData graphData)
{
var serializer = new SerializerBuilder();
var elements = new Dictionary <Type, LinkedList <DrawableElement> >
{
{ typeof(Rectangle), new LinkedList <DrawableElement>() },
{ typeof(Ellipse), new LinkedList <DrawableElement>() },
{ typeof(Pie), new LinkedList <DrawableElement>() },
{ typeof(PolyLine), new LinkedList <DrawableElement>() }
};
foreach (var drawable in graphData.drawables)
{
elements[drawable.GetType()].AddLast(drawable);
}
serializer.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults);
using (StreamWriter writer = new StreamWriter(path))
{
serializer.Build().Serialize(writer, elements);
}
}
public Process Customize()
{
this.OnProgressChanged(new ProgressChangedEventArgs(1, new MarianCustomizationStatus(CustomizationStep.Copying_model, null)));
//First copy the model to new dir
try
{
this.CopyModelDir(this.modelDir, this.customLabel);
//Save model config as soon as the model dir exists
this.customModel.SaveModelConfig();
}
catch (Exception ex)
{
Log.Information($"Customization failed: {ex.Message}");
return(null);
}
//Save the batch to translate after customization to a file (to be batch translated after successful exit)
if (this.postCustomizationBatch != null && this.postCustomizationBatch.Count > 0)
{
FileInfo postCustomizationBatchFile = new FileInfo(Path.Combine(this.customDir.FullName, OpusCatMTEngineSettings.Default.PostFinetuneBatchName));
using (var writer = postCustomizationBatchFile.CreateText())
{
foreach (var sourceString in this.postCustomizationBatch)
{
writer.WriteLine(sourceString);
}
}
}
this.OnProgressChanged(new ProgressChangedEventArgs(2, new MarianCustomizationStatus(CustomizationStep.Copying_training_files, null)));
//Copy raw files to model dir
this.customSource = this.customSource.CopyTo(Path.Combine(this.customDir.FullName, "custom.source"));
this.customTarget = this.customTarget.CopyTo(Path.Combine(this.customDir.FullName, "custom.target"));
this.OnProgressChanged(new ProgressChangedEventArgs(3, new MarianCustomizationStatus(CustomizationStep.Preprocessing_training_files, null)));
//Preprocess input files
this.PreprocessInput();
var decoderYaml = this.customDir.GetFiles("decoder.yml").Single();
var deserializer = new Deserializer();
var decoderSettings = deserializer.Deserialize <MarianDecoderConfig>(decoderYaml.OpenText());
if (this.guidedAlignment)
{
//Generate alignments for fine-tuning corpus
this.alignmentFile = new FileInfo(Path.Combine(this.customDir.FullName, "custom.alignments"));
MarianHelper.GenerateAlignments(this.spSource, this.spTarget, this.alignmentFile, this.model.AlignmentPriorsFile);
//
//Generate alignments for validation set (for evaluating fine-tuning effect on alignment)
this.validAlignmentFile = new FileInfo(Path.Combine(this.customDir.FullName, "combined.alignments"));
MarianHelper.GenerateAlignments(this.spValidSource, this.spValidTarget, this.validAlignmentFile, this.model.AlignmentPriorsFile);
}
this.OnProgressChanged(new ProgressChangedEventArgs(4, new MarianCustomizationStatus(CustomizationStep.Initial_evaluation, null)));
//Do the initial evaluation
var initialValidProcess = this.model.TranslateAndEvaluate(
this.spValidSource,
new FileInfo(Path.Combine(this.customDir.FullName, "valid.0.txt")),
this.spValidTarget,
OpusCatMTEngineSettings.Default.OODValidSetSize,
this.sourceLanguage,
this.targetLanguage,
true
);
//Wait for the initial valid to finish before starting customization
//(TODO: make sure this is not done on UI thread)
initialValidProcess.WaitForExit();
this.OnProgressChanged(new ProgressChangedEventArgs(6, new MarianCustomizationStatus(CustomizationStep.Finetuning, null)));
//Use the initial translation time as basis for estimating the duration of validation file
//translation
this.trainingLog.EstimatedTranslationDuration = Convert.ToInt32((initialValidProcess.ExitTime - initialValidProcess.StartTime).TotalSeconds);
MarianTrainerConfig trainingConfig;
var baseCustomizeYmlPath =
HelperFunctions.GetLocalAppDataPath(
OpusCatMTEngineSettings.Default.CustomizationBaseConfig);
var processDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
//Make sure there's a customization file.
if (!File.Exists(baseCustomizeYmlPath))
{
File.Copy(
Path.Combine(processDir, OpusCatMTEngineSettings.Default.CustomizationBaseConfig),
baseCustomizeYmlPath);
}
//deserialize yaml file
using (var reader = new StreamReader(baseCustomizeYmlPath))
{
trainingConfig = deserializer.Deserialize <MarianTrainerConfig>(reader);
}
trainingConfig.trainSets = new List <string>
{
this.spSource.FullName,
this.spTarget.FullName
};
trainingConfig.ValidSets = new List <string>
{
this.spValidSource.FullName,
this.spValidTarget.FullName
};
trainingConfig.vocabs = new List <string>
{
Path.Combine(this.customDir.FullName, decoderSettings.vocabs[0]),
Path.Combine(this.customDir.FullName, decoderSettings.vocabs[0])
};
switch (this.segmentationMethod)
{
case ".bpe":
string validScriptPath = Path.Combine(this.customDir.FullName, "ValidateBpe.bat");
trainingConfig.validScriptPath =
$"\"{validScriptPath}\"";
File.Copy(
Path.Combine(processDir, "ValidateBpe.bat"), validScriptPath);
break;
case ".spm":
validScriptPath = Path.Combine(this.customDir.FullName, "ValidateSp.bat");
trainingConfig.validScriptPath =
$"\"{validScriptPath}\"";
File.Copy(
Path.Combine(processDir, "ValidateSp.bat"), validScriptPath);
break;
default:
break;
}
trainingConfig.validScriptArgs =
new List <string> {
$"{spValidTarget.FullName}",
$"OOD{OpusCatMTEngineSettings.Default.OODValidSetSize.ToString()}"
};
trainingConfig.validTranslationOutput = Path.Combine(this.customDir.FullName, "valid.{U}.txt");
if (this.guidedAlignment)
{
trainingConfig.guidedAlignment = this.alignmentFile.FullName;
}
trainingConfig.validLog = Path.Combine(this.customDir.FullName, "valid.log");
trainingConfig.log = Path.Combine(this.customDir.FullName, "train.log");
trainingConfig.model = Path.Combine(this.customDir.FullName, decoderSettings.models.Single());
var builder = new SerializerBuilder();
builder.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull);
var serializer = builder.Build();
var configPath = Path.Combine(this.customDir.FullName, OpusCatMTEngineSettings.Default.CustomizationBaseConfig);
using (var writer = File.CreateText(configPath))
{
serializer.Serialize(writer, trainingConfig, typeof(MarianTrainerConfig));
}
Process trainProcess = this.StartTraining();
return(trainProcess);
}
