private void OpenCustomSettingsInEditor_Click(object sender, RoutedEventArgs e)
{
var customizeYml = HelperFunctions.GetLocalAppDataPath(OpusCatMTEngineSettings.Default.CustomizationBaseConfig);
Process.Start("notepad.exe", customizeYml);
}
public Process Customize()
{
this.OnProgressChanged(new ProgressChangedEventArgs(1, new MarianCustomizationStatus(CustomizationStep.Copying_model, null)));
//First copy the model to new dir
try
{
this.CopyModelDir(this.modelDir, this.customLabel);
//Save model config as soon as the model dir exists
this.customModel.SaveModelConfig();
}
catch (Exception ex)
{
Log.Information($"Customization failed: {ex.Message}");
return(null);
}
//Save the batch to translate after customization to a file (to be batch translated after successful exit)
if (this.postCustomizationBatch != null && this.postCustomizationBatch.Count > 0)
{
FileInfo postCustomizationBatchFile = new FileInfo(Path.Combine(this.customDir.FullName, OpusCatMTEngineSettings.Default.PostFinetuneBatchName));
using (var writer = postCustomizationBatchFile.CreateText())
{
foreach (var sourceString in this.postCustomizationBatch)
{
writer.WriteLine(sourceString);
}
}
}
this.OnProgressChanged(new ProgressChangedEventArgs(2, new MarianCustomizationStatus(CustomizationStep.Copying_training_files, null)));
//Copy raw files to model dir
this.customSource = this.customSource.CopyTo(Path.Combine(this.customDir.FullName, "custom.source"));
this.customTarget = this.customTarget.CopyTo(Path.Combine(this.customDir.FullName, "custom.target"));
this.OnProgressChanged(new ProgressChangedEventArgs(3, new MarianCustomizationStatus(CustomizationStep.Preprocessing_training_files, null)));
//Preprocess input files
this.PreprocessInput();
var decoderYaml = this.customDir.GetFiles("decoder.yml").Single();
var deserializer = new Deserializer();
var decoderSettings = deserializer.Deserialize <MarianDecoderConfig>(decoderYaml.OpenText());
if (this.guidedAlignment)
{
//Generate alignments for fine-tuning corpus
this.alignmentFile = new FileInfo(Path.Combine(this.customDir.FullName, "custom.alignments"));
MarianHelper.GenerateAlignments(this.spSource, this.spTarget, this.alignmentFile, this.model.AlignmentPriorsFile);
//
//Generate alignments for validation set (for evaluating fine-tuning effect on alignment)
this.validAlignmentFile = new FileInfo(Path.Combine(this.customDir.FullName, "combined.alignments"));
MarianHelper.GenerateAlignments(this.spValidSource, this.spValidTarget, this.validAlignmentFile, this.model.AlignmentPriorsFile);
}
this.OnProgressChanged(new ProgressChangedEventArgs(4, new MarianCustomizationStatus(CustomizationStep.Initial_evaluation, null)));
//Do the initial evaluation
var initialValidProcess = this.model.TranslateAndEvaluate(
this.spValidSource,
new FileInfo(Path.Combine(this.customDir.FullName, "valid.0.txt")),
this.spValidTarget,
OpusCatMTEngineSettings.Default.OODValidSetSize,
true
);
//Wait for the initial valid to finish before starting customization
//(TODO: make sure this is not done on UI thread)
initialValidProcess.WaitForExit();
this.OnProgressChanged(new ProgressChangedEventArgs(6, new MarianCustomizationStatus(CustomizationStep.Finetuning, null)));
//Use the initial translation time as basis for estimating the duration of validation file
//translation
this.trainingLog.EstimatedTranslationDuration = Convert.ToInt32((initialValidProcess.ExitTime - initialValidProcess.StartTime).TotalSeconds);
MarianTrainerConfig trainingConfig;
var baseCustomizeYmlPath =
HelperFunctions.GetLocalAppDataPath(
OpusCatMTEngineSettings.Default.CustomizationBaseConfig);
var processDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
//Make sure there's a customization file.
if (!File.Exists(baseCustomizeYmlPath))
{
File.Copy(
Path.Combine(processDir, OpusCatMTEngineSettings.Default.CustomizationBaseConfig),
baseCustomizeYmlPath);
}
//deserialize yaml file
using (var reader = new StreamReader(baseCustomizeYmlPath))
{
trainingConfig = deserializer.Deserialize <MarianTrainerConfig>(reader);
}
trainingConfig.trainSets = new List <string>
{
this.spSource.FullName,
this.spTarget.FullName
};
trainingConfig.ValidSets = new List <string>
{
this.spValidSource.FullName,
this.spValidTarget.FullName
};
trainingConfig.vocabs = new List <string>
{
Path.Combine(this.customDir.FullName, decoderSettings.vocabs[0]),
Path.Combine(this.customDir.FullName, decoderSettings.vocabs[0])
};
trainingConfig.validScriptPath = Path.Combine(this.customDir.FullName, "Validate.bat");
File.Copy(
Path.Combine(processDir, "Validate.bat"), trainingConfig.validScriptPath);
trainingConfig.validScriptArgs =
new List <string> {
spValidTarget.FullName, OpusCatMTEngineSettings.Default.OODValidSetSize.ToString()
};
trainingConfig.validTranslationOutput = Path.Combine(this.customDir.FullName, "valid.{U}.txt");
if (this.guidedAlignment)
{
trainingConfig.guidedAlignment = this.alignmentFile.FullName;
}
trainingConfig.validLog = Path.Combine(this.customDir.FullName, "valid.log");
trainingConfig.log = Path.Combine(this.customDir.FullName, "train.log");
trainingConfig.model = Path.Combine(this.customDir.FullName, decoderSettings.models.Single());
var builder = new SerializerBuilder();
builder.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull);
var serializer = builder.Build();
var configPath = Path.Combine(this.customDir.FullName, OpusCatMTEngineSettings.Default.CustomizationBaseConfig);
using (var writer = File.CreateText(configPath))
{
serializer.Serialize(writer, trainingConfig, typeof(MarianTrainerConfig));
}
Process trainProcess = this.StartTraining();
return(trainProcess);
}
