public void TestSeq2SeqTraining()
{
// Build configs for training
Seq2SeqOptions opts = CreateOptions(trainFolderPath, validFolderPath);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
// Load training corpus
var trainCorpus = new Seq2SeqCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize, maxSrcSentLength: opts.MaxTrainSrcSentLength, maxTgtSentLength: opts.MaxTrainTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence);
// Load valid corpus
var validCorpusList = new List <Seq2SeqCorpus>();
if (!opts.ValidCorpusPaths.IsNullOrEmpty())
{
string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';');
foreach (var validCorpusPath in validCorpusPathList)
{
validCorpusList.Add(new Seq2SeqCorpus(validCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence));
}
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create optimizer
IOptimizer optimizer = Misc.CreateOptimizer(opts);
// Build vocabularies for training
(var srcVocab, var tgtVocab) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize, opts.SharedEmbeddings);
// Create metrics
List <IMetric> metrics = new List <IMetric> {
new BleuMetric()
};
//New training
var ss = new Seq2Seq(opts, srcVocab, tgtVocab);
// Add event handler for monitoring
ss.StatusUpdateWatcher += Ss_StatusUpdateWatcher;
ss.EpochEndWatcher += Ss_EpochEndWatcher;
// Kick off training
ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: metrics, decodingOptions: decodingOptions);
ss.SaveModel(suffix: ".test");
// Check if model file exists
Assert.IsTrue(File.Exists(opts.ModelFilePath + ".test"));
}
private static void Main(string[] args)
{
try
{
Logger.LogFile = $"{nameof(Seq2SeqConsole)}_{GetTimeStamp(DateTime.Now)}.log";
ShowOptions(args);
//Parse command line
Options opts = new Options();
ArgParser argParser = new ArgParser(args, opts);
if (string.IsNullOrEmpty(opts.ConfigFilePath) == false)
{
Logger.WriteLine($"Loading config file from '{opts.ConfigFilePath}'");
opts = JsonConvert.DeserializeObject <Options>(File.ReadAllText(opts.ConfigFilePath));
}
AttentionSeq2Seq ss = null;
ProcessorTypeEnums processorType = (ProcessorTypeEnums)Enum.Parse(typeof(ProcessorTypeEnums), opts.ProcessorType);
EncoderTypeEnums encoderType = (EncoderTypeEnums)Enum.Parse(typeof(EncoderTypeEnums), opts.EncoderType);
DecoderTypeEnums decoderType = (DecoderTypeEnums)Enum.Parse(typeof(DecoderTypeEnums), opts.DecoderType);
ModeEnums mode = (ModeEnums)Enum.Parse(typeof(ModeEnums), opts.TaskName);
ShuffleEnums shuffleType = (ShuffleEnums)Enum.Parse(typeof(ShuffleEnums), opts.ShuffleType);
string[] cudaCompilerOptions = String.IsNullOrEmpty(opts.CompilerOptions) ? null : opts.CompilerOptions.Split(' ', StringSplitOptions.RemoveEmptyEntries);
//Parse device ids from options
int[] deviceIds = opts.DeviceIds.Split(',').Select(x => int.Parse(x)).ToArray();
if (mode == ModeEnums.Train)
{
// Load train corpus
ParallelCorpus trainCorpus = new ParallelCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize,
maxSrcSentLength: opts.MaxSrcSentLength, maxTgtSentLength: opts.MaxTgtSentLength, shuffleEnums: shuffleType);
// Load valid corpus
ParallelCorpus validCorpus = string.IsNullOrEmpty(opts.ValidCorpusPath) ? null : new ParallelCorpus(opts.ValidCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxSrcSentLength, opts.MaxTgtSentLength);
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create optimizer
AdamOptimizer optimizer = new AdamOptimizer(opts.GradClip, opts.Beta1, opts.Beta2);
// Create metrics
List <IMetric> metrics = new List <IMetric>
{
new BleuMetric(),
new LengthRatioMetric()
};
if (!String.IsNullOrEmpty(opts.ModelFilePath) && File.Exists(opts.ModelFilePath))
{
//Incremental training
Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'...");
ss = new AttentionSeq2Seq(modelFilePath: opts.ModelFilePath, processorType: processorType, dropoutRatio: opts.DropoutRatio, deviceIds: deviceIds,
isSrcEmbTrainable: opts.IsSrcEmbeddingTrainable, isTgtEmbTrainable: opts.IsTgtEmbeddingTrainable, isEncoderTrainable: opts.IsEncoderTrainable, isDecoderTrainable: opts.IsDecoderTrainable,
maxSrcSntSize: opts.MaxSrcSentLength, maxTgtSntSize: opts.MaxTgtSentLength, memoryUsageRatio: opts.MemoryUsageRatio, shuffleType: shuffleType, compilerOptions: cudaCompilerOptions);
}
else
{
// Load or build vocabulary
Vocab vocab = null;
if (!string.IsNullOrEmpty(opts.SrcVocab) && !string.IsNullOrEmpty(opts.TgtVocab))
{
// Vocabulary files are specified, so we load them
vocab = new Vocab(opts.SrcVocab, opts.TgtVocab);
}
else
{
// We don't specify vocabulary, so we build it from train corpus
vocab = new Vocab(trainCorpus);
}
//New training
ss = new AttentionSeq2Seq(embeddingDim: opts.WordVectorSize, hiddenDim: opts.HiddenSize, encoderLayerDepth: opts.EncoderLayerDepth, decoderLayerDepth: opts.DecoderLayerDepth,
srcEmbeddingFilePath: opts.SrcEmbeddingModelFilePath, tgtEmbeddingFilePath: opts.TgtEmbeddingModelFilePath, vocab: vocab, modelFilePath: opts.ModelFilePath,
dropoutRatio: opts.DropoutRatio, processorType: processorType, deviceIds: deviceIds, multiHeadNum: opts.MultiHeadNum, encoderType: encoderType, decoderType: decoderType,
maxSrcSntSize: opts.MaxSrcSentLength, maxTgtSntSize: opts.MaxTgtSentLength, enableCoverageModel: opts.EnableCoverageModel, memoryUsageRatio: opts.MemoryUsageRatio, shuffleType: shuffleType, compilerOptions: cudaCompilerOptions);
}
// Add event handler for monitoring
ss.IterationDone += ss_IterationDone;
// Kick off training
ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpus: validCorpus, learningRate: learningRate, optimizer: optimizer, metrics: metrics);
}
else if (mode == ModeEnums.Valid)
{
Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPath}'");
// Create metrics
List <IMetric> metrics = new List <IMetric>
{
new BleuMetric(),
new LengthRatioMetric()
};
// Load valid corpus
ParallelCorpus validCorpus = new ParallelCorpus(opts.ValidCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxSrcSentLength, opts.MaxTgtSentLength);
ss = new AttentionSeq2Seq(modelFilePath: opts.ModelFilePath, processorType: processorType, deviceIds: deviceIds, memoryUsageRatio: opts.MemoryUsageRatio, shuffleType: shuffleType, compilerOptions: cudaCompilerOptions);
ss.Valid(validCorpus: validCorpus, metrics: metrics);
}
else if (mode == ModeEnums.Test)
{
Logger.WriteLine($"Test model '{opts.ModelFilePath}' by input corpus '{opts.InputTestFile}'");
//Test trained model
ss = new AttentionSeq2Seq(modelFilePath: opts.ModelFilePath, processorType: processorType, deviceIds: deviceIds, memoryUsageRatio: opts.MemoryUsageRatio,
shuffleType: shuffleType, maxSrcSntSize: opts.MaxSrcSentLength, maxTgtSntSize: opts.MaxTgtSentLength, compilerOptions: cudaCompilerOptions);
List <string> outputLines = new List <string>();
string[] data_sents_raw1 = File.ReadAllLines(opts.InputTestFile);
foreach (string line in data_sents_raw1)
{
if (opts.BeamSearch > 1)
{
// Below support beam search
List <List <string> > outputWordsList = ss.Predict(line.ToLower().Trim().Split(' ').ToList(), opts.BeamSearch);
outputLines.AddRange(outputWordsList.Select(x => string.Join(" ", x)));
}
else
{
var outputTokensBatch = ss.Test(ParallelCorpus.ConstructInputTokens(line.ToLower().Trim().Split(' ').ToList()));
outputLines.AddRange(outputTokensBatch.Select(x => String.Join(" ", x)));
}
}
File.WriteAllLines(opts.OutputTestFile, outputLines);
}
else if (mode == ModeEnums.DumpVocab)
{
ss = new AttentionSeq2Seq(modelFilePath: opts.ModelFilePath, processorType: processorType, deviceIds: deviceIds, compilerOptions: cudaCompilerOptions);
ss.DumpVocabToFiles(opts.SrcVocab, opts.TgtVocab);
}
else
{
argParser.Usage();
}
}
catch (Exception err)
{
Logger.WriteLine($"Exception: '{err.Message}'");
Logger.WriteLine($"Call stack: '{err.StackTrace}'");
}
}
static void Main(string[] args)
{
try
{
//Parse command line
// Seq2SeqOptions opts = new Seq2SeqOptions();
ArgParser argParser = new ArgParser(args, opts);
if (!opts.ConfigFilePath.IsNullOrEmpty())
{
Logger.WriteLine($"Loading config file from '{opts.ConfigFilePath}'");
opts = JsonConvert.DeserializeObject <SeqSimilarityOptions>(File.ReadAllText(opts.ConfigFilePath));
}
Logger.LogFile = $"{nameof(SeqSimilarityConsole)}_{opts.Task}_{Utils.GetTimeStamp(DateTime.Now)}.log";
ShowOptions(args, opts);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
SeqSimilarity ss = null;
if (opts.Task == ModeEnums.Train)
{
// Load train corpus
SeqClassificationMultiTasksCorpus trainCorpus = new SeqClassificationMultiTasksCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize,
maxSentLength: opts.MaxTrainSentLength, shuffleEnums: opts.ShuffleType);
// Load valid corpus
List <SeqClassificationMultiTasksCorpus> validCorpusList = new List <SeqClassificationMultiTasksCorpus>();
if (!opts.ValidCorpusPaths.IsNullOrEmpty())
{
string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';');
foreach (var validCorpusPath in validCorpusPathList)
{
validCorpusList.Add(new SeqClassificationMultiTasksCorpus(opts.ValidCorpusPaths, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSentLength, shuffleEnums: opts.ShuffleType));
}
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create metrics
IMetric metric = null;
if (opts.SimilarityType == "Continuous")
{
metric = new SimilarityMetric();
}
// Create optimizer
IOptimizer optimizer = Misc.CreateOptimizer(opts);
if (!opts.ModelFilePath.IsNullOrEmpty() && File.Exists(opts.ModelFilePath))
{
//Incremental training
Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'...");
ss = new SeqSimilarity(opts);
if (metric == null)
{
metric = new MultiLabelsFscoreMetric("", ss.ClsVocab.GetAllTokens(keepBuildInTokens: false));
}
}
else
{
// Load or build vocabulary
Vocab srcVocab = null;
List <Vocab> tgtVocabs = null;
if (!opts.SrcVocab.IsNullOrEmpty() && !opts.TgtVocab.IsNullOrEmpty())
{
Logger.WriteLine($"Loading source vocabulary from '{opts.SrcVocab}' and target vocabulary from '{opts.TgtVocab}'.");
// Vocabulary files are specified, so we load them
srcVocab = new Vocab(opts.SrcVocab);
tgtVocabs = new List <Vocab>
{
new Vocab(opts.TgtVocab)
};
}
else
{
Logger.WriteLine($"Building vocabulary from training corpus.");
// We don't specify vocabulary, so we build it from train corpus
(srcVocab, tgtVocabs) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize);
}
if (metric == null)
{
metric = new MultiLabelsFscoreMetric("", tgtVocabs[0].GetAllTokens(keepBuildInTokens: false));
}
//New training
ss = new SeqSimilarity(opts, srcVocab, tgtVocabs[0]);
}
// Add event handler for monitoring
ss.StatusUpdateWatcher += Misc.Ss_StatusUpdateWatcher;
ss.EvaluationWatcher += Ss_EvaluationWatcher;
// Kick off training
ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: new List <IMetric>()
{
metric
}, decodingOptions: decodingOptions);
}
//else if (opts.Task == ModeEnums.Valid)
//{
// Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPath}'");
// // Create metrics
// List<IMetric> metrics = new List<IMetric>
//{
// new BleuMetric(),
// new LengthRatioMetric()
//};
// // Load valid corpus
// ParallelCorpus validCorpus = new ParallelCorpus(opts.ValidCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxSrcTestSentLength, opts.MaxTgtTestSentLength, shuffleEnums: shuffleType);
// ss = new Seq2Seq(opts);
// ss.EvaluationWatcher += ss_EvaluationWatcher;
// ss.Valid(validCorpus: validCorpus, metrics: metrics);
//}
else if (opts.Task == ModeEnums.Test)
{
if (File.Exists(opts.OutputFile))
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Yellow, $"Output file '{opts.OutputFile}' exist. Delete it.");
File.Delete(opts.OutputFile);
}
//Test trained model
ss = new SeqSimilarity(opts);
Stopwatch stopwatch = Stopwatch.StartNew();
ss.Test <SeqClassificationMultiTasksCorpusBatch>(opts.InputTestFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath);
stopwatch.Stop();
Logger.WriteLine($"Test mode execution time elapsed: '{stopwatch.Elapsed}'");
}
//else if (opts.Task == ModeEnums.DumpVocab)
//{
// ss = new Seq2Seq(opts);
// ss.DumpVocabToFiles(opts.SrcVocab, opts.TgtVocab);
//}
else
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Task '{opts.Task}' is not supported.");
argParser.Usage();
}
}
catch (Exception err)
{
Logger.WriteLine($"Exception: '{err.Message}'");
Logger.WriteLine($"Call stack: '{err.StackTrace}'");
}
}
private static void Main(string[] args)
{
try
{
//Parse command line
ArgParser argParser = new ArgParser(args, opts);
if (!opts.ConfigFilePath.IsNullOrEmpty())
{
Console.WriteLine($"Loading config file from '{opts.ConfigFilePath}'");
opts = JsonConvert.DeserializeObject <Seq2SeqOptions>(File.ReadAllText(opts.ConfigFilePath));
}
Logger.LogFile = $"{nameof(Seq2SeqConsole)}_{opts.Task}_{Utils.GetTimeStamp(DateTime.Now)}.log";
ShowOptions(args, opts);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
Seq2Seq ss = null;
if (opts.Task == ModeEnums.Train)
{
// Load train corpus
var trainCorpus = new Seq2SeqCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize,
maxSrcSentLength: opts.MaxTrainSrcSentLength, maxTgtSentLength: opts.MaxTrainTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence);
// Load valid corpus
var validCorpusList = new List <Seq2SeqCorpus>();
if (!opts.ValidCorpusPaths.IsNullOrEmpty())
{
string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';');
foreach (var validCorpusPath in validCorpusPathList)
{
validCorpusList.Add(new Seq2SeqCorpus(validCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence));
}
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create optimizer
IOptimizer optimizer = Misc.CreateOptimizer(opts);
// Create metrics
List <IMetric> metrics = CreateMetrics();
if (!opts.ModelFilePath.IsNullOrEmpty() && File.Exists(opts.ModelFilePath))
{
//Incremental training
Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'...");
ss = new Seq2Seq(opts);
}
else
{
// Load or build vocabulary
Vocab srcVocab = null;
Vocab tgtVocab = null;
if (!opts.SrcVocab.IsNullOrEmpty() && !opts.TgtVocab.IsNullOrEmpty())
{
Logger.WriteLine($"Loading source vocabulary from '{opts.SrcVocab}' and target vocabulary from '{opts.TgtVocab}'. Shared vocabulary is '{opts.SharedEmbeddings}'");
if (opts.SharedEmbeddings == true && (opts.SrcVocab != opts.TgtVocab))
{
throw new ArgumentException("The source and target vocabularies must be identical if their embeddings are shared.");
}
// Vocabulary files are specified, so we load them
srcVocab = new Vocab(opts.SrcVocab);
tgtVocab = new Vocab(opts.TgtVocab);
}
else
{
Logger.WriteLine($"Building vocabulary from training corpus. Shared vocabulary is '{opts.SharedEmbeddings}'");
// We don't specify vocabulary, so we build it from train corpus
(srcVocab, tgtVocab) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize, opts.SharedEmbeddings);
}
//New training
ss = new Seq2Seq(opts, srcVocab, tgtVocab);
}
// Add event handler for monitoring
ss.StatusUpdateWatcher += Misc.Ss_StatusUpdateWatcher;
ss.EvaluationWatcher += Ss_EvaluationWatcher;
// Kick off training
ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: metrics, decodingOptions: decodingOptions);
}
else if (opts.Task == ModeEnums.Valid)
{
Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPaths}'");
// Create metrics
List <IMetric> metrics = CreateMetrics();
// Load valid corpus
Seq2SeqCorpus validCorpus = new Seq2SeqCorpus(opts.ValidCorpusPaths, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence);
ss = new Seq2Seq(opts);
ss.EvaluationWatcher += Ss_EvaluationWatcher;
ss.Valid(validCorpus: validCorpus, metrics: metrics, decodingOptions: decodingOptions);
}
else if (opts.Task == ModeEnums.Test)
{
if (File.Exists(opts.OutputFile))
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Yellow, $"Output file '{opts.OutputFile}' exist. Delete it.");
File.Delete(opts.OutputFile);
}
//Test trained model
ss = new Seq2Seq(opts);
Stopwatch stopwatch = Stopwatch.StartNew();
if (String.IsNullOrEmpty(opts.OutputPromptFile))
{
ss.Test <Seq2SeqCorpusBatch>(opts.InputTestFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath);
}
else
{
Logger.WriteLine($"Test with prompt file '{opts.OutputPromptFile}'");
ss.Test <Seq2SeqCorpusBatch>(opts.InputTestFile, opts.OutputPromptFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath);
}
stopwatch.Stop();
Logger.WriteLine($"Test mode execution time elapsed: '{stopwatch.Elapsed}'");
}
else if (opts.Task == ModeEnums.DumpVocab)
{
ss = new Seq2Seq(opts);
ss.DumpVocabToFiles(opts.SrcVocab, opts.TgtVocab);
}
else
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Task '{opts.Task}' is not supported.");
argParser.Usage();
}
}
catch (Exception err)
{
Logger.WriteLine($"Exception: '{err.Message}'");
Logger.WriteLine($"Call stack: '{err.StackTrace}'");
}
}
static void Main(string[] args)
{
ShowOptions(args);
Logger.LogFile = $"{nameof(SeqLabelConsole)}_{GetTimeStamp(DateTime.Now)}.log";
//Parse command line
Options opts = new Options();
ArgParser argParser = new ArgParser(args, opts);
if (String.IsNullOrEmpty(opts.ConfigFilePath) == false)
{
Logger.WriteLine($"Loading config file from '{opts.ConfigFilePath}'");
opts = JsonConvert.DeserializeObject <Options>(File.ReadAllText(opts.ConfigFilePath));
}
SequenceLabel sl = null;
ProcessorTypeEnums processorType = (ProcessorTypeEnums)Enum.Parse(typeof(ProcessorTypeEnums), opts.ProcessorType);
EncoderTypeEnums encoderType = (EncoderTypeEnums)Enum.Parse(typeof(EncoderTypeEnums), opts.EncoderType);
ModeEnums mode = (ModeEnums)Enum.Parse(typeof(ModeEnums), opts.TaskName);
//Parse device ids from options
int[] deviceIds = opts.DeviceIds.Split(',').Select(x => int.Parse(x)).ToArray();
if (mode == ModeEnums.Train)
{
// Load train corpus
ParallelCorpus trainCorpus = new ParallelCorpus(opts.TrainCorpusPath, opts.SrcLang, opts.TgtLang, opts.BatchSize, opts.ShuffleBlockSize, opts.MaxSentLength, addBOSEOS: false);
// Load valid corpus
ParallelCorpus validCorpus = String.IsNullOrEmpty(opts.ValidCorpusPath) ? null : new ParallelCorpus(opts.ValidCorpusPath, opts.SrcLang, opts.TgtLang, opts.BatchSize, opts.ShuffleBlockSize, opts.MaxSentLength, addBOSEOS: false);
// Load or build vocabulary
Vocab vocab = null;
if (!String.IsNullOrEmpty(opts.SrcVocab) && !String.IsNullOrEmpty(opts.TgtVocab))
{
// Vocabulary files are specified, so we load them
vocab = new Vocab(opts.SrcVocab, opts.TgtVocab);
}
else
{
// We don't specify vocabulary, so we build it from train corpus
vocab = new Vocab(trainCorpus);
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create optimizer
AdamOptimizer optimizer = new AdamOptimizer(opts.GradClip, opts.Beta1, opts.Beta2);
// Create metrics
List <IMetric> metrics = new List <IMetric>();
foreach (var word in vocab.TgtVocab)
{
metrics.Add(new SequenceLabelFscoreMetric(word));
}
if (File.Exists(opts.ModelFilePath) == false)
{
//New training
sl = new SequenceLabel(hiddenDim: opts.HiddenSize, embeddingDim: opts.WordVectorSize, encoderLayerDepth: opts.EncoderLayerDepth, multiHeadNum: opts.MultiHeadNum,
encoderType: encoderType,
dropoutRatio: opts.DropoutRatio, deviceIds: deviceIds, processorType: processorType, modelFilePath: opts.ModelFilePath, vocab: vocab);
}
else
{
//Incremental training
Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'...");
sl = new SequenceLabel(modelFilePath: opts.ModelFilePath, processorType: processorType, deviceIds: deviceIds, dropoutRatio: opts.DropoutRatio);
}
// Add event handler for monitoring
sl.IterationDone += ss_IterationDone;
// Kick off training
sl.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpus: validCorpus, learningRate: learningRate, optimizer: optimizer, metrics: metrics);
}
else if (mode == ModeEnums.Valid)
{
Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPath}'");
// Load valid corpus
ParallelCorpus validCorpus = new ParallelCorpus(opts.ValidCorpusPath, opts.SrcLang, opts.TgtLang, opts.BatchSize, opts.ShuffleBlockSize, opts.MaxSentLength, false);
Vocab vocab = new Vocab(validCorpus);
// Create metrics
List <IMetric> metrics = new List <IMetric>();
foreach (var word in vocab.TgtVocab)
{
metrics.Add(new SequenceLabelFscoreMetric(word));
}
sl = new SequenceLabel(modelFilePath: opts.ModelFilePath, processorType: processorType, deviceIds: deviceIds);
sl.Valid(validCorpus: validCorpus, metrics: metrics);
}
else if (mode == ModeEnums.Test)
{
Logger.WriteLine($"Test model '{opts.ModelFilePath}' by input corpus '{opts.InputTestFile}'");
//Test trained model
sl = new SequenceLabel(modelFilePath: opts.ModelFilePath, processorType: processorType, deviceIds: deviceIds);
List <string> outputLines = new List <string>();
var data_sents_raw1 = File.ReadAllLines(opts.InputTestFile);
foreach (string line in data_sents_raw1)
{
var outputTokensBatch = sl.Test(ParallelCorpus.ConstructInputTokens(line.ToLower().Trim().Split(' ').ToList(), false));
outputLines.AddRange(outputTokensBatch.Select(x => String.Join(" ", x)));
}
File.WriteAllLines(opts.OutputTestFile, outputLines);
}
//else if (mode == ModeEnums.VisualizeNetwork)
//{
// ss = new AttentionSeq2Seq(embeddingDim: opts.WordVectorSize, hiddenDim: opts.HiddenSize, encoderLayerDepth: opts.EncoderLayerDepth, decoderLayerDepth: opts.DecoderLayerDepth,
// vocab: new Vocab(), srcEmbeddingFilePath: null, tgtEmbeddingFilePath: null, modelFilePath: opts.ModelFilePath, dropoutRatio: opts.DropoutRatio,
// processorType: processorType, deviceIds: new int[1] { 0 }, multiHeadNum: opts.MultiHeadNum, encoderType: encoderType);
// ss.VisualizeNeuralNetwork(opts.VisualizeNNFilePath);
//}
else
{
argParser.Usage();
}
}
private static void Main(string[] args)
{
ShowOptions(args);
Logger.LogFile = $"{nameof(SeqLabelConsole)}_{Utils.GetTimeStamp(DateTime.Now)}.log";
//Parse command line
SeqLabelOptions opts = new SeqLabelOptions();
ArgParser argParser = new ArgParser(args, opts);
if (!opts.ConfigFilePath.IsNullOrEmpty())
{
Logger.WriteLine($"Loading config file from '{opts.ConfigFilePath}'");
opts = JsonConvert.DeserializeObject<SeqLabelOptions>(File.ReadAllText(opts.ConfigFilePath));
}
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
SeqLabel sl = null;
//Parse device ids from options
int[] deviceIds = opts.DeviceIds.Split(',').Select(x => int.Parse(x)).ToArray();
if ( opts.Task == ModeEnums.Train )
{
// Load train corpus
SeqLabelingCorpus trainCorpus = new SeqLabelingCorpus(opts.TrainCorpusPath, opts.BatchSize, opts.ShuffleBlockSize, maxSentLength: opts.MaxTrainSentLength);
// Load valid corpus
List<SeqLabelingCorpus> validCorpusList = new List<SeqLabelingCorpus>();
if (!opts.ValidCorpusPaths.IsNullOrEmpty())
{
string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';');
foreach (var validCorpusPath in validCorpusPathList)
{
validCorpusList.Add(new SeqLabelingCorpus(opts.ValidCorpusPaths, opts.BatchSize, opts.ShuffleBlockSize, maxSentLength: opts.MaxTestSentLength));
}
}
// Load or build vocabulary
Vocab srcVocab = null;
Vocab tgtVocab = null;
if (!opts.SrcVocab.IsNullOrEmpty() && !opts.TgtVocab.IsNullOrEmpty() )
{
// Vocabulary files are specified, so we load them
srcVocab = new Vocab(opts.SrcVocab);
tgtVocab = new Vocab(opts.TgtVocab);
}
else
{
// We don't specify vocabulary, so we build it from train corpus
(srcVocab, tgtVocab) = trainCorpus.BuildVocabs(opts.SrcVocabSize);
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create optimizer
IOptimizer optimizer = Misc.CreateOptimizer(opts);
// Create metrics
List<IMetric> metrics = new List<IMetric>();
foreach (string word in tgtVocab.Items)
{
if (BuildInTokens.IsPreDefinedToken(word) == false)
{
metrics.Add(new SequenceLabelFscoreMetric(word));
}
}
if (File.Exists(opts.ModelFilePath) == false)
{
//New training
sl = new SeqLabel(opts, srcVocab: srcVocab, clsVocab: tgtVocab);
}
else
{
//Incremental training
Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'...");
sl = new SeqLabel(opts);
}
// Add event handler for monitoring
sl.StatusUpdateWatcher += Misc.Ss_StatusUpdateWatcher;
// Kick off training
sl.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: metrics, decodingOptions: decodingOptions);
}
else if ( opts.Task == ModeEnums.Valid )
{
Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPaths}'");
// Load valid corpus
SeqLabelingCorpus validCorpus = new SeqLabelingCorpus(opts.ValidCorpusPaths, opts.BatchSize, opts.ShuffleBlockSize, opts.MaxTestSentLength);
(Vocab srcVocab, Vocab tgtVocab) = validCorpus.BuildVocabs();
// Create metrics
List<IMetric> metrics = new List<IMetric>();
foreach (string word in tgtVocab.Items)
{
if (BuildInTokens.IsPreDefinedToken(word) == false)
{
metrics.Add(new SequenceLabelFscoreMetric(word));
}
}
sl = new SeqLabel(opts);
sl.Valid(validCorpus: validCorpus, metrics: metrics, decodingOptions: decodingOptions);
}
else if ( opts.Task == ModeEnums.Test )
{
Logger.WriteLine($"Test model '{opts.ModelFilePath}' by input corpus '{opts.InputTestFile}'");
//Test trained model
sl = new SeqLabel(opts);
List<string> outputLines = new List<string>();
string[] data_sents_raw1 = File.ReadAllLines(opts.InputTestFile);
foreach (string line in data_sents_raw1)
{
var nrs = sl.Test<SeqLabelingCorpusBatch>(ConstructInputTokens(line.Trim().Split(' ').ToList()), null, decodingOptions: decodingOptions);
outputLines.AddRange(nrs[0].Output[0].Select(x => string.Join(" ", x)));
}
File.WriteAllLines(opts.OutputFile, outputLines);
}
else
{
argParser.Usage();
}
}
