private void btn_ReadOne_Click(object sender, EventArgs e)
{
//【1】设置读取条形码规格
DecodingOptions decodeOption = new DecodingOptions();
decodeOption.PossibleFormats = new List <BarcodeFormat>()
{
BarcodeFormat.EAN_13,
};
//【2】进行读取操作
BarcodeReader br = new BarcodeReader();
br.Options = decodeOption;
Result rs = br.Decode(this.pictureBox1.Image as Bitmap);
if (rs == null)
{
MessageBox.Show("读取失败");
}
else
{
MessageBox.Show("读取成功，内容为：" + rs.Text);
}
}
/// <summary>
/// 读取编码内容
/// </summary>
/// <param name="filePath"></param>
/// <param name="formats"></param>
/// <returns></returns>
public static string ReadCode(string filePath,
params BarcodeFormat[] formats)
{
var decodeOptions = new DecodingOptions
{
PossibleFormats = new List <BarcodeFormat>
{
BarcodeFormat.EAN_13
}
};
formats?.ToList().ForEach(item =>
{
decodeOptions.PossibleFormats.Add(item);
});
var reader = new BarcodeReader
{
Options = decodeOptions
};
if (!File.Exists(filePath))
{
return(string.Empty);
}
var result = reader.Decode(new Bitmap(filePath));
return(result != null ? result.Text : string.Empty);
}
// 读取条形码
private void Read1DBtn_Click(object sender, EventArgs e)
{
// 1.设置读取条形码规格
DecodingOptions decodeOption = new DecodingOptions();
decodeOption.PossibleFormats = new List <BarcodeFormat>()
{
BarcodeFormat.All_1D,
};
// 2.进行读取操作
ZXing.BarcodeReader br = new BarcodeReader();
br.Options = decodeOption;
ZXing.Result rs = br.Decode(this.barCodeImg.Image as Bitmap);
if (rs == null)
{
this.ContentTxt.Text = "读取失败";
MessageBox.Show("读取失败");
}
else
{
this.ContentTxt.Text = rs.Text;
MessageBox.Show("读取成功，内容：" + rs.Text);
}
}
public void Test_1()
{
// 1.设置读取条形码规格
DecodingOptions decodeOption = new DecodingOptions();
decodeOption.PossibleFormats = new List <BarcodeFormat>()
{
BarcodeFormat.All_1D,
};
// 2.进行读取操作
ZXing.BarcodeReader br = new BarcodeReader();
br.Options = decodeOption;
this.pictureBox2.Image = Bitmap.FromFile(@"e:\BTWork\MachineL\machine_vision\Test\图片处理\123.jpg");
ZXing.Result rs = br.Decode(this.pictureBox2.Image as Bitmap);
if (rs == null)
{
MessageBox.Show("读取失败");
}
else
{
MessageBox.Show("读取成功，内容：" + rs.Text);
}
}
public void TestSeq2SeqInference()
{
var opts = new Seq2SeqOptions();
opts.ModelFilePath = "seq2seq_mt_enu_chs_tiny_test.model";
opts.MaxTestSrcSentLength = 110;
opts.MaxTestTgtSentLength = 110;
opts.ProcessorType = ProcessorTypeEnums.CPU;
opts.DeviceIds = "0";
var seq2seq = new Seq2Seq(opts);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
List <List <List <string> > > groupBatchTokens = BuildInputGroupBatchTokens("▁yes , ▁solutions ▁do ▁exist .");
var nrs = seq2seq.Test <Seq2SeqCorpusBatch>(groupBatchTokens, null, decodingOptions);
var out_tokens = nrs[0].Output[0][0];
var output = string.Join(" ", out_tokens);
Assert.IsTrue(output == "<s> ▁是的 , 解决方案 存在 。 </s>");
groupBatchTokens = BuildInputGroupBatchTokens("▁a ▁question ▁of ▁climate .");
nrs = seq2seq.Test <Seq2SeqCorpusBatch>(groupBatchTokens, null, decodingOptions);
out_tokens = nrs[0].Output[0][0];
output = string.Join(" ", out_tokens);
Assert.IsTrue(output == "<s> ▁ 气候 问题 。 </s>");
}
//读取并解析条形码
private void btnDecodeBarCode_Click(object sender, EventArgs e)
{
if (BarCodeImage != null)
{
// 1.设置读取条形码规格
DecodingOptions decodeOption = new DecodingOptions();
decodeOption.PossibleFormats = new List <BarcodeFormat>()
{
BarcodeFormatHelper.GetFormat(this.cbEncodeType.SelectedItem.ToString()),
};
// 2.进行读取操作
BarcodeReader br = new BarcodeReader();
br.Options = decodeOption;
Result rs = br.Decode(BarCodeImage);
if (rs == null)
{
MessageBox.Show("读取失败");
}
else
{
MessageBox.Show("读取成功，内容：" + rs.Text);
}
}
}
public static string DecoderFromImage(Bitmap image)
{
//use gaussian filter to remove noise
//var gFilter = new GaussianBlur(2);
//image = gFilter.ProcessImage(image);
var options = new DecodingOptions {
PossibleFormats = new List <BarcodeFormat> {
BarcodeFormat.QR_CODE
}, TryHarder = true
};
using (image)
{
//use GlobalHistogramBinarizer for best result
var reader = new BarcodeReader(null, null, ls => new GlobalHistogramBinarizer(ls))
{
AutoRotate = false, TryInverted = false, Options = options
};
var result = reader.Decode(image);
reader = null;
return(result.Text);
}
}
static public (string, string) Call(string key, List <string> inputFeatureGroups)
{
if (m_key2Instance.ContainsKey(key) == false)
{
return("", "");
}
List <List <List <string> > > groupBatchTokens = new List <List <List <string> > >();
foreach (var inputFeatureGroup in inputFeatureGroups)
{
List <string> tokens = inputFeatureGroup.Split(' ').ToList();
List <List <string> > batchTokens = new List <List <string> >();
batchTokens.Add(tokens);
groupBatchTokens.Add(batchTokens);
}
Seq2SeqClassification inst = null;
lock (locker)
{
inst = m_key2Instance[key];
}
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
List <NetworkResult> nrs = inst.Test <Seq2SeqClassificationCorpusBatch>(groupBatchTokens, null, decodingOptions);
var nrCLS = nrs[0];
var nrSeq2Seq = nrs[1];
string tag = nrCLS.Output[0][0][0];
string text = string.Join(" ", nrSeq2Seq.Output[0][0].ToArray(), 1, nrSeq2Seq.Output[0][0].Count - 2);
return(tag, text);
}
static public string Call(string input1, string input2)
{
List <List <List <string> > > groupBatchTokens = new List <List <List <string> > >();
// Build group 1 features for input string 1
List <string> tokens = input1.Split(' ').ToList();
List <List <string> > batchTokens = new List <List <string> >();
batchTokens.Add(tokens);
groupBatchTokens.Add(batchTokens);
// Build group 2 features for input string 2
tokens = input2.Split(' ').ToList();
batchTokens = new List <List <string> >();
batchTokens.Add(tokens);
groupBatchTokens.Add(batchTokens);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
List <NetworkResult> nrs = m_seqSimilarity.Test <SeqClassificationMultiTasksCorpusBatch>(groupBatchTokens, null, decodingOptions);
List <string> tags = new List <string>();
foreach (var nr in nrs)
{
tags.Add(nr.Output[0][0][0]); // shape: (beam_size, batch_size, seq_size)
}
return(string.Join("\t", tags));
}
static public string Call(string srcInput, string tgtInput, int tokenNumToGenerate, bool random, float repeatPenalty)
{
if (opts == null)
{
throw new ArgumentNullException($"The {nameof(Seq2SeqInstance)} may not be initialized, and option instance is null.");
}
if (m_seq2seq == null)
{
throw new ArgumentNullException($"The {nameof(Seq2SeqInstance)} is null.");
}
srcInput = (m_srcSpm != null) ? m_srcSpm.Encode(srcInput) : srcInput;
List <string> tokens = srcInput.Split(' ').ToList();
if (tokens.Count > opts.MaxTestSrcSentLength)
{
tokens = tokens.GetRange(tokens.Count - opts.MaxTestSrcSentLength, opts.MaxTestSrcSentLength);
}
List <List <String> > batchTokens = new List <List <string> >();
batchTokens.Add(tokens);
List <List <List <string> > > srcGroupBatchTokens = new List <List <List <string> > >();
srcGroupBatchTokens.Add(batchTokens);
tgtInput = (m_tgtSpm != null) ? m_tgtSpm.Encode(tgtInput) : tgtInput;
List <string> tokens2 = tgtInput.Split(' ').ToList();
tokenNumToGenerate += tokens2.Count;
List <List <String> > batchTokens2 = new List <List <string> >();
batchTokens2.Add(tokens2);
List <List <List <string> > > tgtGroupBatchTokens = new List <List <List <string> > >();
tgtGroupBatchTokens.Add(batchTokens2);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
decodingOptions.MaxTgtSentLength = tokenNumToGenerate;
decodingOptions.TopPValue = random ? 0.5f : 0.0f;
decodingOptions.RepeatPenalty = repeatPenalty;
var nrs = m_seq2seq.Test <Seq2SeqCorpusBatch>(srcGroupBatchTokens, tgtGroupBatchTokens, decodingOptions);
string rst = String.Join(" ", nrs[0].Output[0][0].ToArray(), 0, nrs[0].Output[0][0].Count);
rst = (m_tgtSpm != null) ? m_tgtSpm.Decode(rst) : rst;
return(rst);
}
/// <summary>
/// Creates a new <see cref="Video"/> instance.
/// </summary>
/// <param name="url">Video source URL. May be a file system path.</param>
/// <param name="decodingOptions">Decoding options.</param>
internal Video([NotNull] string url, [NotNull] DecodingOptions decodingOptions)
{
_decodeContext = new DecodeContext(url, decodingOptions);
_decodingOptions = decodingOptions;
_decodeContext.Ended += decodeContext_Ended;
Duration = TimeSpan.FromSeconds(_decodeContext.GetDurationInSeconds());
}
public Code39BarcodeReader()
{
Options = new DecodingOptions()
{
PossibleFormats = new List <BarcodeFormat>
{
BarcodeFormat.CODE_39
}
};
}
public void TestSeq2SeqTraining()
{
// Build configs for training
Seq2SeqOptions opts = CreateOptions(trainFolderPath, validFolderPath);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
// Load training corpus
var trainCorpus = new Seq2SeqCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize, maxSrcSentLength: opts.MaxTrainSrcSentLength, maxTgtSentLength: opts.MaxTrainTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence);
// Load valid corpus
var validCorpusList = new List <Seq2SeqCorpus>();
if (!opts.ValidCorpusPaths.IsNullOrEmpty())
{
string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';');
foreach (var validCorpusPath in validCorpusPathList)
{
validCorpusList.Add(new Seq2SeqCorpus(validCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence));
}
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create optimizer
IOptimizer optimizer = Misc.CreateOptimizer(opts);
// Build vocabularies for training
(var srcVocab, var tgtVocab) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize, opts.SharedEmbeddings);
// Create metrics
List <IMetric> metrics = new List <IMetric> {
new BleuMetric()
};
//New training
var ss = new Seq2Seq(opts, srcVocab, tgtVocab);
// Add event handler for monitoring
ss.StatusUpdateWatcher += Ss_StatusUpdateWatcher;
ss.EpochEndWatcher += Ss_EpochEndWatcher;
// Kick off training
ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: metrics, decodingOptions: decodingOptions);
ss.SaveModel(suffix: ".test");
// Check if model file exists
Assert.IsTrue(File.Exists(opts.ModelFilePath + ".test"));
}
private void BtnUpload_Click(object sender, EventArgs e)
{
if (FileUpload1.HasFile)
{
DivInfo.Visible = false;
DivWarning.Visible = false;
var ext = Path.GetExtension(FileUpload1.FileName).ToLower();
var tempPath = Path.GetTempFileName() + ext;
FileUpload1.SaveAs(tempPath);
HashSet <String> exts = new HashSet <string> {
".bmp", ".png", ".jpg", ".gif", ".png"
};
if (exts.Contains(ext))
{
ImageQR.ImageUrl = "/handlers/imageloader.ashx?ImgPath=" + tempPath;
var qr = new ZXing.BarcodeReader();
var options = new DecodingOptions
{
CharacterSet = "UTF-8",
};
qr.Options = options;
Bitmap bmp = (Bitmap)Bitmap.FromFile(tempPath);
var res = qr.Decode(bmp);
if (res != null && !string.IsNullOrEmpty(res.Text))
{
try
{
var decrypt = Crypto.Decrypt(res.Text);
LoadHeaderInfo(decrypt.Trim());
}
catch
{
CommonWeb.Alert(this, "Qr dapat dibaca tapi tidak valid.");
}
}
else
{
CommonWeb.Alert(this, "Qr tidak dapat dibaca, atau gambar tidak valid.");
}
}
else
{
CommonWeb.Alert(this, "Silakan upload file dengan tipe gambar");
}
}
else
{
CommonWeb.Alert(this, "Silakan pilih foto QR LHP terlebih dahulu.");
}
}
internal static bool Get2047CharsetName(TextHeader textHeader, out string charsetName)
{
DecodingOptions decodingOptions = new DecodingOptions(DecodingFlags.Rfc2047, null);
DecodingResults decodingResults;
string text;
if (textHeader.TryGetValue(decodingOptions, out decodingResults, out text) && EncodingScheme.Rfc2047 == decodingResults.EncodingScheme)
{
charsetName = decodingResults.CharsetName;
return(true);
}
charsetName = null;
return(false);
}
internal static bool Get2047CharsetName(AddressItem addressItem, out string charsetName)
{
DecodingOptions decodingOptions = new DecodingOptions(DecodingFlags.Rfc2047, null);
DecodingResults decodingResults;
string text;
if (addressItem.TryGetDisplayName(decodingOptions, out decodingResults, out text) && EncodingScheme.Rfc2047 == decodingResults.EncodingScheme)
{
charsetName = decodingResults.CharsetName;
return(true);
}
charsetName = null;
return(false);
}
public DecodingOptions CreateDecodingOptions()
{
DecodingOptions decodingOptions = new DecodingOptions();
decodingOptions.DecodingStrategy = DecodingStrategy;
decodingOptions.TopPValue = DecodingTopPValue;
decodingOptions.RepeatPenalty = DecodingRepeatPenalty;
decodingOptions.BeamSearchSize = BeamSearchSize;
decodingOptions.MaxSrcSentLength = MaxTestSrcSentLength;
decodingOptions.MaxTgtSentLength = MaxTestTgtSentLength;
return(decodingOptions);
}
private string DeCodeImg(Bitmap img)
{
if (img == null)
{
return("");
}
#region 将图片转化成 byte数组
MemoryStream ms = new MemoryStream();
img.Save(ms, System.Drawing.Imaging.ImageFormat.Bmp);
byte[] bt = ms.GetBuffer();
ms.Close();
#endregion
LuminanceSource source = new RGBLuminanceSource(bt, img.Width, img.Height);
BinaryBitmap bitmap = new BinaryBitmap(new ZXing.Common.HybridBinarizer(source));
//【1】设置读取条形码规格
DecodingOptions decodeOption = new DecodingOptions();
decodeOption.PossibleFormats = new List <BarcodeFormat>()
{
BarcodeFormat.EAN_13,
};
//【2】进行读取操作
BarcodeReader br = new BarcodeReader();
br.Options = decodeOption;
Result result2, result1;
try
{
//开始解码
result2 = new MultiFormatReader().decode(bitmap);
result1 = br.Decode(img);
}
catch
{
return("");
}
if (result1 != null)
{
return(result1.Text);
}
if (result2 != null)
{
return(result2.Text);
}
return("");
}
public static string ReadQRCode(Bitmap img)
{
DecodingOptions decodingOptions = new DecodingOptions();
decodingOptions.PossibleFormats = new List <BarcodeFormat>
{
BarcodeFormat.QR_CODE
};
BarcodeReader barcodeReader = new BarcodeReader();
barcodeReader.Options = decodingOptions;
Result result = barcodeReader.Decode(img);
if (result == null)
{
return("");
}
return(result.Text);
}
public static string ReadBarCode(Image img)
{
DecodingOptions decodingOptions = new DecodingOptions();
decodingOptions.PossibleFormats = new List <BarcodeFormat>
{
BarcodeFormat.EAN_13
};
BarcodeReader barcodeReader = new BarcodeReader();
barcodeReader.Options = decodingOptions;
Result result = barcodeReader.Decode(img as Bitmap);
if (result == null)
{
return("");
}
return(result.Text);
}
/// <summary>
/// 解析图片
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
public static string AnalysisImage(string file)
{
Bitmap m1 = (Bitmap)Bitmap.FromFile(file);
BarcodeReader barcodeReader = new BarcodeReader();
DecodingOptions arg_1 = new DecodingOptions();
arg_1.PossibleFormats = new List <BarcodeFormat>()
{
BarcodeFormat.QR_CODE,
BarcodeFormat.CODE_39,
BarcodeFormat.CODE_93,
BarcodeFormat.CODE_128,
};
arg_1.CharacterSet = "UTF-8";//提示已过时？
//arg_1.CharacterSet = "gb2312";//设置了字符集也读不出来中文汉字
barcodeReader.Options = arg_1;
Result result = barcodeReader.Decode(m1);
return(result.Text);
}
public Form1()
{
InitializeComponent();
CheckForIllegalCrossThreadCalls = false;
DecOpt = new DecodingOptions()
{
PossibleFormats = new[] { BarcodeFormat.QR_CODE },
TryHarder = true
};
if (!File.Exists(FileManager.AppFolder + "StaffAPI.txt"))
{
return;
}
ApiToken = FileManager.ReadAllLines(FileManager.AppFolder + "StaffAPI.txt")[0];
Authed = true;
authedLabel.Text = "Authenticated using staff API token.";
authedLabel.ForeColor = Color.DeepSkyBlue;
RegisterHotKey(this.Handle, 1, (int)KeyModifier.Alt, Keys.F12.GetHashCode());
}
public FormM()
{
InitializeComponent();
_workpath = textBoxWorkPath.Text;
_angle = null;
_artemplate = null;
_rundr = null;
_runnameList = null;
_runmsg = "";
_exportdata = "";
_xztpos = -1;
_titlepos = new Dictionary <string, int>();
//for 二维码
DecodingOptions decodeOption = new DecodingOptions();
decodeOption.PossibleFormats = new List <BarcodeFormat>()
{
BarcodeFormat.All_1D
};
_br = new BarcodeReader();
_br.Options = decodeOption;
}
public static string Rfc2047Decode(string encodedValue)
{
if (encodedValue == null)
{
return(null);
}
if (!encodedValue.Contains("=?"))
{
return(encodedValue);
}
MimeString str = new MimeString(encodedValue.Trim());
MimeStringList lines = new MimeStringList(str);
DecodingOptions decodingOptions = new DecodingOptions(DecodingFlags.Rfc2047);
DecodingResults decodingResults;
string result;
if (!MimeCommon.TryDecodeValue(lines, 4026531840U, decodingOptions, out decodingResults, out result))
{
return(encodedValue);
}
return(result);
}
/// <summary>
/// ZXing根据图片读取二维码内容
/// </summary>
/// <param name="img">Image img</param>
/// <param name="BarcodeFormat barcodeFormat">barcodeFormat</param>
/// <returns></returns>
public string ReadQrCode(Image img, BarcodeFormat barcodeFormat)
{
// 1.设置读取条形码规格
DecodingOptions decodeOption = new DecodingOptions();
decodeOption.PossibleFormats = new List <BarcodeFormat>()
{
barcodeFormat
};
// 2.进行读取操作
ZXing.BarcodeReader br = new BarcodeReader();
br.Options = decodeOption;
ZXing.Result rs = br.Decode(img as Bitmap);
if (rs == null)
{
return(string.Empty);
}
else
{
return(rs.Text);
}
}
static public string Call(List <string> inputFeatureGroups)
{
List <List <List <string> > > groupBatchTokens = new List <List <List <string> > >();
foreach (var inputFeatureGroup in inputFeatureGroups)
{
List <string> tokens = inputFeatureGroup.Split(' ').ToList();
List <List <string> > batchTokens = new List <List <string> >();
batchTokens.Add(tokens);
groupBatchTokens.Add(batchTokens);
}
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
List <NetworkResult> nrs = m_seqClassification.Test <SeqClassificationMultiTasksCorpusBatch>(groupBatchTokens, null, decodingOptions);
List <string> tags = new List <string>();
foreach (var nr in nrs)
{
tags.Add(nr.Output[0][0][0]); // shape: (beam_size, batch_size, seq_size)
}
return(string.Join("\t", tags));
}
public static string ReadQRCode(string imgpath)
{
if (!File.Exists(imgpath))
{
return("");
}
Bitmap barcodeBitmap = new Bitmap(imgpath);
DecodingOptions decodingOptions = new DecodingOptions();
decodingOptions.PossibleFormats = new List <BarcodeFormat>
{
BarcodeFormat.QR_CODE
};
BarcodeReader barcodeReader = new BarcodeReader();
barcodeReader.Options = decodingOptions;
Result result = barcodeReader.Decode(barcodeBitmap);
if (result == null)
{
return("");
}
return(result.Text);
}
/// <summary>
/// Creates a new <see cref="VideoDecodingContext"/> instance.
/// </summary>
/// <param name="videoStream">The video stream.</param>
/// <param name="decodingOptions">Decoding options.</param>
internal VideoDecodingContext([NotNull] AVStream *videoStream, [NotNull] DecodingOptions decodingOptions)
{
_codecContext = videoStream->codec;
_videoStream = videoStream;
_decodingOptions = decodingOptions;
}
static void Main(string[] args)
{
try
{
//Parse command line
// Seq2SeqOptions opts = new Seq2SeqOptions();
ArgParser argParser = new ArgParser(args, opts);
if (!opts.ConfigFilePath.IsNullOrEmpty())
{
Logger.WriteLine($"Loading config file from '{opts.ConfigFilePath}'");
opts = JsonConvert.DeserializeObject <SeqSimilarityOptions>(File.ReadAllText(opts.ConfigFilePath));
}
Logger.LogFile = $"{nameof(SeqSimilarityConsole)}_{opts.Task}_{Utils.GetTimeStamp(DateTime.Now)}.log";
ShowOptions(args, opts);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
SeqSimilarity ss = null;
if (opts.Task == ModeEnums.Train)
{
// Load train corpus
SeqClassificationMultiTasksCorpus trainCorpus = new SeqClassificationMultiTasksCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize,
maxSentLength: opts.MaxTrainSentLength, shuffleEnums: opts.ShuffleType);
// Load valid corpus
List <SeqClassificationMultiTasksCorpus> validCorpusList = new List <SeqClassificationMultiTasksCorpus>();
if (!opts.ValidCorpusPaths.IsNullOrEmpty())
{
string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';');
foreach (var validCorpusPath in validCorpusPathList)
{
validCorpusList.Add(new SeqClassificationMultiTasksCorpus(opts.ValidCorpusPaths, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSentLength, shuffleEnums: opts.ShuffleType));
}
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create metrics
IMetric metric = null;
if (opts.SimilarityType == "Continuous")
{
metric = new SimilarityMetric();
}
// Create optimizer
IOptimizer optimizer = Misc.CreateOptimizer(opts);
if (!opts.ModelFilePath.IsNullOrEmpty() && File.Exists(opts.ModelFilePath))
{
//Incremental training
Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'...");
ss = new SeqSimilarity(opts);
if (metric == null)
{
metric = new MultiLabelsFscoreMetric("", ss.ClsVocab.GetAllTokens(keepBuildInTokens: false));
}
}
else
{
// Load or build vocabulary
Vocab srcVocab = null;
List <Vocab> tgtVocabs = null;
if (!opts.SrcVocab.IsNullOrEmpty() && !opts.TgtVocab.IsNullOrEmpty())
{
Logger.WriteLine($"Loading source vocabulary from '{opts.SrcVocab}' and target vocabulary from '{opts.TgtVocab}'.");
// Vocabulary files are specified, so we load them
srcVocab = new Vocab(opts.SrcVocab);
tgtVocabs = new List <Vocab>
{
new Vocab(opts.TgtVocab)
};
}
else
{
Logger.WriteLine($"Building vocabulary from training corpus.");
// We don't specify vocabulary, so we build it from train corpus
(srcVocab, tgtVocabs) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize);
}
if (metric == null)
{
metric = new MultiLabelsFscoreMetric("", tgtVocabs[0].GetAllTokens(keepBuildInTokens: false));
}
//New training
ss = new SeqSimilarity(opts, srcVocab, tgtVocabs[0]);
}
// Add event handler for monitoring
ss.StatusUpdateWatcher += Misc.Ss_StatusUpdateWatcher;
ss.EvaluationWatcher += Ss_EvaluationWatcher;
// Kick off training
ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: new List <IMetric>()
{
metric
}, decodingOptions: decodingOptions);
}
//else if (opts.Task == ModeEnums.Valid)
//{
// Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPath}'");
// // Create metrics
// List<IMetric> metrics = new List<IMetric>
//{
// new BleuMetric(),
// new LengthRatioMetric()
//};
// // Load valid corpus
// ParallelCorpus validCorpus = new ParallelCorpus(opts.ValidCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxSrcTestSentLength, opts.MaxTgtTestSentLength, shuffleEnums: shuffleType);
// ss = new Seq2Seq(opts);
// ss.EvaluationWatcher += ss_EvaluationWatcher;
// ss.Valid(validCorpus: validCorpus, metrics: metrics);
//}
else if (opts.Task == ModeEnums.Test)
{
if (File.Exists(opts.OutputFile))
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Yellow, $"Output file '{opts.OutputFile}' exist. Delete it.");
File.Delete(opts.OutputFile);
}
//Test trained model
ss = new SeqSimilarity(opts);
Stopwatch stopwatch = Stopwatch.StartNew();
ss.Test <SeqClassificationMultiTasksCorpusBatch>(opts.InputTestFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath);
stopwatch.Stop();
Logger.WriteLine($"Test mode execution time elapsed: '{stopwatch.Elapsed}'");
}
//else if (opts.Task == ModeEnums.DumpVocab)
//{
// ss = new Seq2Seq(opts);
// ss.DumpVocabToFiles(opts.SrcVocab, opts.TgtVocab);
//}
else
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Task '{opts.Task}' is not supported.");
argParser.Usage();
}
}
catch (Exception err)
{
Logger.WriteLine($"Exception: '{err.Message}'");
Logger.WriteLine($"Call stack: '{err.StackTrace}'");
}
}
private static void Main(string[] args)
{
try
{
//Parse command line
ArgParser argParser = new ArgParser(args, opts);
if (!opts.ConfigFilePath.IsNullOrEmpty())
{
Console.WriteLine($"Loading config file from '{opts.ConfigFilePath}'");
opts = JsonConvert.DeserializeObject <Seq2SeqOptions>(File.ReadAllText(opts.ConfigFilePath));
}
Logger.LogFile = $"{nameof(Seq2SeqConsole)}_{opts.Task}_{Utils.GetTimeStamp(DateTime.Now)}.log";
ShowOptions(args, opts);
DecodingOptions decodingOptions = opts.CreateDecodingOptions();
Seq2Seq ss = null;
if (opts.Task == ModeEnums.Train)
{
// Load train corpus
var trainCorpus = new Seq2SeqCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize,
maxSrcSentLength: opts.MaxTrainSrcSentLength, maxTgtSentLength: opts.MaxTrainTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence);
// Load valid corpus
var validCorpusList = new List <Seq2SeqCorpus>();
if (!opts.ValidCorpusPaths.IsNullOrEmpty())
{
string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';');
foreach (var validCorpusPath in validCorpusPathList)
{
validCorpusList.Add(new Seq2SeqCorpus(validCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence));
}
}
// Create learning rate
ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount);
// Create optimizer
IOptimizer optimizer = Misc.CreateOptimizer(opts);
// Create metrics
List <IMetric> metrics = CreateMetrics();
if (!opts.ModelFilePath.IsNullOrEmpty() && File.Exists(opts.ModelFilePath))
{
//Incremental training
Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'...");
ss = new Seq2Seq(opts);
}
else
{
// Load or build vocabulary
Vocab srcVocab = null;
Vocab tgtVocab = null;
if (!opts.SrcVocab.IsNullOrEmpty() && !opts.TgtVocab.IsNullOrEmpty())
{
Logger.WriteLine($"Loading source vocabulary from '{opts.SrcVocab}' and target vocabulary from '{opts.TgtVocab}'. Shared vocabulary is '{opts.SharedEmbeddings}'");
if (opts.SharedEmbeddings == true && (opts.SrcVocab != opts.TgtVocab))
{
throw new ArgumentException("The source and target vocabularies must be identical if their embeddings are shared.");
}
// Vocabulary files are specified, so we load them
srcVocab = new Vocab(opts.SrcVocab);
tgtVocab = new Vocab(opts.TgtVocab);
}
else
{
Logger.WriteLine($"Building vocabulary from training corpus. Shared vocabulary is '{opts.SharedEmbeddings}'");
// We don't specify vocabulary, so we build it from train corpus
(srcVocab, tgtVocab) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize, opts.SharedEmbeddings);
}
//New training
ss = new Seq2Seq(opts, srcVocab, tgtVocab);
}
// Add event handler for monitoring
ss.StatusUpdateWatcher += Misc.Ss_StatusUpdateWatcher;
ss.EvaluationWatcher += Ss_EvaluationWatcher;
// Kick off training
ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: metrics, decodingOptions: decodingOptions);
}
else if (opts.Task == ModeEnums.Valid)
{
Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPaths}'");
// Create metrics
List <IMetric> metrics = CreateMetrics();
// Load valid corpus
Seq2SeqCorpus validCorpus = new Seq2SeqCorpus(opts.ValidCorpusPaths, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence);
ss = new Seq2Seq(opts);
ss.EvaluationWatcher += Ss_EvaluationWatcher;
ss.Valid(validCorpus: validCorpus, metrics: metrics, decodingOptions: decodingOptions);
}
else if (opts.Task == ModeEnums.Test)
{
if (File.Exists(opts.OutputFile))
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Yellow, $"Output file '{opts.OutputFile}' exist. Delete it.");
File.Delete(opts.OutputFile);
}
//Test trained model
ss = new Seq2Seq(opts);
Stopwatch stopwatch = Stopwatch.StartNew();
if (String.IsNullOrEmpty(opts.OutputPromptFile))
{
ss.Test <Seq2SeqCorpusBatch>(opts.InputTestFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath);
}
else
{
Logger.WriteLine($"Test with prompt file '{opts.OutputPromptFile}'");
ss.Test <Seq2SeqCorpusBatch>(opts.InputTestFile, opts.OutputPromptFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath);
}
stopwatch.Stop();
Logger.WriteLine($"Test mode execution time elapsed: '{stopwatch.Elapsed}'");
}
else if (opts.Task == ModeEnums.DumpVocab)
{
ss = new Seq2Seq(opts);
ss.DumpVocabToFiles(opts.SrcVocab, opts.TgtVocab);
}
else
{
Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Task '{opts.Task}' is not supported.");
argParser.Usage();
}
}
catch (Exception err)
{
Logger.WriteLine($"Exception: '{err.Message}'");
Logger.WriteLine($"Call stack: '{err.StackTrace}'");
}
}
