private void btn_ReadOne_Click(object sender, EventArgs e) { //【1】设置读取条形码规格 DecodingOptions decodeOption = new DecodingOptions(); decodeOption.PossibleFormats = new List <BarcodeFormat>() { BarcodeFormat.EAN_13, }; //【2】进行读取操作 BarcodeReader br = new BarcodeReader(); br.Options = decodeOption; Result rs = br.Decode(this.pictureBox1.Image as Bitmap); if (rs == null) { MessageBox.Show("读取失败"); } else { MessageBox.Show("读取成功,内容为:" + rs.Text); } }
/// <summary> /// 读取编码内容 /// </summary> /// <param name="filePath"></param> /// <param name="formats"></param> /// <returns></returns> public static string ReadCode(string filePath, params BarcodeFormat[] formats) { var decodeOptions = new DecodingOptions { PossibleFormats = new List <BarcodeFormat> { BarcodeFormat.EAN_13 } }; formats?.ToList().ForEach(item => { decodeOptions.PossibleFormats.Add(item); }); var reader = new BarcodeReader { Options = decodeOptions }; if (!File.Exists(filePath)) { return(string.Empty); } var result = reader.Decode(new Bitmap(filePath)); return(result != null ? result.Text : string.Empty); }
// 读取条形码 private void Read1DBtn_Click(object sender, EventArgs e) { // 1.设置读取条形码规格 DecodingOptions decodeOption = new DecodingOptions(); decodeOption.PossibleFormats = new List <BarcodeFormat>() { BarcodeFormat.All_1D, }; // 2.进行读取操作 ZXing.BarcodeReader br = new BarcodeReader(); br.Options = decodeOption; ZXing.Result rs = br.Decode(this.barCodeImg.Image as Bitmap); if (rs == null) { this.ContentTxt.Text = "读取失败"; MessageBox.Show("读取失败"); } else { this.ContentTxt.Text = rs.Text; MessageBox.Show("读取成功,内容:" + rs.Text); } }
public void Test_1() { // 1.设置读取条形码规格 DecodingOptions decodeOption = new DecodingOptions(); decodeOption.PossibleFormats = new List <BarcodeFormat>() { BarcodeFormat.All_1D, }; // 2.进行读取操作 ZXing.BarcodeReader br = new BarcodeReader(); br.Options = decodeOption; this.pictureBox2.Image = Bitmap.FromFile(@"e:\BTWork\MachineL\machine_vision\Test\图片处理\123.jpg"); ZXing.Result rs = br.Decode(this.pictureBox2.Image as Bitmap); if (rs == null) { MessageBox.Show("读取失败"); } else { MessageBox.Show("读取成功,内容:" + rs.Text); } }
public void TestSeq2SeqInference() { var opts = new Seq2SeqOptions(); opts.ModelFilePath = "seq2seq_mt_enu_chs_tiny_test.model"; opts.MaxTestSrcSentLength = 110; opts.MaxTestTgtSentLength = 110; opts.ProcessorType = ProcessorTypeEnums.CPU; opts.DeviceIds = "0"; var seq2seq = new Seq2Seq(opts); DecodingOptions decodingOptions = opts.CreateDecodingOptions(); List <List <List <string> > > groupBatchTokens = BuildInputGroupBatchTokens("▁yes , ▁solutions ▁do ▁exist ."); var nrs = seq2seq.Test <Seq2SeqCorpusBatch>(groupBatchTokens, null, decodingOptions); var out_tokens = nrs[0].Output[0][0]; var output = string.Join(" ", out_tokens); Assert.IsTrue(output == "<s> ▁是的 , 解决方案 存在 。 </s>"); groupBatchTokens = BuildInputGroupBatchTokens("▁a ▁question ▁of ▁climate ."); nrs = seq2seq.Test <Seq2SeqCorpusBatch>(groupBatchTokens, null, decodingOptions); out_tokens = nrs[0].Output[0][0]; output = string.Join(" ", out_tokens); Assert.IsTrue(output == "<s> ▁ 气候 问题 。 </s>"); }
//读取并解析条形码 private void btnDecodeBarCode_Click(object sender, EventArgs e) { if (BarCodeImage != null) { // 1.设置读取条形码规格 DecodingOptions decodeOption = new DecodingOptions(); decodeOption.PossibleFormats = new List <BarcodeFormat>() { BarcodeFormatHelper.GetFormat(this.cbEncodeType.SelectedItem.ToString()), }; // 2.进行读取操作 BarcodeReader br = new BarcodeReader(); br.Options = decodeOption; Result rs = br.Decode(BarCodeImage); if (rs == null) { MessageBox.Show("读取失败"); } else { MessageBox.Show("读取成功,内容:" + rs.Text); } } }
public static string DecoderFromImage(Bitmap image) { //use gaussian filter to remove noise //var gFilter = new GaussianBlur(2); //image = gFilter.ProcessImage(image); var options = new DecodingOptions { PossibleFormats = new List <BarcodeFormat> { BarcodeFormat.QR_CODE }, TryHarder = true }; using (image) { //use GlobalHistogramBinarizer for best result var reader = new BarcodeReader(null, null, ls => new GlobalHistogramBinarizer(ls)) { AutoRotate = false, TryInverted = false, Options = options }; var result = reader.Decode(image); reader = null; return(result.Text); } }
static public (string, string) Call(string key, List <string> inputFeatureGroups) { if (m_key2Instance.ContainsKey(key) == false) { return("", ""); } List <List <List <string> > > groupBatchTokens = new List <List <List <string> > >(); foreach (var inputFeatureGroup in inputFeatureGroups) { List <string> tokens = inputFeatureGroup.Split(' ').ToList(); List <List <string> > batchTokens = new List <List <string> >(); batchTokens.Add(tokens); groupBatchTokens.Add(batchTokens); } Seq2SeqClassification inst = null; lock (locker) { inst = m_key2Instance[key]; } DecodingOptions decodingOptions = opts.CreateDecodingOptions(); List <NetworkResult> nrs = inst.Test <Seq2SeqClassificationCorpusBatch>(groupBatchTokens, null, decodingOptions); var nrCLS = nrs[0]; var nrSeq2Seq = nrs[1]; string tag = nrCLS.Output[0][0][0]; string text = string.Join(" ", nrSeq2Seq.Output[0][0].ToArray(), 1, nrSeq2Seq.Output[0][0].Count - 2); return(tag, text); }
static public string Call(string input1, string input2) { List <List <List <string> > > groupBatchTokens = new List <List <List <string> > >(); // Build group 1 features for input string 1 List <string> tokens = input1.Split(' ').ToList(); List <List <string> > batchTokens = new List <List <string> >(); batchTokens.Add(tokens); groupBatchTokens.Add(batchTokens); // Build group 2 features for input string 2 tokens = input2.Split(' ').ToList(); batchTokens = new List <List <string> >(); batchTokens.Add(tokens); groupBatchTokens.Add(batchTokens); DecodingOptions decodingOptions = opts.CreateDecodingOptions(); List <NetworkResult> nrs = m_seqSimilarity.Test <SeqClassificationMultiTasksCorpusBatch>(groupBatchTokens, null, decodingOptions); List <string> tags = new List <string>(); foreach (var nr in nrs) { tags.Add(nr.Output[0][0][0]); // shape: (beam_size, batch_size, seq_size) } return(string.Join("\t", tags)); }
static public string Call(string srcInput, string tgtInput, int tokenNumToGenerate, bool random, float repeatPenalty) { if (opts == null) { throw new ArgumentNullException($"The {nameof(Seq2SeqInstance)} may not be initialized, and option instance is null."); } if (m_seq2seq == null) { throw new ArgumentNullException($"The {nameof(Seq2SeqInstance)} is null."); } srcInput = (m_srcSpm != null) ? m_srcSpm.Encode(srcInput) : srcInput; List <string> tokens = srcInput.Split(' ').ToList(); if (tokens.Count > opts.MaxTestSrcSentLength) { tokens = tokens.GetRange(tokens.Count - opts.MaxTestSrcSentLength, opts.MaxTestSrcSentLength); } List <List <String> > batchTokens = new List <List <string> >(); batchTokens.Add(tokens); List <List <List <string> > > srcGroupBatchTokens = new List <List <List <string> > >(); srcGroupBatchTokens.Add(batchTokens); tgtInput = (m_tgtSpm != null) ? m_tgtSpm.Encode(tgtInput) : tgtInput; List <string> tokens2 = tgtInput.Split(' ').ToList(); tokenNumToGenerate += tokens2.Count; List <List <String> > batchTokens2 = new List <List <string> >(); batchTokens2.Add(tokens2); List <List <List <string> > > tgtGroupBatchTokens = new List <List <List <string> > >(); tgtGroupBatchTokens.Add(batchTokens2); DecodingOptions decodingOptions = opts.CreateDecodingOptions(); decodingOptions.MaxTgtSentLength = tokenNumToGenerate; decodingOptions.TopPValue = random ? 0.5f : 0.0f; decodingOptions.RepeatPenalty = repeatPenalty; var nrs = m_seq2seq.Test <Seq2SeqCorpusBatch>(srcGroupBatchTokens, tgtGroupBatchTokens, decodingOptions); string rst = String.Join(" ", nrs[0].Output[0][0].ToArray(), 0, nrs[0].Output[0][0].Count); rst = (m_tgtSpm != null) ? m_tgtSpm.Decode(rst) : rst; return(rst); }
/// <summary> /// Creates a new <see cref="Video"/> instance. /// </summary> /// <param name="url">Video source URL. May be a file system path.</param> /// <param name="decodingOptions">Decoding options.</param> internal Video([NotNull] string url, [NotNull] DecodingOptions decodingOptions) { _decodeContext = new DecodeContext(url, decodingOptions); _decodingOptions = decodingOptions; _decodeContext.Ended += decodeContext_Ended; Duration = TimeSpan.FromSeconds(_decodeContext.GetDurationInSeconds()); }
public Code39BarcodeReader() { Options = new DecodingOptions() { PossibleFormats = new List <BarcodeFormat> { BarcodeFormat.CODE_39 } }; }
public void TestSeq2SeqTraining() { // Build configs for training Seq2SeqOptions opts = CreateOptions(trainFolderPath, validFolderPath); DecodingOptions decodingOptions = opts.CreateDecodingOptions(); // Load training corpus var trainCorpus = new Seq2SeqCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize, maxSrcSentLength: opts.MaxTrainSrcSentLength, maxTgtSentLength: opts.MaxTrainTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence); // Load valid corpus var validCorpusList = new List <Seq2SeqCorpus>(); if (!opts.ValidCorpusPaths.IsNullOrEmpty()) { string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';'); foreach (var validCorpusPath in validCorpusPathList) { validCorpusList.Add(new Seq2SeqCorpus(validCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence)); } } // Create learning rate ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount); // Create optimizer IOptimizer optimizer = Misc.CreateOptimizer(opts); // Build vocabularies for training (var srcVocab, var tgtVocab) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize, opts.SharedEmbeddings); // Create metrics List <IMetric> metrics = new List <IMetric> { new BleuMetric() }; //New training var ss = new Seq2Seq(opts, srcVocab, tgtVocab); // Add event handler for monitoring ss.StatusUpdateWatcher += Ss_StatusUpdateWatcher; ss.EpochEndWatcher += Ss_EpochEndWatcher; // Kick off training ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: metrics, decodingOptions: decodingOptions); ss.SaveModel(suffix: ".test"); // Check if model file exists Assert.IsTrue(File.Exists(opts.ModelFilePath + ".test")); }
private void BtnUpload_Click(object sender, EventArgs e) { if (FileUpload1.HasFile) { DivInfo.Visible = false; DivWarning.Visible = false; var ext = Path.GetExtension(FileUpload1.FileName).ToLower(); var tempPath = Path.GetTempFileName() + ext; FileUpload1.SaveAs(tempPath); HashSet <String> exts = new HashSet <string> { ".bmp", ".png", ".jpg", ".gif", ".png" }; if (exts.Contains(ext)) { ImageQR.ImageUrl = "/handlers/imageloader.ashx?ImgPath=" + tempPath; var qr = new ZXing.BarcodeReader(); var options = new DecodingOptions { CharacterSet = "UTF-8", }; qr.Options = options; Bitmap bmp = (Bitmap)Bitmap.FromFile(tempPath); var res = qr.Decode(bmp); if (res != null && !string.IsNullOrEmpty(res.Text)) { try { var decrypt = Crypto.Decrypt(res.Text); LoadHeaderInfo(decrypt.Trim()); } catch { CommonWeb.Alert(this, "Qr dapat dibaca tapi tidak valid."); } } else { CommonWeb.Alert(this, "Qr tidak dapat dibaca, atau gambar tidak valid."); } } else { CommonWeb.Alert(this, "Silakan upload file dengan tipe gambar"); } } else { CommonWeb.Alert(this, "Silakan pilih foto QR LHP terlebih dahulu."); } }
internal static bool Get2047CharsetName(TextHeader textHeader, out string charsetName) { DecodingOptions decodingOptions = new DecodingOptions(DecodingFlags.Rfc2047, null); DecodingResults decodingResults; string text; if (textHeader.TryGetValue(decodingOptions, out decodingResults, out text) && EncodingScheme.Rfc2047 == decodingResults.EncodingScheme) { charsetName = decodingResults.CharsetName; return(true); } charsetName = null; return(false); }
internal static bool Get2047CharsetName(AddressItem addressItem, out string charsetName) { DecodingOptions decodingOptions = new DecodingOptions(DecodingFlags.Rfc2047, null); DecodingResults decodingResults; string text; if (addressItem.TryGetDisplayName(decodingOptions, out decodingResults, out text) && EncodingScheme.Rfc2047 == decodingResults.EncodingScheme) { charsetName = decodingResults.CharsetName; return(true); } charsetName = null; return(false); }
public DecodingOptions CreateDecodingOptions() { DecodingOptions decodingOptions = new DecodingOptions(); decodingOptions.DecodingStrategy = DecodingStrategy; decodingOptions.TopPValue = DecodingTopPValue; decodingOptions.RepeatPenalty = DecodingRepeatPenalty; decodingOptions.BeamSearchSize = BeamSearchSize; decodingOptions.MaxSrcSentLength = MaxTestSrcSentLength; decodingOptions.MaxTgtSentLength = MaxTestTgtSentLength; return(decodingOptions); }
private string DeCodeImg(Bitmap img) { if (img == null) { return(""); } #region 将图片转化成 byte数组 MemoryStream ms = new MemoryStream(); img.Save(ms, System.Drawing.Imaging.ImageFormat.Bmp); byte[] bt = ms.GetBuffer(); ms.Close(); #endregion LuminanceSource source = new RGBLuminanceSource(bt, img.Width, img.Height); BinaryBitmap bitmap = new BinaryBitmap(new ZXing.Common.HybridBinarizer(source)); //【1】设置读取条形码规格 DecodingOptions decodeOption = new DecodingOptions(); decodeOption.PossibleFormats = new List <BarcodeFormat>() { BarcodeFormat.EAN_13, }; //【2】进行读取操作 BarcodeReader br = new BarcodeReader(); br.Options = decodeOption; Result result2, result1; try { //开始解码 result2 = new MultiFormatReader().decode(bitmap); result1 = br.Decode(img); } catch { return(""); } if (result1 != null) { return(result1.Text); } if (result2 != null) { return(result2.Text); } return(""); }
public static string ReadQRCode(Bitmap img) { DecodingOptions decodingOptions = new DecodingOptions(); decodingOptions.PossibleFormats = new List <BarcodeFormat> { BarcodeFormat.QR_CODE }; BarcodeReader barcodeReader = new BarcodeReader(); barcodeReader.Options = decodingOptions; Result result = barcodeReader.Decode(img); if (result == null) { return(""); } return(result.Text); }
public static string ReadBarCode(Image img) { DecodingOptions decodingOptions = new DecodingOptions(); decodingOptions.PossibleFormats = new List <BarcodeFormat> { BarcodeFormat.EAN_13 }; BarcodeReader barcodeReader = new BarcodeReader(); barcodeReader.Options = decodingOptions; Result result = barcodeReader.Decode(img as Bitmap); if (result == null) { return(""); } return(result.Text); }
/// <summary> /// 解析图片 /// </summary> /// <param name="file"></param> /// <returns></returns> public static string AnalysisImage(string file) { Bitmap m1 = (Bitmap)Bitmap.FromFile(file); BarcodeReader barcodeReader = new BarcodeReader(); DecodingOptions arg_1 = new DecodingOptions(); arg_1.PossibleFormats = new List <BarcodeFormat>() { BarcodeFormat.QR_CODE, BarcodeFormat.CODE_39, BarcodeFormat.CODE_93, BarcodeFormat.CODE_128, }; arg_1.CharacterSet = "UTF-8";//提示已过时? //arg_1.CharacterSet = "gb2312";//设置了字符集也读不出来中文汉字 barcodeReader.Options = arg_1; Result result = barcodeReader.Decode(m1); return(result.Text); }
public Form1() { InitializeComponent(); CheckForIllegalCrossThreadCalls = false; DecOpt = new DecodingOptions() { PossibleFormats = new[] { BarcodeFormat.QR_CODE }, TryHarder = true }; if (!File.Exists(FileManager.AppFolder + "StaffAPI.txt")) { return; } ApiToken = FileManager.ReadAllLines(FileManager.AppFolder + "StaffAPI.txt")[0]; Authed = true; authedLabel.Text = "Authenticated using staff API token."; authedLabel.ForeColor = Color.DeepSkyBlue; RegisterHotKey(this.Handle, 1, (int)KeyModifier.Alt, Keys.F12.GetHashCode()); }
public FormM() { InitializeComponent(); _workpath = textBoxWorkPath.Text; _angle = null; _artemplate = null; _rundr = null; _runnameList = null; _runmsg = ""; _exportdata = ""; _xztpos = -1; _titlepos = new Dictionary <string, int>(); //for 二维码 DecodingOptions decodeOption = new DecodingOptions(); decodeOption.PossibleFormats = new List <BarcodeFormat>() { BarcodeFormat.All_1D }; _br = new BarcodeReader(); _br.Options = decodeOption; }
public static string Rfc2047Decode(string encodedValue) { if (encodedValue == null) { return(null); } if (!encodedValue.Contains("=?")) { return(encodedValue); } MimeString str = new MimeString(encodedValue.Trim()); MimeStringList lines = new MimeStringList(str); DecodingOptions decodingOptions = new DecodingOptions(DecodingFlags.Rfc2047); DecodingResults decodingResults; string result; if (!MimeCommon.TryDecodeValue(lines, 4026531840U, decodingOptions, out decodingResults, out result)) { return(encodedValue); } return(result); }
/// <summary> /// ZXing根据图片读取二维码内容 /// </summary> /// <param name="img">Image img</param> /// <param name="BarcodeFormat barcodeFormat">barcodeFormat</param> /// <returns></returns> public string ReadQrCode(Image img, BarcodeFormat barcodeFormat) { // 1.设置读取条形码规格 DecodingOptions decodeOption = new DecodingOptions(); decodeOption.PossibleFormats = new List <BarcodeFormat>() { barcodeFormat }; // 2.进行读取操作 ZXing.BarcodeReader br = new BarcodeReader(); br.Options = decodeOption; ZXing.Result rs = br.Decode(img as Bitmap); if (rs == null) { return(string.Empty); } else { return(rs.Text); } }
static public string Call(List <string> inputFeatureGroups) { List <List <List <string> > > groupBatchTokens = new List <List <List <string> > >(); foreach (var inputFeatureGroup in inputFeatureGroups) { List <string> tokens = inputFeatureGroup.Split(' ').ToList(); List <List <string> > batchTokens = new List <List <string> >(); batchTokens.Add(tokens); groupBatchTokens.Add(batchTokens); } DecodingOptions decodingOptions = opts.CreateDecodingOptions(); List <NetworkResult> nrs = m_seqClassification.Test <SeqClassificationMultiTasksCorpusBatch>(groupBatchTokens, null, decodingOptions); List <string> tags = new List <string>(); foreach (var nr in nrs) { tags.Add(nr.Output[0][0][0]); // shape: (beam_size, batch_size, seq_size) } return(string.Join("\t", tags)); }
public static string ReadQRCode(string imgpath) { if (!File.Exists(imgpath)) { return(""); } Bitmap barcodeBitmap = new Bitmap(imgpath); DecodingOptions decodingOptions = new DecodingOptions(); decodingOptions.PossibleFormats = new List <BarcodeFormat> { BarcodeFormat.QR_CODE }; BarcodeReader barcodeReader = new BarcodeReader(); barcodeReader.Options = decodingOptions; Result result = barcodeReader.Decode(barcodeBitmap); if (result == null) { return(""); } return(result.Text); }
/// <summary> /// Creates a new <see cref="VideoDecodingContext"/> instance. /// </summary> /// <param name="videoStream">The video stream.</param> /// <param name="decodingOptions">Decoding options.</param> internal VideoDecodingContext([NotNull] AVStream *videoStream, [NotNull] DecodingOptions decodingOptions) { _codecContext = videoStream->codec; _videoStream = videoStream; _decodingOptions = decodingOptions; }
static void Main(string[] args) { try { //Parse command line // Seq2SeqOptions opts = new Seq2SeqOptions(); ArgParser argParser = new ArgParser(args, opts); if (!opts.ConfigFilePath.IsNullOrEmpty()) { Logger.WriteLine($"Loading config file from '{opts.ConfigFilePath}'"); opts = JsonConvert.DeserializeObject <SeqSimilarityOptions>(File.ReadAllText(opts.ConfigFilePath)); } Logger.LogFile = $"{nameof(SeqSimilarityConsole)}_{opts.Task}_{Utils.GetTimeStamp(DateTime.Now)}.log"; ShowOptions(args, opts); DecodingOptions decodingOptions = opts.CreateDecodingOptions(); SeqSimilarity ss = null; if (opts.Task == ModeEnums.Train) { // Load train corpus SeqClassificationMultiTasksCorpus trainCorpus = new SeqClassificationMultiTasksCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize, maxSentLength: opts.MaxTrainSentLength, shuffleEnums: opts.ShuffleType); // Load valid corpus List <SeqClassificationMultiTasksCorpus> validCorpusList = new List <SeqClassificationMultiTasksCorpus>(); if (!opts.ValidCorpusPaths.IsNullOrEmpty()) { string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';'); foreach (var validCorpusPath in validCorpusPathList) { validCorpusList.Add(new SeqClassificationMultiTasksCorpus(opts.ValidCorpusPaths, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSentLength, shuffleEnums: opts.ShuffleType)); } } // Create learning rate ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount); // Create metrics IMetric metric = null; if (opts.SimilarityType == "Continuous") { metric = new SimilarityMetric(); } // Create optimizer IOptimizer optimizer = Misc.CreateOptimizer(opts); if (!opts.ModelFilePath.IsNullOrEmpty() && File.Exists(opts.ModelFilePath)) { //Incremental training Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'..."); ss = new SeqSimilarity(opts); if (metric == null) { metric = new MultiLabelsFscoreMetric("", ss.ClsVocab.GetAllTokens(keepBuildInTokens: false)); } } else { // Load or build vocabulary Vocab srcVocab = null; List <Vocab> tgtVocabs = null; if (!opts.SrcVocab.IsNullOrEmpty() && !opts.TgtVocab.IsNullOrEmpty()) { Logger.WriteLine($"Loading source vocabulary from '{opts.SrcVocab}' and target vocabulary from '{opts.TgtVocab}'."); // Vocabulary files are specified, so we load them srcVocab = new Vocab(opts.SrcVocab); tgtVocabs = new List <Vocab> { new Vocab(opts.TgtVocab) }; } else { Logger.WriteLine($"Building vocabulary from training corpus."); // We don't specify vocabulary, so we build it from train corpus (srcVocab, tgtVocabs) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize); } if (metric == null) { metric = new MultiLabelsFscoreMetric("", tgtVocabs[0].GetAllTokens(keepBuildInTokens: false)); } //New training ss = new SeqSimilarity(opts, srcVocab, tgtVocabs[0]); } // Add event handler for monitoring ss.StatusUpdateWatcher += Misc.Ss_StatusUpdateWatcher; ss.EvaluationWatcher += Ss_EvaluationWatcher; // Kick off training ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: new List <IMetric>() { metric }, decodingOptions: decodingOptions); } //else if (opts.Task == ModeEnums.Valid) //{ // Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPath}'"); // // Create metrics // List<IMetric> metrics = new List<IMetric> //{ // new BleuMetric(), // new LengthRatioMetric() //}; // // Load valid corpus // ParallelCorpus validCorpus = new ParallelCorpus(opts.ValidCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxSrcTestSentLength, opts.MaxTgtTestSentLength, shuffleEnums: shuffleType); // ss = new Seq2Seq(opts); // ss.EvaluationWatcher += ss_EvaluationWatcher; // ss.Valid(validCorpus: validCorpus, metrics: metrics); //} else if (opts.Task == ModeEnums.Test) { if (File.Exists(opts.OutputFile)) { Logger.WriteLine(Logger.Level.err, ConsoleColor.Yellow, $"Output file '{opts.OutputFile}' exist. Delete it."); File.Delete(opts.OutputFile); } //Test trained model ss = new SeqSimilarity(opts); Stopwatch stopwatch = Stopwatch.StartNew(); ss.Test <SeqClassificationMultiTasksCorpusBatch>(opts.InputTestFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath); stopwatch.Stop(); Logger.WriteLine($"Test mode execution time elapsed: '{stopwatch.Elapsed}'"); } //else if (opts.Task == ModeEnums.DumpVocab) //{ // ss = new Seq2Seq(opts); // ss.DumpVocabToFiles(opts.SrcVocab, opts.TgtVocab); //} else { Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Task '{opts.Task}' is not supported."); argParser.Usage(); } } catch (Exception err) { Logger.WriteLine($"Exception: '{err.Message}'"); Logger.WriteLine($"Call stack: '{err.StackTrace}'"); } }
private static void Main(string[] args) { try { //Parse command line ArgParser argParser = new ArgParser(args, opts); if (!opts.ConfigFilePath.IsNullOrEmpty()) { Console.WriteLine($"Loading config file from '{opts.ConfigFilePath}'"); opts = JsonConvert.DeserializeObject <Seq2SeqOptions>(File.ReadAllText(opts.ConfigFilePath)); } Logger.LogFile = $"{nameof(Seq2SeqConsole)}_{opts.Task}_{Utils.GetTimeStamp(DateTime.Now)}.log"; ShowOptions(args, opts); DecodingOptions decodingOptions = opts.CreateDecodingOptions(); Seq2Seq ss = null; if (opts.Task == ModeEnums.Train) { // Load train corpus var trainCorpus = new Seq2SeqCorpus(corpusFilePath: opts.TrainCorpusPath, srcLangName: opts.SrcLang, tgtLangName: opts.TgtLang, batchSize: opts.BatchSize, shuffleBlockSize: opts.ShuffleBlockSize, maxSrcSentLength: opts.MaxTrainSrcSentLength, maxTgtSentLength: opts.MaxTrainTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence); // Load valid corpus var validCorpusList = new List <Seq2SeqCorpus>(); if (!opts.ValidCorpusPaths.IsNullOrEmpty()) { string[] validCorpusPathList = opts.ValidCorpusPaths.Split(';'); foreach (var validCorpusPath in validCorpusPathList) { validCorpusList.Add(new Seq2SeqCorpus(validCorpusPath, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence)); } } // Create learning rate ILearningRate learningRate = new DecayLearningRate(opts.StartLearningRate, opts.WarmUpSteps, opts.WeightsUpdateCount); // Create optimizer IOptimizer optimizer = Misc.CreateOptimizer(opts); // Create metrics List <IMetric> metrics = CreateMetrics(); if (!opts.ModelFilePath.IsNullOrEmpty() && File.Exists(opts.ModelFilePath)) { //Incremental training Logger.WriteLine($"Loading model from '{opts.ModelFilePath}'..."); ss = new Seq2Seq(opts); } else { // Load or build vocabulary Vocab srcVocab = null; Vocab tgtVocab = null; if (!opts.SrcVocab.IsNullOrEmpty() && !opts.TgtVocab.IsNullOrEmpty()) { Logger.WriteLine($"Loading source vocabulary from '{opts.SrcVocab}' and target vocabulary from '{opts.TgtVocab}'. Shared vocabulary is '{opts.SharedEmbeddings}'"); if (opts.SharedEmbeddings == true && (opts.SrcVocab != opts.TgtVocab)) { throw new ArgumentException("The source and target vocabularies must be identical if their embeddings are shared."); } // Vocabulary files are specified, so we load them srcVocab = new Vocab(opts.SrcVocab); tgtVocab = new Vocab(opts.TgtVocab); } else { Logger.WriteLine($"Building vocabulary from training corpus. Shared vocabulary is '{opts.SharedEmbeddings}'"); // We don't specify vocabulary, so we build it from train corpus (srcVocab, tgtVocab) = trainCorpus.BuildVocabs(opts.SrcVocabSize, opts.TgtVocabSize, opts.SharedEmbeddings); } //New training ss = new Seq2Seq(opts, srcVocab, tgtVocab); } // Add event handler for monitoring ss.StatusUpdateWatcher += Misc.Ss_StatusUpdateWatcher; ss.EvaluationWatcher += Ss_EvaluationWatcher; // Kick off training ss.Train(maxTrainingEpoch: opts.MaxEpochNum, trainCorpus: trainCorpus, validCorpusList: validCorpusList.ToArray(), learningRate: learningRate, optimizer: optimizer, metrics: metrics, decodingOptions: decodingOptions); } else if (opts.Task == ModeEnums.Valid) { Logger.WriteLine($"Evaluate model '{opts.ModelFilePath}' by valid corpus '{opts.ValidCorpusPaths}'"); // Create metrics List <IMetric> metrics = CreateMetrics(); // Load valid corpus Seq2SeqCorpus validCorpus = new Seq2SeqCorpus(opts.ValidCorpusPaths, opts.SrcLang, opts.TgtLang, opts.ValBatchSize, opts.ShuffleBlockSize, opts.MaxTestSrcSentLength, opts.MaxTestTgtSentLength, shuffleEnums: opts.ShuffleType, tooLongSequence: opts.TooLongSequence); ss = new Seq2Seq(opts); ss.EvaluationWatcher += Ss_EvaluationWatcher; ss.Valid(validCorpus: validCorpus, metrics: metrics, decodingOptions: decodingOptions); } else if (opts.Task == ModeEnums.Test) { if (File.Exists(opts.OutputFile)) { Logger.WriteLine(Logger.Level.err, ConsoleColor.Yellow, $"Output file '{opts.OutputFile}' exist. Delete it."); File.Delete(opts.OutputFile); } //Test trained model ss = new Seq2Seq(opts); Stopwatch stopwatch = Stopwatch.StartNew(); if (String.IsNullOrEmpty(opts.OutputPromptFile)) { ss.Test <Seq2SeqCorpusBatch>(opts.InputTestFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath); } else { Logger.WriteLine($"Test with prompt file '{opts.OutputPromptFile}'"); ss.Test <Seq2SeqCorpusBatch>(opts.InputTestFile, opts.OutputPromptFile, opts.OutputFile, opts.BatchSize, decodingOptions, opts.SrcSentencePieceModelPath, opts.TgtSentencePieceModelPath); } stopwatch.Stop(); Logger.WriteLine($"Test mode execution time elapsed: '{stopwatch.Elapsed}'"); } else if (opts.Task == ModeEnums.DumpVocab) { ss = new Seq2Seq(opts); ss.DumpVocabToFiles(opts.SrcVocab, opts.TgtVocab); } else { Logger.WriteLine(Logger.Level.err, ConsoleColor.Red, $"Task '{opts.Task}' is not supported."); argParser.Usage(); } } catch (Exception err) { Logger.WriteLine($"Exception: '{err.Message}'"); Logger.WriteLine($"Call stack: '{err.StackTrace}'"); } }