/// <summary> /// Create Language Detector /// </summary> /// <param name="model_fn">language detector model file name</param> /// <returns></returns> protected FastTextLauncher CreateLangDetector(string model_fn = "lid.176.bin") { var fmod = FastTextPath(model_fn); AssertFileExists(fmod, "FastText language detector model file"); var fexe = FastTextBin; AssertFileExists(fexe, "FastText executable"); var lang_detector = FTCmd.CreateLangDetect(fexe, fmod); return(lang_detector); }
FastTextLauncher CreateLangDetector() { var fmod = DataArcPath("lid.176.bin"); AssertFileExists(fmod, "FastText model file"); var fexe = FastTextBin; AssertFileExists(fexe, "FastText executable"); var lang_detector = FTCmd.CreateLangDetect(fexe, fmod); return(lang_detector); }
/// <summary> /// Fill Empty Add-in Dictionary Vectors /// </summary> /// <param name="ft_bin_fn">FastText bin model filename</param> /// <param name="dbf_w2v_fn">DB word to vector filename</param> protected void SubProcFillEmptyVectDict( string ft_bin_fn, string dbf_w2v_fn, FTLangLabel lang) { using (var dbx = new FastTextProcessDB(dbf_w2v_fn)) { var words = dbx.Dict(DictDbSet.DictKind.Addin).GetWordsWithEmptyVect(); if (words.Any()) { var fmod = DataArcPath(ft_bin_fn); AssertFileExists(fmod, "FastText model file"); var fexe = FastTextBin; AssertFileExists(fexe, "FastText executable"); var trans = dbx.BeginTransaction(); try { var dict = dbx.Dict(DictDbSet.DictKind.Addin); using (var ftl = FTCmd.CreateW2V(fexe, fmod)) { ftl.RunByLineAsync( (txt_src, res_txt) => dict.UpdateVectOfWord(Dict.CreateParseFT(res_txt, lang)) ); foreach (var w in words) { ftl.Push(w); } } trans.Commit(); } catch { trans.Rollback(); throw; } } } }