/// <summary>
/// Create Language Detector
/// </summary>
/// <param name="model_fn">language detector model file name</param>
/// <returns></returns>
protected FastTextLauncher CreateLangDetector(string model_fn = "lid.176.bin")
{
var fmod = FastTextPath(model_fn);
AssertFileExists(fmod, "FastText language detector model file");
var fexe = FastTextBin;
AssertFileExists(fexe, "FastText executable");
var lang_detector = FTCmd.CreateLangDetect(fexe, fmod);
return(lang_detector);
}
FastTextLauncher CreateLangDetector()
{
var fmod = DataArcPath("lid.176.bin");
AssertFileExists(fmod, "FastText model file");
var fexe = FastTextBin;
AssertFileExists(fexe, "FastText executable");
var lang_detector = FTCmd.CreateLangDetect(fexe, fmod);
return(lang_detector);
}
/// <summary>
/// Fill Empty Add-in Dictionary Vectors
/// </summary>
/// <param name="ft_bin_fn">FastText bin model filename</param>
/// <param name="dbf_w2v_fn">DB word to vector filename</param>
protected void SubProcFillEmptyVectDict(
string ft_bin_fn, string dbf_w2v_fn, FTLangLabel lang)
{
using (var dbx = new FastTextProcessDB(dbf_w2v_fn))
{
var words = dbx.Dict(DictDbSet.DictKind.Addin).GetWordsWithEmptyVect();
if (words.Any())
{
var fmod = DataArcPath(ft_bin_fn);
AssertFileExists(fmod, "FastText model file");
var fexe = FastTextBin;
AssertFileExists(fexe, "FastText executable");
var trans = dbx.BeginTransaction();
try
{
var dict = dbx.Dict(DictDbSet.DictKind.Addin);
using (var ftl = FTCmd.CreateW2V(fexe, fmod))
{
ftl.RunByLineAsync(
(txt_src, res_txt) =>
dict.UpdateVectOfWord(Dict.CreateParseFT(res_txt, lang))
);
foreach (var w in words)
{
ftl.Push(w);
}
}
trans.Commit();
}
catch
{
trans.Rollback();
throw;
}
}
}
}
