unsafe public static void Compare(NativeTextMMFModelBinary model_1, ManagedTextModel model_2)
{
var set = NativeTextMMFModelBinary.FuskingTraitor.GetSet(model_1);
var dict = ManagedTextModel.FuskingTraitor.GetDictionary(model_2);
Debug.Assert(set.Count == dict.Count);
using (var set_enm = set.GetEnumerator())
using (var dict_enm = dict.GetEnumerator())
{
for ( ; set_enm.MoveNext() && dict_enm.MoveNext();)
{
var set_m = NativeTextMMFModelBinary.ToModelRecord(set_enm.Current);
var dict_ngram = dict_enm.Current.Key;
Debug.Assert(set_m.Ngram == dict_ngram);
fixed(char *dict_ngram_ptr = dict_ngram)
{
if (!set.TryGetValue((IntPtr)dict_ngram_ptr, out var exitstsPtr))
{
Debugger.Break();
}
if (!dict.TryGetValue(set_m.Ngram, out var dict_prob))
{
Debugger.Break();
}
Debug.Assert(set_m.Probability == dict_prob);
}
}
}
}
private static void Comare_ModelBinaryNative_And_ModelClassic(IEnumerable <string> modelFilenames)
{
#if DEBUG
var bmcfg = new BinaryModelConfig(modelFilenames)
{
ModelDictionaryCapacity = Config.Inst.MODEL_DICTIONARY_CAPACITY
};
using (var model_1 = new NativeTextMMFModelBinary(bmcfg))
using (var model_2 = new ManagedTextModel(Config.Inst.GetTextModelConfig()))
{
ModelComparer.Compare(model_1, model_2);
}
#else
throw (new NotImplementedException("Allowed only in DEBUG mode"));
#endif
}
private static void Test__ManagedTextModel_RU()
{
Console.Write("load \"-text-\" language model...");
var sw = Stopwatch.StartNew();
using (var model = new ManagedTextModel(Config.Inst.GetTextModelConfig_RU()))
{
var count = model.RecordCount;
Console.WriteLine($"=> elapsed: {sw.StopAndElapsed()}, record-count: {count}\r\n");
Test__TextSegmenter(model);
Console.Write("\r\ndisposing language model...");
}
Console.WriteLine("=> end\r\n");
}
private static IEnumerable <string> ConvertFromTxt2Bin()
{
var tmcfg = Config.Inst.GetTextModelConfig();
using (var model = new ManagedTextModel(tmcfg))
{
var config = new Txt2BinModelConverterConfig()
{
Model = model,
OutputFileName = Config.Inst.OUTPUT_FILENAME,
OutputFileSizeInBytes = Config.Inst.OUTPUT_FILE_SIZE_IN_BYTES,
};
var outputFileNames = Txt2BinModelConverter.Run(config);
Console.WriteLine($"{Environment.NewLine} output-files: ");
Console.WriteLine(" --------------");
for (var i = 0; i < outputFileNames.Count; i++)
{
Console.WriteLine($" {(i + 1).ToString()}). '{outputFileNames[ i ]}'");
}
Console.WriteLine(" --------------\r\n");
return(outputFileNames);
}
}
