unsafe public static void Compare(NativeTextMMFModelBinary model_1, ManagedTextModel model_2)
{
var set = NativeTextMMFModelBinary.FuskingTraitor.GetSet(model_1);
var dict = ManagedTextModel.FuskingTraitor.GetDictionary(model_2);
Debug.Assert(set.Count == dict.Count);
using (var set_enm = set.GetEnumerator())
using (var dict_enm = dict.GetEnumerator())
{
for ( ; set_enm.MoveNext() && dict_enm.MoveNext();)
{
var set_m = NativeTextMMFModelBinary.ToModelRecord(set_enm.Current);
var dict_ngram = dict_enm.Current.Key;
Debug.Assert(set_m.Ngram == dict_ngram);
fixed(char *dict_ngram_ptr = dict_ngram)
{
if (!set.TryGetValue((IntPtr)dict_ngram_ptr, out var exitstsPtr))
{
Debugger.Break();
}
if (!dict.TryGetValue(set_m.Ngram, out var dict_prob))
{
Debugger.Break();
}
Debug.Assert(set_m.Probability == dict_prob);
}
}
}
}
private static void Comare_ModelBinaryNative_And_ModelClassic(IEnumerable <string> modelFilenames)
{
#if DEBUG
var bmcfg = new BinaryModelConfig(modelFilenames)
{
ModelDictionaryCapacity = Config.Inst.MODEL_DICTIONARY_CAPACITY
};
using (var model_1 = new NativeTextMMFModelBinary(bmcfg))
using (var model_2 = new ManagedTextModel(Config.Inst.GetTextModelConfig()))
{
ModelComparer.Compare(model_1, model_2);
}
#else
throw (new NotImplementedException("Allowed only in DEBUG mode"));
#endif
}
private static void Test__NativeTextMMFModelBinary_RU()
{
Console.Write("load \"-binary-\" language model...");
var sw = Stopwatch.StartNew();
using (var model = new NativeTextMMFModelBinary(Config.Inst.GetBinaryModelConfig_RU()))
{
var count = model.RecordCount;
Console.WriteLine($"=> elapsed: {sw.StopAndElapsed()}, record-count: {count}\r\n");
Test__TextSegmenter(model);
Console.Write("\r\ndisposing language model...");
}
Console.WriteLine("=> end\r\n");
}