コード例 #1
0
        unsafe public static void Compare(NativeTextMMFModelBinary model_1, ManagedTextModel model_2)
        {
            var set  = NativeTextMMFModelBinary.FuskingTraitor.GetSet(model_1);
            var dict = ManagedTextModel.FuskingTraitor.GetDictionary(model_2);

            Debug.Assert(set.Count == dict.Count);

            using (var set_enm = set.GetEnumerator())
                using (var dict_enm = dict.GetEnumerator())
                {
                    for ( ; set_enm.MoveNext() && dict_enm.MoveNext();)
                    {
                        var set_m      = NativeTextMMFModelBinary.ToModelRecord(set_enm.Current);
                        var dict_ngram = dict_enm.Current.Key;

                        Debug.Assert(set_m.Ngram == dict_ngram);

                        fixed(char *dict_ngram_ptr = dict_ngram)
                        {
                            if (!set.TryGetValue((IntPtr)dict_ngram_ptr, out var exitstsPtr))
                            {
                                Debugger.Break();
                            }
                            if (!dict.TryGetValue(set_m.Ngram, out var dict_prob))
                            {
                                Debugger.Break();
                            }

                            Debug.Assert(set_m.Probability == dict_prob);
                        }
                    }
                }
        }
コード例 #2
0
        private static void Comare_ModelBinaryNative_And_ModelClassic(IEnumerable <string> modelFilenames)
        {
#if DEBUG
            var bmcfg = new BinaryModelConfig(modelFilenames)
            {
                ModelDictionaryCapacity = Config.Inst.MODEL_DICTIONARY_CAPACITY
            };

            using (var model_1 = new NativeTextMMFModelBinary(bmcfg))
                using (var model_2 = new ManagedTextModel(Config.Inst.GetTextModelConfig()))
                {
                    ModelComparer.Compare(model_1, model_2);
                }
#else
            throw (new NotImplementedException("Allowed only in DEBUG mode"));
#endif
        }
コード例 #3
0
        private static void Test__NativeTextMMFModelBinary_RU()
        {
            Console.Write("load \"-binary-\" language model...");

            var sw = Stopwatch.StartNew();

            using (var model = new NativeTextMMFModelBinary(Config.Inst.GetBinaryModelConfig_RU()))
            {
                var count = model.RecordCount;

                Console.WriteLine($"=> elapsed: {sw.StopAndElapsed()}, record-count: {count}\r\n");

                Test__TextSegmenter(model);

                Console.Write("\r\ndisposing language model...");
            }
            Console.WriteLine("=> end\r\n");
        }