private static void Test__MModelClassic()
{
var sw = Stopwatch.StartNew();
var model = new MModelClassic(Config.Inst.GetMModelConfig());
var count = model.RecordCount;
sw.Stop();
GCCollect();
Console.WriteLine("elapsed: " + sw.Elapsed + ", count: " + count);
Console.ForegroundColor = ConsoleColor.DarkGray; Console.WriteLine("\r\n[.....push enter for continue.....]"); Console.ResetColor();
Console.ReadLine();
//*
var detector = new MDetector(Config.Inst.GetMDetectorConfig(), model);
var languageInfos = detector.DetectLanguage("\r\n[.....push enter for continue.....]");
//*/
Console.Write("disposing language model...");
model.Dispose();
model = null;
detector = null;
GCCollect();
Console.WriteLine("end");
}
private static void Comare_ModelBinaryNative_And_ModelClassic()
{
#if DEBUG
using (var model_1 = new MModelBinaryNative(Config.Inst.GetModelBinaryNativeConfig()))
using (var model_2 = new MModelClassic(Config.Inst.GetModelConfig()))
{
ModelComparer.Compare(model_1, model_2);
}
#else
throw (new NotImplementedException("Allowed only in DEBUG mode"));
#endif
}
unsafe public static void Compare(MModelBinaryNative model_1, MModelClassic model_2)
{
var dict1 = MModelBinaryNative.FuskingTraitor.GetDictionary(model_1);
var dict2 = MModelClassic.FuskingTraitor.GetDictionary(model_2);
Debug.Assert(dict1.Count == dict2.Count);
using (var e1 = dict1.GetEnumerator())
using (var e2 = dict2.GetEnumerator())
{
BucketValue bucketVal;
IntPtr weighByLanguagesBasePtr;
for ( ; e1.MoveNext() && e2.MoveNext();)
{
var s1 = e1.Current.Key;
var s2 = e2.Current.Key;
//Debug.Assert( s1.ToText() == s2 );
fixed(char *s2_ptr = s2)
if (!dict1.TryGetValue((IntPtr)s2_ptr, out weighByLanguagesBasePtr))
{
Debugger.Break();
}
if (!dict2.TryGetValue(s1.ToText(), out bucketVal))
{
Debugger.Break();
}
{
var ar1 = e1.Current.Value.ToWeighByLanguageNative().ToLanguageWeightPairs();
var ar2 = bucketVal.GetAllBucketRef().ToLanguageWeightPairs();
Debug.Assert(ar1.Length == ar2.Length);
var isect = Intersect(ar1, ar2);
Debug.Assert(isect.Length == ar2.Length);
}
{
var ar1 = weighByLanguagesBasePtr.ToWeighByLanguageNative().ToLanguageWeightPairs();
var ar2 = e2.Current.Value.GetAllBucketRef().ToLanguageWeightPairs();
Debug.Assert(ar1.Length == ar2.Length);
var isect = Intersect(ar1, ar2);
Debug.Assert(isect.Length == ar2.Length);
}
}
}
}
private static void ConvertFromTxt2Bin()
{
var modelConfig = Config.Inst.GetModelConfig();
var model = new MModelClassic(modelConfig);
var config = new Txt2BinModelConverterConfig()
{
Model = model,
OutputFileName = Config.Inst.OUTPUT_FILE_NAME,
OutputFileSizeInBytes = Config.Inst.OUTPUT_FILE_SIZE_IN_BYTES, //1024 * 1024 * 99,
};
var outputFileNames = Txt2BinModelConverter.Run(config);
Console.WriteLine();
Console.WriteLine(" output-files: ");
Console.WriteLine(" --------------");
for (var i = 0; i < outputFileNames.Count; i++)
{
Console.WriteLine(' ' + (i + 1).ToString() + "). '" + outputFileNames[i] + '\'');
}
Console.WriteLine(" --------------\r\n");
}