unsafe private static void LoadFromBinFile(string modelFilename, SetNative set)
{
const int BUFFER_SIZE = 0x2000;
using (var fs = new FileStream(modelFilename, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE, FileOptions.SequentialScan))
using (var mmf = MemoryMappedFile.CreateFromFile(fs, null, 0L, MemoryMappedFileAccess.Read, new MemoryMappedFileSecurity(), HandleInheritability.None, true))
using (var accessor = mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read))
{
byte *buffer = null;
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref buffer);
for (byte *endBuffer = buffer + fs.Length; buffer < endBuffer;)
{
#region [.read 'textPtr' as C#-chars (with double-byte-zero '\0').]
var bufferCharPtr = (char *)buffer;
for (var idx = 0; ; idx++)
{
if (BUFFER_SIZE < idx)
{
throw (new InvalidDataException("WTF?!?!: [BUFFER_SIZE < idx]"));
}
if (bufferCharPtr[idx] == '\0')
{
#region [.alloc term-&-probability and copy term.]
//alloc with include zero-'\0' end-of-string
var source = bufferCharPtr;
var sourceLength = idx;
var recordSize = ((sourceLength + 1) * sizeof(char)) + sizeof(double);
var destPtr = Marshal.AllocHGlobal(recordSize);
var destination = (char *)destPtr;
for ( ; 0 < sourceLength; sourceLength--)
{
*(destination++) = *(source++);
}
*destination = '\0';
#endregion
#region [.read probability.]
*(double *)(destination + 1) = *(double *)(source + 1);
#endregion
#if DEBUG
var m = ToModelRecord(destPtr);
var prob = *(double *)(destination + 1);
Debug.Assert(m.Probability == prob);
#endif
set.Add(destPtr);
#region [.move to next record.]
buffer += recordSize;
#endregion
break;
}
}
#endregion
}
}
}
private static SetNative LoadBinaryModel(BinaryModelConfig config)
{
var set = new SetNative(config.ModelDictionaryCapacity);
foreach (var modelFilename in config.ModelFilenames)
{
LoadFromBinFile(modelFilename, set);
}
return(set);
}
private void DisposeNativeResources()
{
if (_Set != null)
{
foreach (var ptr in _Set)
{
Marshal.FreeHGlobal(ptr);
}
_Set = null;
}
}
public NativeTextMMFModelBinary(BinaryModelConfig config)
{
_Set = LoadBinaryModel(config);
}
public NativeTextMMFModelBinary(BinaryModelConfig config)
{
_NativeMemAllocator = new NativeMemAllocationMediator(nativeBlockAllocSize: 1024 * 1024);
_Set = LoadBinaryModel(config, _NativeMemAllocator);
}
