unsafe private static void LoadFromBinFile(string modelFilename, SetNative set) { const int BUFFER_SIZE = 0x2000; using (var fs = new FileStream(modelFilename, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE, FileOptions.SequentialScan)) using (var mmf = MemoryMappedFile.CreateFromFile(fs, null, 0L, MemoryMappedFileAccess.Read, new MemoryMappedFileSecurity(), HandleInheritability.None, true)) using (var accessor = mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read)) { byte *buffer = null; accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref buffer); for (byte *endBuffer = buffer + fs.Length; buffer < endBuffer;) { #region [.read 'textPtr' as C#-chars (with double-byte-zero '\0').] var bufferCharPtr = (char *)buffer; for (var idx = 0; ; idx++) { if (BUFFER_SIZE < idx) { throw (new InvalidDataException("WTF?!?!: [BUFFER_SIZE < idx]")); } if (bufferCharPtr[idx] == '\0') { #region [.alloc term-&-probability and copy term.] //alloc with include zero-'\0' end-of-string var source = bufferCharPtr; var sourceLength = idx; var recordSize = ((sourceLength + 1) * sizeof(char)) + sizeof(double); var destPtr = Marshal.AllocHGlobal(recordSize); var destination = (char *)destPtr; for ( ; 0 < sourceLength; sourceLength--) { *(destination++) = *(source++); } *destination = '\0'; #endregion #region [.read probability.] *(double *)(destination + 1) = *(double *)(source + 1); #endregion #if DEBUG var m = ToModelRecord(destPtr); var prob = *(double *)(destination + 1); Debug.Assert(m.Probability == prob); #endif set.Add(destPtr); #region [.move to next record.] buffer += recordSize; #endregion break; } } #endregion } } }
private static SetNative LoadBinaryModel(BinaryModelConfig config) { var set = new SetNative(config.ModelDictionaryCapacity); foreach (var modelFilename in config.ModelFilenames) { LoadFromBinFile(modelFilename, set); } return(set); }
private void DisposeNativeResources() { if (_Set != null) { foreach (var ptr in _Set) { Marshal.FreeHGlobal(ptr); } _Set = null; } }
public NativeTextMMFModelBinary(BinaryModelConfig config) { _Set = LoadBinaryModel(config); }
public NativeTextMMFModelBinary(BinaryModelConfig config) { _NativeMemAllocator = new NativeMemAllocationMediator(nativeBlockAllocSize: 1024 * 1024); _Set = LoadBinaryModel(config, _NativeMemAllocator); }