private static SetNative LoadBinaryModel(BinaryModelConfig config, NativeMemAllocationMediator nativeMemAllocator) { var set = new SetNative(config.ModelDictionaryCapacity); foreach (var modelFilename in config.ModelFilenames) { LoadFromBinFile(modelFilename, set, nativeMemAllocator); } return(set); }
unsafe private static void LoadFromBinFile(string modelFilename, SetNative set, NativeMemAllocationMediator nativeMemAllocator) { const int BUFFER_SIZE = 0x2000; using (var fs = new FileStream(modelFilename, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE, FileOptions.SequentialScan)) using (var mmf = MemoryMappedFile.CreateFromFile(fs, null, 0L, MemoryMappedFileAccess.Read, new MemoryMappedFileSecurity(), HandleInheritability.None, true)) using (var accessor = mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read)) { byte *buffer = null; accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref buffer); for (byte *endBuffer = buffer + fs.Length; buffer < endBuffer;) { #region [.read 'textPtr' as C#-chars (with double-byte-zero '\0').] var bufferCharPtr = (char *)buffer; for (var idx = 0; ; idx++) { if (BUFFER_SIZE < idx) { throw (new InvalidDataException("WTF?!?!: [BUFFER_SIZE < idx]")); } if (bufferCharPtr[idx] == '\0') { #region [.alloc term-&-probability and copy term.] //alloc with include zero-'\0' end-of-string var source = bufferCharPtr; var sourceLength = idx; var sourceLenInBytes = sourceLength * sizeof(char); var recordSize = (sourceLenInBytes + sizeof(char)) + sizeof(double); var destPtr = nativeMemAllocator.Alloc(recordSize); Buffer.MemoryCopy(source, (void *)destPtr, sourceLenInBytes, sourceLenInBytes); source += sourceLength; var destination = ((char *)destPtr) + sourceLength; * destination = '\0'; #region comm. prev. //* //var recordSize = ((sourceLength + 1) * sizeof(char)) + sizeof(double); //var destPtr = Marshal.AllocHGlobal( recordSize ); //var destination = (char*) destPtr; //for ( ; 0 < sourceLength; sourceLength-- ) //{ // *(destination++) = *(source++); //} //*destination = '\0'; //*/ #endregion #endregion #region [.read probability.] *(double *)(destination + 1) = *(double *)(source + 1); #endregion #if DEBUG var m = ToModelRecord(destPtr); var prob = *(double *)(destination + 1); Debug.Assert(m.Probability == prob); #endif set.Add(destPtr); #region [.move to next record.] buffer += recordSize; #endregion break; } } #endregion } } }
public NativeTextMMFModelBinary(BinaryModelConfig config) { _NativeMemAllocator = new NativeMemAllocationMediator(nativeBlockAllocSize: 1024 * 1024); _Set = LoadBinaryModel(config, _NativeMemAllocator); }