/*unsafe private static Dictionary< IntPtr, WeighByLanguage[] > LoadDictionaryNativeFromBinFile_v1( string fileName, int capacity ) * { * var dict = new Dictionary< IntPtr, WeighByLanguage[] >( capacity ); * * const int BUFFER_SIZE = 0x2000; * * using ( var fs = new FileStream( fileName, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE, FileOptions.SequentialScan ) ) * using ( var mmf = MemoryMappedFile.CreateFromFile( fs, null, 0L, MemoryMappedFileAccess.Read, new MemoryMappedFileSecurity(), HandleInheritability.None, true ) ) * using ( var accessor = mmf.CreateViewAccessor( 0L, 0L, MemoryMappedFileAccess.Read ) ) * { * byte* buffer = null; * accessor.SafeMemoryMappedViewHandle.AcquirePointer( ref buffer ); * * IntPtr textPtr; * for ( byte* endBuffer = buffer + fs.Length; buffer < endBuffer; ) * { #region [.read 'textPtr' as C#-chars (with double-byte-zero '\0').] * var bufferCharPtr = (char*) buffer; * for ( var idx = 0; ; idx++ ) * { * if ( BUFFER_SIZE < idx ) * { * throw (new InvalidDataException( "WTF?!?!: [BUFFER_SIZE < idx]" )); * } * if ( bufferCharPtr[ idx ] == '\0' ) * { * textPtr = StringsHelper.AllocHGlobalAndCopy( bufferCharPtr, idx ); * buffer = (byte*) (bufferCharPtr + idx + 1); * break; * } * } #endregion * #region [.read buckets.] * var countBuckets = *buffer++; * var pairs = new WeighByLanguage[ countBuckets ]; * for ( var i = 0; i < countBuckets; i++ ) * { * pairs[ i ] = new WeighByLanguage() * { * Language = (Language) (*buffer++), * Weight = *((float*) buffer), * }; * buffer += sizeof(float); * } #endregion * * dict.Add( textPtr, pairs ); * } * } * * return (dict); * }*/ /*unsafe private static Dictionary< IntPtr, IntPtr > LoadDictionaryNativeFromBinFile_v2( string fileName, int capacity ) * { * var dict = new Dictionary< IntPtr, IntPtr >( capacity ); * * const int BUFFER_SIZE = 0x2000; * * using ( var fs = new FileStream( fileName, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE, FileOptions.SequentialScan ) ) * using ( var mmf = MemoryMappedFile.CreateFromFile( fs, null, 0L, MemoryMappedFileAccess.Read, new MemoryMappedFileSecurity(), HandleInheritability.None, true ) ) * using ( var accessor = mmf.CreateViewAccessor( 0L, 0L, MemoryMappedFileAccess.Read ) ) * { * byte* buffer = null; * accessor.SafeMemoryMappedViewHandle.AcquirePointer( ref buffer ); * * IntPtr textPtr; * for ( byte* endBuffer = buffer + fs.Length; buffer < endBuffer; ) * { #region [.read 'textPtr' as C#-chars (with double-byte-zero '\0').] * var bufferCharPtr = (char*) buffer; * for ( var idx = 0; ; idx++ ) * { * if ( BUFFER_SIZE < idx ) * { * throw (new InvalidDataException( "WTF?!?!: [BUFFER_SIZE < idx]" )); * } * if ( bufferCharPtr[ idx ] == '\0' ) * { * textPtr = StringsHelper.AllocHGlobalAndCopy( bufferCharPtr, idx ); * buffer = (byte*) (bufferCharPtr + idx + 1); * break; * } * } #endregion * #region [.read buckets.] * var countBuckets = *buffer++; * var pairsPtr = Marshal.AllocHGlobal( sizeof(byte) + countBuckets * sizeof(WeighByLanguage) ); * var pairsBytePtr = (byte*) pairsPtr; * pairsBytePtr++ = countBuckets; * var pairsLanguageWeightPtr = (WeighByLanguage*) pairsBytePtr; * for ( var i = 0; i < countBuckets; i++ ) * { * var ptr = &pairsLanguageWeightPtr[ i ]; * ptr->Language = (Language) (*buffer++); * ptr->Weight = *((float*) buffer); * //pairsLanguageWeightPtr[ i ] = new WeighByLanguage() * // { * // Language = (Language) (*buffer++), * // Weight = *((float*) buffer), * // }; * buffer += sizeof(float); * } #endregion * * dict.Add( textPtr, pairsPtr ); * } * } * * return (dict); * }*/ private static Dictionary <IntPtr, IntPtr> LoadBinaryModel(MModelBinaryNativeConfig config) { var dict = new Dictionary <IntPtr, IntPtr>(config.ModelDictionaryCapacity, default(IntPtrEqualityComparer)); foreach (var modelFilename in config.ModelFilenames) { LoadFromBinFile(modelFilename, dict); } return(dict); }
public MModelBinaryNative(MModelBinaryNativeConfig config) { _Dictionary = LoadBinaryModel(config); }