unsafe private static void LoadFromBinFile(string modelFilename, Dictionary <IntPtr, IntPtr> dict) { const int BUFFER_SIZE = 0x2000; using (var fs = new FileStream(modelFilename, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE, FileOptions.SequentialScan)) using (var mmf = MemoryMappedFile.CreateFromFile(fs, null, 0L, MemoryMappedFileAccess.Read, new MemoryMappedFileSecurity(), HandleInheritability.None, true)) using (var accessor = mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read)) { byte *buffer = null; accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref buffer); IntPtr textPtr; for (byte *endBuffer = buffer + fs.Length; buffer < endBuffer;) { #region [.read 'textPtr' as C#-chars (with double-byte-zero '\0').] var bufferCharPtr = (char *)buffer; for (var idx = 0; ; idx++) { if (BUFFER_SIZE < idx) { throw (new InvalidDataException("WTF?!?!: [BUFFER_SIZE < idx]")); } if (bufferCharPtr[idx] == '\0') { textPtr = StringsHelper.AllocHGlobalAndCopy(bufferCharPtr, idx); buffer = (byte *)(bufferCharPtr + idx + 1); break; } } #endregion #region [.read buckets.] var countBuckets = *buffer++; var pairsPtr = Marshal.AllocHGlobal(sizeof(byte) + countBuckets * sizeof(WeighByLanguage)); var pairsBytePtr = (byte *)pairsPtr; * pairsBytePtr++ = countBuckets; var pairsLanguageWeightPtr = (WeighByLanguage *)pairsBytePtr; for (var i = 0; i < countBuckets; i++) { var ptr = &pairsLanguageWeightPtr[i]; ptr->Language = (Language)(*buffer++); ptr->Weight = *((float *)buffer); //pairsLanguageWeightPtr[ i ] = new WeighByLanguage() // { // Language = (Language) (*buffer++), // Weight = *((float*) buffer), // }; buffer += sizeof(float); } #endregion dict.Add(textPtr, pairsPtr); } } }
/*public bool TryAddToEndOfChain( ref LanguageNativeTextMMFConfig.Pair pair ) { int i = FindEntry( pair.TextPtr ); if ( i >= 0 ) { AddToHead( ref _Entries[ i ].value, ref pair ); return (true); } return (false); }*/ unsafe internal void AddNewOrToEndOfChain( ref MModelNativeTextMMFBase.Pair pair ) { int hashCode = _Comparer.GetHashCode( pair.TextPtr ) & 0x7FFFFFFF; int targetBucket = hashCode % _Buckets.Length; #region [.try merge with exists.] for ( int i = _Buckets[ targetBucket ]; i >= 0; i = _Entries[ i ].next ) { if ( _Entries[ i ].hashCode == hashCode && _Comparer.Equals( _Entries[ i ].key, pair.TextPtr ) ) { //add to end of chain AddToHead( ref _Entries[ i ].value, ref pair ); return; } } #endregion #region [.add new.] int index; if ( _FreeCount > 0 ) { index = _FreeList; _FreeList = _Entries[ index ].next; _FreeCount--; } else { if ( _Count == _Entries.Length ) { Resize(); targetBucket = hashCode % _Buckets.Length; } index = _Count; _Count++; } var textPtr = StringsHelper.AllocHGlobalAndCopy( pair.TextPtr, pair.TextLength ); _Entries[ index ].hashCode = hashCode; _Entries[ index ].next = _Buckets[ targetBucket ]; _Entries[ index ].key = textPtr; _Entries[ index ].value = new BucketValue( pair.Language, pair.Weight ); _Buckets[ targetBucket ] = index; #endregion }
unsafe private void LoadMMFCallbackRoutine(ref MModelNativeTextMMFBase.Pair pair) { BucketValue bucketVal; if (Dictionary.TryGetValue(pair.TextPtr, out bucketVal)) { var bucketRef = new BucketRef() { Language = pair.Language, Weight = pair.Weight, NextBucket = bucketVal.NextBucket, }; bucketVal.NextBucket = bucketRef; Dictionary[pair.TextPtr] = bucketVal; #region commented. previous /* * var bucketRef = new BucketRef() { Language = pair.Language, Weight = pair.Weight }; * if ( bucketVal.NextBucket == null ) * { * bucketVal.NextBucket = bucketRef; * * DictionaryIntptr[ pair.TextPtr ] = bucketVal; * } * else * { * var br = bucketVal.NextBucket; * for (; br.NextBucket != null; br = br.NextBucket ); * br.NextBucket = bucketRef; * } */ #endregion } else { var textPtr = StringsHelper.AllocHGlobalAndCopy(pair.TextPtr, pair.TextLength); Dictionary.Add(textPtr, new BucketValue(pair.Language, pair.Weight)); } }