unsafe private static void LoadFromBinFile(string modelFilename, Dictionary <IntPtr, IntPtr> dict)
{
const int BUFFER_SIZE = 0x2000;
using (var fs = new FileStream(modelFilename, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE, FileOptions.SequentialScan))
using (var mmf = MemoryMappedFile.CreateFromFile(fs, null, 0L, MemoryMappedFileAccess.Read, new MemoryMappedFileSecurity(), HandleInheritability.None, true))
using (var accessor = mmf.CreateViewAccessor(0L, 0L, MemoryMappedFileAccess.Read))
{
byte *buffer = null;
accessor.SafeMemoryMappedViewHandle.AcquirePointer(ref buffer);
IntPtr textPtr;
for (byte *endBuffer = buffer + fs.Length; buffer < endBuffer;)
{
#region [.read 'textPtr' as C#-chars (with double-byte-zero '\0').]
var bufferCharPtr = (char *)buffer;
for (var idx = 0; ; idx++)
{
if (BUFFER_SIZE < idx)
{
throw (new InvalidDataException("WTF?!?!: [BUFFER_SIZE < idx]"));
}
if (bufferCharPtr[idx] == '\0')
{
textPtr = StringsHelper.AllocHGlobalAndCopy(bufferCharPtr, idx);
buffer = (byte *)(bufferCharPtr + idx + 1);
break;
}
}
#endregion
#region [.read buckets.]
var countBuckets = *buffer++;
var pairsPtr = Marshal.AllocHGlobal(sizeof(byte) + countBuckets * sizeof(WeighByLanguage));
var pairsBytePtr = (byte *)pairsPtr;
* pairsBytePtr++ = countBuckets;
var pairsLanguageWeightPtr = (WeighByLanguage *)pairsBytePtr;
for (var i = 0; i < countBuckets; i++)
{
var ptr = &pairsLanguageWeightPtr[i];
ptr->Language = (Language)(*buffer++);
ptr->Weight = *((float *)buffer);
//pairsLanguageWeightPtr[ i ] = new WeighByLanguage()
// {
// Language = (Language) (*buffer++),
// Weight = *((float*) buffer),
// };
buffer += sizeof(float);
}
#endregion
dict.Add(textPtr, pairsPtr);
}
}
}
/*public bool TryAddToEndOfChain( ref LanguageNativeTextMMFConfig.Pair pair )
{
int i = FindEntry( pair.TextPtr );
if ( i >= 0 )
{
AddToHead( ref _Entries[ i ].value, ref pair );
return (true);
}
return (false);
}*/
unsafe internal void AddNewOrToEndOfChain( ref MModelNativeTextMMFBase.Pair pair )
{
int hashCode = _Comparer.GetHashCode( pair.TextPtr ) & 0x7FFFFFFF;
int targetBucket = hashCode % _Buckets.Length;
#region [.try merge with exists.]
for ( int i = _Buckets[ targetBucket ]; i >= 0; i = _Entries[ i ].next )
{
if ( _Entries[ i ].hashCode == hashCode && _Comparer.Equals( _Entries[ i ].key, pair.TextPtr ) )
{
//add to end of chain
AddToHead( ref _Entries[ i ].value, ref pair );
return;
}
}
#endregion
#region [.add new.]
int index;
if ( _FreeCount > 0 )
{
index = _FreeList;
_FreeList = _Entries[ index ].next;
_FreeCount--;
}
else
{
if ( _Count == _Entries.Length )
{
Resize();
targetBucket = hashCode % _Buckets.Length;
}
index = _Count;
_Count++;
}
var textPtr = StringsHelper.AllocHGlobalAndCopy( pair.TextPtr, pair.TextLength );
_Entries[ index ].hashCode = hashCode;
_Entries[ index ].next = _Buckets[ targetBucket ];
_Entries[ index ].key = textPtr;
_Entries[ index ].value = new BucketValue( pair.Language, pair.Weight );
_Buckets[ targetBucket ] = index;
#endregion
}
unsafe private void LoadMMFCallbackRoutine(ref MModelNativeTextMMFBase.Pair pair)
{
BucketValue bucketVal;
if (Dictionary.TryGetValue(pair.TextPtr, out bucketVal))
{
var bucketRef = new BucketRef()
{
Language = pair.Language,
Weight = pair.Weight,
NextBucket = bucketVal.NextBucket,
};
bucketVal.NextBucket = bucketRef;
Dictionary[pair.TextPtr] = bucketVal;
#region commented. previous
/*
* var bucketRef = new BucketRef() { Language = pair.Language, Weight = pair.Weight };
* if ( bucketVal.NextBucket == null )
* {
* bucketVal.NextBucket = bucketRef;
*
* DictionaryIntptr[ pair.TextPtr ] = bucketVal;
* }
* else
* {
* var br = bucketVal.NextBucket;
* for (; br.NextBucket != null; br = br.NextBucket );
* br.NextBucket = bucketRef;
* }
*/
#endregion
}
else
{
var textPtr = StringsHelper.AllocHGlobalAndCopy(pair.TextPtr, pair.TextLength);
Dictionary.Add(textPtr, new BucketValue(pair.Language, pair.Weight));
}
}