private void ConsecutivelyLoadMMFCallback(ref Pair pair)
{
BucketValue bucketVal;
if (_Dictionary.TryGetValue(pair.Text, out bucketVal))
{
var bucketRef = new BucketRef()
{
Language = pair.Language, Weight = pair.Weight
};
if (bucketVal.NextBucket == null)
{
bucketVal.NextBucket = bucketRef;
_Dictionary[pair.Text] = bucketVal;
}
else
{
var br = bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
_Dictionary.Add(pair.Text, new BucketValue(pair.Language, pair.Weight));
}
}
private void AddNewOrMergeWithExists( ref Entry otherEntry )
{
int hashCode = _Comparer.GetHashCode( otherEntry.key ) & 0x7FFFFFFF;
int targetBucket = hashCode % _Buckets.Length;
#region [.try merge with exists.]
for ( int i = _Buckets[ targetBucket ]; i >= 0; i = _Entries[ i ].next )
{
if ( _Entries[ i ].hashCode == hashCode && _Comparer.Equals( _Entries[ i ].key, otherEntry.key ) )
{
//merge with exists
var bucketRef = new BucketRef()
{
Language = otherEntry.value.Language,
Weight = otherEntry.value.Weight,
NextBucket = otherEntry.value.NextBucket,
};
var next = _Entries[ i ].value.NextBucket;
if ( next == null )
{
_Entries[ i ].value.NextBucket = bucketRef;
}
else
{
for ( ; next.NextBucket != null; next = next.NextBucket ) ;
next.NextBucket = bucketRef;
}
return;
}
}
#endregion
#region [.add new.]
int index;
if ( _FreeCount > 0 )
{
index = _FreeList;
_FreeList = _Entries[ index ].next;
_FreeCount--;
}
else
{
if ( _Count == _Entries.Length )
{
Resize();
targetBucket = hashCode % _Buckets.Length;
}
index = _Count;
_Count++;
}
_Entries[ index ].hashCode = hashCode;
_Entries[ index ].next = _Buckets[ targetBucket ];
_Entries[ index ].key = otherEntry.key;
_Entries[ index ].value = otherEntry.value;
_Buckets[ targetBucket ] = index;
#endregion
}
private void ParallelLoad_v2(MModelConfig config)
{
var concurrencyLevel = Environment.ProcessorCount;
var cdict = (0 < config.ModelDictionaryCapacity)
? new ConcurrentDictionary <string, BucketValue>(concurrencyLevel, config.ModelDictionaryCapacity)
: new ConcurrentDictionary <string, BucketValue>();
Parallel.ForEach(config.LanguageConfigs,
new ParallelOptions()
{
MaxDegreeOfParallelism = concurrencyLevel
},
(languageConfig) =>
{
var _bucketVal = default(BucketValue);
foreach (var pair in languageConfig.GetModelFilenameContent())
{
var text = pair.Key.ToUpperInvariant();
var weight = pair.Value;
if (cdict.TryGetValue(text, out _bucketVal))
{
lock ( cdict )
{
var bucketRef = new BucketRef()
{
Language = languageConfig.Language, Weight = weight
};
if (_bucketVal.NextBucket == null)
{
_bucketVal.NextBucket = bucketRef;
cdict[text] = _bucketVal;
}
else
{
var br = _bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
}
else
{
cdict.TryAdd(text, new BucketValue(languageConfig.Language, weight));
}
}
}
);
_Dictionary = new Dictionary <string, BucketValue>(cdict);
cdict = null;
GC.Collect();
}
public WeighByLanguageEnumerator(ref BucketValue bucketVal)
{
_IsStarted = false;
_Next = bucketVal.NextBucket;
_Current = new WeighByLanguage()
{
Language = bucketVal.Language, Weight = bucketVal.Weight
};
}
private void ConsecutivelyLoad(MModelConfig config)
{
_Dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <string, BucketValue>(config.ModelDictionaryCapacity)
: new Dictionary <string, BucketValue>();
var bucketVal = default(BucketValue);
foreach (var languageConfig in config.LanguageConfigs)
{
foreach (var pair in languageConfig.GetModelFilenameContent())
{
var text = pair.Key.ToUpperInvariant();
var weight = pair.Value;
if (_Dictionary.TryGetValue(text, out bucketVal))
{
#region
/*
* if ( bucketVal.Language == languageConfig.Language )
* {
* ...
* _Dictionary[ text ] = bucketVal;
* }
*/
#endregion
var bucketRef = new BucketRef()
{
Language = languageConfig.Language, Weight = weight
};
if (bucketVal.NextBucket == null)
{
bucketVal.NextBucket = bucketRef;
_Dictionary[text] = bucketVal;
}
else
{
var br = bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
_Dictionary.Add(text, new BucketValue(languageConfig.Language, weight));
}
}
}
}
public bool MoveNext()
{
if (!_IsStarted)
{
_IsStarted = true;
return(true);
}
if (_Next != null)
{
_Current = new WeighByLanguage()
{
Language = _Next.Language, Weight = _Next.Weight
};
_Next = _Next.NextBucket;
return(true);
}
return(false);
}
unsafe private void LoadMMFCallbackRoutine(ref MModelNativeTextMMFBase.Pair pair)
{
BucketValue bucketVal;
if (Dictionary.TryGetValue(pair.TextPtr, out bucketVal))
{
var bucketRef = new BucketRef()
{
Language = pair.Language,
Weight = pair.Weight,
NextBucket = bucketVal.NextBucket,
};
bucketVal.NextBucket = bucketRef;
Dictionary[pair.TextPtr] = bucketVal;
#region commented. previous
/*
* var bucketRef = new BucketRef() { Language = pair.Language, Weight = pair.Weight };
* if ( bucketVal.NextBucket == null )
* {
* bucketVal.NextBucket = bucketRef;
*
* DictionaryIntptr[ pair.TextPtr ] = bucketVal;
* }
* else
* {
* var br = bucketVal.NextBucket;
* for (; br.NextBucket != null; br = br.NextBucket );
* br.NextBucket = bucketRef;
* }
*/
#endregion
}
else
{
var textPtr = StringsHelper.AllocHGlobalAndCopy(pair.TextPtr, pair.TextLength);
Dictionary.Add(textPtr, new BucketValue(pair.Language, pair.Weight));
}
}
private void ParallelLoad_v1(MModelConfig config)
{
var dictBag = new ConcurrentBag <Dictionary <string, BucketValue> >();
Parallel.ForEach(config.LanguageConfigs,
() => new Dictionary <string, BucketValue>(),
(languageConfig, loopState, i, dict) =>
{
var _bucketVal = default(BucketValue);
foreach (var pair in languageConfig.GetModelFilenameContent())
{
var text = pair.Key.ToUpperInvariant();
var weight = pair.Value;
if (dict.TryGetValue(text, out _bucketVal))
{
var bucketRef = new BucketRef()
{
Language = languageConfig.Language, Weight = weight
};
if (_bucketVal.NextBucket == null)
{
_bucketVal.NextBucket = bucketRef;
dict[text] = _bucketVal;
}
else
{
var br = _bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
dict.Add(text, new BucketValue(languageConfig.Language, weight));
}
}
return(dict);
},
(dict) =>
{
if (dict.Count != 0)
{
dictBag.Add(dict);
}
}
);
var bucketVal = default(BucketValue);
_Dictionary = dictBag.First();
foreach (var dict in dictBag.Skip(1))
{
foreach (var pair in dict)
{
var text = pair.Key;
var bucketValElse = pair.Value;
if (_Dictionary.TryGetValue(text, out bucketVal))
{
var bucketRef = new BucketRef()
{
Language = bucketValElse.Language,
Weight = bucketValElse.Weight,
NextBucket = bucketValElse.NextBucket,
};
if (bucketVal.NextBucket == null)
{
bucketVal.NextBucket = bucketRef;
_Dictionary[text] = bucketVal;
}
else
{
var br = bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
_Dictionary.Add(text, bucketValElse);
}
}
}
dictBag = null;
GC.Collect();
}
private void ParallelLoadMMF(MModelConfig config)
{
#region [.parallel load by partitions.]
var processorCount = Environment.ProcessorCount;
//var partitions = CreatePartitions( config.LanguageConfigs, processorCount );
var partitions = config.LanguageConfigs.SplitByPartitionCountOrGreater(processorCount);
var unitBag = new ConcurrentBag <ParallelLoadUnit>();
Parallel.ForEach(partitions,
new ParallelOptions()
{
MaxDegreeOfParallelism = processorCount
},
() => default(ParallelLoadUnit),
(partition, loopState, i, unit) =>
{
const int EMPIRICALLY_CHOSEN_FUSKING_NUMBER = 27;
var capacity = (int)(partition.TotalModelFilenameLengths / EMPIRICALLY_CHOSEN_FUSKING_NUMBER);
unit.Initialize(capacity);
foreach (var languageConfig in partition.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, unit.LoadMMFCallback);
}
return(unit);
},
(unit) =>
{
if (unit.Dictionary.Count != 0)
{
unitBag.Add(unit);
}
}
);
#endregion
#region [.merge.]
var bucketVal = default(BucketValue);
var dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <IntPtr, BucketValue>(config.ModelDictionaryCapacity, default(IntPtrEqualityComparer))
: new Dictionary <IntPtr, BucketValue>(default(IntPtrEqualityComparer));
foreach (var dict in unitBag.Select(unit => unit.Dictionary))
{
foreach (var pair in dict)
{
var textPtr = pair.Key;
var bucketValElse = pair.Value;
if (dictionary.TryGetValue(textPtr, out bucketVal))
{
var bucketRef = new BucketRef()
{
Language = bucketValElse.Language,
Weight = bucketValElse.Weight,
NextBucket = bucketValElse.NextBucket,
};
if (bucketVal.NextBucket == null)
{
bucketVal.NextBucket = bucketRef;
dictionary[textPtr] = bucketVal;
}
else
{
var br = bucketVal.NextBucket;
for ( ; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
dictionary.Add(textPtr, bucketValElse);
}
}
//dict.Clear(); //--- too slow => TODO: вытащить словарь/pull dictionary ---//
}
_Dictionary = dictionary;
unitBag = null;
#endregion
}
