private void ParallelLoadMMF(MModelConfig config)
{
#region [.parallel load by partitions.]
var processorCount = Environment.ProcessorCount;
var partitions = config.LanguageConfigs.SplitByPartitionCountOrGreater(processorCount);
var unitBag = new ConcurrentBag <ParallelLoadUnit>();
Parallel.ForEach(partitions,
new ParallelOptions()
{
MaxDegreeOfParallelism = processorCount
},
() => default(ParallelLoadUnit),
(partition, loopState, i, unit) =>
{
const int EMPIRICALLY_CHOSEN_FUSKING_NUMBER = 27;
var capacity = (int)(partition.TotalModelFilenameLengths / EMPIRICALLY_CHOSEN_FUSKING_NUMBER);
unit.Initialize(capacity);
foreach (var languageConfig in partition.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, unit.LoadMMFCallback);
}
return(unit);
},
(unit) =>
{
if (unit.DictionaryNative != null && unit.DictionaryNative.Count != 0)
{
unitBag.Add(unit);
}
}
);
#endregion
#region [.merge.]
var dictionary = new DictionaryNative(config.ModelDictionaryCapacity);
foreach (var dict in unitBag.Select(unit => unit.DictionaryNative))
{
dictionary.MergeWith(dict);
dict.Dispose();
}
_Dictionary = dictionary;
unitBag = null;
#endregion
}
