public MModelClassicMMF(MModelConfig config)
{
//var sw = Stopwatch.StartNew();
ConsecutivelyLoadMMF(config);
//sw.Stop();
//Console.WriteLine( "Elapsed: " + sw.Elapsed );
}
public MModelDictionaryNativeMMF(MModelConfig config)
{
//var sw = Stopwatch.StartNew();
ParallelLoadMMF(config);
//sw.Stop();
//Console.WriteLine( "Elapsed: " + sw.Elapsed );
}
private void ParallelLoad_v2(MModelConfig config)
{
var concurrencyLevel = Environment.ProcessorCount;
var cdict = (0 < config.ModelDictionaryCapacity)
? new ConcurrentDictionary <string, BucketValue>(concurrencyLevel, config.ModelDictionaryCapacity)
: new ConcurrentDictionary <string, BucketValue>();
Parallel.ForEach(config.LanguageConfigs,
new ParallelOptions()
{
MaxDegreeOfParallelism = concurrencyLevel
},
(languageConfig) =>
{
var _bucketVal = default(BucketValue);
foreach (var pair in languageConfig.GetModelFilenameContent())
{
var text = pair.Key.ToUpperInvariant();
var weight = pair.Value;
if (cdict.TryGetValue(text, out _bucketVal))
{
lock ( cdict )
{
var bucketRef = new BucketRef()
{
Language = languageConfig.Language, Weight = weight
};
if (_bucketVal.NextBucket == null)
{
_bucketVal.NextBucket = bucketRef;
cdict[text] = _bucketVal;
}
else
{
var br = _bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
}
else
{
cdict.TryAdd(text, new BucketValue(languageConfig.Language, weight));
}
}
}
);
_Dictionary = new Dictionary <string, BucketValue>(cdict);
cdict = null;
GC.Collect();
}
private void ConsecutivelyLoad(MModelConfig config)
{
_Dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <string, BucketValue>(config.ModelDictionaryCapacity)
: new Dictionary <string, BucketValue>();
var bucketVal = default(BucketValue);
foreach (var languageConfig in config.LanguageConfigs)
{
foreach (var pair in languageConfig.GetModelFilenameContent())
{
var text = pair.Key.ToUpperInvariant();
var weight = pair.Value;
if (_Dictionary.TryGetValue(text, out bucketVal))
{
#region
/*
* if ( bucketVal.Language == languageConfig.Language )
* {
* ...
* _Dictionary[ text ] = bucketVal;
* }
*/
#endregion
var bucketRef = new BucketRef()
{
Language = languageConfig.Language, Weight = weight
};
if (bucketVal.NextBucket == null)
{
bucketVal.NextBucket = bucketRef;
_Dictionary[text] = bucketVal;
}
else
{
var br = bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
_Dictionary.Add(text, new BucketValue(languageConfig.Language, weight));
}
}
}
}
private void ParallelLoadMMF(MModelConfig config)
{
#region [.parallel load by partitions.]
var processorCount = Environment.ProcessorCount;
var partitions = config.LanguageConfigs.SplitByPartitionCountOrGreater(processorCount);
var unitBag = new ConcurrentBag <ParallelLoadUnit>();
Parallel.ForEach(partitions,
new ParallelOptions()
{
MaxDegreeOfParallelism = processorCount
},
() => default(ParallelLoadUnit),
(partition, loopState, i, unit) =>
{
const int EMPIRICALLY_CHOSEN_FUSKING_NUMBER = 27;
var capacity = (int)(partition.TotalModelFilenameLengths / EMPIRICALLY_CHOSEN_FUSKING_NUMBER);
unit.Initialize(capacity);
foreach (var languageConfig in partition.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, unit.LoadMMFCallback);
}
return(unit);
},
(unit) =>
{
if (unit.DictionaryNative != null && unit.DictionaryNative.Count != 0)
{
unitBag.Add(unit);
}
}
);
#endregion
#region [.merge.]
var dictionary = new DictionaryNative(config.ModelDictionaryCapacity);
foreach (var dict in unitBag.Select(unit => unit.DictionaryNative))
{
dictionary.MergeWith(dict);
dict.Dispose();
}
_Dictionary = dictionary;
unitBag = null;
#endregion
}
private void ConsecutivelyLoadMMF(MModelConfig config)
{
_Dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <string, BucketValue>(config.ModelDictionaryCapacity)
: new Dictionary <string, BucketValue>();
var callback = new LoadModelFilenameContentCallback(ConsecutivelyLoadMMFCallback);
foreach (var languageConfig in config.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, callback);
}
}
public MModelClassic(MModelConfig config)
{
//var sw = Stopwatch.StartNew();
//ParallelLoad_v1( config );
//ParallelLoad_v2( config );
//sw.Stop();
//Console.WriteLine( "ParallelLoad: " + sw.Elapsed );
//sw.Restart();
ConsecutivelyLoad(config);
//sw.Stop();
//Console.WriteLine( "ConsecutivelyLoad: " + sw.Elapsed );
}
/*private static LanguageConfigPartition[] CreatePartitions( IEnumerable< LanguageConfig > languageConfigs, int processorCount )
* {
* var array = languageConfigs.Select( lc => new { LanguageConfig = lc, ModelFilenameLength = (new FileInfo( lc.ModelFilename )).Length } )
* .OrderByDescending( a => a.ModelFilenameLength )
* .ToArray();
* var partSize = (int) (1.0 * array.Length / processorCount + 0.5);
*
* var partitions = new LanguageConfigPartition[ processorCount ];
* for ( var i = 0; i < array.Length; i++ )
* {
* int partIndex;
* int itemIndexInPart = Math.DivRem( i, partSize - 1, out partIndex );
*
* if ( partitions[ partIndex ].LanguageConfigs == null )
* {
* partitions[ partIndex ].LanguageConfigs = new LanguageConfig[ partSize ];
* }
* //Debug.Assert( part[ itemIndexInPart ] == null );
* var a = array[ i ];
* partitions[ partIndex ].LanguageConfigs[ itemIndexInPart ] = a.LanguageConfig;
* partitions[ partIndex ].TotalModelFilenameLengths += a.ModelFilenameLength;
* }
* return (partitions);
* }*/
private void ConsecutivelyLoadMMF(MModelConfig config)
{
_Dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <IntPtr, BucketValue>(config.ModelDictionaryCapacity, default(IntPtrEqualityComparer))
: new Dictionary <IntPtr, BucketValue>(default(IntPtrEqualityComparer));
var callback = new LoadModelFilenameContentCallback(ConsecutivelyLoadMMFCallback);
foreach (var languageConfig in config.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, callback);
}
}
public MModelNativeTextMMF(MModelConfig config, ModelLoadTypeEnum modelLoadType)
{
//var sw = Stopwatch.StartNew();
switch (modelLoadType)
{
case ModelLoadTypeEnum.Consecutively:
ConsecutivelyLoadMMF(config);
break;
case ModelLoadTypeEnum.Parallel:
ParallelLoadMMF(config);
break;
default:
throw (new ArgumentException(modelLoadType.ToString()));
}
//sw.Stop();
//Console.WriteLine( "Elapsed: " + sw.Elapsed );
}
private void ParallelLoad_v1(MModelConfig config)
{
var dictBag = new ConcurrentBag <Dictionary <string, BucketValue> >();
Parallel.ForEach(config.LanguageConfigs,
() => new Dictionary <string, BucketValue>(),
(languageConfig, loopState, i, dict) =>
{
var _bucketVal = default(BucketValue);
foreach (var pair in languageConfig.GetModelFilenameContent())
{
var text = pair.Key.ToUpperInvariant();
var weight = pair.Value;
if (dict.TryGetValue(text, out _bucketVal))
{
var bucketRef = new BucketRef()
{
Language = languageConfig.Language, Weight = weight
};
if (_bucketVal.NextBucket == null)
{
_bucketVal.NextBucket = bucketRef;
dict[text] = _bucketVal;
}
else
{
var br = _bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
dict.Add(text, new BucketValue(languageConfig.Language, weight));
}
}
return(dict);
},
(dict) =>
{
if (dict.Count != 0)
{
dictBag.Add(dict);
}
}
);
var bucketVal = default(BucketValue);
_Dictionary = dictBag.First();
foreach (var dict in dictBag.Skip(1))
{
foreach (var pair in dict)
{
var text = pair.Key;
var bucketValElse = pair.Value;
if (_Dictionary.TryGetValue(text, out bucketVal))
{
var bucketRef = new BucketRef()
{
Language = bucketValElse.Language,
Weight = bucketValElse.Weight,
NextBucket = bucketValElse.NextBucket,
};
if (bucketVal.NextBucket == null)
{
bucketVal.NextBucket = bucketRef;
_Dictionary[text] = bucketVal;
}
else
{
var br = bucketVal.NextBucket;
for (; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
_Dictionary.Add(text, bucketValElse);
}
}
}
dictBag = null;
GC.Collect();
}
private void ParallelLoadMMF(MModelConfig config)
{
#region [.parallel load by partitions.]
var processorCount = Environment.ProcessorCount;
//var partitions = CreatePartitions( config.LanguageConfigs, processorCount );
var partitions = config.LanguageConfigs.SplitByPartitionCountOrGreater(processorCount);
var unitBag = new ConcurrentBag <ParallelLoadUnit>();
Parallel.ForEach(partitions,
new ParallelOptions()
{
MaxDegreeOfParallelism = processorCount
},
() => default(ParallelLoadUnit),
(partition, loopState, i, unit) =>
{
const int EMPIRICALLY_CHOSEN_FUSKING_NUMBER = 27;
var capacity = (int)(partition.TotalModelFilenameLengths / EMPIRICALLY_CHOSEN_FUSKING_NUMBER);
unit.Initialize(capacity);
foreach (var languageConfig in partition.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, unit.LoadMMFCallback);
}
return(unit);
},
(unit) =>
{
if (unit.Dictionary.Count != 0)
{
unitBag.Add(unit);
}
}
);
#endregion
#region [.merge.]
var bucketVal = default(BucketValue);
var dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <IntPtr, BucketValue>(config.ModelDictionaryCapacity, default(IntPtrEqualityComparer))
: new Dictionary <IntPtr, BucketValue>(default(IntPtrEqualityComparer));
foreach (var dict in unitBag.Select(unit => unit.Dictionary))
{
foreach (var pair in dict)
{
var textPtr = pair.Key;
var bucketValElse = pair.Value;
if (dictionary.TryGetValue(textPtr, out bucketVal))
{
var bucketRef = new BucketRef()
{
Language = bucketValElse.Language,
Weight = bucketValElse.Weight,
NextBucket = bucketValElse.NextBucket,
};
if (bucketVal.NextBucket == null)
{
bucketVal.NextBucket = bucketRef;
dictionary[textPtr] = bucketVal;
}
else
{
var br = bucketVal.NextBucket;
for ( ; br.NextBucket != null; br = br.NextBucket)
{
;
}
br.NextBucket = bucketRef;
}
}
else
{
dictionary.Add(textPtr, bucketValElse);
}
}
//dict.Clear(); //--- too slow => TODO: вытащить словарь/pull dictionary ---//
}
_Dictionary = dictionary;
unitBag = null;
#endregion
}
