private void ParallelLoadMMF(MModelConfig config) { #region [.parallel load by partitions.] var processorCount = Environment.ProcessorCount; var partitions = config.LanguageConfigs.SplitByPartitionCountOrGreater(processorCount); var unitBag = new ConcurrentBag <ParallelLoadUnit>(); Parallel.ForEach(partitions, new ParallelOptions() { MaxDegreeOfParallelism = processorCount }, () => default(ParallelLoadUnit), (partition, loopState, i, unit) => { const int EMPIRICALLY_CHOSEN_FUSKING_NUMBER = 27; var capacity = (int)(partition.TotalModelFilenameLengths / EMPIRICALLY_CHOSEN_FUSKING_NUMBER); unit.Initialize(capacity); foreach (var languageConfig in partition.LanguageConfigs) { LanguageModelFileReaderMMF.Read(languageConfig, unit.LoadMMFCallback); } return(unit); }, (unit) => { if (unit.DictionaryNative != null && unit.DictionaryNative.Count != 0) { unitBag.Add(unit); } } ); #endregion #region [.merge.] var dictionary = new DictionaryNative(config.ModelDictionaryCapacity); foreach (var dict in unitBag.Select(unit => unit.DictionaryNative)) { dictionary.MergeWith(dict); dict.Dispose(); } _Dictionary = dictionary; unitBag = null; #endregion }
private void ConsecutivelyLoadMMF(MModelConfig config) { _Dictionary = (0 < config.ModelDictionaryCapacity) ? new Dictionary <string, BucketValue>(config.ModelDictionaryCapacity) : new Dictionary <string, BucketValue>(); var callback = new LoadModelFilenameContentCallback(ConsecutivelyLoadMMFCallback); foreach (var languageConfig in config.LanguageConfigs) { LanguageModelFileReaderMMF.Read(languageConfig, callback); } }
/*private static LanguageConfigPartition[] CreatePartitions( IEnumerable< LanguageConfig > languageConfigs, int processorCount ) * { * var array = languageConfigs.Select( lc => new { LanguageConfig = lc, ModelFilenameLength = (new FileInfo( lc.ModelFilename )).Length } ) * .OrderByDescending( a => a.ModelFilenameLength ) * .ToArray(); * var partSize = (int) (1.0 * array.Length / processorCount + 0.5); * * var partitions = new LanguageConfigPartition[ processorCount ]; * for ( var i = 0; i < array.Length; i++ ) * { * int partIndex; * int itemIndexInPart = Math.DivRem( i, partSize - 1, out partIndex ); * * if ( partitions[ partIndex ].LanguageConfigs == null ) * { * partitions[ partIndex ].LanguageConfigs = new LanguageConfig[ partSize ]; * } * //Debug.Assert( part[ itemIndexInPart ] == null ); * var a = array[ i ]; * partitions[ partIndex ].LanguageConfigs[ itemIndexInPart ] = a.LanguageConfig; * partitions[ partIndex ].TotalModelFilenameLengths += a.ModelFilenameLength; * } * return (partitions); * }*/ private void ConsecutivelyLoadMMF(MModelConfig config) { _Dictionary = (0 < config.ModelDictionaryCapacity) ? new Dictionary <IntPtr, BucketValue>(config.ModelDictionaryCapacity, default(IntPtrEqualityComparer)) : new Dictionary <IntPtr, BucketValue>(default(IntPtrEqualityComparer)); var callback = new LoadModelFilenameContentCallback(ConsecutivelyLoadMMFCallback); foreach (var languageConfig in config.LanguageConfigs) { LanguageModelFileReaderMMF.Read(languageConfig, callback); } }
private void ParallelLoadMMF(MModelConfig config) { #region [.parallel load by partitions.] var processorCount = Environment.ProcessorCount; //var partitions = CreatePartitions( config.LanguageConfigs, processorCount ); var partitions = config.LanguageConfigs.SplitByPartitionCountOrGreater(processorCount); var unitBag = new ConcurrentBag <ParallelLoadUnit>(); Parallel.ForEach(partitions, new ParallelOptions() { MaxDegreeOfParallelism = processorCount }, () => default(ParallelLoadUnit), (partition, loopState, i, unit) => { const int EMPIRICALLY_CHOSEN_FUSKING_NUMBER = 27; var capacity = (int)(partition.TotalModelFilenameLengths / EMPIRICALLY_CHOSEN_FUSKING_NUMBER); unit.Initialize(capacity); foreach (var languageConfig in partition.LanguageConfigs) { LanguageModelFileReaderMMF.Read(languageConfig, unit.LoadMMFCallback); } return(unit); }, (unit) => { if (unit.Dictionary.Count != 0) { unitBag.Add(unit); } } ); #endregion #region [.merge.] var bucketVal = default(BucketValue); var dictionary = (0 < config.ModelDictionaryCapacity) ? new Dictionary <IntPtr, BucketValue>(config.ModelDictionaryCapacity, default(IntPtrEqualityComparer)) : new Dictionary <IntPtr, BucketValue>(default(IntPtrEqualityComparer)); foreach (var dict in unitBag.Select(unit => unit.Dictionary)) { foreach (var pair in dict) { var textPtr = pair.Key; var bucketValElse = pair.Value; if (dictionary.TryGetValue(textPtr, out bucketVal)) { var bucketRef = new BucketRef() { Language = bucketValElse.Language, Weight = bucketValElse.Weight, NextBucket = bucketValElse.NextBucket, }; if (bucketVal.NextBucket == null) { bucketVal.NextBucket = bucketRef; dictionary[textPtr] = bucketVal; } else { var br = bucketVal.NextBucket; for ( ; br.NextBucket != null; br = br.NextBucket) { ; } br.NextBucket = bucketRef; } } else { dictionary.Add(textPtr, bucketValElse); } } //dict.Clear(); //--- too slow => TODO: вытащить словарь/pull dictionary ---// } _Dictionary = dictionary; unitBag = null; #endregion }