public void Initialize(int capacity)
{
if (DictionaryNative == null)
{
Capacity = capacity;
DictionaryNative = new DictionaryNative(capacity);
LoadMMFCallback = new LoadModelFilenameContentCallback(DictionaryNative.AddNewOrToEndOfChain);
}
}
public void Initialize(int capacity)
{
if (Dictionary == null)
{
Capacity = capacity;
Dictionary = new Dictionary <IntPtr, BucketValue>(capacity, default(IntPtrEqualityComparer));
LoadMMFCallback = new LoadModelFilenameContentCallback(LoadMMFCallbackRoutine);
}
}
private void ConsecutivelyLoadMMF(MModelConfig config)
{
_Dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <string, BucketValue>(config.ModelDictionaryCapacity)
: new Dictionary <string, BucketValue>();
var callback = new LoadModelFilenameContentCallback(ConsecutivelyLoadMMFCallback);
foreach (var languageConfig in config.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, callback);
}
}
/*private static LanguageConfigPartition[] CreatePartitions( IEnumerable< LanguageConfig > languageConfigs, int processorCount )
* {
* var array = languageConfigs.Select( lc => new { LanguageConfig = lc, ModelFilenameLength = (new FileInfo( lc.ModelFilename )).Length } )
* .OrderByDescending( a => a.ModelFilenameLength )
* .ToArray();
* var partSize = (int) (1.0 * array.Length / processorCount + 0.5);
*
* var partitions = new LanguageConfigPartition[ processorCount ];
* for ( var i = 0; i < array.Length; i++ )
* {
* int partIndex;
* int itemIndexInPart = Math.DivRem( i, partSize - 1, out partIndex );
*
* if ( partitions[ partIndex ].LanguageConfigs == null )
* {
* partitions[ partIndex ].LanguageConfigs = new LanguageConfig[ partSize ];
* }
* //Debug.Assert( part[ itemIndexInPart ] == null );
* var a = array[ i ];
* partitions[ partIndex ].LanguageConfigs[ itemIndexInPart ] = a.LanguageConfig;
* partitions[ partIndex ].TotalModelFilenameLengths += a.ModelFilenameLength;
* }
* return (partitions);
* }*/
private void ConsecutivelyLoadMMF(MModelConfig config)
{
_Dictionary = (0 < config.ModelDictionaryCapacity)
? new Dictionary <IntPtr, BucketValue>(config.ModelDictionaryCapacity, default(IntPtrEqualityComparer))
: new Dictionary <IntPtr, BucketValue>(default(IntPtrEqualityComparer));
var callback = new LoadModelFilenameContentCallback(ConsecutivelyLoadMMFCallback);
foreach (var languageConfig in config.LanguageConfigs)
{
LanguageModelFileReaderMMF.Read(languageConfig, callback);
}
}
private void LoadModelFilenameContent(LoadModelFilenameContentCallback callbackAction)
{
using (var emmf = EnumeratorMMF.Create(ModelFilename))
{
var lineCount = 0;
var text = default(string);
var weight = default(float);
var pair = new Pair()
{
Language = this.Language
};
#region [.read first line.]
if (!emmf.MoveNext())
{
return;
}
#endregion
#region [.skip beginning comments.]
for ( ; ;)
{
#region [.check on comment.]
if (*emmf.Current.Start != '#')
{
break;
}
#endregion
#region [.move to next line.]
if (!emmf.MoveNext())
{
return;
}
#endregion
}
#endregion
#region [.read all lines.]
for ( ; ;)
{
lineCount++;
var ns = emmf.Current;
#region [.first-value in string.]
int startIndex_1 = 0;
int finishIndex_2 = ns.Length - 1;
#region commented
//skip starts white-spaces
/*for ( ; ; )
* {
* if ( ((_CTM[ ns.Start[ startIndex_1 ] ] & CharType.IsWhiteSpace) != CharType.IsWhiteSpace) ||
* (finishIndex_2 <= ++startIndex_1)
* )
* {
* break;
* }
* }*/
#endregion
//search '\t'
int startIndex_2 = 0;
int finishIndex_1 = 0;
for ( ; ;)
{
if (ns.Start[finishIndex_1] == '\t')
{
startIndex_2 = finishIndex_1 + 1;
finishIndex_1--;
break;
}
//not found '\t'
if (finishIndex_2 <= ++finishIndex_1)
{
throw (new InvalidDataException(string.Format(INVALIDDATAEXCEPTION_FORMAT_MESSAGE, ModelFilename, lineCount, ns.ToString())));
}
}
//skip ends white-spaces
for ( ; ;)
{
if (((_CTM[ns.Start[finishIndex_1]] & CharType.IsWhiteSpace) != CharType.IsWhiteSpace) ||
(--finishIndex_1 <= startIndex_1)
)
{
break;
}
}
if (finishIndex_1 < startIndex_1)
{
throw (new InvalidDataException(string.Format(INVALIDDATAEXCEPTION_FORMAT_MESSAGE, ModelFilename, lineCount, ns.ToString())));
}
#endregion
#region [.second-value in string.]
//skip starts white-spaces
for ( ; ;)
{
if (((_CTM[ns.Start[startIndex_2]] & CharType.IsWhiteSpace) != CharType.IsWhiteSpace) ||
(finishIndex_2 <= ++startIndex_2)
)
{
break;
}
}
#region commented
//skip ends white-spaces
/*for ( ; ; )
* {
* if ( ((_CTM[ ns.Start[ finishIndex_2 ] ] & CharType.IsWhiteSpace) != CharType.IsWhiteSpace) ||
* (--finishIndex_2 <= startIndex_1)
* )
* {
* break;
* }
* }*/
#endregion
#endregion
#region [.fill 'Pair_v1' & calling 'callbackAction()'.]
var len = (finishIndex_2 - startIndex_2) + 1;
text = StringsHelper.ToString(ns.Start + startIndex_2, len);
if (!float.TryParse(text, NS, NFI, out weight)) //if ( !Number.TryParseSingle( text, NS, NFI, out weight ) )
{
throw (new InvalidDataException(string.Format(INVALIDDATAEXCEPTION_FORMAT_MESSAGE, ModelFilename, lineCount, ns.ToString())));
}
len = (finishIndex_1 - startIndex_1) + 1;
text = StringsHelper.ToString(ns.Start + startIndex_1, len);
StringsHelper.ToUpperInvariantInPlace(text);
pair.Text = text;
pair.Weight = weight;
callbackAction(ref pair);
#endregion
#region [.move to next line.]
if (!emmf.MoveNext())
{
break;
}
#endregion
}
#endregion
}
}
public static void Read(LanguageConfig languageConfig, LoadModelFilenameContentCallback callbackAction)
{
var _this = new LanguageModelFileReaderMMF(languageConfig);
_this.LoadModelFilenameContent(callbackAction);
}
