public void AddToken(Hashtable targetTable, string nGram)
{
TokenStats stats = targetTable[nGram] as TokenStats;
if (stats != null)
{
stats.AddOccurence();
return;
}
// max 2 mio entries... then abort
if (targetTable.Count > 1000 * 1000)
return;
stats = new TokenStats(nGram);
targetTable.Add(nGram,stats);
}
private void BuildCharTable()
{
this.m_CharsetTable.Clear();
int rank = 0;
int pos = 0;
foreach (ITokenStats stats in this.Values)
{
if (stats.Token.Length == 1)
{
TokenStats newStats = new TokenStats(stats.Token);
newStats.Occurences = stats.Occurences;
newStats.Position = ++pos;
newStats.Rank = pos;
this.m_CharsetTable.Add(stats.Token,newStats);
if (this.m_CharsetTable.Count == 35)
break;
}
}
m_CharsetTable.Ranks = pos;
}
public void GetObjectData(SerializationInfo info, StreamingContext context)
{
info.AddValue("MaxTokens",this.m_MaxTokens);
info.AddValue("MaxWordLen",this.m_MaxWordLen);
info.AddValue("Ranks",this.m_Ranks);
info.AddValue("WordTable",this.m_WordTable);
String[] objArray1 = new String[this.Count];
TokenStats[] objArray2 = new TokenStats[this.Count];
int i=0;
foreach(String key in this.Keys)
{
objArray1[i]=key;
i++;
}
i=0;
foreach(TokenStats val in this.Values)
{
objArray2[i]=val;
i++;
}
info.AddValue("Keys", objArray1, typeof(String[]));
info.AddValue("Values", objArray2, typeof(TokenStats[]));
}
