private float GetTokenWeight(string tokenText)
{
float?weight = null;
if (this.Config.WeightedTokens?.Any() == true)
{
if (this.weightedTokens == null)
{
this.weightedTokens = new Dictionary <Regex, float>();
foreach (KeyValuePair <string, float> matchWeight in this.Config.WeightedTokens)
{
Regex regex = ClusteringConfig.PlatformRegexInit("^" + matchWeight.Key.TrimStart('^').TrimEnd('$') + "$", false, TimeSpan.FromMilliseconds(100));
this.weightedTokens[regex] = matchWeight.Value;
}
}
try
{
foreach (KeyValuePair <Regex, float> matchWeight in this.weightedTokens)
{
if (matchWeight.Key.IsMatch(tokenText))
{
if (weight.HasValue)
{
weight = Math.Max(weight.Value, matchWeight.Value);
}
else
{
weight = matchWeight.Value;
}
}
}
if (weight.HasValue)
{
weight *= (float)Math.Max(1, Math.Log(tokenText.Length));
}
}
catch (RegexMatchTimeoutException)
{
Debug.WriteLine($"'{tokenText}' token weight evaluation timed out, assigning weight=2.0");
weight = 2.0f;
}
}
return(weight.GetValueOrDefault(1));
}
public ClusteringVocabulary(ClusteringConfig config) : this()
{
this.Config = config;
}
public Clustering(ClusteringConfig config)
{
this.Config = config;
this.m_vocabulary = new ClusteringVocabulary(config);
}
protected ClusterBase(ClusteringConfig config)
{
this.Config = config;
}
