|
public static MakeTokens ( IList |
||
| strings | IList |
|
| return | IList |
|
internal static void ParseRules(IEnumerable <string> rules, SlowSynonymMap map, string mappingSep, string synSep, bool expansion, TokenizerFactory tokFactory)
{
int count = 0;
foreach (string rule in rules)
{
// To use regexes, we need an expression that specifies an odd number of chars.
// This can't really be done with string.split(), and since we need to
// do unescaping at some point anyway, we wouldn't be saving any effort
// by using regexes.
IList <string> mapping = SplitSmart(rule, mappingSep, false);
IList <IList <string> > source;
IList <IList <string> > target;
if (mapping.Count > 2)
{
throw new ArgumentException("Invalid Synonym Rule:" + rule);
}
else if (mapping.Count == 2)
{
source = GetSynList(mapping[0], synSep, tokFactory);
target = GetSynList(mapping[1], synSep, tokFactory);
}
else
{
source = GetSynList(mapping[0], synSep, tokFactory);
if (expansion)
{
// expand to all arguments
target = source;
}
else
{
// reduce to first argument
target = new List <IList <string> >(1)
{
source[0]
};
}
}
bool includeOrig = false;
foreach (IList <string> fromToks in source)
{
count++;
foreach (IList <string> toToks in target)
{
map.Add(fromToks, SlowSynonymMap.MakeTokens(toToks), includeOrig, true);
}
}
}
}
