public List <Token> GetTokens(IPatternProducer p)
{
var tokenStrings = parser.PreprocessAndSplit(p.ToString());
return(tokenStrings.Select(tokenString => new Token(tokenString, GetTokenType(tokenString)))
.ToList());
}
public static Pattern ReplaceDollarsWithCandidates(string sequence, IList <IPatternProducer> candidates,
IPatternProducer underscoreReplacement, IDictionary <string, IPatternProducer> specialReplacers,
string inputSeparator, string outputSeparator, string finalThingToAppend)
{
var buddy = new StringBuilder();
// Understand the contents of the special replacer map.
// Specifically, find the longest-length item in the map.
int maxSpecialReplacerKeyLength = 0;
if (specialReplacers != null)
{
foreach (string replacerKey in specialReplacers.Keys)
{
if (replacerKey.Length > maxSpecialReplacerKeyLength)
{
maxSpecialReplacerKeyLength = replacerKey.Length;
}
}
}
// Split the input into individual elements and parse each one.
foreach (var elementWithSeparator in SplitElementsWithSeparators(sequence))
{
string element = elementWithSeparator.Element;
string innerSeparator = elementWithSeparator.Separator.ToString();
string distributionNeeded = null;
string appender = "";
bool replacementFound = false;
if (element.StartsWith("$"))
{
int indexForAppender = element.Length;
for (int len = Math.Min(maxSpecialReplacerKeyLength + 1, element.Length); len > 1 && !replacementFound; len--)
{
string selectionString = element.Substring(1, len - 1);
if (specialReplacers.ContainsKey(selectionString))
{
distributionNeeded = specialReplacers[selectionString].ToString();
indexForAppender = len;
replacementFound = true;
}
}
if (!replacementFound)
{
if (element[1] >= '0' && element[1] <= '9')
{
int index = int.Parse(element.Substring(1, 1));
distributionNeeded = candidates[index].ToString();
indexForAppender = 2;
replacementFound = true;
}
else if (element[1] == '!')
{
distributionNeeded = underscoreReplacement.ToString();
indexForAppender = 2;
replacementFound = true;
}
}
if (indexForAppender < element.Length)
{
appender = element.Substring(indexForAppender, element.Length - indexForAppender);
replacementFound = true;
}
}
else
{
distributionNeeded = element;
replacementFound = true;
}
if (replacementFound)
{
if (finalThingToAppend != null)
{
appender = appender + finalThingToAppend;
}
Distribute(buddy, distributionNeeded, appender);
}
buddy.Append(innerSeparator.Equals(",") ? " " : innerSeparator);
}
buddy.Remove(buddy.Length - 1, 1);
return(new Pattern(buddy.ToString()));
}
public string Preprocess(IPatternProducer producer)
{
return(Preprocess(producer.ToString()));
}
