public void TestAddToken()
{
var token = new Token("foo")
{
Preamble = "bar"
};
var list = new CandidateTokenList();
list.Add(token);
Assert.AreEqual(1, list.Count);
Assert.AreEqual("bar", list.Preamble);
}
private void Tokenize(TokenizeResultBase result, object value, Template template, string input)
{
log.Verbose($"Start: Processing: {template.Name}");
using (new LogIndentation())
{
var candidates = new CandidateTokenList();
var enumerator = new TokenEnumerator(input);
var replacement = new StringBuilder();
var matchIds = new HashSet <int>();
var disabledRepeatingTokens = new HashSet <int>();
var replacementLocation = new FileLocation();
var hintsMissing = FindHints(template, enumerator, result);
while (enumerator.IsEmpty == false && hintsMissing == false)
{
var next = enumerator.Peek();
// Handle Windows new lines (normalize to Unix)
if (next == "\r" && enumerator.Peek(1) == "\n")
{
enumerator.Next();
next = "\n";
}
// Check for repeated current token
if (candidates.Any && enumerator.Match(candidates.Preamble) && candidates.Preamble.Length > 0)
{
// Can't assign, so clear current context and move to next match
if (candidates.CanAnyAssign(replacement.ToString()) == false)
{
for (var i = 0; i < candidates.Tokens.Count; i++)
{
// If repeated token was the last match, then this non-match will stop it
// matching any future results
var token = candidates.Tokens[i];
if (WasLastMatchedToken(result, token) && string.IsNullOrWhiteSpace(token.Preamble) && string.IsNullOrWhiteSpace(replacement.ToString()))
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
using (new LogIndentation())
{
log.Verbose(("-> Disabled this repeating token."));
disabledRepeatingTokens.Add(token.Id);
candidates.Remove(token);
i--;
}
}
else if (token.ConsiderOnce)
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping & removing {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
candidates.Remove(token);
result.Tokens.AddMiss(token);
matchIds.Add(token.Id);
}
else
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
}
}
replacement.Clear();
enumerator.Advance(candidates.Preamble.Length);
replacementLocation = enumerator.Location;
continue;
}
}
// Assign newline terminated token
if (candidates.Any && candidates.TerminateOnNewLine && next == "\n")
{
if (candidates.Tokens.First().Repeating&&
string.IsNullOrWhiteSpace(candidates.Preamble) &&
result.Tokens.HasMatches)
{
if (result.Tokens.Matches.Last().Token.Id == candidates.Tokens.First().Id)
{
if (enumerator.Location.Line > result.Tokens.Matches.Last().Location.Line + 1)
{
disabledRepeatingTokens.Add(candidates.Tokens.First().Id);
candidates.Remove(candidates.Tokens.First());
}
}
}
using (new LogIndentation())
{
try
{
if (candidates.TryAssign(value, replacement, template.Options, replacementLocation, out var assigned, out var assignedValue))
{
result.Tokens.AddMatch(assigned, assignedValue, enumerator.Location);
AddMatchedTokenIds(template, assigned, matchIds);
}
else
{
foreach (var token in candidates.Tokens)
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
}
}
}
