private void Tokenize(TokenizeResultBase result, object value, Template template, string input)
{
log.Verbose($"Start: Processing: {template.Name}");
using (new LogIndentation())
{
var candidates = new CandidateTokenList();
var enumerator = new TokenEnumerator(input);
var replacement = new StringBuilder();
var matchIds = new HashSet <int>();
var disabledRepeatingTokens = new HashSet <int>();
var replacementLocation = new FileLocation();
var hintsMissing = FindHints(template, enumerator, result);
while (enumerator.IsEmpty == false && hintsMissing == false)
{
var next = enumerator.Peek();
// Handle Windows new lines (normalize to Unix)
if (next == "\r" && enumerator.Peek(1) == "\n")
{
enumerator.Next();
next = "\n";
}
// Check for repeated current token
if (candidates.Any && enumerator.Match(candidates.Preamble) && candidates.Preamble.Length > 0)
{
// Can't assign, so clear current context and move to next match
if (candidates.CanAnyAssign(replacement.ToString()) == false)
{
for (var i = 0; i < candidates.Tokens.Count; i++)
{
// If repeated token was the last match, then this non-match will stop it
// matching any future results
var token = candidates.Tokens[i];
if (WasLastMatchedToken(result, token) && string.IsNullOrWhiteSpace(token.Preamble) && string.IsNullOrWhiteSpace(replacement.ToString()))
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
using (new LogIndentation())
{
log.Verbose(("-> Disabled this repeating token."));
disabledRepeatingTokens.Add(token.Id);
candidates.Remove(token);
i--;
}
}
else if (token.ConsiderOnce)
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping & removing {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
candidates.Remove(token);
result.Tokens.AddMiss(token);
matchIds.Add(token.Id);
}
else
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
}
}
replacement.Clear();
enumerator.Advance(candidates.Preamble.Length);
replacementLocation = enumerator.Location;
continue;
}
}
// Assign newline terminated token
if (candidates.Any && candidates.TerminateOnNewLine && next == "\n")
{
if (candidates.Tokens.First().Repeating&&
string.IsNullOrWhiteSpace(candidates.Preamble) &&
result.Tokens.HasMatches)
{
if (result.Tokens.Matches.Last().Token.Id == candidates.Tokens.First().Id)
{
if (enumerator.Location.Line > result.Tokens.Matches.Last().Location.Line + 1)
{
disabledRepeatingTokens.Add(candidates.Tokens.First().Id);
candidates.Remove(candidates.Tokens.First());
}
}
}
using (new LogIndentation())
{
try
{
if (candidates.TryAssign(value, replacement, template.Options, replacementLocation, out var assigned, out var assignedValue))
{
result.Tokens.AddMatch(assigned, assignedValue, enumerator.Location);
AddMatchedTokenIds(template, assigned, matchIds);
}
else
{
foreach (var token in candidates.Tokens)
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
}
}
}
public bool TryParse <T>(Template template, string input, out int matches, out T result) where T : class, new()
{
log.Debug($"Start: Processing: {template.Name}");
Token current = null;
matches = 0;
var value = new T();
var enumerator = new TokenEnumerator(input);
var replacement = new StringBuilder();
var matchIds = new List <int>();
while (enumerator.IsEmpty == false)
{
var next = enumerator.Peek();
// Handle Windows new lines (normalize to Unix)
if (next == "\r" && enumerator.Peek(1) == "\n")
{
enumerator.Next();
next = "\n";
}
// Check for repeated current token
if (current != null && enumerator.Match(current.Preamble))
{
// Can't assign, so clear current context and move to next match
if (current.CanAssign(replacement.ToString()) == false)
{
replacement.Clear();
enumerator.Advance(current.Preamble.Length);
continue;
}
}
// Check for next token
if (enumerator.Match(template.TokensExcluding(matchIds), out var match))
{
if (current == null)
{
current = match;
replacement.Clear();
enumerator.Advance(match.Preamble.Length);
matchIds.AddRange(template.GetTokenIdsUpTo(match));
}
else if (replacement.Length > 0 && current.Assign(value, replacement.ToString(), template.Options, log))
{
matches++;
current = match;
replacement.Clear();
enumerator.Advance(match.Preamble.Length);
matchIds.AddRange(template.GetTokenIdsUpTo(match));
}
else
{
replacement.Append(next);
enumerator.Next();
}
}
// Append to replacement
else
{
replacement.Append(next);
enumerator.Next();
}
}
if (current != null && replacement.Length > 0 && !string.IsNullOrEmpty(current.Name))
{
current.Assign(value, replacement.ToString(), template.Options, log);
matches++;
}
result = value;
log.Debug($" Found {matches} matches.");
log.Debug($"Finished: Processing: {template.Name}");
return(matches > 0);
}
