public void TestAddToken() { var token = new Token("foo") { Preamble = "bar" }; var list = new CandidateTokenList(); list.Add(token); Assert.AreEqual(1, list.Count); Assert.AreEqual("bar", list.Preamble); }
private void Tokenize(TokenizeResultBase result, object value, Template template, string input) { log.Verbose($"Start: Processing: {template.Name}"); using (new LogIndentation()) { var candidates = new CandidateTokenList(); var enumerator = new TokenEnumerator(input); var replacement = new StringBuilder(); var matchIds = new HashSet <int>(); var disabledRepeatingTokens = new HashSet <int>(); var replacementLocation = new FileLocation(); var hintsMissing = FindHints(template, enumerator, result); while (enumerator.IsEmpty == false && hintsMissing == false) { var next = enumerator.Peek(); // Handle Windows new lines (normalize to Unix) if (next == "\r" && enumerator.Peek(1) == "\n") { enumerator.Next(); next = "\n"; } // Check for repeated current token if (candidates.Any && enumerator.Match(candidates.Preamble) && candidates.Preamble.Length > 0) { // Can't assign, so clear current context and move to next match if (candidates.CanAnyAssign(replacement.ToString()) == false) { for (var i = 0; i < candidates.Tokens.Count; i++) { // If repeated token was the last match, then this non-match will stop it // matching any future results var token = candidates.Tokens[i]; if (WasLastMatchedToken(result, token) && string.IsNullOrWhiteSpace(token.Preamble) && string.IsNullOrWhiteSpace(replacement.ToString())) { log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString()); using (new LogIndentation()) { log.Verbose(("-> Disabled this repeating token.")); disabledRepeatingTokens.Add(token.Id); candidates.Remove(token); i--; } } else if (token.ConsiderOnce) { log.Verbose("-> Ln: {0} Col: {1} : Skipping & removing {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString()); candidates.Remove(token); result.Tokens.AddMiss(token); matchIds.Add(token.Id); } else { log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString()); } } replacement.Clear(); enumerator.Advance(candidates.Preamble.Length); replacementLocation = enumerator.Location; continue; } } // Assign newline terminated token if (candidates.Any && candidates.TerminateOnNewLine && next == "\n") { if (candidates.Tokens.First().Repeating&& string.IsNullOrWhiteSpace(candidates.Preamble) && result.Tokens.HasMatches) { if (result.Tokens.Matches.Last().Token.Id == candidates.Tokens.First().Id) { if (enumerator.Location.Line > result.Tokens.Matches.Last().Location.Line + 1) { disabledRepeatingTokens.Add(candidates.Tokens.First().Id); candidates.Remove(candidates.Tokens.First()); } } } using (new LogIndentation()) { try { if (candidates.TryAssign(value, replacement, template.Options, replacementLocation, out var assigned, out var assignedValue)) { result.Tokens.AddMatch(assigned, assignedValue, enumerator.Location); AddMatchedTokenIds(template, assigned, matchIds); } else { foreach (var token in candidates.Tokens) { log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString()); } } }