Пример #1
0
        private void Tokenize(TokenizeResultBase result, object value, Template template, string input)
        {
            log.Verbose($"Start: Processing: {template.Name}");

            using (new LogIndentation())
            {
                var candidates = new CandidateTokenList();

                var enumerator              = new TokenEnumerator(input);
                var replacement             = new StringBuilder();
                var matchIds                = new HashSet <int>();
                var disabledRepeatingTokens = new HashSet <int>();
                var replacementLocation     = new FileLocation();

                var hintsMissing = FindHints(template, enumerator, result);

                while (enumerator.IsEmpty == false && hintsMissing == false)
                {
                    var next = enumerator.Peek();

                    // Handle Windows new lines (normalize to Unix)
                    if (next == "\r" && enumerator.Peek(1) == "\n")
                    {
                        enumerator.Next();
                        next = "\n";
                    }

                    // Check for repeated current token
                    if (candidates.Any && enumerator.Match(candidates.Preamble) && candidates.Preamble.Length > 0)
                    {
                        // Can't assign, so clear current context and move to next match
                        if (candidates.CanAnyAssign(replacement.ToString()) == false)
                        {
                            for (var i = 0; i < candidates.Tokens.Count; i++)
                            {
                                // If repeated token was the last match, then this non-match will stop it
                                // matching any future results
                                var token = candidates.Tokens[i];
                                if (WasLastMatchedToken(result, token) && string.IsNullOrWhiteSpace(token.Preamble) && string.IsNullOrWhiteSpace(replacement.ToString()))
                                {
                                    log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
                                    using (new LogIndentation())
                                    {
                                        log.Verbose(("-> Disabled this repeating token."));
                                        disabledRepeatingTokens.Add(token.Id);
                                        candidates.Remove(token);
                                        i--;
                                    }
                                }
                                else if (token.ConsiderOnce)
                                {
                                    log.Verbose("-> Ln: {0} Col: {1} : Skipping & removing {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());

                                    candidates.Remove(token);
                                    result.Tokens.AddMiss(token);
                                    matchIds.Add(token.Id);
                                }
                                else
                                {
                                    log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
                                }
                            }

                            replacement.Clear();
                            enumerator.Advance(candidates.Preamble.Length);
                            replacementLocation = enumerator.Location;
                            continue;
                        }
                    }

                    // Assign newline terminated token
                    if (candidates.Any && candidates.TerminateOnNewLine && next == "\n")
                    {
                        if (candidates.Tokens.First().Repeating&&
                            string.IsNullOrWhiteSpace(candidates.Preamble) &&
                            result.Tokens.HasMatches)
                        {
                            if (result.Tokens.Matches.Last().Token.Id == candidates.Tokens.First().Id)
                            {
                                if (enumerator.Location.Line > result.Tokens.Matches.Last().Location.Line + 1)
                                {
                                    disabledRepeatingTokens.Add(candidates.Tokens.First().Id);
                                    candidates.Remove(candidates.Tokens.First());
                                }
                            }
                        }

                        using (new LogIndentation())
                        {
                            try
                            {
                                if (candidates.TryAssign(value, replacement, template.Options, replacementLocation, out var assigned, out var assignedValue))
                                {
                                    result.Tokens.AddMatch(assigned, assignedValue, enumerator.Location);
                                    AddMatchedTokenIds(template, assigned, matchIds);
                                }
                                else
                                {
                                    foreach (var token in candidates.Tokens)
                                    {
                                        log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
                                    }
                                }
                            }
Пример #2
0
        public bool TryParse <T>(Template template, string input, out int matches, out T result) where T : class, new()
        {
            log.Debug($"Start: Processing: {template.Name}");

            Token current = null;

            matches = 0;

            var value       = new T();
            var enumerator  = new TokenEnumerator(input);
            var replacement = new StringBuilder();
            var matchIds    = new List <int>();

            while (enumerator.IsEmpty == false)
            {
                var next = enumerator.Peek();

                // Handle Windows new lines (normalize to Unix)
                if (next == "\r" && enumerator.Peek(1) == "\n")
                {
                    enumerator.Next();
                    next = "\n";
                }

                // Check for repeated current token
                if (current != null && enumerator.Match(current.Preamble))
                {
                    // Can't assign, so clear current context and move to next match
                    if (current.CanAssign(replacement.ToString()) == false)
                    {
                        replacement.Clear();
                        enumerator.Advance(current.Preamble.Length);
                        continue;
                    }
                }

                // Check for next token
                if (enumerator.Match(template.TokensExcluding(matchIds), out var match))
                {
                    if (current == null)
                    {
                        current = match;
                        replacement.Clear();
                        enumerator.Advance(match.Preamble.Length);
                        matchIds.AddRange(template.GetTokenIdsUpTo(match));
                    }
                    else if (replacement.Length > 0 && current.Assign(value, replacement.ToString(), template.Options, log))
                    {
                        matches++;
                        current = match;
                        replacement.Clear();
                        enumerator.Advance(match.Preamble.Length);
                        matchIds.AddRange(template.GetTokenIdsUpTo(match));
                    }
                    else
                    {
                        replacement.Append(next);
                        enumerator.Next();
                    }
                }

                // Append to replacement
                else
                {
                    replacement.Append(next);
                    enumerator.Next();
                }
            }

            if (current != null && replacement.Length > 0 && !string.IsNullOrEmpty(current.Name))
            {
                current.Assign(value, replacement.ToString(), template.Options, log);
                matches++;
            }

            result = value;

            log.Debug($"  Found {matches} matches.");
            log.Debug($"Finished: Processing: {template.Name}");

            return(matches > 0);
        }