private int Compare(UncategorizedToken left, UncategorizedToken right) { if (left == null || right == null) { throw new InvalidOperationException("Failed to compare because the specified arguments contain null."); } return(String.Compare(left.Value, right.Value)); }
protected override IEnumerable <IToken> Tokenize(UncategorizedToken token) { Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName); var result = new List <IToken>(); var value = token.Value; var current = 0; var startIndex = 0; while (current < token.Value.Length) { Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex); if (value[current] == '\'' || value[current] == '"') { if (startIndex < current) { result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex))); Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value); } var endIndex = value.IndexOf(value[current], current + 1); if (endIndex != -1) { result.Add(BuildUpToken(value[current], value.Substring(current + 1, endIndex - (current + 1)))); current = endIndex + 1; startIndex = endIndex + 1; Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value); } else { result.Add(BuildUpToken(value[current], value.Substring(current + 1))); current = value.Length; startIndex = value.Length; Logger.Verbose("Extracted a quoted token '{0}'.", result.Last().Value); } } else { current++; } } if (startIndex < current) { Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex); result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex))); Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value); } return(result); }
protected override IEnumerable <IToken> Tokenize(UncategorizedToken token) { Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName); var result = new List <IToken>(); var value = token.Value; var current = 0; var startIndex = 0; while (current < value.Length) { Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex); if (char.IsWhiteSpace(value[current])) { if (startIndex < current) { var operand = value.Substring(startIndex, current - startIndex); var symbol = _mappings.Keys.FirstOrDefault(o => o.Equals(operand, StringComparison.OrdinalIgnoreCase)); if (symbol != null) { result.Add(_mappings[symbol]); Logger.Verbose("Extracted an operator token '{0}'.", symbol); } else { result.Add(new OperandToken(operand)); Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value); } } startIndex = current + 1; } current++; } if (startIndex < current) { Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex); result.Add(new OperandToken(value.Substring(startIndex, current - startIndex))); Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value); } return(result); }
protected override IEnumerable <IToken> Tokenize(UncategorizedToken token) { Logger.Info("Tokenizing the specified uncategorized token '{0}' with '{1}'.", token.Value, GetType().FullName); var result = new List <IToken>(); var value = token.Value; var current = 0; var startIndex = 0; while (current < value.Length) { Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex); foreach (var symbol in _mappings.Keys.Where(o => o.Length <= value.Length - current).OrderByDescending(o => o.Length)) { if (value.Substring(current, symbol.Length).Equals(symbol, StringComparison.OrdinalIgnoreCase)) { if (startIndex < current) { result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex))); Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value); } current += symbol.Length - 1; startIndex = current + 1; result.Add(_mappings[symbol]); Logger.Verbose("Extracted an operand token '{0}'.", result.Last().Value); break; } } current++; } if (startIndex < current) { Logger.Verbose("Progressing tokenization (current index = '{0}', start index = '{1}').", current, startIndex); result.Add(new UncategorizedToken(value.Substring(startIndex, current - startIndex))); Logger.Verbose("Extracted an uncategorized token '{0}'.", result.Last().Value); } return(result); }
protected abstract IEnumerable <IToken> Tokenize(UncategorizedToken token);
internal IEnumerable <IToken> InvokeTokenize(UncategorizedToken token) { return(base.Tokenize(token)); }
protected override IEnumerable <IToken> Tokenize(UncategorizedToken token) { PassedUncategorizedTokens.Add(token); return(new List <IToken>()); }