public void Tokenize_IsInvalidZeroToken()
{
List <string> listValue = new List <string>();
listValue.Add(".00");
ITokenizer tokenizer = new RegexTokenizer();
IEnumerable <Token> tokens = tokenizer.Tokenize(listValue);
}
public TokenDefintion(string pattern)
{
Priority = -1;
Ignore = true;
var tokenizer = new RegexTokenizer();
var tokens = tokenizer.Tokenize(pattern);
var compiler = new RegexParser(tokens);
Regex = compiler.Parse();
}
/// <summary>
///
/// </summary>
/// <param name="tokenizer"></param>
/// <param name="sqlStatements"></param>
public void OutputTokenList(RegexTokenizer tokenizer, SqlStatement sqlStatement)
{
sqlStatement.tokenList.AddRange(tokenizer.Tokenize(sqlStatement.StatementValue));
sqlStatement.tokenList.ForEach(x => Console.WriteLine($"Token Type: {x.TokenType}, Token Value: {x.Value}"));
//var temp = parser.Parse(sqlStatement.tokenList);
OutputParseTree(sqlStatement);
Console.WriteLine($"End of statement.{Environment.NewLine}");
}
public TokenDefintion(TType type, string pattern, int priority)
{
Type = type;
Priority = priority;
Ignore = false;
var tokenizer = new RegexTokenizer();
var tokens = tokenizer.Tokenize(pattern);
var compiler = new RegexParser(tokens);
Regex = compiler.Parse();
}
public void CanTokenizeAndParseQueryString()
{
string query = @"
MATCH app = 'MyTestApp'
AND ex IN ('System.NullReferenceException', 'System.FormatException')
BETWEEN 2016-01-01 00:00:00 AND 2016-02-01 00:00:00
LIMIT 100
";
var tokenizer = new RegexTokenizer();
var tokenSequence = tokenizer.Tokenize(query).ToList();
var parser = new QueryParser();
var dataRepresentation = parser.Parse(tokenSequence);
var json = JsonConvert.SerializeObject(dataRepresentation, Formatting.Indented, SerializerSettings);
Assert.NotNull(json);
}
public void ValidateFile()
{
try
{
//Get all lines of the file as tokens
ITokenizer tokenizer = new RegexTokenizer();
var tokenSequence = tokenizer.Tokenize(_fileContent).ToList();
//Parse all tokens
Parser p = new Parser();
_expenses = p.Parse(tokenSequence);
if (_expenses.Count == 0)
{
throw new Exception("Nothing to Process");
}
}
catch (Exception)
{
throw;
}
}
