static string GetTokens(string sourceCode) { var sw = new Stopwatch(); sw.Start(); var lexer = new LexicalAnalyser(sourceCode); LinkedList <Token> tokens = lexer.Tokenize(); sw.Stop(); ElapsedTime = sw.Elapsed; return(tokens.Aggregate(string.Empty, (current, i) => current + $"Type: {i.Type}\n" + $"Value: {i.Value}\n\n")); }
public QueryOutFields(string outFieldsText) { outFieldsText = outFieldsText?.Trim(); if (String.IsNullOrEmpty(outFieldsText)) { // do nothing } else if (outFieldsText.Contains(";")) { // Parse, Lexer if (!outFieldsText.EndsWith(";")) { outFieldsText = $"{ outFieldsText };"; } var lexicalAnalyser = new LexicalAnalyser(new LuceneServerOutFieldsSyntax()); var tokens = lexicalAnalyser.Tokenize(outFieldsText); Fields = tokens.GetStatements() .Select(s => new QueryOutField(s)) .ToArray(); } else { // Simple Fields = outFieldsText.Split(',') .Select(f => f.Trim()) .Select(f => new QueryOutField(f)) .ToArray(); } _names = this.Fields?.Select(f => f.Name).ToArray() ?? new string[0]; if (this.Fields == null || Fields.Count() == 0) { this.Fields = new QueryOutField[] { new QueryOutField("*") } } ; }