private Ast.SqlStmt ParseSqlStmt(TokenQueue q, string rootProdName = "sql-stmt")
{
var start = q.GetLocation();
var tok = q.SourceToken;
Ast.SqliteSyntaxProduction syntaxNode;
var result = SqliteParser.ReadStmt(q, out syntaxNode, rootProdName);
if (result.IsValid)
{
var stmt = new Ast.SqlStmt {
Sql = q.Substring(start, result.NumValidTokens),
SourceToken = tok,
SqliteSyntax = syntaxNode
};
_preprocessor.PreprocessStmt(stmt);
return(stmt);
}
else if (result.InvalidMessage != null)
{
throw new SyntaxException(result.InvalidMessage);
}
else
{
throw new SyntaxException(q);
}
}
private Ast.Expr ParseExpr(TokenQueue q)
{
var start = q.GetLocation();
var tok = q.SourceToken;
Ast.SqliteSyntaxProduction syntaxNode;
var result = SqliteParser.ReadExpr(q, out syntaxNode);
if (result.IsValid)
{
return(new Ast.Expr {
Sql = q.Substring(start, result.NumValidTokens),
SourceToken = tok,
SqliteSyntax = syntaxNode
});
}
else if (result.InvalidMessage != null)
{
throw new SyntaxException(result.InvalidMessage);
}
else
{
throw new SyntaxException(q);
}
}
private Ast.ImportTxtStmt ParseImportTxtStmt(TokenQueue q) // or null
{
var stmt = new Ast.ImportTxtStmt {
SourceToken = q.SourceToken
};
var start = q.GetLocation();
if (!q.TakeMaybe("import"))
{
q.Jump(start); return(null);
}
if (!q.TakeMaybe("txt", "text"))
{
q.Jump(start); return(null);
}
stmt.FilenameExpr = ParseExpr(q);
q.Take("into");
stmt.TableName = Check(q, ParseIdentifierOrExpr(q));
if (q.Peek() == "(")
{
q.Take("(");
stmt.LineNumberColumnName = Check(q, ParseIdentifierOrExpr(q));
q.Take(",");
stmt.TextColumnName = Check(q, ParseIdentifierOrExpr(q));
q.Take(")");
}
if (q.TakeMaybe("options"))
{
stmt.OptionsList = Check(q, ParseOptionsList(q));
}
ConsumeSemicolon(q);
return(stmt);
}
private Ast.ExportTxtStmt ParseExportTxtStmt(TokenQueue q) // or null
{
var stmt = new Ast.ExportTxtStmt {
SourceToken = q.SourceToken
};
var start = q.GetLocation();
if (!q.TakeMaybe("export"))
{
q.Jump(start); return(null);
}
if (!q.TakeMaybe("txt", "text"))
{
q.Jump(start); return(null);
}
stmt.FilenameExpr = ParseExpr(q);
q.Take("from");
q.Take("(");
stmt.SelectStmt = ParseSqlStmt(q, "select-stmt");
q.Take(")");
if (q.TakeMaybe("options"))
{
stmt.OptionsList = Check(q, ParseOptionsList(q));
}
ConsumeSemicolon(q);
return(stmt);
}
private Ast.ImportXlsStmt ParseImportXlsStmt(TokenQueue q) // or null
{
var stmt = new Ast.ImportXlsStmt {
SourceToken = q.SourceToken
};
var start = q.GetLocation();
if (!q.TakeMaybe("import"))
{
q.Jump(start); return(null);
}
if (!q.TakeMaybe("xls", "xlsx"))
{
q.Jump(start); return(null);
}
stmt.FilenameExpr = ParseExpr(q);
if (q.TakeMaybe("worksheet"))
{
stmt.WhichSheetExpr = ParseExpr(q);
}
q.Take("into");
stmt.ImportTable = Check(q, ParseImportTable(q));
if (q.TakeMaybe("options"))
{
stmt.OptionsList = Check(q, ParseOptionsList(q));
}
ConsumeSemicolon(q);
return(stmt);
}
private bool PeekExpr(TokenQueue q)
{
var start = q.GetLocation();
var tok = q.SourceToken;
Ast.SqliteSyntaxProduction ast;
var result = SqliteParser.ReadExpr(q, out ast);
q.Jump(start);
return(result.IsValid);
}
public override MatchResult?MatchStep(MatchStack stack, MatchFrame frame, TokenQueue q)
{
if (frame.OrState == OrTermState.Start)
{
// try to match the first sub-production
stack.Push(Prods[0]);
frame.OrState = OrTermState.Match;
frame.OrProdIndex = 0;
frame.OrStartLoc = q.GetLocation();
return(null);
}
else if (frame.OrState == OrTermState.Match)
{
// we have finished matching one of the productions. if it matched, then we're done. if not, move on
// to the next production.
var result = frame.SubResult;
if (result.IsMatch)
{
return(MatchResult.Matched);
}
else if (result.ErrorMessage == null)
{
// no match. rewind to the beginning and retry with the next one.
q.Jump(frame.OrStartLoc);
frame.OrProdIndex++;
if (frame.OrProdIndex >= Prods.Length)
{
// we have exhausted all of the possibilities and none of them matched.
return(MatchResult.NoMatch);
}
stack.Push(Prods[frame.OrProdIndex]);
return(null);
}
else
{
// started to match but mismatched past the point of no return.
return(result);
}
}
else
{
throw new Exception($"Unrecognized state: {frame.OrState}");
}
}
public override MatchResult?MatchStep(MatchStack stack, MatchFrame frame, TokenQueue q)
{
if (frame.OptionalState == OptionalTermState.Start)
{
// try to match the sub-production
stack.Push(Prod);
frame.OptionalState = OptionalTermState.Match;
frame.OptionalStartLoc = q.GetLocation();
return(null);
}
else if (frame.OptionalState == OptionalTermState.Match)
{
// done matching the sub-production
var result = frame.SubResult;
if (result.IsMatch)
{
// the optional term is indeed present.
return(MatchResult.Matched);
}
else if (result.ErrorMessage == null)
{
// it didn't match but wasn't an error. this is fine, but we do have to walk the cursor back to
// where it started since we effectively "matched" zero tokens.
q.Jump(frame.OptionalStartLoc);
return(MatchResult.Matched);
}
else
{
// it started to match but then mismatched past the point of no return. that's an error.
return(result);
}
}
else
{
throw new Exception($"Unrecognized state: {frame.OptionalState}");
}
}
public static MatchResult Match(string rootProdName, TokenQueue q, out Ast.SqliteSyntaxProduction ast)
{
// we use an explicit stack rather than function call recursion because our BNF grammar is deeply nested,
// particularly the productions for 'expr'.
var stack = new MatchStack {
Queue = q
};
stack.Push(SqliteGrammar.Prods[rootProdName]);
MatchResult?rootResult = null;
Ast.SqliteSyntaxProduction rootAst = null;
Action <MatchResult, Ast.SqliteSyntaxProduction> finishFrame = (frameResult, frameAstProd) => {
stack.Pop();
var parentFrame = stack.Peek();
if (parentFrame == null)
{
rootResult = frameResult;
rootAst = frameAstProd;
}
else
{
parentFrame.SubResult = frameResult;
if (frameResult.IsMatch)
{
parentFrame.AstProd.Items.Add(frameAstProd);
}
}
};
#if MATCHER_LOG
var matcherLogWriter = File.CreateText(@"C:\temp\matcher.log");
int matcherLogPreviousDepth = 0;
#endif
// trampoline loop
while (!rootResult.HasValue && stack.Any())
{
#if MATCHER_LOG
stack.DebugDump(matcherLogWriter, q.GetLocation(), q.Substring(q.GetLocation(), 1),
matcherLogPreviousDepth > stack.Count);
matcherLogPreviousDepth = stack.Count;
#endif
var frame = stack.Peek();
var result = frame.Prod.Terms[frame.TermIndex].MatchStep(stack, frame, q);
if (result.HasValue)
{
// we are done matching this term
if (result.Value.IsMatch)
{
// move to the next term in the production.
frame.Clear(all: false);
frame.TermIndex++;
if (frame.TermIndex >= frame.Prod.Terms.Length)
{
// we have matched this full production
var prodEndLoc = q.GetLocation();
frame.AstProd.StartToken = frame.ProdStartLoc;
frame.AstProd.NumTokens = prodEndLoc - frame.ProdStartLoc;
frame.AstProd.Text = q.Substring(frame.ProdStartLoc, prodEndLoc - frame.ProdStartLoc);
finishFrame(MatchResult.Matched, frame.AstProd);
}
}
else
{
// we needed a match and didn't find one. we have to abandon this production.
finishFrame(result.Value, null);
}
}
}
#if MATCHER_LOG
matcherLogWriter.Close();
#endif
if (!rootResult.HasValue && !stack.Any()) // detect bugs
{
throw new Exception("Expected a MatchResult but one was not set.");
}
ast = rootAst;
return(rootResult.Value);
}
public override MatchResult?MatchStep(MatchStack stack, MatchFrame frame, TokenQueue q)
{
if (frame.ListState == ListTermState.Start)
{
frame.ListState = ListTermState.MatchItem;
stack.Push(ItemProd);
return(null); // -> MatchItem
}
else if (frame.ListState == ListTermState.MatchSeparator)
{
var result = frame.SubResult;
if (result.IsMatch)
{
// we have a separator. now try to match the item following it.
frame.ListState = ListTermState.MatchItem;
stack.Push(ItemProd);
return(null); // -> MatchItem
}
else if (result.ErrorMessage == null)
{
// we didn't find a separator. this list is done. back up to the beginning of where
// the not-separator started and we're done.
q.Jump(frame.ListSeparatorStartLoc);
if (frame.ListCount < Min)
{
return(MatchResult.Error(
$"At least {Min} list item{(Min == 1 ? " is" : "s are")} required, but only " +
$"{frame.ListCount} {(frame.ListCount == 1 ? "was" : "were")} provided. " +
$"Expected list item: {ItemProd.GetExpected()}"));
}
else
{
return(MatchResult.Matched);
}
}
else
{
return(result); // error
}
}
else if (frame.ListState == ListTermState.MatchItem)
{
var result = frame.SubResult;
if (result.IsMatch)
{
// we have an item. is there another?
frame.ListCount++;
if (SeparatorProd == null)
{
// there is no separator, so match the next item
frame.ListState = ListTermState.MatchItem;
frame.ListSeparatorStartLoc = q.GetLocation();
stack.Push(ItemProd);
return(null); // -> MatchItem
}
else
{
// match separator + item
frame.ListState = ListTermState.MatchSeparator;
frame.ListSeparatorStartLoc = q.GetLocation();
stack.Push(SeparatorProd);
return(null); // -> MatchSeparator
}
}
else if (result.ErrorMessage == null)
{
if (frame.ListCount == 0)
{
// the first item might be missing because the list can potentially be optional.
return(Min == 0 ? MatchResult.Matched : MatchResult.NoMatch);
}
else if (SeparatorProd == null)
{
// there's no separator, so eventually we'll end up here when the list ends.
q.Jump(frame.ListSeparatorStartLoc);
if (frame.ListCount < Min)
{
return(MatchResult.Error(
$"At least {Min} list item{(Min == 1 ? " is" : "s are")} required, but only " +
$"{frame.ListCount} {(frame.ListCount == 1 ? "was" : "were")} provided. " +
$"Expected list item: {ItemProd.GetExpected()}"));
}
else
{
return(MatchResult.Matched);
}
}
else
{
// subsequent items must be present because, in the MatchItem state, we've already consumed a
// separator so there must be an item following it.
return(MatchResult.Error($"Expected list item: {ItemProd.GetExpected()}"));
}
}
else
{
return(result); // error
}
}
else
{
throw new Exception($"Unrecognized state: {frame.ListState}");
}
}