private SqlWithCteNode ParseCte(ITokenizer t)
{
// "RECURSIVE"? <identifier> ("(" <columnList> ")")? "AS" "(" <QueryExpression> ")"
bool isRecursive = t.NextIs(SqlTokenType.Keyword, "RECURSIVE", true);
var name = t.Expect(SqlTokenType.Identifier);
var cteNode = new SqlWithCteNode
{
Location = name.Location,
Name = new SqlIdentifierNode(name),
Recursive = isRecursive
};
var lookahead = t.Peek();
if (lookahead.IsSymbol("("))
{
cteNode.ColumnNames = ParseParenthesis(t, x => ParseList(x, ParseIdentifier)).Expression;
}
t.Expect(SqlTokenType.Keyword, "AS");
// TODO: The CTE can contain INSERT, UPDATE and DELETE statements as well (usually with the RETURNING clause)
cteNode.Select = ParseParenthesis(t, ParseQueryExpression).Expression;
return(cteNode);
}
private ISqlNode ParseSelectColumnExpression(ITokenizer t)
{
var expr = ParseScalarExpression(t);
var lookahead = t.Peek();
if (lookahead.IsKeyword("OVER"))
{
// "OVER" "(" <OverPartitionBy>? <OverOrderBy>? <OverRows>? ")"
var overToken = t.GetNext();
t.Expect(SqlTokenType.Symbol, "(");
var overNode = new SqlOverNode
{
Location = overToken.Location,
Expression = expr,
PartitionBy = ParseOverPartitionBy(t),
OrderBy = ParseOverOrderBy(t),
RowsRange = ParseOverRows(t)
};
t.Expect(SqlTokenType.Symbol, ")");
return(overNode);
}
return(expr);
}
private ISqlNode ParseScalarExpression1(ITokenizer t)
{
// "NULL" | ("-" | "+" | "~") <Expression0> | <Expression0>
var next = t.Peek();
if (next.IsKeyword("NULL"))
{
var nullToken = t.GetNext();
return(new SqlNullNode(nullToken));
}
if (next.IsSymbol("-", "+", "~"))
{
var op = t.GetNext();
var expr = ParseScalarExpression1(t);
return(new SqlPrefixOperationNode
{
Location = op.Location,
Operator = new SqlOperatorNode(op),
Right = expr
});
}
return(ParseScalarExpression0(t));
}
/// <exception cref="Java.Util.NoSuchElementException"/>
private Tree GetTreeFromInputStream()
{
int wordIndex = 1;
// FSA
while (tokenizer.MoveNext())
{
string token = tokenizer.Current;
switch (token)
{
case leftParen:
{
// cdm 20100225: This next line used to have "" instead of null, but the traditional and current tree normalizers depend on the label being null not "" when there is no label on a tree (like the outermost English PTB level)
string label = (tokenizer.Peek().Equals(leftParen)) ? null : tokenizer.Current;
if (rightParen.Equals(label))
{
//Skip past empty trees
continue;
}
else
{
if (treeNormalizer != null)
{
label = treeNormalizer.NormalizeNonterminal(label);
}
}
if (label != null)
{
label = StarPattern.Matcher(label).ReplaceAll("*");
label = SlashPattern.Matcher(label).ReplaceAll("/");
}
Tree newTree = treeFactory.NewTreeNode(label, null);
// dtrs are added below
if (currentTree == null)
{
stack.Add(newTree);
}
else
{
currentTree.AddChild(newTree);
stack.Add(currentTree);
}
currentTree = newTree;
break;
}
case rightParen:
{
if (stack.IsEmpty())
{
// Warn that file has too many right parentheses
log.Info("PennTreeReader: warning: file has extra non-matching right parenthesis [ignored]");
goto label_break;
}
//Accept
currentTree = stack.Remove(stack.Count - 1);
// i.e., stack.pop()
if (stack.IsEmpty())
{
return(currentTree);
}
break;
}
default:
{
if (currentTree == null)
{
// A careful Reader should warn here, but it's kind of useful to
// suppress this because then the TreeReader doesn't print a ton of
// messages if there is a README file in a directory of Trees.
// log.info("PennTreeReader: warning: file has extra token not in a s-expression tree: " + token + " [ignored]");
goto label_break;
}
string terminal = (treeNormalizer == null) ? token : treeNormalizer.NormalizeTerminal(token);
terminal = StarPattern.Matcher(terminal).ReplaceAll("*");
terminal = SlashPattern.Matcher(terminal).ReplaceAll("/");
Tree leaf = treeFactory.NewLeaf(terminal);
if (leaf.Label() is IHasIndex)
{
IHasIndex hi = (IHasIndex)leaf.Label();
hi.SetIndex(wordIndex);
}
if (leaf.Label() is IHasWord)
{
IHasWord hw = (IHasWord)leaf.Label();
hw.SetWord(leaf.Label().Value());
}
if (leaf.Label() is IHasTag)
{
IHasTag ht = (IHasTag)leaf.Label();
ht.SetTag(currentTree.Label().Value());
}
wordIndex++;
currentTree.AddChild(leaf);
// cdm: Note: this implementation just isn't as efficient as the old recursive descent parser (see 2008 code), where all the daughters are gathered before the tree is made....
break;
}
}
label_continue :;
}
label_break :;
//Reject
if (currentTree != null)
{
log.Info("PennTreeReader: warning: incomplete tree (extra left parentheses in input): " + currentTree);
}
return(null);
}
/// <summary>
/// Reads a prefix-expression from the input.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="token">The token to append the total token onto.</param>
/// <returns>The expression that was read.</returns>
protected virtual IParseExp ReadPrefixExp(ITokenizer input, ref Token token)
{
Stack<UnaryInfo> ex = new Stack<UnaryInfo>();
IParseExp o = null;
Token last, debug = input.Peek();
debug.Value = "";
// check for unary operators
last = input.Peek();
while (last.Value == "-" || last.Value == "not" || last.Value == "#")
{
Read(input, ref debug);
if (last.Value == "-")
{
ex.Push(new UnaryInfo(1, last.StartPos, last.StartLine));
}
else if (last.Value == "not")
{
ex.Push(new UnaryInfo(2, last.StartPos, last.StartLine));
}
else
{
Contract.Assert(last.Value == "#");
ex.Push(new UnaryInfo(3, last.StartPos, last.StartLine));
}
last = input.Peek();
}
// check for literals
last = input.Peek();
int over = -1;
if (last.Value != null)
{
NumberFormatInfo ni = CultureInfo.CurrentCulture.NumberFormat;
if (last.Value != "..." && (char.IsNumber(last.Value, 0) || last.Value.StartsWith(ni.NumberDecimalSeparator, StringComparison.CurrentCulture)))
{
Read(input, ref debug); // read the number.
try
{
o = new LiteralItem(double.Parse(last.Value, CultureInfo.CurrentCulture)) { Debug = last };
}
catch (FormatException e)
{
throw new SyntaxException(Resources.BadNumberFormat, input.Name, last, e);
}
}
else if (last.Value.StartsWith("&", StringComparison.Ordinal))
{
Read(input, ref debug);
try
{
o = new LiteralItem(Convert.ToDouble(long.Parse(last.Value.Substring(1),
NumberStyles.AllowHexSpecifier, CultureInfo.CurrentCulture))) { Debug = last };
}
catch (FormatException e)
{
throw new SyntaxException(Resources.BadNumberFormat, input.Name, last, e);
}
}
else if (last.Value.StartsWith("\"", StringComparison.Ordinal))
{
Read(input, ref debug);
o = new LiteralItem(last.Value.Substring(1)) { Debug = last };
}
else if (last.Value.StartsWith("{", StringComparison.Ordinal))
{
o = ReadTable(input, ref debug);
}
else if (last.Value == "(")
{
Read(input, ref debug);
o = ReadExp(input, ref debug);
last = Read(input, ref debug);
if (last.Value != ")")
throw new SyntaxException(string.Format(Resources.TokenInvalidExpecting, last.Value, "expression", ")"),
input.Name, last);
}
else if (last.Value == "true")
{
Read(input, ref debug);
o = new LiteralItem(true) { Debug = last };
}
else if (last.Value == "false")
{
Read(input, ref debug);
o = new LiteralItem(false) { Debug = last };
}
else if (last.Value == "nil")
{
Read(input, ref debug);
o = new LiteralItem(null) { Debug = last };
}
else if (last.Value == "function")
{
o = ReadFunctionHelper(input, ref debug, false, false);
}
else
{
// allow for specifying overloads on global variables
if (last.Value.IndexOf('`') != -1)
{
if (!int.TryParse(last.Value.Substring(last.Value.IndexOf('`') + 1), out over))
throw new InvalidOperationException(Resources.OnlyNumbersInOverload);
last.Value = last.Value.Substring(0, last.Value.IndexOf('`'));
}
Read(input, ref debug);
o = new NameItem(last.Value) { Debug = last };
}
}
// read function calls and indexers
{
string inst = null;
bool cont = true;
while (cont)
{
last = input.Peek();
last.Value = last.Value ?? "";
if (last.Value == ".")
{
Read(input, ref debug);
if (over != -1)
throw new SyntaxException(Resources.FunctionCallAfterOverload, input.Name, last);
if (inst != null)
throw new SyntaxException(Resources.IndexerAfterInstance, input.Name, last);
last = Read(input, ref debug);
// allow for specifying an overload
if (last.Value.IndexOf('`') != -1)
{
if (!int.TryParse(last.Value.Substring(last.Value.IndexOf('`') + 1), out over))
throw new InvalidOperationException(Resources.OnlyNumbersInOverload);
last.Value = last.Value.Substring(0, last.Value.IndexOf('`'));
}
if (!IsName(last.Value))
throw new SyntaxException(string.Format(Resources.TokenNotAName, "indexer", last.Value),
input.Name, last);
if (!(o is IParsePrefixExp))
throw new SyntaxException(Resources.IndexAfterExpression, input.Name, last);
o = new IndexerItem(o, new LiteralItem(last.Value) { Debug = last }) { Debug = debug };
}
else if (last.Value == ":")
{
Read(input, ref debug);
if (over != -1)
throw new SyntaxException(Resources.FunctionCallAfterOverload, input.Name, last);
if (inst != null)
throw new SyntaxException(Resources.OneInstanceCall, input.Name, last);
inst = Read(input, ref debug).Value;
if (!IsName(inst))
throw new SyntaxException(string.Format(Resources.TokenNotAName, "indexer", last.Value),
input.Name, last);
}
else if (last.Value == "[")
{
Read(input, ref debug);
if (over != -1)
throw new SyntaxException(Resources.FunctionCallAfterOverload,
input.Name, last);
if (inst != null)
throw new SyntaxException(Resources.IndexerAfterInstance, input.Name, last);
var temp = ReadExp(input, ref debug);
last = Read(input, ref debug);
o = new IndexerItem(o, temp) { Debug = debug };
if (last.Value != "]")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, last.Value, "indexer", "]"),
input.Name, last);
}
else if (last.Value.StartsWith("\"", StringComparison.Ordinal))
{
Read(input, ref debug);
FuncCallItem temp = new FuncCallItem(o, inst, over) { Debug = debug };
o = temp;
temp.AddItem(new LiteralItem(last.Value.Substring(1)), false);
inst = null;
over = -1;
}
else if (last.Value == "{")
{
var temp = ReadTable(input, ref debug);
FuncCallItem func = new FuncCallItem(o, inst, over) { Debug = debug };
o = func;
func.AddItem(temp, false);
inst = null;
over = -1;
}
else if (last.Value == "(")
{
Read(input, ref debug);
FuncCallItem func = new FuncCallItem(o, inst, over);
o = func;
inst = null;
over = -1;
while (input.Peek().Value != ")" && input.Peek().Value != null)
{
bool? byRef = null;
if (input.Peek().Value == "@")
{
byRef = false;
Read(input, ref debug);
}
else if (input.Peek().Value == "ref")
{
Read(input, ref debug);
if (input.Peek().Value == "(")
{
Read(input, ref debug);
byRef = true;
}
else
byRef = false;
}
var temp = ReadExp(input, ref debug);
if (byRef != null && !(temp is NameItem) && !(temp is IndexerItem))
throw new SyntaxException(Resources.OnlyVarByReference, input.Name, last);
if (temp == null)
throw new SyntaxException(string.Format(Resources.InvalidDefinition, "function call"),
input.Name, last);
func.AddItem(temp, byRef != null);
if (byRef == true && (last = input.Read()).Value != ")")
throw new SyntaxException(Resources.RefOneArgument,
input.Name, last);
if (input.Peek().Value == ",")
Read(input, ref debug);
else if (input.Peek().Value == ")")
break;
else
throw new SyntaxException(string.Format(Resources.TokenInvalidExpecting2, input.Peek().Value, "function call", ",", ")"), input.Name, last);
}
if (input.Peek() == null)
throw new SyntaxException(string.Format(Resources.UnexpectedEOF, "function call"),
input.Name, last);
Read(input, ref debug);
func.Debug = debug;
}
else
{
if (inst != null)
throw new SyntaxException(Resources.InstanceMissingArgs, input.Name, last);
if (over != -1)
throw new SyntaxException(Resources.OverloadMissingArgs, input.Name, last);
cont = false;
}
}
}
// read exponents
// HACK: This is needed here because the power operator has
// higher precedence than the unary operators. Rather than
// have unary operators handled in ReadExp, they are handled
// so exponents need to be handled before we apply the
// unary operators.
if (input.Peek().Value == "^")
{
Read(input, ref debug);
var temp = ReadPrefixExp(input, ref debug);
BinOpItem item = new BinOpItem(o, BinaryOperationType.Power, temp) { Debug = debug };
o = item;
}
// now apply the unary operators
while (ex.Count > 0)
{
var loc = ex.Pop();
Token tok = new Token(debug.Value, loc.StartPos, debug.EndPos, loc.StartLine, debug.EndLine);
switch (loc.Version)
{
case 1: // neg
if (o is LiteralItem)
{
object oo = (o as LiteralItem).Value;
if (!(oo is double))
throw new SyntaxException(Resources.InvalidUnary,
input.Name, debug);
o = new LiteralItem(-(double)oo) { Debug = tok };
}
else
o = new UnOpItem(o, UnaryOperationType.Minus) { Debug = tok };
break;
case 2: // not
o = new UnOpItem(o, UnaryOperationType.Not) { Debug = tok };
break;
case 3: // len
o = new UnOpItem(o, UnaryOperationType.Length) { Debug = tok };
break;
}
}
// finaly return
token.Append(debug);
return o;
}
/// <summary>
/// Reads a function from the input. Input must either be on the word 'function' or
/// on the next token. If it is on 'function' and canName is true, it will give
/// the function the read name; otherwise it will give it a null name.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="token">The token to append the read Token to.</param>
/// <param name="canName">True if the function can have a name, otherwise false.</param>
/// <param name="local">True if this function is a local definition, otherwise false.</param>
/// <returns>The function definition that was read.</returns>
protected virtual FuncDefItem ReadFunctionHelper(ITokenizer input, ref Token token, bool canName, bool local)
{
IParseVariable name = null;
string inst = null;
Token last = input.Peek(), debug = last;
if (last.Value == "function")
{
input.Read(); // read 'function'
last = input.Peek();
if (IsName(last.Value))
{
Token nameTok = input.Read(); // read name
name = new NameItem(last.Value) { Debug = last };
// handle indexers
last = input.Peek();
while (last.Value == ".")
{
Read(input, ref nameTok); // read '.'
last = input.Peek();
if (!IsName(last.Value))
break;
name = new IndexerItem(name, new LiteralItem(last.Value) { Debug = last }) { Debug = nameTok };
Read(input, ref nameTok);
}
if (input.Peek().Value == ":")
{
Read(input, ref nameTok);
inst = Read(input, ref nameTok).Value;
if (!IsName(inst))
throw new SyntaxException(string.Format(Resources.TokenInvalid, last.Value, "function"),
input.Name, last);
}
debug.Append(nameTok);
}
}
if (name != null && !canName)
throw new SyntaxException(Resources.FunctionCantHaveName, input.Name, debug);
FuncDefItem ret = new FuncDefItem(name, local);
ret.InstanceName = inst;
last = Read(input, ref debug);
if (last.Value != "(")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, last.Value, "function", "("),
input.Name, last);
last = input.Peek();
while (last.Value != ")")
{
Token temp = Read(input, ref debug); // read the name
if (!IsName(last.Value) && last.Value != "...")
throw new SyntaxException(string.Format(Resources.TokenInvalid, last.Value, "function"),
input.Name, temp);
ret.AddArgument(new NameItem(last.Value) { Debug = last });
last = input.Peek();
if (last.Value == ",")
Read(input, ref debug);
else if (last.Value != ")")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting2, last.Value, "function", ",", ")"),
input.Name, last);
last = input.Peek();
}
if (last.Value != ")")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, last.Value, "function", ")"),
input.Name, last);
Read(input, ref debug); // read ')'
BlockItem chunk = ReadBlock(input, ref debug);
chunk.Return = chunk.Return ?? new ReturnItem();
ret.Block = chunk;
last = Read(input, ref debug);
if (last.Value != "end")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, last.Value, "function", "end"),
input.Name, last);
token.Append(debug);
ret.Debug = debug;
return ret;
}
private ISqlNode ParseScalarExpression0(ITokenizer t)
{
// Terminal expression
// <MethodCall> | <Identifier> | <Variable> | <String> | <Number> | "(" <Expression> ")"
var next = t.Peek();
if (next.IsKeyword("TARGET", "SOURCE"))
{
return(ParseQualifiedIdentifier(t));
}
if (next.IsKeyword("CAST", "CONVERT", "COUNT"))
{
var name = t.GetNext();
if (t.Peek().IsSymbol("("))
{
t.PutBack(name);
return(ParseFunctionCall(t));
}
throw ParsingException.CouldNotParseRule(nameof(ParseScalarExpression0), next);
}
if (next.IsType(SqlTokenType.Identifier))
{
var name = t.GetNext();
if (t.Peek().IsSymbol("("))
{
t.PutBack(name);
return(ParseFunctionCall(t));
}
t.PutBack(name);
return(ParseQualifiedIdentifier(t));
}
if (next.IsType(SqlTokenType.Variable))
{
return(new SqlVariableNode(t.GetNext()));
}
if (next.IsType(SqlTokenType.QuotedString))
{
return(new SqlStringNode(t.GetNext()));
}
if (next.IsType(SqlTokenType.Number))
{
return(new SqlNumberNode(t.GetNext()));
}
if (next.IsSymbol("("))
{
// "(" (<QueryExpression> | <ScalarExpression>) ")"
// e.g. SET @x = (select 5) or INSERT INTO x(num) VALUES (1, 2, (select 3))
var value = ParseParenthesis(t, x =>
{
if (x.Peek().IsKeyword("SELECT"))
{
return(ParseQueryExpression(t));
}
return(ParseScalarExpression(t));
});
if (value.Expression is SqlSelectNode)
{
return(value);
}
return(value.Expression);
}
throw ParsingException.CouldNotParseRule(nameof(ParseScalarExpression0), next);
}
public SqlExecuteArgumentNode ParseExecArgument(ITokenizer t)
{
// (<parameter> "=")? (<Expression> | "DEFAULT" | <Variable> ("OUT"|"OUTPUT"))
var next = t.GetNext();
if (next.IsSymbol(";") || next.IsType(SqlTokenType.EndOfInput))
{
return(null);
}
if (next.IsKeyword("DEFAULT"))
{
var keyword = new SqlKeywordNode(next);
return(new SqlExecuteArgumentNode
{
Location = keyword.Location,
Value = keyword
});
}
if (next.IsType(SqlTokenType.Variable))
{
var lookahead = t.Peek();
if (lookahead.IsSymbol("="))
{
t.GetNext();
return(new SqlExecuteArgumentNode
{
Location = next.Location,
AssignVariable = new SqlVariableNode(next),
Value = ParseScalarExpression(t),
IsOut = t.NextIs(SqlTokenType.Keyword, "OUT", true) || t.NextIs(SqlTokenType.Keyword, "OUTPUT", true)
});
}
}
ISqlNode expression = null;
if (next.IsType(SqlTokenType.Identifier, SqlTokenType.Number, SqlTokenType.QuotedString, SqlTokenType.Variable))
{
t.PutBack(next);
expression = ParseScalarExpression(t);
}
else if (next.IsType(SqlTokenType.Keyword))
{
if (t.Peek().IsSymbol("("))
{
t.PutBack(next);
expression = ParseScalarExpression(t);
}
}
if (expression == null)
{
t.PutBack(next);
return(null);
}
return(new SqlExecuteArgumentNode
{
Location = next.Location,
Value = expression,
IsOut = t.NextIs(SqlTokenType.Keyword, "OUT", true) || t.NextIs(SqlTokenType.Keyword, "OUTPUT", true)
});
}
private ISqlNode ParseUnterminatedStatement(ITokenizer t)
{
t.Skip(SqlTokenType.Whitespace);
var keyword = t.Peek();
if (keyword.Type == SqlTokenType.Keyword)
{
if (keyword.Value == "SELECT")
{
return(ParseQueryExpression(t));
}
if (keyword.Value == "WITH")
{
return(ParseWithStatement(t));
}
if (keyword.Value == "INSERT")
{
return(ParseInsertStatement(t));
}
if (keyword.Value == "UPDATE")
{
return(ParseUpdateStatement(t));
}
if (keyword.Value == "DELETE")
{
return(ParseDeleteStatement(t));
}
if (keyword.Value == "DECLARE")
{
return(ParseDeclare(t));
}
if (keyword.Value == "EXEC" || keyword.Value == "EXECUTE")
{
return(ParseExecute(t));
}
if (keyword.Value == "BEGIN")
{
return(ParseBeginEndStatementList(t));
}
if (keyword.Value == "IF")
{
return(ParseIf(t));
}
if (keyword.Value == "MERGE")
{
return(ParseMergeStatement(t));
}
}
if (keyword.Type == SqlTokenType.Identifier || keyword.Type == SqlTokenType.Variable)
{
var id = t.GetNext();
var op = t.Peek();
t.PutBack(id);
if (op.IsSymbol(":="))
{
return(ParseSet(t));
}
}
// TODO: RETURN?
// TODO: THROW/TRY/CATCH
// TODO: WHILE/CONTINUE/BREAK
// TODO: CREATE/DROP/ALTER? Do we want to handle DDL statments here?
return(null);
}
/// <summary>
/// Reads a for statement from the input.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="prev">The token to append what is read into.</param>
/// <returns>The object that was read.</returns>
protected virtual IParseStatement ReadFor(ITokenizer input, ref Token prev)
{
var debug = input.Read(); // read 'for'
if (debug.Value != "for")
throw new InvalidOperationException(string.Format(Resources.MustBeOn, "for", "ReadFor"));
// read a name
var name = Read(input, ref debug);
if (!IsName(name.Value))
throw new SyntaxException(
string.Format(Resources.TokenNotAName, "for", name.Value),
input.Name, name);
if (_reserved.Contains(name.Value))
throw new SyntaxException(
string.Format(Resources.TokenReserved, name.Value),
input.Name, name);
// numeric for
if (input.Peek().Value == "=")
{
var ret = ReadNumberFor(input, ref debug, name);
prev.Append(debug);
return ret;
}
// generic for statement
else
{
var ret = ReadGenericFor(input, ref debug, name);
prev.Append(debug);
return ret;
}
}
/// <summary>
/// Reads a return statement from the input.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="prev">The token to append what is read into.</param>
/// <returns>The object that was read.</returns>
protected virtual ReturnItem ReadReturn(ITokenizer input, ref Token prev)
{
var debug = input.Read(); // read 'return'
ReturnItem r = new ReturnItem();
if (debug.Value != "return")
throw new InvalidOperationException(string.Format(Resources.MustBeOn, "return", "ReadReturn"));
var name = input.Peek();
if (name.Value != "end" && name.Value != "until" && name.Value != "elseif" &&
name.Value != "else")
{
r.AddExpression(ReadExp(input, ref debug));
while (input.Peek().Value == ",")
{
Read(input, ref debug); // read ','
r.AddExpression(ReadExp(input, ref debug));
}
if (input.Peek().Value == ";")
{
Read(input, ref debug); // read ';'
}
// look at the next token for validation but keep it in the
// reader for the parrent.
name = input.Peek();
if (name.Value != "end" && name.Value != "until" && name.Value != "elseif" &&
name.Value != "else" && !IsNullOrWhiteSpace(name.Value))
throw new SyntaxException(
Resources.ReturnAtEnd,
input.Name, debug);
}
prev.Append(debug);
r.Debug = debug;
return r;
}
/// <summary>
/// Reads a class statement from the input.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="prev">The token to append what is read into.</param>
/// <returns>The object that was read.</returns>
protected virtual IParseStatement ReadClass(ITokenizer input, ref Token prev)
{
var debug = input.Read(); // read 'class'
string sname = null;
List<string> imp = new List<string>();
if (debug.Value != "class")
throw new InvalidOperationException(string.Format(Resources.MustBeOn, "class", "ReadClass"));
if (input.Peek().Value.StartsWith("'", StringComparison.Ordinal) ||
input.Peek().Value.StartsWith("\"", StringComparison.Ordinal))
{
var name = Read(input, ref debug);
sname = name.Value.Substring(1);
if (input.Peek().Value == "(")
{
Read(input, ref debug); // read '('
while (input.Peek().Value != ")")
{
// read the name
name = Read(input, ref debug);
if (!IsName(name.Value))
throw new SyntaxException(
string.Format(Resources.TokenNotAName, "class", name.Value),
input.Name, name);
imp.Add(name.Value);
// read ','
name = Read(input, ref debug);
if (name.Value != ",")
throw new SyntaxException(
string.Format(Resources.TokenInvalid, name.Value, "class"),
input.Name, name);
}
Read(input, ref debug); // read ')'
}
}
else
{
var name = Read(input, ref debug);
sname = name.Value;
if (!IsName(sname))
throw new SyntaxException(
string.Format(Resources.TokenNotAName, "class", name.Value),
input.Name, name);
if (input.Peek().Value == ":")
{
do
{
// simply include the '.' in the name.
string n = "";
do
{
Read(input, ref debug); // read ':' or ','
n += (n == "" ? "" : ".") + Read(input, ref debug).Value;
} while (input.Peek().Value == ".");
imp.Add(n);
} while (input.Peek().Value == ",");
}
}
prev.Append(debug);
return new ClassDefItem(sname, imp.ToArray()) { Debug = debug };
}
/// <summary>
/// Reads a local statement from the input.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="prev">The token to append what is read into.</param>
/// <returns>The object that was read.</returns>
protected virtual IParseStatement ReadLocal(ITokenizer input, ref Token prev)
{
var debug = input.Read(); // read 'local'
if (debug.Value != "local")
throw new InvalidOperationException(string.Format(Resources.MustBeOn, "local", "ReadLocal"));
var name = input.Peek();
if (name.Value == "function")
{
prev.Append(debug);
return ReadFunctionHelper(input, ref prev, true, true);
}
else
{
Read(input, ref debug); // read name
if (!IsName(name.Value))
throw new SyntaxException(
string.Format(Resources.TokenNotAName, "local", name.Value),
input.Name, name);
if (_reserved.Contains(name.Value))
throw new SyntaxException(
string.Format(Resources.TokenReserved, name.Value),
input.Name, name);
var i = ReadAssignment(input, ref debug, true, new NameItem(name.Value) { Debug = name });
prev.Append(debug);
return i;
}
}
/// <summary>
/// Reads a block of code from the input. Any end tokens
/// should not be read and are handled by the parrent call
/// (e.g. 'end' or 'until').
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="prev">The token to append the total token onto.</param>
/// <returns>The item that was read.</returns>
protected virtual BlockItem ReadBlock(ITokenizer input, ref Token prev)
{
BlockItem ret = new BlockItem();
Token total = input.Peek();
total.Value = "";
Token name;
while ((name = input.Peek()).Value != null)
{
if (Functions.ContainsKey(name.Value))
{
var temp = Functions[name.Value](input, ref total);
ret.AddItem(temp);
}
else if (name.Value == "return")
{
ret.Return = ReadReturn(input, ref total);
return ret;
}
else if (name.Value == ";")
{
Read(input, ref total); // read ';'
}
else if (name.Value == "end" || name.Value == "else" || name.Value == "elseif" ||
name.Value == "until")
{
// don'type read as it will be handled by the parrent
prev.Append(total); // don't add 'end' to the prev
ret.Debug = total; // or the current block, this
return ret; // end belongs to the parrent.
}
else
{
Token debug = name;
debug.Value = "";
var exp = ReadExp(input, ref debug);
if (exp is FuncCallItem)
{
(exp as FuncCallItem).Statement = true;
ret.AddItem((FuncCallItem)exp);
}
else if (exp is LiteralItem)
{
throw new SyntaxException(
"A literal is not a variable.",
input.Name, debug);
}
else if (exp is NameItem || exp is IndexerItem)
{
var i = ReadAssignment(input, ref debug, false, (IParseVariable)exp);
ret.AddItem(i);
}
else
throw new SyntaxException(
string.Format(Resources.TokenStatement, name.Value),
input.Name, debug);
total.Append(debug);
}
} // end While
// only gets here if this is the global function
ret.Debug = total;
ret.Return = ret.Return ?? new ReturnItem();
return ret;
}
/// <summary>
/// Reads a table from the input. Input must be either on the starting '{'.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="token">The token to append the read Tokenm to.</param>
/// <returns>The table that was read.</returns>
protected virtual TableItem ReadTable(ITokenizer input, ref Token token)
{
Token debug = input.Read();
if (debug.Value != "{")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, debug.Value, "table", "{"),
input.Name, debug);
TableItem ret = new TableItem();
Token last = input.Peek();
while (last.Value != "}")
{
if (last.Value == "[")
{
Read(input, ref debug); // read the "["
var temp = ReadExp(input, ref debug);
if (temp == null)
throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
input.Name, debug);
// read ']'
last = Read(input, ref debug);
if (last.Value != "]")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, last.Value, "table", "]"),
input.Name, last);
// read '='
last = Read(input, ref debug);
if (last.Value != "=")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, last.Value, "table", "="),
input.Name, last);
// read the expression
var val = ReadExp(input, ref debug);
if (val == null)
throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
input.Name, debug);
ret.AddItem(temp, val);
}
else
{
var val = ReadExp(input, ref debug);
if (input.Peek().Value == "=")
{
Read(input, ref debug); // read '='
NameItem name = val as NameItem;
if (name == null)
throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
input.Name, debug);
// read the expression
var exp = ReadExp(input, ref debug);
ret.AddItem(new LiteralItem(name.Name), exp);
}
else
{
ret.AddItem(null, val);
}
}
if (input.Peek().Value != "," && input.Peek().Value != ";")
break;
else
Read(input, ref debug);
last = input.Peek();
} // end While
Token end = Read(input, ref debug); // read the "}"
if (end.Value != "}")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, end.Value, "table", "}"),
input.Name, end);
ret.Debug = debug;
token.Append(debug);
return ret;
}
private ISqlNode ParseBooleanExpression1(ITokenizer t)
{
// <ScalarExpression> <ComparisonOperator> <ScalarExpression>
// <ScalarExpression> "IS" "NOT"? "NULL"
// <ScalarExpression> "NOT"? "BETWEEN" <ScalarExpression> "AND" <ScalarExpression>
// <ScalarExpression> "NOT"? "IN" "(" <ValueList> ")"
// <ScalarExpression> "NOT"? "LIKE" <String>
var left = ParseScalarExpression(t);
var operatorToken = t.Peek();
if (operatorToken.IsSymbol(">", "<", "=", "<=", ">=", "!=", "<>"))
{
t.GetNext();
if (t.Peek().IsKeyword("ALL", "ANY", "SOME"))
{
var prefix = t.GetNext();
var query = ParseParenthesis(t, ParseQueryExpression);
return(new SqlInfixOperationNode
{
Location = operatorToken.Location,
Left = left,
Right = query,
Operator = new SqlOperatorNode(operatorToken.Value + " " + prefix.Value)
});
}
var right = ParseScalarExpression(t);
return(new SqlInfixOperationNode
{
Location = operatorToken.Location,
Left = left,
Right = right,
Operator = new SqlOperatorNode(operatorToken)
});
}
if (operatorToken.IsKeyword("IS"))
{
t.GetNext();
var not = t.NextIs(SqlTokenType.Keyword, "NOT", true);
var nullToken = t.Expect(SqlTokenType.Keyword, "NULL");
return(new SqlInfixOperationNode
{
Location = operatorToken.Location,
Left = left,
Operator = new SqlOperatorNode
{
Location = operatorToken.Location,
Operator = not ? "IS NOT" : "IS"
},
Right = new SqlNullNode(nullToken)
});
}
var isNot = false;
if (operatorToken.IsKeyword("NOT"))
{
t.GetNext();
isNot = true;
operatorToken = t.Peek();
}
if (operatorToken.IsKeyword("BETWEEN"))
{
t.GetNext();
var first = ParseScalarExpression(t);
t.Expect(SqlTokenType.Keyword, "AND");
var second = ParseScalarExpression(t);
return(new SqlBetweenOperationNode
{
Location = operatorToken.Location,
Not = isNot,
Left = left,
Low = first,
High = second
});
}
if (operatorToken.IsKeyword("IN"))
{
t.GetNext();
// TODO: "IN" "(" <QueryExpression> ")"
var list = ParseParenthesis(t, t2 => ParseList(t2, ParseVariableOrConstant)).Expression;
return(new SqlInNode
{
Not = isNot,
Search = left,
Location = operatorToken.Location,
Items = list
});
}
if (operatorToken.IsKeyword("LIKE"))
{
t.GetNext();
var right = ParseMaybeParenthesis(t, ParseString);
return(new SqlInfixOperationNode
{
Left = left,
Operator = new SqlOperatorNode
{
Location = operatorToken.Location,
Operator = (isNot ? "NOT " : "") + "LIKE"
},
Right = right
});
}
// There's no operator, it's just a scalar expression in a boolean context.
// This happens in CASE statements, for example. Return the scalar expression
// as-is and let the AST deal with it.
return(left);
}
/// <summary>
/// Reads an if statement from the input.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="prev">The token to append what is read into.</param>
/// <returns>The object that was read.</returns>
protected virtual IParseStatement ReadIf(ITokenizer input, ref Token prev)
{
var debug = input.Read(); // read 'if'
IfItem i = new IfItem();
if (debug.Value != "if")
throw new InvalidOperationException(string.Format(Resources.MustBeOn, "if", "ReadIf"));
// read the initial expression
i.Exp = ReadExp(input, ref debug);
// read 'then'
var name = Read(input, ref debug);
if (name.Value != "then")
throw new SyntaxException(
string.Format(Resources.TokenInvalid, name.Value, "if"),
input.Name, debug);
// read the block
var readBlock = ReadBlock(input, ref debug);
i.Block = readBlock;
// handle elseif(s)
while ((name = input.Peek()).Value == "elseif")
{
Read(input, ref debug); // read 'elseif'
// read the expression
var readExp = ReadExp(input, ref debug);
// read 'then'
name = Read(input, ref debug);
if (name.Value != "then")
throw new SyntaxException(
string.Format(Resources.TokenInvalid, name.Value, "elseif"),
input.Name, debug);
// read the block
readBlock = ReadBlock(input, ref debug);
i.AddElse(readExp, readBlock);
}
// handle else
if (name.Value != "else" && name.Value != "end")
throw new SyntaxException(
string.Format(Resources.TokenInvalid, name.Value, "if"),
input.Name, debug);
if (name.Value == "else")
{
Read(input, ref debug); // read 'else'
// read the block
readBlock = ReadBlock(input, ref debug);
i.ElseBlock = readBlock;
}
// read 'end'
name = Read(input, ref debug);
if (name.Value != "end")
throw new SyntaxException(
string.Format(Resources.TokenInvalid, name.Value, "if"),
input.Name, debug);
prev.Append(debug);
i.Debug = debug;
return i;
}
private Tree GetTreeFromInputStream()
{
int wordIndex = 0;
// FSA
//label:
while (tokenizer.HasNext())
{
string token = tokenizer.Next();
switch (token)
{
case LeftParen:
// cdm 20100225: This next line used to have "" instead of null, but the traditional and current tree normalizers depend on the label being null not "" when there is no label on a tree (like the outermost English PTB level)
string label = (tokenizer.Peek().Equals(LeftParen)) ? null : tokenizer.Next();
if (RightParen.Equals(label))
{
//Skip past empty trees
continue;
}
else if (treeNormalizer != null)
{
label = treeNormalizer.NormalizeNonterminal(label);
}
if (label != null)
{
label = StarPattern.Replace(label, "*");
label = SlashPattern.Replace(label, "/");
}
Tree newTree = treeFactory.NewTreeNode(label, null); // dtrs are added below
if (currentTree == null)
{
stack.Add(newTree);
}
else
{
currentTree.AddChild(newTree);
stack.Add(currentTree);
}
currentTree = newTree;
break;
case RightParen:
if (!stack.Any())
{
// Warn that file has too many right parens
//break label;
goto post_while_label;
}
//Accept
currentTree = stack.Last();
stack.RemoveAt(stack.Count - 1); // i.e., stack.pop()
if (!stack.Any())
{
return(currentTree);
}
break;
default:
if (currentTree == null)
{
// A careful Reader should warn here, but it's kind of useful to
// suppress this because then the TreeReader doesn't print a ton of
// messages if there is a README file in a directory of Trees.
//break label;
goto post_while_label;
}
string terminal = (treeNormalizer == null) ? token : treeNormalizer.NormalizeTerminal(token);
terminal = StarPattern.Replace(terminal, "*");
terminal = SlashPattern.Replace(terminal, "/");
Tree leaf = treeFactory.NewLeaf(terminal);
if (leaf.Label() is IHasIndex)
{
var hi = (IHasIndex)leaf.Label();
hi.SetIndex(wordIndex);
}
if (leaf.Label() is IHasWord)
{
var hw = (IHasWord)leaf.Label();
hw.SetWord(leaf.Label().Value());
}
wordIndex++;
currentTree.AddChild(leaf);
// cdm: Note: this implementation just isn't as efficient as the old recursive descent parser (see 2008 code), where all the daughters are gathered before the tree is made....
break;
}
}
post_while_label:
{
}
//Reject
return(null);
}
/// <summary>
/// Reads an assignment statement from the input. The input is currently
/// after the first name, on the comma or equal sign. The debug token
/// contains the name and should contain the entire statement.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="debug">Currently contains the first name, and after
/// should contain the entire statement.</param>
/// <param name="local">True if this is a local definition, otherwise false.</param>
/// <param name="variable">The first variable that was read.</param>
/// <returns>The statement that was read.</returns>
protected virtual AssignmentItem ReadAssignment(ITokenizer input, ref Token debug, bool local, IParseVariable variable)
{
// read each of the variable names
AssignmentItem assign = new AssignmentItem(local);
assign.AddName(variable);
while (input.Peek().Value == ",")
{
Read(input, ref debug); // read ','
// read the left-hand-expression
var exp = ReadExp(input, ref debug);
if ((local && !(exp is NameItem)) || (!local && !(exp is IParseVariable)))
throw new SyntaxException(Resources.NameOrExpForVar, input.Name, debug);
assign.AddName((IParseVariable)exp);
}
// read the initial values
if (input.Peek().Value == "=")
{
Read(input, ref debug); // read '='
assign.AddItem(ReadExp(input, ref debug));
while (input.Peek().Value == ",")
{
Read(input, ref debug); // read ','
assign.AddItem(ReadExp(input, ref debug));
}
}
else if (!local)
throw new SyntaxException(
string.Format(Resources.InvalidDefinition, "assignment"),
input.Name, debug);
assign.Debug = debug;
return assign;
}
/// <summary>
/// Reads part of a generic for loop from the input. The input is
/// currently on the token after the first name and debug contains
/// the parts read for the 'for' loop. 'name' contains the name of
/// the first variable.
/// </summary>
/// <param name="input">Where to read input from/</param>
/// <param name="debug">The token that currently holds what was read
/// so far in the for statement and should after contain the entire loop.</param>
/// <param name="name">The token that contains the name of the variable.</param>
/// <returns>The loop object that was read.</returns>
protected virtual ForGenItem ReadGenericFor(ITokenizer input, ref Token debug, Token name)
{
// read the variables
List<NameItem> names = new List<NameItem>();
names.Add(new NameItem(name.Value) { Debug = name });
while (input.Peek().Value == ",")
{
Read(input, ref debug); // read ','
// read the name
name = Read(input, ref debug);
if (!IsName(name.Value))
throw new SyntaxException(
string.Format(Resources.TokenNotAName, "for", name.Value),
input.Name, name);
if (_reserved.Contains(name.Value))
throw new SyntaxException(
string.Format(Resources.TokenReserved, name.Value),
input.Name, name);
names.Add(new NameItem(name.Value) { Debug = name });
}
// check for 'in'
name = Read(input, ref debug);
if (name.Value != "in")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "in"),
input.Name, name);
// read the expression-list
ForGenItem f = new ForGenItem(names);
f.AddExpression(ReadExp(input, ref debug));
while (input.Peek().Value == ",")
{
Read(input, ref debug); // read ","
f.AddExpression(ReadExp(input, ref debug));
}
// check for 'do'
name = Read(input, ref debug);
if (name.Value != "do")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "do"),
input.Name, name);
// read the chunk
f.Block = ReadBlock(input, ref debug);
// read 'end'
name = Read(input, ref debug);
if (name.Value != "end")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "end"),
input.Name, name);
f.Debug = debug;
return f;
}
/// <summary>
/// Reads part of a numerical for loop from the input. The input is
/// currently on the equals sign '=' and the debug token currently
/// contains the parts read for the 'for' loop. 'name' contains the
/// name of the variable.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="debug">The token that currently holds what was read
/// so far in the for statement and should after contain the entire loop.</param>
/// <param name="name">The token that contains the name of the variable.</param>
/// <returns>The loop object that was read.</returns>
protected virtual ForNumItem ReadNumberFor(ITokenizer input, ref Token debug, Token name)
{
// read "="
var temp = Read(input, ref debug);
if (temp.Value != "=")
throw new InvalidOperationException(string.Format(Resources.MustBeOn, "=", "ReadNumberFor"));
// get the 'start' value
var start = ReadExp(input, ref debug);
// read ','
temp = Read(input, ref debug);
if (temp.Value != ",")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, temp.Value, "for", ","),
input.Name, temp);
// get the 'limit'
var limit = ReadExp(input, ref debug);
// read ','
IParseExp step = null;
if (input.Peek().Value == ",")
{
Read(input, ref debug);
// read the 'step'
step = ReadExp(input, ref debug);
}
ForNumItem i = new ForNumItem(new NameItem(name.Value) { Debug = name }, start, limit, step);
// check for 'do'
name = Read(input, ref debug);
if (name.Value != "do")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "do"),
input.Name, name);
// read the block
i.Block = ReadBlock(input, ref debug);
// read 'end'
name = Read(input, ref debug);
if (name.Value != "end")
throw new SyntaxException(
string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "end"),
input.Name, name);
i.Debug = debug;
return i;
}
/// <summary>
/// Read parse trees from a Reader.
/// </summary>
/// <param name="input">The Reader</param>
/// <param name="tf">TreeFactory -- factory to create some kind of Tree</param>
/// <param name="tn">the method of normalizing trees</param>
/// <param name="st">Tokenizer that divides up Reader</param>
public PennTreeReader(TextReader input, ITreeFactory tf, TreeNormalizer tn, ITokenizer<string> st)
{
reader = input;
treeFactory = tf;
treeNormalizer = tn;
tokenizer = st;
// check for whacked out headers still present in Brown corpus in Treebank 3
string first = (st.HasNext() ? st.Peek() : null);
if (first != null && first.StartsWith("*x*x*x"))
{
int foundCount = 0;
while (foundCount < 4 && st.HasNext())
{
first = st.Next();
if (first != null && first.StartsWith("*x*x*x"))
{
foundCount++;
}
}
}
}
/// <summary>
/// Reads an expression from the input and returns the
/// item that was read.
/// </summary>
/// <param name="input">Where to read input from.</param>
/// <param name="precedence">The precedence of the previous expression
/// or -1 if a root.</param>
/// <param name="token">The Token that represents the entire expression
/// should be appended to this variable.</param>
/// <returns>The expression that was read.</returns>
protected virtual IParseExp ReadExp(ITokenizer input, ref Token token, int precedence = -1)
{
Token debug = input.Peek();
debug.Value = "";
IParseExp cur = ReadPrefixExp(input, ref debug);
BinOpItem ret = null;
start:
Token last = input.Peek();
BinaryOperationType type = GetOperationType(last.Value);
int nPrec = GetPrecedence(type);
if (nPrec != -1 && (precedence == -1 || precedence > nPrec))
{
Read(input, ref debug); // read the exp
var temp = ReadExp(input, ref debug, nPrec);
ret = new BinOpItem(ret ?? cur, type, temp);
ret.Debug = debug;
goto start;
}
token.Append(debug);
return ret ?? cur;
}