internal static ParsedStrongAlias Parse(ParseSession parser) { if (!parser.CheckToken(0, "type")) { return(null); } var nametoken = parser.PeekToken(1); if (nametoken == null) { return(null); } if (!parser.CheckToken(2, ":")) { return(null); } parser.ConsumeTokens(4); var aliastype = new StrongAlias { }; return(new ParsedStrongAlias { Name = nametoken, Type = aliastype }); }
internal static Expression Parse(ParseSession parser, int starttoken, out int consumedtokens) { int totaltokens = starttoken; consumedtokens = totaltokens; bool matchedstatement = false; if (!ParseExpressionTerm(parser, true, totaltokens, out totaltokens, out matchedstatement)) { return(null); } if (matchedstatement && parser.CheckToken(totaltokens, ")")) { } else { while (ParseExpressionOperator(parser, totaltokens)) { ++totaltokens; if (!ParseExpressionTerm(parser, false, totaltokens, out totaltokens, out matchedstatement)) { return(null); } } } consumedtokens = totaltokens; return(new Expression()); }
// // Helper routine for parsing a function return from a token stream. // // Passes back the index of the next token to examine. Returns null if // no function return is provided or if there is a syntax error. // internal static FunctionReturn Parse(ParseSession parser, int starttoken, out int consumedtokens) { consumedtokens = starttoken; int totaltokens = starttoken; if (!parser.CheckToken(totaltokens, "->")) { return(null); } ++totaltokens; var variable = Variable.Parse(parser, totaltokens, out totaltokens, Variable.Origins.Return); if (variable == null) { var expr = Expression.Parse(parser, totaltokens, out totaltokens); if (expr == null) { return(null); } consumedtokens = totaltokens; return(new FunctionReturn { ReturnExpression = expr }); } consumedtokens = totaltokens; return(new FunctionReturn { ReturnVariable = variable }); }
private static Variable ParseFunctionReturn(ParseSession parser, int starttoken, out int consumedtokens) { consumedtokens = starttoken; int totaltokens = starttoken; if (!parser.CheckToken(totaltokens, "->")) { return(null); } ++totaltokens; var variable = Variable.Parse(parser, totaltokens, out totaltokens, Variable.Origins.Return); if (variable == null) { var expr = Expression.Parse(parser, totaltokens, out totaltokens); if (expr == null) { return(null); } // TODO - return something sane for expressions consumedtokens = totaltokens; return(null); } consumedtokens = totaltokens; return(variable); }
private static List <FunctionOverload.Tag> ParseFunctionTags(ParseSession parser, int starttoken, out int consumedtokens) { consumedtokens = starttoken; int totaltokens = starttoken; if (!parser.CheckToken(totaltokens, "[")) { return(null); } ++totaltokens; var ret = new List <FunctionOverload.Tag>(); while (!parser.CheckToken(totaltokens, "]")) { var tag = new FunctionOverload.Tag { Name = parser.PeekToken(totaltokens) }; if (parser.CheckToken(totaltokens + 1, "(")) { totaltokens += 2; tag.Params = new List <FunctionOverload.Tag.TagParam>(); var p = new FunctionOverload.Tag.TagParam { AssociatedTokens = new List <Token>() }; while (!parser.CheckToken(totaltokens, ")")) { p.AssociatedTokens.Add(parser.PeekToken(totaltokens)); ++totaltokens; if (parser.CheckToken(totaltokens, ",")) { tag.Params.Add(p); ++totaltokens; p = new FunctionOverload.Tag.TagParam { AssociatedTokens = new List <Token>() }; } } } ++totaltokens; ret.Add(tag); if (parser.CheckToken(totaltokens, ",")) { ++totaltokens; } } ++totaltokens; consumedtokens = totaltokens; return(ret); }
private void AugmentProject(Project project, string filecontents) { project.RegisterSourceFile(FileFullPath, this); var lexer = new LexSession(this, filecontents); var parser = new ParseSession(lexer); parser.AugmentProject(project); }
private static bool ParseAssignment(ParseSession parser, int starttoken, out int consumedtokens) { consumedtokens = starttoken; int totaltokens = starttoken; ++totaltokens; while (parser.CheckToken(totaltokens, ".")) { totaltokens += 2; } if (!parser.CheckToken(totaltokens, "=") && !parser.CheckToken(totaltokens, "+=") && !parser.CheckToken(totaltokens, "-=")) { return(false); } var assignmenttoken = parser.PeekToken(totaltokens); ++totaltokens; bool haschain = true; while (haschain) { while (parser.CheckToken(totaltokens + 1, ".")) { totaltokens += 2; } if (parser.CheckToken(totaltokens + 1, "=")) { if (assignmenttoken.Text != "=") { return(false); } ++totaltokens; } else { haschain = false; } } var expr = Expression.Parse(parser, totaltokens, out totaltokens); if (expr == null) { return(false); } consumedtokens = totaltokens; return(true); }
internal static Variable Parse(ParseSession parser, int starttoken, out int consumedtokens, Origins origin) { consumedtokens = starttoken; if (parser.CheckToken(starttoken + 1, ".")) { return(null); } var basetypename = parser.PeekToken(starttoken); int totaltokens = starttoken + 1; if (parser.CheckToken(totaltokens, "<")) { ++totaltokens; if (!parser.ParseTemplateArguments(totaltokens, basetypename, out totaltokens)) { return(null); } } var varname = parser.PeekToken(totaltokens); if (!parser.CheckToken(totaltokens + 1, "=")) { return(null); } var type = TypeSignatureInstantiated.Construct(parser, starttoken, totaltokens); var variable = new Variable { Name = varname, Origin = origin, Type = type }; totaltokens += 2; Expression.Parse(parser, totaltokens, out totaltokens); while (parser.CheckToken(totaltokens, ",")) { ++totaltokens; Expression.Parse(parser, totaltokens, out totaltokens); } consumedtokens = totaltokens; return(variable); }
public static TypeSignature Construct(ParseSession parser, int begintoken, int endtoken) { var signature = new TypeSignature(); signature.TypeName = parser.PeekToken(begintoken).Text; if (parser.CheckToken(begintoken + 1, "<")) { signature.TypeParameters = new List <TypeParameter>(); int totaltokens = begintoken + 2; while (!parser.CheckToken(totaltokens, ">")) { var tokentype = parser.PeekToken(totaltokens); var tokenname = parser.PeekToken(totaltokens + 1); var param = new TypeParameter { Name = tokenname.Text, ArgumentType = new TypeSignature { TypeName = tokentype.Text } }; signature.TypeParameters.Add(param); totaltokens += 2; if (parser.CheckToken(totaltokens, ",")) { ++totaltokens; } } } int reftoken = endtoken - 1; if (reftoken > begintoken) { signature.TypeIsReference = parser.CheckToken(reftoken, "ref"); } return(signature); }
public static TypeSignatureInstantiated Construct(ParseSession parser, int begintoken, int endtoken) { var signature = new TypeSignatureInstantiated(); signature.TypeName = parser.PeekToken(begintoken).Text; if (parser.CheckToken(begintoken + 1, "<")) { signature.TypeArguments = new List <TypeArgument>(); int totaltokens = begintoken + 2; while (!parser.CheckToken(totaltokens, ">")) { var argtoken = parser.PeekToken(totaltokens); signature.TypeArguments.Add(new TypeArgument { SpecifiedType = new TypeSignatureInstantiated { TypeName = argtoken.Text } }); ++totaltokens; if (parser.CheckToken(totaltokens, ",")) { ++totaltokens; } } } int reftoken = endtoken - 1; if (reftoken > begintoken) { signature.TypeIsReference = parser.CheckToken(reftoken, "ref"); } return(signature); }
internal static GlobalBlock Parse(ParseSession parser) { if (!parser.CheckToken(0, "global")) { return(null); } if (!parser.CheckToken(1, "{")) { return(null); } var block = new GlobalBlock(); int totaltokens = 2; do { var variable = Variable.Parse(parser, totaltokens, out totaltokens, Variable.Origins.Global); if (variable != null) { block.Variables.Add(variable); } else { return(null); } parser.ConsumeTokens(totaltokens); totaltokens = 0; } while (!parser.CheckToken(totaltokens, "}")); ++totaltokens; parser.ConsumeTokens(totaltokens); return(block); }
// // Helper routine to parse a lexical scope in its entirety given a token stream. // internal static LexicalScope Parse(ParseSession parser, LexicalScope parentscope, int starttoken, out int consumedtokens) { consumedtokens = starttoken; int totaltokens = starttoken; Token afterStartBrace = parser.PeekToken(totaltokens); if (afterStartBrace == null) { parser.ConsumeTokens(starttoken); throw new SyntaxError("Missing closing }", parser.ReversePeekToken()); } var ret = new LexicalScope(); ret.File = afterStartBrace.File; ret.StartLine = afterStartBrace.Line; ret.StartColumn = afterStartBrace.Column; ret.ParentScope = parentscope; if (parentscope != null) { if (parentscope.ChildScopes == null) { parentscope.ChildScopes = new List <LexicalScope>(); } parentscope.ChildScopes.Add(ret); } while (!parser.CheckToken(totaltokens, "}")) { if (CodeHelpers.ParseEntity(parser, ret, totaltokens, out totaltokens) || parser.ParsePreopStatement(totaltokens, out totaltokens) || parser.ParsePostopStatement(totaltokens, out totaltokens) || parser.ParseStatement(totaltokens, out totaltokens)) { parser.ConsumeTokens(totaltokens); totaltokens = 0; continue; } var variable = Variable.Parse(parser, totaltokens, out totaltokens, Variable.Origins.Local); if (variable != null) { ret.Variables.Add(variable); parser.ConsumeTokens(totaltokens); totaltokens = 0; continue; } if (CodeHelpers.ParseAssignment(parser, totaltokens, out totaltokens)) { parser.ConsumeTokens(totaltokens); totaltokens = 0; continue; } consumedtokens = totaltokens; return(null); } Token endBrace = parser.PeekToken(totaltokens); ++totaltokens; consumedtokens = totaltokens; ret.EndLine = endBrace.Line; ret.EndColumn = endBrace.Column; return(ret); }
private static List <FunctionOverload.Parameter> ParseFunctionParams(ParseSession parser, int starttoken, out int consumedtokens) { var ret = new List <FunctionOverload.Parameter>(); consumedtokens = starttoken; int totaltokens = starttoken; while (!parser.CheckToken(totaltokens, "{") && !parser.CheckToken(totaltokens, "->")) { if (parser.CheckToken(totaltokens, "nothing")) { var signature = new TypeSignatureInstantiated(); ret.Add(new FunctionOverload.Parameter { Name = parser.PeekToken(totaltokens), Type = signature }); ++totaltokens; } else if (parser.CheckToken(totaltokens, "(")) { ++totaltokens; var higherordername = parser.PeekToken(totaltokens); if (parser.CheckToken(totaltokens + 1, ":")) { totaltokens += 2; bool moreparams = true; if (parser.CheckToken(totaltokens, ")")) { moreparams = false; } while (moreparams) { ++totaltokens; if (parser.CheckToken(totaltokens, "ref")) { ++totaltokens; } if (!parser.CheckToken(totaltokens, ",")) { break; } ++totaltokens; } } if (parser.CheckToken(totaltokens, "->")) { totaltokens += 2; } if (!parser.CheckToken(totaltokens, ")")) { return(null); } ++totaltokens; var signature = new TypeSignatureInstantiated(); // TODO - implement higher order function signatures ret.Add(new FunctionOverload.Parameter { Name = higherordername, Type = signature }); } else if (IsLiteralFunctionParam(parser.PeekToken(totaltokens))) { // TODO - better literal support var signature = new TypeSignatureInstantiated(); ret.Add(new FunctionOverload.Parameter { Name = parser.PeekToken(totaltokens), Type = signature }); ++totaltokens; } else { int begintoken = totaltokens; if (parser.CheckToken(totaltokens + 1, "<")) { var paramtype = parser.PeekToken(totaltokens); totaltokens += 2; if (!parser.ParseTemplateArguments(totaltokens, paramtype, out totaltokens)) { return(null); } } else { ++totaltokens; } var paramname = parser.PeekToken(totaltokens); if (parser.CheckToken(totaltokens, "ref")) { ++totaltokens; paramname = parser.PeekToken(totaltokens); } var signature = TypeSignatureInstantiated.Construct(parser, begintoken, totaltokens); ret.Add(new FunctionOverload.Parameter { Name = paramname, Type = signature }); ++totaltokens; } if (!parser.CheckToken(totaltokens, ",")) { consumedtokens = totaltokens; return(ret); } ++totaltokens; } return(ret); }
public static FunctionSignature Parse(ParseSession parser) { var nametoken = parser.PeekToken(0); if (nametoken == null || string.IsNullOrEmpty(nametoken.Text)) { return(null); } int totaltokens = 1; if (parser.CheckToken(totaltokens, "<")) { ++totaltokens; if (!parser.ParseTemplateParameters(totaltokens, nametoken, out totaltokens)) { return(null); } } if (!parser.CheckToken(totaltokens, ":")) { return(null); } ++totaltokens; var overload = new FunctionOverload(); if (!parser.CheckToken(totaltokens, "[")) { var paramlist = ParseFunctionParams(parser, totaltokens, out totaltokens); var funcreturn = ParseFunctionReturn(parser, totaltokens, out totaltokens); overload.Parameters = paramlist; overload.ReturnType = funcreturn?.Type; } var tags = ParseFunctionTags(parser, totaltokens, out totaltokens); overload.Tags = tags; if (parser.CheckToken(totaltokens, "{")) { ++totaltokens; var scope = ParseCodeBlock(parser, null, totaltokens, out totaltokens); overload.Scope = scope; if (overload.Scope != null && overload.Parameters != null) { foreach (var p in overload.Parameters) { var v = new Variable { Name = p.Name, Type = p.Type, Origin = Variable.Origins.Parameter }; overload.Scope.Variables.Add(v); } } } parser.ConsumeTokens(totaltokens); var ret = new FunctionSignature { Name = nametoken, Overloads = new List <FunctionOverload>() }; ret.Overloads.Add(overload); return(ret); }
private static bool ParseEntity(ParseSession parser, LexicalScope parentscope, int starttoken, out int consumedtokens) { consumedtokens = starttoken; int totaltokens = starttoken; if (parser.CheckToken(totaltokens, "if")) { if (!parser.CheckToken(totaltokens + 1, "(")) { return(false); } totaltokens += 2; var expr = Expression.Parse(parser, totaltokens, out totaltokens); if (expr == null) { return(false); } while (parser.CheckToken(totaltokens, ")")) { ++totaltokens; } if (!parser.CheckToken(totaltokens, "{")) { return(false); } ++totaltokens; ParseCodeBlock(parser, parentscope, totaltokens, out totaltokens); while (parser.CheckToken(totaltokens, "elseif")) { totaltokens += 2; var condexpr = Expression.Parse(parser, totaltokens, out totaltokens); if (condexpr == null) { return(false); } while (parser.CheckToken(totaltokens, ")")) { ++totaltokens; } if (!parser.CheckToken(totaltokens, "{")) { return(false); } ++totaltokens; ParseCodeBlock(parser, parentscope, totaltokens, out totaltokens); } if (parser.CheckToken(totaltokens, "else")) { ++totaltokens; if (!parser.CheckToken(totaltokens, "{")) { return(false); } ++totaltokens; ParseCodeBlock(parser, parentscope, totaltokens, out totaltokens); } consumedtokens = totaltokens; return(true); } else if (parser.CheckToken(totaltokens, "while")) { if (!parser.CheckToken(totaltokens + 1, "(")) { return(false); } totaltokens += 2; while (parser.CheckToken(totaltokens, ")")) { ++totaltokens; } var expr = Expression.Parse(parser, totaltokens, out totaltokens); ++totaltokens; if (!parser.CheckToken(totaltokens, "{")) { return(false); } ++totaltokens; ParseCodeBlock(parser, parentscope, totaltokens, out totaltokens); consumedtokens = totaltokens; return(true); } return(false); }
// // Helper routine for parsing a set of function parameters from a token stream. // // Passes back the index of the next token to inspect, as well as a list of parameters // extracted from the token stream, if applicable. May return null in case of a syntax // error. // private static List <FunctionOverload.Parameter> ParseFunctionParams(ParseSession parser, int starttoken, out int consumedtokens) { var ret = new List <FunctionOverload.Parameter>(); consumedtokens = starttoken; int totaltokens = starttoken; while (!parser.CheckToken(totaltokens, "{") && !parser.CheckToken(totaltokens, "->")) { // // Handle the "nothing" case since we don't need to parse any identifiers in addition // to the actual "nothing" keyword. This should come first to avoid backtracking in a // later parse step. // if (parser.CheckToken(totaltokens, "nothing")) { var signature = new TypeSignatureInstantiated(); ret.Add(new FunctionOverload.Parameter { Name = parser.PeekToken(totaltokens), Type = signature }); ++totaltokens; } // // Handle higher-order functions. // else if (parser.CheckToken(totaltokens, "(")) { ++totaltokens; var higherordername = parser.PeekToken(totaltokens); // TODO - don't treat syntax elements as optional if they are technically required! if (parser.CheckToken(totaltokens + 1, ":")) { totaltokens += 2; bool moreparams = true; if (parser.CheckToken(totaltokens, ")")) { moreparams = false; } while (moreparams) { ++totaltokens; if (parser.CheckToken(totaltokens, "ref")) { ++totaltokens; } if (!parser.CheckToken(totaltokens, ",")) { break; } ++totaltokens; } } if (parser.CheckToken(totaltokens, "->")) { totaltokens += 2; } if (!parser.CheckToken(totaltokens, ")")) { return(null); } ++totaltokens; var signature = new TypeSignatureInstantiated(); // TODO - implement higher order function signatures ret.Add(new FunctionOverload.Parameter { Name = higherordername, Type = signature }); } // // Handle functions with literal parameters (for pattern matching). // else if (IsLiteralFunctionParam(parser.PeekToken(totaltokens))) { // TODO - better literal support var signature = new TypeSignatureInstantiated(); ret.Add(new FunctionOverload.Parameter { Name = parser.PeekToken(totaltokens), Type = signature }); ++totaltokens; } // // Handle the general case of "type [ref] identifier[,]" format parameters. // // Also handles the presence of template arguments attached to the parameter // type, as necessary. // else { int begintoken = totaltokens; if (parser.CheckToken(totaltokens + 1, "<")) { var paramtype = parser.PeekToken(totaltokens); totaltokens += 2; if (!parser.ParseTemplateArguments(totaltokens, paramtype, out totaltokens)) { return(null); } } else { ++totaltokens; } var paramname = parser.PeekToken(totaltokens); if (parser.CheckToken(totaltokens, "ref")) { ++totaltokens; paramname = parser.PeekToken(totaltokens); } var signature = TypeSignatureInstantiated.Construct(parser, begintoken, totaltokens); ret.Add(new FunctionOverload.Parameter { Name = paramname, Type = signature }); ++totaltokens; } if (!parser.CheckToken(totaltokens, ",")) { consumedtokens = totaltokens; return(ret); } ++totaltokens; } return(ret); }
private static bool ParseExpressionOperator(ParseSession parser, int starttoken) { var token = parser.PeekToken(starttoken); if (token == null) { return(false); } string op = token.Text; if (op == ")") { return(false); } if (op == ",") { return(false); } if (op == "") { return(false); } if (op.Length > 2) { return(false); } if (op == ".") { return(true); } else if (op == "+") { return(true); } else if (op == "-") { return(true); } else if (op == "*") { return(true); } else if (op == "/") { return(true); } else if (op == "==") { return(true); } else if (op == "!=") { return(true); } else if (op == ";") { return(true); } else if (op == ">") { return(true); } else if (op == "<") { return(true); } else if (op == "&") { return(true); } else if (op == "&&") { return(true); } return(false); }
internal static ParsedSumType Parse(ParseSession parser) { int totaltokens = 0; if (!parser.CheckToken(0, "type")) { return(null); } Token sumtypename = parser.PeekToken(1); if (sumtypename == null || string.IsNullOrEmpty(sumtypename.Text)) { return(null); } if (parser.CheckToken(2, "<")) { if (!parser.ParseTemplateParameters(3, sumtypename, out totaltokens)) { return(null); } if (!parser.CheckToken(totaltokens, ":")) { return(null); } } else if (!parser.CheckToken(2, ":")) { return(null); } else if (!parser.CheckToken(4, "|")) { return(null); } else { totaltokens = 2; } do { ++totaltokens; if (parser.CheckToken(totaltokens + 1, "<")) { var token = parser.PeekToken(totaltokens); if (token == null) { return(null); } if (!parser.ParseTemplateArguments(totaltokens + 2, token, out totaltokens)) { return(null); } } else { ++totaltokens; } } while (parser.CheckToken(totaltokens, "|")); // Success! Consume everything and return the constructed result parser.ConsumeTokens(totaltokens); var sumtype = new SumType { }; return(new ParsedSumType { Name = sumtypename, Type = sumtype }); }
private static bool ParseExpressionTerm(ParseSession parser, bool atstart, int starttoken, out int consumedtokens, out bool matchedstatement) { matchedstatement = false; int totaltokens = starttoken; consumedtokens = totaltokens; if (parser.CheckToken(totaltokens, ")")) { return(false); } if (parser.CheckToken(totaltokens, ",")) { return(false); } if (parser.PeekToken(totaltokens) == null) { return(false); } if (parser.CheckToken(totaltokens, "(")) { ++totaltokens; if (Parse(parser, totaltokens, out totaltokens) != null) { ++totaltokens; consumedtokens = totaltokens; return(true); } return(false); } if (parser.CheckToken(totaltokens, "!")) { ++totaltokens; var ret = ParseExpressionTerm(parser, atstart, totaltokens, out totaltokens, out matchedstatement); if (matchedstatement && parser.CheckToken(totaltokens, ")")) { ++totaltokens; } consumedtokens = totaltokens; return(ret); } if (parser.CheckToken(totaltokens, "false") || parser.CheckToken(totaltokens, "true") || parser.CheckToken(totaltokens, "0") || parser.CheckToken(totaltokens, "0.0")) { ++totaltokens; } else if (parser.ParsePreopStatement(totaltokens, out totaltokens)) { consumedtokens = totaltokens; return(true); } else if (parser.ParseStatement(totaltokens, out totaltokens)) { matchedstatement = true; consumedtokens = totaltokens; return(true); } else { ++totaltokens; } consumedtokens = totaltokens; return(true); }
// // Helper routine for parsing a structure definition. // // Consumes tokens and returns a wrapped Structure on success. // Returns null on parsing failures, either due to syntactical // mistakes, or due to legitimate code that isn't a structure. // internal static ParsedObject <Structure> Parse(ParseSession parser) { int totaltokens = 0; if (!parser.CheckToken(0, "structure")) { return(null); } var nametoken = parser.PeekToken(1); if (nametoken == null) { return(null); } if (parser.CheckToken(2, "<")) { if (!parser.ParseTemplateParameters(3, nametoken, out totaltokens)) { return(null); } } else { totaltokens = 2; } if (!parser.CheckToken(totaltokens, ":")) { return(null); } ++totaltokens; var structure = new Structure { Name = nametoken }; structure.Members = new List <Member>(); var parsed = new ParsedObject <Structure> { Name = nametoken, Object = structure }; bool moremembers = true; while (moremembers) { if (parser.CheckToken(totaltokens, "(")) { ++totaltokens; var membername = parser.PeekToken(totaltokens); ++totaltokens; if (!parser.CheckToken(totaltokens, ":")) { return(null); } ++totaltokens; bool moreparams = true; while (moreparams) { ++totaltokens; if (!parser.CheckToken(totaltokens, ",")) { moreparams = false; } else { ++totaltokens; } } if (parser.CheckToken(totaltokens, "->")) { totaltokens += 2; } if (!parser.CheckToken(totaltokens, ")")) { return(null); } ++totaltokens; // TODO - register function-typed structure members } else { int typestarttoken = totaltokens; var membertype = parser.PeekToken(totaltokens); ++totaltokens; int typeendtoken = totaltokens; var membername = parser.PeekToken(totaltokens); ++totaltokens; if (membername.Text.Equals("<")) { int starttotal = totaltokens; if (!parser.ParseTemplateArguments(totaltokens, membertype, out totaltokens)) { return(null); } if (totaltokens <= starttotal) { return(null); } typeendtoken = totaltokens; membername = parser.PeekToken(totaltokens); ++totaltokens; } if (membername.Text.Equals("ref")) { typeendtoken = totaltokens; membername = parser.PeekToken(totaltokens); ++totaltokens; } parsed.Object.Members.Add(new Member { Name = membername, Type = TypeSignatureInstantiated.Construct(parser, typestarttoken, typeendtoken) }); } if (!parser.CheckToken(totaltokens, ",")) { moremembers = false; } else { ++totaltokens; } } parser.ConsumeTokens(totaltokens); return(parsed); }