internal static ParsedStrongAlias Parse(ParseSession parser)
{
if (!parser.CheckToken(0, "type"))
{
return(null);
}
var nametoken = parser.PeekToken(1);
if (nametoken == null)
{
return(null);
}
if (!parser.CheckToken(2, ":"))
{
return(null);
}
parser.ConsumeTokens(4);
var aliastype = new StrongAlias {
};
return(new ParsedStrongAlias {
Name = nametoken, Type = aliastype
});
}
internal static Variable Parse(ParseSession parser, int starttoken, out int consumedtokens, Origins origin)
{
consumedtokens = starttoken;
if (parser.CheckToken(starttoken + 1, "."))
{
return(null);
}
var basetypename = parser.PeekToken(starttoken);
int totaltokens = starttoken + 1;
if (parser.CheckToken(totaltokens, "<"))
{
++totaltokens;
if (!parser.ParseTemplateArguments(totaltokens, basetypename, out totaltokens))
{
return(null);
}
}
var varname = parser.PeekToken(totaltokens);
if (!parser.CheckToken(totaltokens + 1, "="))
{
return(null);
}
var type = TypeSignatureInstantiated.Construct(parser, starttoken, totaltokens);
var variable = new Variable {
Name = varname, Origin = origin, Type = type
};
totaltokens += 2;
Expression.Parse(parser, totaltokens, out totaltokens);
while (parser.CheckToken(totaltokens, ","))
{
++totaltokens;
Expression.Parse(parser, totaltokens, out totaltokens);
}
consumedtokens = totaltokens;
return(variable);
}
private static bool ParseAssignment(ParseSession parser, int starttoken, out int consumedtokens)
{
consumedtokens = starttoken;
int totaltokens = starttoken;
++totaltokens;
while (parser.CheckToken(totaltokens, "."))
{
totaltokens += 2;
}
if (!parser.CheckToken(totaltokens, "=") &&
!parser.CheckToken(totaltokens, "+=") &&
!parser.CheckToken(totaltokens, "-="))
{
return(false);
}
var assignmenttoken = parser.PeekToken(totaltokens);
++totaltokens;
bool haschain = true;
while (haschain)
{
while (parser.CheckToken(totaltokens + 1, "."))
{
totaltokens += 2;
}
if (parser.CheckToken(totaltokens + 1, "="))
{
if (assignmenttoken.Text != "=")
{
return(false);
}
++totaltokens;
}
else
{
haschain = false;
}
}
var expr = Expression.Parse(parser, totaltokens, out totaltokens);
if (expr == null)
{
return(false);
}
consumedtokens = totaltokens;
return(true);
}
public static TypeSignature Construct(ParseSession parser, int begintoken, int endtoken)
{
var signature = new TypeSignature();
signature.TypeName = parser.PeekToken(begintoken).Text;
if (parser.CheckToken(begintoken + 1, "<"))
{
signature.TypeParameters = new List <TypeParameter>();
int totaltokens = begintoken + 2;
while (!parser.CheckToken(totaltokens, ">"))
{
var tokentype = parser.PeekToken(totaltokens);
var tokenname = parser.PeekToken(totaltokens + 1);
var param = new TypeParameter {
Name = tokenname.Text, ArgumentType = new TypeSignature {
TypeName = tokentype.Text
}
};
signature.TypeParameters.Add(param);
totaltokens += 2;
if (parser.CheckToken(totaltokens, ","))
{
++totaltokens;
}
}
}
int reftoken = endtoken - 1;
if (reftoken > begintoken)
{
signature.TypeIsReference = parser.CheckToken(reftoken, "ref");
}
return(signature);
}
public static TypeSignatureInstantiated Construct(ParseSession parser, int begintoken, int endtoken)
{
var signature = new TypeSignatureInstantiated();
signature.TypeName = parser.PeekToken(begintoken).Text;
if (parser.CheckToken(begintoken + 1, "<"))
{
signature.TypeArguments = new List <TypeArgument>();
int totaltokens = begintoken + 2;
while (!parser.CheckToken(totaltokens, ">"))
{
var argtoken = parser.PeekToken(totaltokens);
signature.TypeArguments.Add(new TypeArgument {
SpecifiedType = new TypeSignatureInstantiated {
TypeName = argtoken.Text
}
});
++totaltokens;
if (parser.CheckToken(totaltokens, ","))
{
++totaltokens;
}
}
}
int reftoken = endtoken - 1;
if (reftoken > begintoken)
{
signature.TypeIsReference = parser.CheckToken(reftoken, "ref");
}
return(signature);
}
private static bool ParseExpressionTerm(ParseSession parser, bool atstart, int starttoken, out int consumedtokens, out bool matchedstatement)
{
matchedstatement = false;
int totaltokens = starttoken;
consumedtokens = totaltokens;
if (parser.CheckToken(totaltokens, ")"))
{
return(false);
}
if (parser.CheckToken(totaltokens, ","))
{
return(false);
}
if (parser.PeekToken(totaltokens) == null)
{
return(false);
}
if (parser.CheckToken(totaltokens, "("))
{
++totaltokens;
if (Parse(parser, totaltokens, out totaltokens) != null)
{
++totaltokens;
consumedtokens = totaltokens;
return(true);
}
return(false);
}
if (parser.CheckToken(totaltokens, "!"))
{
++totaltokens;
var ret = ParseExpressionTerm(parser, atstart, totaltokens, out totaltokens, out matchedstatement);
if (matchedstatement && parser.CheckToken(totaltokens, ")"))
{
++totaltokens;
}
consumedtokens = totaltokens;
return(ret);
}
if (parser.CheckToken(totaltokens, "false") || parser.CheckToken(totaltokens, "true") || parser.CheckToken(totaltokens, "0") || parser.CheckToken(totaltokens, "0.0"))
{
++totaltokens;
}
else if (parser.ParsePreopStatement(totaltokens, out totaltokens))
{
consumedtokens = totaltokens;
return(true);
}
else if (parser.ParseStatement(totaltokens, out totaltokens))
{
matchedstatement = true;
consumedtokens = totaltokens;
return(true);
}
else
{
++totaltokens;
}
consumedtokens = totaltokens;
return(true);
}
private static bool ParseExpressionOperator(ParseSession parser, int starttoken)
{
var token = parser.PeekToken(starttoken);
if (token == null)
{
return(false);
}
string op = token.Text;
if (op == ")")
{
return(false);
}
if (op == ",")
{
return(false);
}
if (op == "")
{
return(false);
}
if (op.Length > 2)
{
return(false);
}
if (op == ".")
{
return(true);
}
else if (op == "+")
{
return(true);
}
else if (op == "-")
{
return(true);
}
else if (op == "*")
{
return(true);
}
else if (op == "/")
{
return(true);
}
else if (op == "==")
{
return(true);
}
else if (op == "!=")
{
return(true);
}
else if (op == ";")
{
return(true);
}
else if (op == ">")
{
return(true);
}
else if (op == "<")
{
return(true);
}
else if (op == "&")
{
return(true);
}
else if (op == "&&")
{
return(true);
}
return(false);
}
//
// Helper routine to parse a lexical scope in its entirety given a token stream.
//
internal static LexicalScope Parse(ParseSession parser, LexicalScope parentscope, int starttoken, out int consumedtokens)
{
consumedtokens = starttoken;
int totaltokens = starttoken;
Token afterStartBrace = parser.PeekToken(totaltokens);
if (afterStartBrace == null)
{
parser.ConsumeTokens(starttoken);
throw new SyntaxError("Missing closing }", parser.ReversePeekToken());
}
var ret = new LexicalScope();
ret.File = afterStartBrace.File;
ret.StartLine = afterStartBrace.Line;
ret.StartColumn = afterStartBrace.Column;
ret.ParentScope = parentscope;
if (parentscope != null)
{
if (parentscope.ChildScopes == null)
{
parentscope.ChildScopes = new List <LexicalScope>();
}
parentscope.ChildScopes.Add(ret);
}
while (!parser.CheckToken(totaltokens, "}"))
{
if (CodeHelpers.ParseEntity(parser, ret, totaltokens, out totaltokens) ||
parser.ParsePreopStatement(totaltokens, out totaltokens) ||
parser.ParsePostopStatement(totaltokens, out totaltokens) ||
parser.ParseStatement(totaltokens, out totaltokens))
{
parser.ConsumeTokens(totaltokens);
totaltokens = 0;
continue;
}
var variable = Variable.Parse(parser, totaltokens, out totaltokens, Variable.Origins.Local);
if (variable != null)
{
ret.Variables.Add(variable);
parser.ConsumeTokens(totaltokens);
totaltokens = 0;
continue;
}
if (CodeHelpers.ParseAssignment(parser, totaltokens, out totaltokens))
{
parser.ConsumeTokens(totaltokens);
totaltokens = 0;
continue;
}
consumedtokens = totaltokens;
return(null);
}
Token endBrace = parser.PeekToken(totaltokens);
++totaltokens;
consumedtokens = totaltokens;
ret.EndLine = endBrace.Line;
ret.EndColumn = endBrace.Column;
return(ret);
}
internal static ParsedSumType Parse(ParseSession parser)
{
int totaltokens = 0;
if (!parser.CheckToken(0, "type"))
{
return(null);
}
Token sumtypename = parser.PeekToken(1);
if (sumtypename == null || string.IsNullOrEmpty(sumtypename.Text))
{
return(null);
}
if (parser.CheckToken(2, "<"))
{
if (!parser.ParseTemplateParameters(3, sumtypename, out totaltokens))
{
return(null);
}
if (!parser.CheckToken(totaltokens, ":"))
{
return(null);
}
}
else if (!parser.CheckToken(2, ":"))
{
return(null);
}
else if (!parser.CheckToken(4, "|"))
{
return(null);
}
else
{
totaltokens = 2;
}
do
{
++totaltokens;
if (parser.CheckToken(totaltokens + 1, "<"))
{
var token = parser.PeekToken(totaltokens);
if (token == null)
{
return(null);
}
if (!parser.ParseTemplateArguments(totaltokens + 2, token, out totaltokens))
{
return(null);
}
}
else
{
++totaltokens;
}
} while (parser.CheckToken(totaltokens, "|"));
// Success! Consume everything and return the constructed result
parser.ConsumeTokens(totaltokens);
var sumtype = new SumType {
};
return(new ParsedSumType {
Name = sumtypename, Type = sumtype
});
}
private static List <FunctionOverload.Parameter> ParseFunctionParams(ParseSession parser, int starttoken, out int consumedtokens)
{
var ret = new List <FunctionOverload.Parameter>();
consumedtokens = starttoken;
int totaltokens = starttoken;
while (!parser.CheckToken(totaltokens, "{") && !parser.CheckToken(totaltokens, "->"))
{
if (parser.CheckToken(totaltokens, "nothing"))
{
var signature = new TypeSignatureInstantiated();
ret.Add(new FunctionOverload.Parameter {
Name = parser.PeekToken(totaltokens), Type = signature
});
++totaltokens;
}
else if (parser.CheckToken(totaltokens, "("))
{
++totaltokens;
var higherordername = parser.PeekToken(totaltokens);
if (parser.CheckToken(totaltokens + 1, ":"))
{
totaltokens += 2;
bool moreparams = true;
if (parser.CheckToken(totaltokens, ")"))
{
moreparams = false;
}
while (moreparams)
{
++totaltokens;
if (parser.CheckToken(totaltokens, "ref"))
{
++totaltokens;
}
if (!parser.CheckToken(totaltokens, ","))
{
break;
}
++totaltokens;
}
}
if (parser.CheckToken(totaltokens, "->"))
{
totaltokens += 2;
}
if (!parser.CheckToken(totaltokens, ")"))
{
return(null);
}
++totaltokens;
var signature = new TypeSignatureInstantiated(); // TODO - implement higher order function signatures
ret.Add(new FunctionOverload.Parameter {
Name = higherordername, Type = signature
});
}
else if (IsLiteralFunctionParam(parser.PeekToken(totaltokens)))
{
// TODO - better literal support
var signature = new TypeSignatureInstantiated();
ret.Add(new FunctionOverload.Parameter {
Name = parser.PeekToken(totaltokens), Type = signature
});
++totaltokens;
}
else
{
int begintoken = totaltokens;
if (parser.CheckToken(totaltokens + 1, "<"))
{
var paramtype = parser.PeekToken(totaltokens);
totaltokens += 2;
if (!parser.ParseTemplateArguments(totaltokens, paramtype, out totaltokens))
{
return(null);
}
}
else
{
++totaltokens;
}
var paramname = parser.PeekToken(totaltokens);
if (parser.CheckToken(totaltokens, "ref"))
{
++totaltokens;
paramname = parser.PeekToken(totaltokens);
}
var signature = TypeSignatureInstantiated.Construct(parser, begintoken, totaltokens);
ret.Add(new FunctionOverload.Parameter {
Name = paramname, Type = signature
});
++totaltokens;
}
if (!parser.CheckToken(totaltokens, ","))
{
consumedtokens = totaltokens;
return(ret);
}
++totaltokens;
}
return(ret);
}
public static FunctionSignature Parse(ParseSession parser)
{
var nametoken = parser.PeekToken(0);
if (nametoken == null || string.IsNullOrEmpty(nametoken.Text))
{
return(null);
}
int totaltokens = 1;
if (parser.CheckToken(totaltokens, "<"))
{
++totaltokens;
if (!parser.ParseTemplateParameters(totaltokens, nametoken, out totaltokens))
{
return(null);
}
}
if (!parser.CheckToken(totaltokens, ":"))
{
return(null);
}
++totaltokens;
var overload = new FunctionOverload();
if (!parser.CheckToken(totaltokens, "["))
{
var paramlist = ParseFunctionParams(parser, totaltokens, out totaltokens);
var funcreturn = ParseFunctionReturn(parser, totaltokens, out totaltokens);
overload.Parameters = paramlist;
overload.ReturnType = funcreturn?.Type;
}
var tags = ParseFunctionTags(parser, totaltokens, out totaltokens);
overload.Tags = tags;
if (parser.CheckToken(totaltokens, "{"))
{
++totaltokens;
var scope = ParseCodeBlock(parser, null, totaltokens, out totaltokens);
overload.Scope = scope;
if (overload.Scope != null && overload.Parameters != null)
{
foreach (var p in overload.Parameters)
{
var v = new Variable {
Name = p.Name, Type = p.Type, Origin = Variable.Origins.Parameter
};
overload.Scope.Variables.Add(v);
}
}
}
parser.ConsumeTokens(totaltokens);
var ret = new FunctionSignature {
Name = nametoken, Overloads = new List <FunctionOverload>()
};
ret.Overloads.Add(overload);
return(ret);
}
//
// Helper routine for parsing a set of function parameters from a token stream.
//
// Passes back the index of the next token to inspect, as well as a list of parameters
// extracted from the token stream, if applicable. May return null in case of a syntax
// error.
//
private static List <FunctionOverload.Parameter> ParseFunctionParams(ParseSession parser, int starttoken, out int consumedtokens)
{
var ret = new List <FunctionOverload.Parameter>();
consumedtokens = starttoken;
int totaltokens = starttoken;
while (!parser.CheckToken(totaltokens, "{") && !parser.CheckToken(totaltokens, "->"))
{
//
// Handle the "nothing" case since we don't need to parse any identifiers in addition
// to the actual "nothing" keyword. This should come first to avoid backtracking in a
// later parse step.
//
if (parser.CheckToken(totaltokens, "nothing"))
{
var signature = new TypeSignatureInstantiated();
ret.Add(new FunctionOverload.Parameter {
Name = parser.PeekToken(totaltokens), Type = signature
});
++totaltokens;
}
//
// Handle higher-order functions.
//
else if (parser.CheckToken(totaltokens, "("))
{
++totaltokens;
var higherordername = parser.PeekToken(totaltokens);
// TODO - don't treat syntax elements as optional if they are technically required!
if (parser.CheckToken(totaltokens + 1, ":"))
{
totaltokens += 2;
bool moreparams = true;
if (parser.CheckToken(totaltokens, ")"))
{
moreparams = false;
}
while (moreparams)
{
++totaltokens;
if (parser.CheckToken(totaltokens, "ref"))
{
++totaltokens;
}
if (!parser.CheckToken(totaltokens, ","))
{
break;
}
++totaltokens;
}
}
if (parser.CheckToken(totaltokens, "->"))
{
totaltokens += 2;
}
if (!parser.CheckToken(totaltokens, ")"))
{
return(null);
}
++totaltokens;
var signature = new TypeSignatureInstantiated(); // TODO - implement higher order function signatures
ret.Add(new FunctionOverload.Parameter {
Name = higherordername, Type = signature
});
}
//
// Handle functions with literal parameters (for pattern matching).
//
else if (IsLiteralFunctionParam(parser.PeekToken(totaltokens)))
{
// TODO - better literal support
var signature = new TypeSignatureInstantiated();
ret.Add(new FunctionOverload.Parameter {
Name = parser.PeekToken(totaltokens), Type = signature
});
++totaltokens;
}
//
// Handle the general case of "type [ref] identifier[,]" format parameters.
//
// Also handles the presence of template arguments attached to the parameter
// type, as necessary.
//
else
{
int begintoken = totaltokens;
if (parser.CheckToken(totaltokens + 1, "<"))
{
var paramtype = parser.PeekToken(totaltokens);
totaltokens += 2;
if (!parser.ParseTemplateArguments(totaltokens, paramtype, out totaltokens))
{
return(null);
}
}
else
{
++totaltokens;
}
var paramname = parser.PeekToken(totaltokens);
if (parser.CheckToken(totaltokens, "ref"))
{
++totaltokens;
paramname = parser.PeekToken(totaltokens);
}
var signature = TypeSignatureInstantiated.Construct(parser, begintoken, totaltokens);
ret.Add(new FunctionOverload.Parameter {
Name = paramname, Type = signature
});
++totaltokens;
}
if (!parser.CheckToken(totaltokens, ","))
{
consumedtokens = totaltokens;
return(ret);
}
++totaltokens;
}
return(ret);
}
//
// Helper routine for parsing a series of function tags from a token stream.
//
// Function tag lists are denoted by square brackets. Each tag is a single token
// followed by an optional parenthesized list of parameters. Both sorts of lists
// are delimited by commas. Parameters may be literal values only, including but
// not limited to string literals. Expressions are not permitted.
//
// Passes back the index of the next token to inspect. Returns null if there are
// no tags present. Presently does not handle syntax errors nicely.
//
private static List <FunctionOverload.Tag> ParseFunctionTags(ParseSession parser, int starttoken, out int consumedtokens)
{
consumedtokens = starttoken;
int totaltokens = starttoken;
if (!parser.CheckToken(totaltokens, "["))
{
return(null);
}
++totaltokens;
var ret = new List <FunctionOverload.Tag>();
while (!parser.CheckToken(totaltokens, "]"))
{
var tag = new FunctionOverload.Tag {
Name = parser.PeekToken(totaltokens)
};
if (parser.CheckToken(totaltokens + 1, "("))
{
totaltokens += 2;
tag.Params = new List <FunctionOverload.Tag.TagParam>();
var p = new FunctionOverload.Tag.TagParam {
AssociatedTokens = new List <Token>()
};
while (!parser.CheckToken(totaltokens, ")"))
{
p.AssociatedTokens.Add(parser.PeekToken(totaltokens));
++totaltokens;
if (parser.CheckToken(totaltokens, ","))
{
tag.Params.Add(p);
++totaltokens;
p = new FunctionOverload.Tag.TagParam {
AssociatedTokens = new List <Token>()
};
}
}
tag.Params.Add(p);
}
++totaltokens;
ret.Add(tag);
if (parser.CheckToken(totaltokens, ","))
{
++totaltokens;
}
}
++totaltokens;
consumedtokens = totaltokens;
return(ret);
}
//
// Helper routine for parsing a structure definition.
//
// Consumes tokens and returns a wrapped Structure on success.
// Returns null on parsing failures, either due to syntactical
// mistakes, or due to legitimate code that isn't a structure.
//
internal static ParsedObject <Structure> Parse(ParseSession parser)
{
int totaltokens = 0;
if (!parser.CheckToken(0, "structure"))
{
return(null);
}
var nametoken = parser.PeekToken(1);
if (nametoken == null)
{
return(null);
}
if (parser.CheckToken(2, "<"))
{
if (!parser.ParseTemplateParameters(3, nametoken, out totaltokens))
{
return(null);
}
}
else
{
totaltokens = 2;
}
if (!parser.CheckToken(totaltokens, ":"))
{
return(null);
}
++totaltokens;
var structure = new Structure {
Name = nametoken
};
structure.Members = new List <Member>();
var parsed = new ParsedObject <Structure> {
Name = nametoken, Object = structure
};
bool moremembers = true;
while (moremembers)
{
if (parser.CheckToken(totaltokens, "("))
{
++totaltokens;
var membername = parser.PeekToken(totaltokens);
++totaltokens;
if (!parser.CheckToken(totaltokens, ":"))
{
return(null);
}
++totaltokens;
bool moreparams = true;
while (moreparams)
{
++totaltokens;
if (!parser.CheckToken(totaltokens, ","))
{
moreparams = false;
}
else
{
++totaltokens;
}
}
if (parser.CheckToken(totaltokens, "->"))
{
totaltokens += 2;
}
if (!parser.CheckToken(totaltokens, ")"))
{
return(null);
}
++totaltokens;
// TODO - register function-typed structure members
}
else
{
int typestarttoken = totaltokens;
var membertype = parser.PeekToken(totaltokens);
++totaltokens;
int typeendtoken = totaltokens;
var membername = parser.PeekToken(totaltokens);
++totaltokens;
if (membername.Text.Equals("<"))
{
int starttotal = totaltokens;
if (!parser.ParseTemplateArguments(totaltokens, membertype, out totaltokens))
{
return(null);
}
if (totaltokens <= starttotal)
{
return(null);
}
typeendtoken = totaltokens;
membername = parser.PeekToken(totaltokens);
++totaltokens;
}
if (membername.Text.Equals("ref"))
{
typeendtoken = totaltokens;
membername = parser.PeekToken(totaltokens);
++totaltokens;
}
parsed.Object.Members.Add(new Member {
Name = membername, Type = TypeSignatureInstantiated.Construct(parser, typestarttoken, typeendtoken)
});
}
if (!parser.CheckToken(totaltokens, ","))
{
moremembers = false;
}
else
{
++totaltokens;
}
}
parser.ConsumeTokens(totaltokens);
return(parsed);
}