protected abstract void ParseClassMember(
TokenStream tokens,
FileScope fileScope,
ClassDefinition classDef,
IList <FunctionDefinition> methodsOut,
IList <FieldDefinition> fieldsOut,
IList <PropertyDefinition> propertiesOut);
protected virtual FunctionDefinition MaybeParseFunctionDefinition(
TokenStream tokens,
Node owner,
FileScope fileScope,
AnnotationCollection annotations,
ModifierCollection modifiers)
{
return(null);
}
protected virtual Namespace ParseNamespace(
TokenStream tokens,
Node owner,
FileScope fileScope,
AnnotationCollection annotations)
{
Token namespaceToken = tokens.PopExpected(this.parser.Keywords.NAMESPACE);
Token first = tokens.Pop();
this.parser.VerifyIdentifier(first);
List <Token> namespacePieces = new List <Token>()
{
first
};
string namespaceBuilder = first.Value;
parser.RegisterNamespace(namespaceBuilder);
while (tokens.PopIfPresent("."))
{
Token nsToken = tokens.Pop();
this.parser.VerifyIdentifier(nsToken);
namespacePieces.Add(nsToken);
namespaceBuilder += "." + nsToken.Value;
parser.RegisterNamespace(namespaceBuilder);
}
string name = string.Join(".", namespacePieces.Select <Token, string>(t => t.Value));
Namespace namespaceInstance = new Namespace(namespaceToken, name, owner, fileScope, ModifierCollection.EMPTY, annotations);
tokens.PopExpected("{");
List <TopLevelEntity> namespaceMembers = new List <TopLevelEntity>();
while (!tokens.PopIfPresent("}"))
{
TopLevelEntity executable = this.Parse(tokens, namespaceInstance, fileScope);
if (executable is FunctionDefinition ||
executable is ClassDefinition ||
executable is EnumDefinition ||
executable is ConstDefinition ||
executable is Namespace)
{
namespaceMembers.Add(executable);
}
else
{
throw new ParserException(executable, "Only function, class, and nested namespace declarations may exist as direct members of a namespace.");
}
}
namespaceInstance.Code = namespaceMembers.ToArray();
return(namespaceInstance);
}
protected virtual EnumDefinition ParseEnumDefinition(
TokenStream tokens,
Node owner,
FileScope fileScope,
ModifierCollection modifiers,
AnnotationCollection annotations)
{
Token enumToken = tokens.PopExpected(this.parser.Keywords.ENUM);
Token nameToken = tokens.Pop();
this.parser.VerifyIdentifier(nameToken);
string name = nameToken.Value;
EnumDefinition ed = new EnumDefinition(enumToken, nameToken, owner, fileScope, modifiers, annotations);
tokens.PopExpected("{");
bool nextForbidden = false;
List <Token> items = new List <Token>();
List <Expression> values = new List <Expression>();
while (!tokens.PopIfPresent("}"))
{
if (nextForbidden)
{
tokens.PopExpected("}"); // crash
}
Token enumItem = tokens.Pop();
this.parser.VerifyIdentifier(enumItem);
if (tokens.PopIfPresent("="))
{
values.Add(this.parser.ExpressionParser.Parse(tokens, ed));
}
else
{
values.Add(null);
}
nextForbidden = !tokens.PopIfPresent(",");
items.Add(enumItem);
}
ed.SetItems(items, values);
return(ed);
}
internal virtual ImportStatement ParseImport(TokenStream tokens, FileScope fileScope)
{
Token importToken = tokens.PopExpected(parser.Keywords.IMPORT);
List <string> importPathBuilder = new List <string>();
while (!tokens.PopIfPresent(";"))
{
if (importPathBuilder.Count > 0)
{
tokens.PopExpected(".");
}
Token pathToken = tokens.Pop();
parser.VerifyIdentifier(pathToken);
importPathBuilder.Add(pathToken.Value);
}
string importPath = string.Join(".", importPathBuilder);
return(new ImportStatement(importToken, importPath, fileScope));
}
public async Task ParseFile(string filename, string code)
{
FileScope fileScope = new FileScope(filename, code, this.CurrentScope, this.GetNextFileId());
this.RegisterFileUsed(fileScope, code);
TokenStream tokens = new TokenStream(fileScope);
List <string> namespaceImportsBuilder = new List <string>();
// Implicitly import the Core library for the current locale.
LocalizedAssemblyView implicitCoreImport = await this.ScopeManager.GetCoreLibrary(this);
namespaceImportsBuilder.Add(implicitCoreImport.Name);
fileScope.Imports.Add(new ImportStatement(null, implicitCoreImport.Name, fileScope));
while (tokens.HasMore && tokens.IsNext(this.Keywords.IMPORT))
{
ImportStatement importStatement = this.TopLevelParser.ParseImport(tokens, fileScope);
if (importStatement == null)
{
throw new Exception();
}
namespaceImportsBuilder.Add(importStatement.ImportPath);
LocalizedAssemblyView localizedAssemblyView = await this.ScopeManager.GetOrImportAssembly(this, importStatement.FirstToken, importStatement.ImportPath);
if (localizedAssemblyView == null)
{
this.unresolvedImports.Add(importStatement);
}
}
string[] namespaceImports = namespaceImportsBuilder.ToArray();
while (tokens.HasMore)
{
this.CurrentScope.AddExecutable(this.TopLevelParser.Parse(tokens, null, fileScope));
}
}
internal TokenStream(FileScope file)
{
this.file = file;
this.innerStream = new InnerTokenStream(Tokenizer.Tokenize(file));
}
private void RegisterFileUsed(FileScope file, string code)
{
this.filesUsed.Add(file.ID, file.Name + "\n" + code);
}
public static Token[] Tokenize(FileScope file)
{
Localization.Locale locale = file.CompilationScope.Locale;
string code = file.Content;
// Add a newline and a dummy character at the end.
// Set the length equal to the code with the newline but without the null terminator.
// This makes dereferencing the index + 1 code simpler and all makes the check for the end
// of word tokens and single-line comments easy.
code += "\n\0";
int length = code.Length - 1;
int[] lineByIndex = new int[code.Length];
int[] colByIndex = new int[code.Length];
char c;
int line = 0;
int col = 0;
for (int i = 0; i < code.Length; ++i)
{
c = code[i];
lineByIndex[i] = line;
colByIndex[i] = col;
if (c == '\n')
{
++line;
col = -1;
}
++col;
}
List <Token> tokens = new List <Token>();
TokenMode mode = TokenMode.NORMAL;
char modeSubtype = ' ';
int tokenStart = 0;
string tokenValue;
char c2;
bool isTokenEnd = false;
bool stringIsRaw = false;
for (int i = 0; i < length; ++i)
{
c = code[i];
switch (mode)
{
case TokenMode.COMMENT:
if (modeSubtype == '*')
{
if (c == '*' && code[i + 1] == '/')
{
++i;
mode = TokenMode.NORMAL;
}
}
else
{
if (c == '\n')
{
mode = TokenMode.NORMAL;
}
}
break;
case TokenMode.NORMAL:
if (WHITESPACE.Contains(c))
{
// do nothing
}
else if (c == '/' && (code[i + 1] == '/' || code[i + 1] == '*'))
{
mode = TokenMode.COMMENT;
modeSubtype = code[++i];
}
else if (IsIdentifierChar(c))
{
tokenStart = i;
mode = TokenMode.WORD;
}
else if (c == '"' | c == '\'')
{
tokenStart = i;
mode = TokenMode.STRING;
modeSubtype = c;
stringIsRaw = tokens.Count > 0 && tokens[tokens.Count - 1].Value == "@";
}
else
{
if (c == '.')
{
c2 = code[i + 1];
if (c2 >= '0' && c2 <= '9')
{
mode = TokenMode.WORD;
tokenStart = i++;
}
}
if (mode == TokenMode.NORMAL)
{
tokens.Add(new Token(c.ToString(), TokenType.PUNCTUATION, file, lineByIndex[i], colByIndex[i]));
}
}
break;
case TokenMode.STRING:
if (c == modeSubtype)
{
tokenValue = code.Substring(tokenStart, i - tokenStart + 1);
tokens.Add(new Token(tokenValue, TokenType.STRING, file, lineByIndex[i], colByIndex[i]));
mode = TokenMode.NORMAL;
}
else if (!stringIsRaw && c == '\\')
{
++i;
}
break;
case TokenMode.WORD:
isTokenEnd = false;
if (IsIdentifierChar(c))
{
// do nothing
}
else if (c == '.')
{
if (code[tokenStart] >= '0' && code[tokenStart] <= '9')
{
// do nothing
}
else
{
isTokenEnd = true;
}
}
else
{
isTokenEnd = true;
}
if (isTokenEnd)
{
tokenValue = code.Substring(tokenStart, i - tokenStart);
c = tokenValue[0];
TokenType type = TokenType.WORD;
if ((c >= '0' && c <= '9') || c == '.')
{
type = TokenType.NUMBER;
}
else if (!locale.Keywords.IsValidVariable(tokenValue))
{
type = TokenType.KEYWORD;
}
tokens.Add(new Token(tokenValue, type, file, lineByIndex[tokenStart], colByIndex[tokenStart]));
mode = TokenMode.NORMAL;
--i;
}
break;
}
}
switch (mode)
{
case TokenMode.COMMENT:
throw new ParserException(file, "There is an unclosed comment in this file.");
case TokenMode.STRING:
throw new ParserException(file, "There is an unclosed string in this file.");
case TokenMode.WORD:
throw new System.InvalidOperationException();
default:
break;
}
return(tokens.ToArray());
}
internal virtual TopLevelEntity Parse(
TokenStream tokens,
TopLevelEntity owner,
FileScope fileScope)
{
AnnotationCollection annotations = this.parser.AnnotationParser.ParseAnnotations(tokens);
ModifierCollection modifiers = ModifierCollection.Parse(tokens);
string value = tokens.PeekValue();
if (value == this.parser.Keywords.IMPORT)
{
throw this.parser.GenerateParseError(
ErrorMessages.ALL_IMPORTS_MUST_OCCUR_AT_BEGINNING_OF_FILE,
tokens.Pop());
}
// TODO: check for annotations that aren't used.
// https://github.com/blakeohare/crayon/issues/305
if (value == this.parser.Keywords.NAMESPACE)
{
return(this.ParseNamespace(tokens, owner, fileScope, annotations));
}
if (value == this.parser.Keywords.CONST)
{
return(this.ParseConst(tokens, owner, fileScope, modifiers, annotations));
}
if (value == this.parser.Keywords.FUNCTION)
{
return(this.ParseFunction(tokens, owner, fileScope, modifiers, annotations));
}
if (value == this.parser.Keywords.CLASS)
{
return(this.ParseClassDefinition(tokens, owner, fileScope, modifiers, annotations));
}
if (value == this.parser.Keywords.ENUM)
{
return(this.ParseEnumDefinition(tokens, owner, fileScope, modifiers, annotations));
}
if (value == this.parser.Keywords.CONSTRUCTOR && owner is ClassDefinition)
{
return(this.ParseConstructor(tokens, (ClassDefinition)owner, modifiers, annotations));
}
FunctionDefinition nullableFunctionDef = this.MaybeParseFunctionDefinition(tokens, owner, fileScope, annotations, modifiers);
if (nullableFunctionDef != null)
{
return(nullableFunctionDef);
}
tokens.EnsureNotEof();
Token token = tokens.Peek();
throw ParserException.ThrowException(
this.parser.CurrentLocale,
ErrorMessages.UNEXPECTED_TOKEN_NO_SPECIFIC_EXPECTATIONS,
token,
token.Value);
}
protected abstract FunctionDefinition ParseFunction(
TokenStream tokens,
TopLevelEntity nullableOwner,
FileScope fileScope,
ModifierCollection modifiers,
AnnotationCollection annotations);
protected virtual ClassDefinition ParseClassDefinition(
TokenStream tokens,
Node owner,
FileScope fileScope,
ModifierCollection modifiers,
AnnotationCollection classAnnotations)
{
Token classToken = tokens.PopExpected(this.parser.Keywords.CLASS);
Token classNameToken = tokens.Pop();
if (classNameToken.Type != TokenType.WORD)
{
throw new ParserException(classNameToken, "This is not a valid class name.");
}
List <Token> baseClassTokens = new List <Token>();
List <string> baseClassStrings = new List <string>();
if (tokens.PopIfPresent(":"))
{
if (baseClassTokens.Count > 0)
{
tokens.PopExpected(",");
}
Token baseClassToken = tokens.Pop();
string baseClassName = baseClassToken.Value;
this.parser.VerifyIdentifier(baseClassToken);
while (tokens.PopIfPresent("."))
{
Token baseClassTokenNext = tokens.Pop();
this.parser.VerifyIdentifier(baseClassTokenNext);
baseClassName += "." + baseClassTokenNext.Value;
}
baseClassTokens.Add(baseClassToken);
baseClassStrings.Add(baseClassName);
}
ClassDefinition cd = new ClassDefinition(
classToken,
classNameToken,
baseClassTokens,
baseClassStrings,
owner,
fileScope,
modifiers,
classAnnotations,
this.parser);
tokens.PopExpected("{");
List <FunctionDefinition> methods = new List <FunctionDefinition>();
List <FieldDefinition> fields = new List <FieldDefinition>();
List <PropertyDefinition> properties = new List <PropertyDefinition>();
while (!tokens.PopIfPresent("}"))
{
this.ParseClassMember(tokens, fileScope, cd, methods, fields, properties);
}
cd.Methods = methods.ToArray();
cd.Fields = fields.ToArray();
if (cd.Constructor == null)
{
// This should be empty if there is no base class, or just pass along the base class' args if there is.
cd.Constructor = new ConstructorDefinition(
cd,
ModifierCollection.EMPTY,
new AnnotationCollection(parser));
if (cd.BaseClassTokens.Length > 0)
{
cd.Constructor.BaseToken = cd.FirstToken;
cd.Constructor.SetBaseArgs(new Expression[0]);
}
}
return(cd);
}
protected abstract ConstDefinition ParseConst(
TokenStream tokens,
Node owner,
FileScope fileScope,
ModifierCollection modifiers,
AnnotationCollection annotations);
public FileScopedEntityLookup SetFileScope(FileScope fileScope)
{
this.fileScope = fileScope;
return(this);
}
internal ParserException(FileScope file, string message)
: base(file.Name + ": " + message)
{
this.file = file;
this.OriginalMessage = message;
}