public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) {
foreach (Token token in tokens) {
if (!token.Terminal.IsSet(TermOptions.IsBrace)) {
yield return token;
continue;
}
//open brace symbol
if (token.Terminal.IsSet(TermOptions.IsOpenBrace)) {
_braces.Push(token);
yield return token;
continue;
}
//We have closing brace
if (_braces.Count == 0) {
yield return context.CreateErrorTokenAndReportError( token.Location, token.Text, "Unmatched closing brace '{0}'", token.Text);
continue;
}
//check match
Token last = _braces.Pop();
if (last.AsSymbol.IsPairFor != token.AsSymbol) {
yield return context.CreateErrorTokenAndReportError(token.Location, token.Text,
"Unmatched closing brace '{0}' - expected '{1}'", last.AsSymbol.IsPairFor.Name);
continue;
}
//everything is ok, there's matching brace on top of the stack
Token.LinkMatchingBraces(last, token);
context.CurrentParseTree.OpenBraces.Add(last);
yield return token; //return this token
}//foreach token
yield break;
}//method
public override Token TryMatch(CompilerContext context, ISourceStream source) {
Match m = _expression.Match(source.Text, source.Position);
if (!m.Success || m.Index != source.Position)
return null;
source.Position += m.Length;
string text = source.GetLexeme();
return new Token(this, source.TokenStart, text, null);
}
public ParseTree Parse(CompilerContext context, string sourceText, string fileName) {
context.CurrentParseTree = new ParseTree(sourceText, fileName);
Scanner.SetSource(sourceText);
Scanner.BeginScan(context);
CoreParser.Parse(context);
if (context.CurrentParseTree.Errors.Count > 0)
context.CurrentParseTree.Errors.Sort(SyntaxErrorList.ByLocation);
return context.CurrentParseTree;
}
private void Parse(Grammar grammar, string script, bool expectError) {
var compiler = new Compiler(grammar);
var context = new CompilerContext(compiler);
var source = new SourceStream(script, "source");
AstNode root = compiler.Parse(context, source);
compiler.AnalyzeCode(root, context);
if (!expectError && context.Errors.Count > 0)
Assert.Fail("Unexpected source error(s) found: " + context.Errors[0].Message);
}
public EditorAdapter(Compiler compiler) {
_compiler = compiler;
_context = new CompilerContext(_compiler);
_scanner = compiler.Parser.Scanner;
_scanner.BeginScan(_context);
_parseTree = new ParseTree(string.Empty, "Source");
_colorizerThread = new Thread(ColorizerLoop);
_colorizerThread.IsBackground = true;
_parserThread = new Thread(ParserLoop);
_parserThread.IsBackground = true;
}
public void BeginScan(CompilerContext context) {
_context = context;
_bufferedTokens.Clear();
//create streams
FilteredStream = UnfilteredStream = CreateUnfilteredTokenStream();
//chain all token filters
foreach (TokenFilter filter in _data.TokenFilters) {
FilteredStream = filter.BeginFiltering(context, FilteredStream);
}
FilteredTokenEnumerator = FilteredStream.GetEnumerator();
}
public override IEnumerable<Token> BeginFiltering(CompilerContext context, IEnumerable<Token> tokens) {
//TODO: fix this.
// This is a temporary workaround, to "undo" the change to whitespace made by NewLineTerminal.
// if we have both NewLineTerminal in the grammar and CodeOutlineFilter, then NewLines should be generated by filter,
// not by terminal.
context.Compiler.Language.Grammar.WhitespaceChars = " \t\r\n\v";
_prevLine = 0;
_indents.Clear();
foreach (Token token in tokens) {
if (token.Terminal == base.Grammar.Eof) {
yield return CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location); //this is necessary, because grammar rules reference newLine terminator
//unindent all buffered indents
if (_trackIndents)
foreach (int i in _indents)
yield return CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location);
_indents.Clear();
//return EOF token
yield return token;
yield break;
}//if Eof
//Now deal with normal, non-EOF tokens
//We intercept only content tokens on new lines
if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine) {
yield return token;
continue;
}
//if we are here, we have content token on new line; produce newLine token and possibly indents
yield return CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location);
_prevLine = token.Location.Line;
if (!_trackIndents) {
yield return token;
continue;
}
//Now take care of indents
int currIndent = token.Location.Column;
int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek();
if (currIndent > prevIndent) {
_indents.Push(currIndent);
yield return CreateSpecialToken(GrammarData.Grammar.Indent, token.Location);
} else if (currIndent < prevIndent) {
//produce one or more dedent tokens while popping indents from stack
while (_indents.Count > 0 && _indents.Peek() > currIndent) {
_indents.Pop();
yield return CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location);
}
if (_indents.Count == 0 || _indents.Peek() != currIndent) {
yield return context.CreateErrorTokenAndReportError (token.Location, string.Empty, "Invalid dedent level, no previous matching indent found.");
//TODO: add error recovery here
}
}//else if currIndent < prevIndent
yield return token;
} //foreach token
}//method
public override Token TryMatch(CompilerContext context, ISourceStream source) {
char current = source.CurrentChar;
if (!LineTerminators.Contains(current)) return null;
//Treat \r\n as a single terminator
bool doExtraShift = (current == '\r' && source.NextChar == '\n');
source.Position++; //main shift
if (doExtraShift)
source.Position++;
Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);
return result;
}
public void Parse(CompilerContext context) {
_context = context;
_traceOn = _context.OptionIsSet(CompilerOptions.TraceParser);
_currentInput = null;
InputStack.Clear();
Stack.Clear();
_currentState = Data.InitialState; //set the current state to InitialState
Stack.Push(new ParseTreeNode(Data.InitialState));
//main loop
while (ExecuteAction()) {}
}//Parse
private void Evaluate(Grammar grammar, string script, bool expectError, object result) {
var compiler = new Compiler(grammar);
var context = new CompilerContext(compiler);
var source = new SourceStream(script, "source");
AstNode root = compiler.Parse(context, source);
compiler.AnalyzeCode(root, context);
if (!expectError && context.Errors.Count > 0)
Assert.Fail("Unexpected source error(s) found: " + context.Errors[0].Message);
var evalContext = new EvaluationContext(context.Runtime, root);
root.Evaluate(evalContext);
Assert.AreEqual(evalContext.CurrentResult, result, "Evaluation result is null, expected " + result);
}
public virtual string GetSyntaxErrorMessage(CompilerContext context, ParserState state, ParseTreeNode currentInput) {
return null; //Irony then would construct default message
}
public virtual void CreateAstNode(CompilerContext context, ParseTreeNode nodeInfo) {
var term = nodeInfo.Term;
if (term.NodeCreator != null) {
term.NodeCreator(context, nodeInfo);
//We assume that Node creator method creates node and initializes it, so parser does not need to call
// IAstNodeInit.InitNode() method on node object.
return;
}
Type nodeType = term.NodeType ?? this.DefaultNodeType;
if (nodeType == null) return;
nodeInfo.AstNode = Activator.CreateInstance(nodeType);
//Initialize node
var iInit = nodeInfo.AstNode as IAstNodeInit;
if (iInit != null)
iInit.Init(context, nodeInfo);
}
//This method is called if Scanner failed to produce token; it offers custom method a chance
public virtual Token TryMatch(CompilerContext context, ISourceStream source) {
return null;
}
public override IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens)
{
//TODO: fix this.
// This is a temporary workaround, to "undo" the change to whitespace made by NewLineTerminal.
// if we have both NewLineTerminal in the grammar and CodeOutlineFilter, then NewLines should be generated by filter,
// not by terminal.
context.Compiler.Language.Grammar.WhitespaceChars = " \t\r\n\v";
_prevLine = 0;
_indents.Clear();
foreach (Token token in tokens)
{
if (token.Terminal == base.Grammar.Eof)
{
yield return(CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location)); //this is necessary, because grammar rules reference newLine terminator
//unindent all buffered indents
if (_trackIndents)
{
foreach (int i in _indents)
{
yield return(CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location));
}
}
_indents.Clear();
//return EOF token
yield return(token);
yield break;
}//if Eof
//Now deal with normal, non-EOF tokens
//We intercept only content tokens on new lines
if (token.Terminal.Category != TokenCategory.Content || token.Location.Line == _prevLine)
{
yield return(token);
continue;
}
//if we are here, we have content token on new line; produce newLine token and possibly indents
yield return(CreateSpecialToken(GrammarData.Grammar.NewLine, token.Location));
_prevLine = token.Location.Line;
if (!_trackIndents)
{
yield return(token);
continue;
}
//Now take care of indents
int currIndent = token.Location.Column;
int prevIndent = _indents.Count == 0 ? 0 : _indents.Peek();
if (currIndent > prevIndent)
{
_indents.Push(currIndent);
yield return(CreateSpecialToken(GrammarData.Grammar.Indent, token.Location));
}
else if (currIndent < prevIndent)
{
//produce one or more dedent tokens while popping indents from stack
while (_indents.Count > 0 && _indents.Peek() > currIndent)
{
_indents.Pop();
yield return(CreateSpecialToken(GrammarData.Grammar.Dedent, token.Location));
}
if (_indents.Count == 0 || _indents.Peek() != currIndent)
{
yield return(context.CreateErrorTokenAndReportError(token.Location, string.Empty, "Invalid dedent level, no previous matching indent found."));
//TODO: add error recovery here
}
} //else if currIndent < prevIndent
yield return(token);
} //foreach token
} //method
protected override void InitDetails(CompilerContext context, CompoundTokenDetails details) {
base.InitDetails(context, details);
details.Flags = (short) this.Flags;
}
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9
// so we try to do a quick parse for these, without starting the whole general process
protected override Token QuickParse(CompilerContext context, ISourceStream source) {
if (IsSet(NumberFlags.DisableQuickParse)) return null;
char current = source.CurrentChar;
if (char.IsDigit(current) && QuickParseTerminators.IndexOf(source.NextChar) >= 0) {
int iValue = current - '0';
object value = null;
switch (DefaultIntTypes[0]) {
case TypeCode.Int32: value = iValue; break;
case TypeCode.UInt32: value = (UInt32)iValue; break;
case TypeCode.Byte: value = (byte)iValue; break;
case TypeCode.SByte: value = (sbyte) iValue; break;
case TypeCode.Int16: value = (Int16)iValue; break;
case TypeCode.UInt16: value = (UInt16)iValue; break;
default: return null;
}
Token token = new Token(this, source.TokenStart, current.ToString(), value);
source.Position++;
return token;
} else
return null;
}
public virtual IEnumerable <Token> BeginFiltering(CompilerContext context, IEnumerable <Token> tokens)
{
yield break;
}
public override Token TryMatch(CompilerContext context, ISourceStream source) {
return _handler(this, context, source);
}
protected override void InitDetails(CompilerContext context, CompoundTokenDetails details)
{
base.InitDetails(context, details);
details.Flags = (short)this.Flags;
}
protected virtual Token QuickParse(CompilerContext context, ISourceStream source)
{
return(null);
}
public override Token TryMatch(CompilerContext context, ISourceStream source)
{
return(_handler(this, context, source));
}
//This method is called if Scanner failed to produce token; it offers custom method a chance
public virtual Token TryMatch(CompilerContext context, ISourceStream source)
{
return(null);
}
protected virtual void InitDetails(CompilerContext context, CompoundTokenDetails details)
{
details.PartialOk = (context.Mode == CompileMode.VsLineScan);
details.PartialContinues = (context.ScannerState.Value != 0);
}
public virtual string GetSyntaxErrorMessage(CompilerContext context, ParserState state, ParseTreeNode currentInput)
{
return(null); //Irony then would construct default message
}