/// <summary> /// Finds the last import statement and returns its end location (the position after the semicolon). /// If no import but module statement was found, the end location of this module statement will be returned. /// </summary> public static CodeLocation FindLastImportStatementEndLocation(DModule m, string moduleCode = null) { IStatement lastStmt = null; foreach (var s in m.StaticStatements) if (s is ImportStatement) lastStmt = s; else if (lastStmt != null) break; if (lastStmt != null) return lastStmt.EndLocation; if (m.OptionalModuleStatement != null) return m.OptionalModuleStatement.EndLocation; if (moduleCode != null) using(var sr = new StringReader(moduleCode)) using (var lx = new Lexer(sr) { OnlyEnlistDDocComments = false }) { lx.NextToken(); if (lx.Comments.Count != 0) return lx.Comments[lx.Comments.Count - 1].EndPosition; } return new CodeLocation(1, 1); }
static DToken SearchBackward(TextDocument doc, int caretOffset, CodeLocation caret,out DToken lastToken) { var ttp = doc.GetText(0, caretOffset); var sr = new StringReader(ttp); var lexer = new Lexer(sr); lexer.NextToken(); var stk=new Stack<DToken>(); while (lexer.LookAhead.Kind!=DTokens.EOF) { if (lexer.LookAhead.Kind == DTokens.OpenParenthesis || lexer.LookAhead.Kind==DTokens.OpenSquareBracket || lexer.LookAhead.Kind==DTokens.OpenCurlyBrace) stk.Push(lexer.LookAhead); else if (lexer.LookAhead.Kind == DTokens.CloseParenthesis || lexer.LookAhead.Kind == DTokens.CloseSquareBracket || lexer.LookAhead.Kind == DTokens.CloseCurlyBrace) { if (stk.Peek().Kind == getOppositeBracketToken( lexer.LookAhead.Kind)) stk.Pop(); } lexer.NextToken(); } lastToken = lexer.CurrentToken; sr.Close(); lexer.Dispose(); if (stk.Count < 1) return null; return stk.Pop(); }
public void Unicode1() { var lex = new Lexer (new StringReader ("'ߞ'")); lex.NextToken (); Assert.That (lex.LexerErrors.Count, Is.EqualTo(0)); }
public void Dispose() { doc = null; BlockAttributes.Clear(); BlockAttributes = null; DeclarationAttributes.Clear(); DeclarationAttributes = null; Lexer.Dispose(); Lexer = null; ParseErrors = null; }
public void CharLiteralBug() { string code = "'@@'"; var sr = new StringReader (code); var lex = new Lexer (sr); lex.NextToken (); Assert.That (lex.LexerErrors.Count, Is.EqualTo(1)); Assert.That (lex.LookAhead.Kind, Is.EqualTo(DTokens.Literal)); Assert.That (lex.LookAhead.LiteralFormat, Is.EqualTo(LiteralFormat.CharLiteral)); }
public DParser(Lexer lexer) { this.Lexer = lexer; Lexer.LexerErrors = ParseErrors; }
//[Test] public void LexingPerformance() { var f = File.ReadAllText(Environment.OSVersion.Platform == PlatformID.Win32NT ? @"D:\D\dmd2\src\phobos\std\string.d" : "/usr/include/dlang/std/string.d"); var lx = new Lexer(new StringReader(f)); var sw = new Stopwatch(); sw.Start(); while(true) { lx.NextToken(); if(lx.IsEOF) break; } sw.Stop(); Console.WriteLine(sw.ElapsedMilliseconds); }
public CodeBlock CalculateIndentation(TextReader code, int line) { block = null; Lexer = new Lexer(code); maxLine = line; Lexer.NextToken(); DToken lastToken = null; while (!Lexer.IsEOF) { if (t != null && la.line > t.line && t.line < maxLine) { RemoveNextLineUnindentBlocks(); } lastToken = t; Lexer.NextToken(); if (IsEOF) { if (la.line > maxLine || Lexer.IsEOF) lastLineIndent = null; if (t.line>maxLine) break; } /* * if(..) * for(...) * while(...) * foo(); * // No indentation anymore! */ if (t.Kind == DTokens.Comma || t.Kind == DTokens.Semicolon && maxLine>t.line && la.line > t.line) { if (block == null) continue; if (block.Reason == CodeBlock.IndentReason.UnfinishedStatement) PopBlock(); while ( block != null && block.Reason == CodeBlock.IndentReason.SingleLineStatement && !IsSemicolonContainingStatement) PopBlock(); } // (,[,{ else if (t.Kind == DTokens.OpenParenthesis || t.Kind == DTokens.OpenSquareBracket || t.Kind == DTokens.OpenCurlyBrace) { var tBlock = block; if (block != null && ( block.Reason == CodeBlock.IndentReason.SingleLineStatement || block.Reason == CodeBlock.IndentReason.UnfinishedStatement)) { PopBlock(); } PushBlock(tBlock).BlockStartToken = t.Kind; } // ),],} else if (t.Kind == DTokens.CloseParenthesis || t.Kind == DTokens.CloseSquareBracket || t.Kind == DTokens.CloseCurlyBrace) { if (t.Kind == DTokens.CloseCurlyBrace) { while (block != null && !block.IsClampBlock) PopBlock(); /* * If the last token was on this line OR if it's eof but on the following line, * decrement indent on next line only. */ if (lastToken!=null && lastToken.line == t.line && block != null) { block.PopOnNextLine = true; } else PopBlock(); } else { while (block != null && !block.IsClampBlock) PopBlock(); if (lastLineIndent == null && (block == null || block.StartLocation.Line < t.line)) lastLineIndent = block; if (t.Kind == DTokens.CloseParenthesis && block != null && block.BlockStartToken == DTokens.OpenParenthesis && la.Kind!=DTokens.OpenCurlyBrace) { block=block.previousBlock; continue; } else PopBlock(); if (t.Kind == DTokens.CloseParenthesis && block != null && block.BlockStartToken == DTokens.OpenParenthesis) { if (la.Kind == DTokens.OpenCurlyBrace && la.line > t.line) PopBlock(); else if (block!=null && block.LastPreBlockIdentifier!=null && IsPreStatementToken(block.LastPreBlockIdentifier.Kind)) block = block.previousBlock; } } } else if ((DParser.IsAttributeSpecifier(t.Kind, la.Kind) && la.Kind==DTokens.Colon) || t.Kind == DTokens.Case || t.Kind==DTokens.Default) { while (block != null && block.BlockStartToken!=DTokens.OpenCurlyBrace) PopBlock(); PushBlock().Reason = CodeBlock.IndentReason.StatementLabel; HadCaseStatementBegin = true; } else if (t.Kind == DTokens.Colon) { if (HadCaseStatementBegin) { while (block != null && block.Reason != CodeBlock.IndentReason.StatementLabel) PopBlock(); HadCaseStatementBegin = false; } } // Don't indent these in front of function bodies else if (t.Kind == DTokens.In || t.Kind == DTokens.Out || t.Kind == DTokens.Body) { if (block != null && block.Reason == CodeBlock.IndentReason.UnfinishedStatement) PopBlock(); } else if (block == null || block.Reason != CodeBlock.IndentReason.UnfinishedStatement && block.Reason != CodeBlock.IndentReason.SingleLineStatement) PushBlock().Reason = CodeBlock.IndentReason.UnfinishedStatement; } if (t!=null && la.line > t.line) RemoveNextLineUnindentBlocks(); return lastLineIndent ?? block; }
public static DParser Create(TextReader tr, params TokenTracker[] trackers) { var lx = new Lexer(tr); if(trackers!=null && trackers.Length>0) lx.TokenTracker.Trackers.AddRange(trackers); return new DParser(lx); }
/// <summary> /// Checks if an identifier is about to be typed. Therefore, we assume that this identifier hasn't been typed yet. /// So, we also will assume that the caret location is the start of the identifier; /// </summary> public static bool IsTypeIdentifier(string code, int caret) { //try{ if (caret < 1) return false; code = code.Insert(caret, " "); // To ensure correct behaviour, insert a phantom ws after the caret // Check for preceding letters if (char.IsLetter(code[caret])) return true; int precedingExpressionOrTypeStartOffset = ReverseParsing.SearchExpressionStart(code, caret); if (precedingExpressionOrTypeStartOffset >= caret) return false; var expressionCode = code.Substring(precedingExpressionOrTypeStartOffset, caret - precedingExpressionOrTypeStartOffset); if (string.IsNullOrEmpty(expressionCode) || expressionCode.Trim() == string.Empty) return false; var lx = new Lexer(new StringReader(expressionCode)); var firstToken = lx.NextToken(); if (DTokens.ClassLike[firstToken.Kind]) return true; while (lx.LookAhead.Kind != DTokens.EOF) lx.NextToken(); var lastToken = lx.CurrentToken; if (lastToken.Kind == DTokens.Times) return false; // TODO: Check if it's an expression or not if (lastToken.Kind == DTokens.CloseSquareBracket || lastToken.Kind == DTokens.Identifier) return true; if (lastToken.Kind == DTokens.CloseParenthesis) { lx.CurrentToken = firstToken; while (lx.LookAhead.Kind != DTokens.OpenParenthesis && lx.LookAhead.Kind != DTokens.EOF) lx.NextToken(); if (sigTokens[lx.CurrentToken.Kind]) return false; else return true; } //}catch(Exception ex) { } return false; }
static DToken SearchForward(TextDocument doc, int caretOffset, CodeLocation caret, int searchedBracketToken) { var code = doc.GetText(caretOffset, doc.TextLength - caretOffset); var lexer = new Lexer(new System.IO.StringReader(code)); lexer.SetInitialLocation(caret); lexer.NextToken(); var stk = new Stack<DToken>(); while (lexer.LookAhead.Kind!=DTokens.EOF) { if (lexer.LookAhead.Kind == DTokens.OpenParenthesis || lexer.LookAhead.Kind == DTokens.OpenSquareBracket || lexer.LookAhead.Kind == DTokens.OpenCurlyBrace) stk.Push(lexer.LookAhead); else if (lexer.LookAhead.Kind == DTokens.CloseParenthesis || lexer.LookAhead.Kind == DTokens.CloseSquareBracket || lexer.LookAhead.Kind == DTokens.CloseCurlyBrace) { if(stk.Count != 0) stk.Pop(); else if (lexer.LookAhead.Kind == searchedBracketToken) return lexer.LookAhead; } lexer.NextToken(); } lexer.Dispose(); if (stk.Count < 1) return null; return stk.Pop(); }