public Token RealCompleteDirective() { Token tok = this; bool no_control = true; while (!tok.IsEnd() && !tok.MakesNewLine() && no_control) { if (tok.IsQuote()) { tok = tok.RealCompleteText(); if (tok.RealPrev().MakesNewLine()) { tok = tok.RealPrev(); break; } continue; } if (tok.IsControl()) { no_control = false; } if (tok.MakesNewLine()) { no_control = true; } tok = tok.RealNext(); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
public Token RealCompleteText() { Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.IsQuote()) { if (tok.IsControl() && tok.Whitespaces.Length == 0) { tok++; } else if (tok.IsQuote() && tok.Whitespaces.Length == 0 && tok.RealNext().IsQuote()) { tok++; } tok++; } if (tok.IsEnd()) { Program.Log("Text block not finished?"); return(tok); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
public Token RealCompleteLineComment() { Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.MakesNewLine()) { tok = tok.RealNext(); } if (tok.IsEnd()) { Program.Log("Line comment without newline."); return(tok); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
public Token RealCompleteBlockComment() { //Console.WriteLine("Starting block comment from "+this.Value); Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.IsBlockCommentEnd()) { tok = tok.RealNext(); } if (tok.IsEnd()) { Console.Write("Block comment not finished?\n"); return(tok); } Tokenizer.SetUnimportantRange(this, tok.RealNext()); //Console.WriteLine("Ending block comment at " + tok.Value+tok.RealNext().Value+tok.RealNext().RealNext().Value); return(tok.RealNext().RealNext()); }
public Token Next() { Token tok = this.RealNext(); while (tok != null && tok.IsUnimportant()) { tok = tok.RealNext(); } return(tok); }
private void parseLines() { parseWhitespaces(root); Token tok = root; while (lines.Count > 0) { tok.RealNextTok = new Token(); tok.RealNext().RealPrevTok = tok; tok = tok.RealNext(); if (!parseSymbol(tok)) { parseIdentifier(tok); } } Token end = new Token(); end.Type = Token.Typeenum.End; end.RealNextTok = end; tok.RealNextTok = end; end.RealPrevTok = tok; }
public Token RealCompleteChar() { //Console.WriteLine("starting at " + this.Value + "," + this.RealNext().Value); Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.IsCharDelimiter()) { //Console.Write(tok.Value + "|"); tok++; } if (tok.IsEnd()) { Program.Log("Char block not finished?"); Console.ReadKey(); return(tok); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
private void replaceWithArray(Token from, Token to) { Token arr = new Token(); arr.Type = Token.Typeenum.Identifier; arr.Value = "array"; Token left = new Token(); left.Type = Token.Typeenum.Symbol; left.Value = "<"; Token right = new Token(); right.Type = Token.Typeenum.TemplateClosure; right.Value = ">"; linkTokens(arr, left); linkTokens(from.RealPrev(), arr); linkTokens(left, from); linkTokens(right, to.RealNext()); if (right.Next().IsIdentifier() || right.Next().Type == Token.Typeenum.TemplateClosure) { right.Whitespaces = " "; } linkTokens(to, right); }
public bool IsArrayStart() { if (!(IsLeft() && Next() != null && Next().IsRight())) { return(false); } for (Token n = RealNext(); !n.IsRight(); n = n.RealNext()) { if (n.IsLineCommentStart()) { continue; } if (n.IsBlockCommentStart()) { continue; } if (n.IsBlockCommentEnd()) { continue; } return(false); } return(true); }
private void process() { // real tokens, cut out comments and such Token prev = root; Token tok = root.RealNext(); while (!tok.IsEnd()) { if (tok.IsQuote()) { tok = tok.RealCompleteText(); } else if (tok.IsCharDelimiter()) { tok = tok.RealCompleteChar(); } else if (tok.IsBlockCommentStart()) { tok = tok.RealCompleteBlockComment(); } else if (tok.IsLineCommentStart()) { tok = tok.RealCompleteLineComment(); } else if (tok.IsDirectiveStart()) { tok = tok.RealCompleteDirective(); } else { prev = tok; tok++; } } // add semicolons to class definitions tok = root; for (tok = root; !tok.IsEnd(); tok = tok.Next()) { if (tok.IsEnum() || tok.IsClass() || tok.IsInterface()) { tok = tok.PreCompleteClass(); if (tok.IsEnd()) { Token semi = new Token(); semi.Type = Token.Typeenum.Symbol; semi.Value = ";"; Token tail = tok.Prev(); tail.InsertAfter(semi); tail.Whitespaces = ""; tok = semi; } else if (tok.Next().IsEnd() || !tok.Next().IsSemi()) { Token semi = new Token(); semi.Type = Token.Typeenum.Symbol; semi.Value = ";"; semi.Whitespaces = tok.Whitespaces; tok.Whitespaces = ""; tok.InsertAfter(semi); tok = semi; } } } // expand arrays enumstate state = enumstate.Clear; tok = root; while (!tok.Next().IsEnd()) { tok = tok.Next(); } for (; !tok.IsBegin(); tok = tok.Prev()) { if (tok.IsLeftBracket()) { state = enumstate.FoundBracket; continue; } else if (tok.IsAssign()) { if (state == enumstate.FoundBracket) { state = enumstate.FoundAssign; } else { state = enumstate.Clear; } continue; } else if (tok.IsIdentifier()) { if (state == enumstate.FoundAssign) { state = enumstate.FoundIdentifier; } else { state = enumstate.Clear; } continue; } if (tok.IsArrayEnd()) { if (state == enumstate.FoundIdentifier) { // go back until identifier while (!tok.IsEnd() && !tok.IsIdentifier()) { tok = tok.Prev(); } if (tok.IsEnd()) { Program.Log("Assignment in beginning of the file?"); continue; } continue; } // make an array tok.RemoveSelf(); tok = tok.Prev(); tok.RemoveSelf(); tok = tok.Prev(); Token ident = tok; while (!ident.IsBegin() && !ident.IsIdentifier()) { ident = ident.Prev(); } if (ident.IsBegin()) { Program.Log("Array without a type, am I parsing something inlined?"); continue; } replaceWithArray(ident, tok); tok = tok.Next(); } state = enumstate.Clear; } tok = root; for (tok = root; !tok.IsEnd(); tok = tok.Next()) { if (tok.IsNot()) { tok.Value = "!"; tok.Whitespaces = ""; } } if (Program.Fixargs) { fixAllArguments(); } }
private void replaceWithArray(Token from, Token to) { Token arr = new Token(); arr.Type = Token.Typeenum.Identifier; arr.Value = "array"; Token left = new Token(); left.Type = Token.Typeenum.Symbol; left.Value = "<"; Token right = new Token(); right.Type = Token.Typeenum.TemplateClosure; right.Value = ">"; linkTokens(arr, left); linkTokens(from.RealPrev(), arr); linkTokens(left,from); linkTokens(right,to.RealNext()); if (right.Next().IsIdentifier() || right.Next().Type == Token.Typeenum.TemplateClosure) right.Whitespaces = " "; linkTokens(to, right); }