public Token RealCompleteText() { Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.IsQuote()) { if (tok.IsControl() && tok.Whitespaces.Length == 0) { tok++; } else if (tok.IsQuote() && tok.Whitespaces.Length == 0 && tok.RealNext().IsQuote()) { tok++; } tok++; } if (tok.IsEnd()) { Program.Log("Text block not finished?"); return(tok); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
public Token PreCompleteClass() { Token tok = this.Next(); while (!tok.IsEnd() && !tok.IsLeftBracket()) { tok = tok.Next(); } if (tok.IsEnd()) { Program.Log("Bad class definition?\n"); return(tok); } int depth = 1; while (depth > 0) { tok = tok.Next(); if (tok.IsEnd()) { Program.Log("Failure on class closure: " + this.Next().Value); return(tok); } if (tok.IsLeftBracket()) { depth++; } else if (tok.IsRightBracket()) { depth--; } } return(tok); }
public Token RealCompleteDirective() { Token tok = this; bool no_control = true; while (!tok.IsEnd() && !tok.MakesNewLine() && no_control) { if (tok.IsQuote()) { tok = tok.RealCompleteText(); if (tok.RealPrev().MakesNewLine()) { tok = tok.RealPrev(); break; } continue; } if (tok.IsControl()) { no_control = false; } if (tok.MakesNewLine()) { no_control = true; } tok = tok.RealNext(); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
public Token RealCompleteLineComment() { Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.MakesNewLine()) { tok = tok.RealNext(); } if (tok.IsEnd()) { Program.Log("Line comment without newline."); return(tok); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
public Token RealCompleteBlockComment() { //Console.WriteLine("Starting block comment from "+this.Value); Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.IsBlockCommentEnd()) { tok = tok.RealNext(); } if (tok.IsEnd()) { Console.Write("Block comment not finished?\n"); return(tok); } Tokenizer.SetUnimportantRange(this, tok.RealNext()); //Console.WriteLine("Ending block comment at " + tok.Value+tok.RealNext().Value+tok.RealNext().RealNext().Value); return(tok.RealNext().RealNext()); }
public Token RealCompleteChar() { //Console.WriteLine("starting at " + this.Value + "," + this.RealNext().Value); Token tok = this.RealNext(); while (!tok.IsEnd() && !tok.IsCharDelimiter()) { //Console.Write(tok.Value + "|"); tok++; } if (tok.IsEnd()) { Program.Log("Char block not finished?"); Console.ReadKey(); return(tok); } Tokenizer.SetUnimportantRange(this, tok); return(tok.RealNext()); }
public bool Save(String filename) { StreamWriter file = new StreamWriter(filename, false, System.Text.Encoding.Default); file.NewLine = newline; for (Token tok = root; !tok.IsEnd(); tok++) { file.Write(tok.Value); file.Write(tok.Whitespaces); } file.Close(); return(true); }
public static void SetUnimportantRange(Token from, Token to) { if (from.IsEdge()) { return; } for (Token tok = from; tok != to && !tok.IsEnd(); tok++) { tok.SetUnimportant(); } if (!to.IsEdge()) { to.SetUnimportant(); } }
private void fixArguments(Token tok) { String funcname = tok.Prev().Prev().Value; List <String> nonbasic = new List <String>(); while (!tok.IsRightPar()) { Type type = new Type(); tok = tok.CompleteType(type); String ident = tok.IsIdentifier() ? tok.Value : "anonymous"; if (tok.IsIdentifier() && !type.IsBasic() && type.ModName == "") { nonbasic.Add(ident); } if (tok.IsIdentifier()) { tok = tok.Next(); } if (tok.IsEnd() || tok.IsRightPar()) { break; } tok = tok.Next(); if (tok.IsEnd()) { break; } } tok = tok.Next(); if (!tok.IsLeftBracket()) { return; } tok = tok.Next(); if (tok.IsEnd()) { return; } int level = tok.ScopeLevel; Token next; Token prev; while (nonbasic.Count > 0 && !tok.IsRightBracket() && !tok.IsEnd() && tok.ScopeLevel == level) { next = tok.Next(); prev = tok.Prev(); if (tok.IsIdentifier() && !next.IsEnd() && prev.IsSemi() && next.IsAssign() && nonbasic.Contains(tok.Value)) { nonbasic.Remove(tok.Value); } tok = tok.Next(); } if (nonbasic.Count > 0) { Program.LogNoLine(funcname + ": nonbasic value-args without assignment found: "); bool comma = false; foreach (String s in nonbasic) { if (comma) { Program.LogNoLine(", ", false); } else { comma = true; } Program.LogNoLine(s, false); } Program.Log(".", false); } }
private void process() { // real tokens, cut out comments and such Token prev = root; Token tok = root.RealNext(); while (!tok.IsEnd()) { if (tok.IsQuote()) { tok = tok.RealCompleteText(); } else if (tok.IsCharDelimiter()) { tok = tok.RealCompleteChar(); } else if (tok.IsBlockCommentStart()) { tok = tok.RealCompleteBlockComment(); } else if (tok.IsLineCommentStart()) { tok = tok.RealCompleteLineComment(); } else if (tok.IsDirectiveStart()) { tok = tok.RealCompleteDirective(); } else { prev = tok; tok++; } } // add semicolons to class definitions tok = root; for (tok = root; !tok.IsEnd(); tok = tok.Next()) { if (tok.IsEnum() || tok.IsClass() || tok.IsInterface()) { tok = tok.PreCompleteClass(); if (tok.IsEnd()) { Token semi = new Token(); semi.Type = Token.Typeenum.Symbol; semi.Value = ";"; Token tail = tok.Prev(); tail.InsertAfter(semi); tail.Whitespaces = ""; tok = semi; } else if (tok.Next().IsEnd() || !tok.Next().IsSemi()) { Token semi = new Token(); semi.Type = Token.Typeenum.Symbol; semi.Value = ";"; semi.Whitespaces = tok.Whitespaces; tok.Whitespaces = ""; tok.InsertAfter(semi); tok = semi; } } } // expand arrays enumstate state = enumstate.Clear; tok = root; while (!tok.Next().IsEnd()) { tok = tok.Next(); } for (; !tok.IsBegin(); tok = tok.Prev()) { if (tok.IsLeftBracket()) { state = enumstate.FoundBracket; continue; } else if (tok.IsAssign()) { if (state == enumstate.FoundBracket) { state = enumstate.FoundAssign; } else { state = enumstate.Clear; } continue; } else if (tok.IsIdentifier()) { if (state == enumstate.FoundAssign) { state = enumstate.FoundIdentifier; } else { state = enumstate.Clear; } continue; } if (tok.IsArrayEnd()) { if (state == enumstate.FoundIdentifier) { // go back until identifier while (!tok.IsEnd() && !tok.IsIdentifier()) { tok = tok.Prev(); } if (tok.IsEnd()) { Program.Log("Assignment in beginning of the file?"); continue; } continue; } // make an array tok.RemoveSelf(); tok = tok.Prev(); tok.RemoveSelf(); tok = tok.Prev(); Token ident = tok; while (!ident.IsBegin() && !ident.IsIdentifier()) { ident = ident.Prev(); } if (ident.IsBegin()) { Program.Log("Array without a type, am I parsing something inlined?"); continue; } replaceWithArray(ident, tok); tok = tok.Next(); } state = enumstate.Clear; } tok = root; for (tok = root; !tok.IsEnd(); tok = tok.Next()) { if (tok.IsNot()) { tok.Value = "!"; tok.Whitespaces = ""; } } if (Program.Fixargs) { fixAllArguments(); } }
private void fixArguments(Token tok) { String funcname = tok.Prev().Prev().Value; List<String> nonbasic = new List<String>(); while (!tok.IsRightPar()) { Type type = new Type(); tok = tok.CompleteType(type); String ident = tok.IsIdentifier() ? tok.Value : "anonymous"; if (tok.IsIdentifier() && !type.IsBasic() && type.ModName=="") nonbasic.Add(ident); if (tok.IsIdentifier()) tok = tok.Next(); if (tok.IsEnd() || tok.IsRightPar()) break; tok = tok.Next(); if (tok.IsEnd()) break; } tok = tok.Next(); if (!tok.IsLeftBracket()) return; tok=tok.Next(); if (tok.IsEnd()) return; int level = tok.ScopeLevel; Token next; Token prev; while (nonbasic.Count>0 && !tok.IsRightBracket() && !tok.IsEnd() && tok.ScopeLevel == level) { next=tok.Next(); prev = tok.Prev(); if (tok.IsIdentifier() && !next.IsEnd() && prev.IsSemi() && next.IsAssign() && nonbasic.Contains(tok.Value)) { nonbasic.Remove(tok.Value); } tok = tok.Next(); } if (nonbasic.Count > 0) { Program.LogNoLine(funcname + ": nonbasic value-args without assignment found: "); bool comma = false; foreach (String s in nonbasic) { if (comma) Program.LogNoLine(", ",false); else comma = true; Program.LogNoLine(s,false); } Program.Log(".",false); } }