public Token RealCompleteDirective()
{
Token tok = this;
bool no_control = true;
while (!tok.IsEnd() && !tok.MakesNewLine() && no_control)
{
if (tok.IsQuote())
{
tok = tok.RealCompleteText();
if (tok.RealPrev().MakesNewLine())
{
tok = tok.RealPrev();
break;
}
continue;
}
if (tok.IsControl())
{
no_control = false;
}
if (tok.MakesNewLine())
{
no_control = true;
}
tok = tok.RealNext();
}
Tokenizer.SetUnimportantRange(this, tok);
return(tok.RealNext());
}
public Token RealCompleteText()
{
Token tok = this.RealNext();
while (!tok.IsEnd() && !tok.IsQuote())
{
if (tok.IsControl() && tok.Whitespaces.Length == 0)
{
tok++;
}
else if (tok.IsQuote() && tok.Whitespaces.Length == 0 && tok.RealNext().IsQuote())
{
tok++;
}
tok++;
}
if (tok.IsEnd())
{
Program.Log("Text block not finished?");
return(tok);
}
Tokenizer.SetUnimportantRange(this, tok);
return(tok.RealNext());
}
public Token RealCompleteLineComment()
{
Token tok = this.RealNext();
while (!tok.IsEnd() && !tok.MakesNewLine())
{
tok = tok.RealNext();
}
if (tok.IsEnd())
{
Program.Log("Line comment without newline.");
return(tok);
}
Tokenizer.SetUnimportantRange(this, tok);
return(tok.RealNext());
}
public Token RealCompleteBlockComment()
{
//Console.WriteLine("Starting block comment from "+this.Value);
Token tok = this.RealNext();
while (!tok.IsEnd() && !tok.IsBlockCommentEnd())
{
tok = tok.RealNext();
}
if (tok.IsEnd())
{
Console.Write("Block comment not finished?\n");
return(tok);
}
Tokenizer.SetUnimportantRange(this, tok.RealNext());
//Console.WriteLine("Ending block comment at " + tok.Value+tok.RealNext().Value+tok.RealNext().RealNext().Value);
return(tok.RealNext().RealNext());
}
public Token Next()
{
Token tok = this.RealNext(); while (tok != null && tok.IsUnimportant())
{
tok = tok.RealNext();
}
return(tok);
}
private void parseLines()
{
parseWhitespaces(root);
Token tok = root;
while (lines.Count > 0)
{
tok.RealNextTok = new Token();
tok.RealNext().RealPrevTok = tok;
tok = tok.RealNext();
if (!parseSymbol(tok))
{
parseIdentifier(tok);
}
}
Token end = new Token();
end.Type = Token.Typeenum.End;
end.RealNextTok = end;
tok.RealNextTok = end;
end.RealPrevTok = tok;
}
public Token RealCompleteChar()
{
//Console.WriteLine("starting at " + this.Value + "," + this.RealNext().Value);
Token tok = this.RealNext();
while (!tok.IsEnd() && !tok.IsCharDelimiter())
{
//Console.Write(tok.Value + "|");
tok++;
}
if (tok.IsEnd())
{
Program.Log("Char block not finished?");
Console.ReadKey();
return(tok);
}
Tokenizer.SetUnimportantRange(this, tok);
return(tok.RealNext());
}
private void replaceWithArray(Token from, Token to)
{
Token arr = new Token();
arr.Type = Token.Typeenum.Identifier;
arr.Value = "array";
Token left = new Token();
left.Type = Token.Typeenum.Symbol;
left.Value = "<";
Token right = new Token();
right.Type = Token.Typeenum.TemplateClosure;
right.Value = ">";
linkTokens(arr, left);
linkTokens(from.RealPrev(), arr);
linkTokens(left, from);
linkTokens(right, to.RealNext());
if (right.Next().IsIdentifier() || right.Next().Type == Token.Typeenum.TemplateClosure)
{
right.Whitespaces = " ";
}
linkTokens(to, right);
}
public bool IsArrayStart()
{
if (!(IsLeft() && Next() != null && Next().IsRight()))
{
return(false);
}
for (Token n = RealNext(); !n.IsRight(); n = n.RealNext())
{
if (n.IsLineCommentStart())
{
continue;
}
if (n.IsBlockCommentStart())
{
continue;
}
if (n.IsBlockCommentEnd())
{
continue;
}
return(false);
}
return(true);
}
private void process()
{
// real tokens, cut out comments and such
Token prev = root;
Token tok = root.RealNext();
while (!tok.IsEnd())
{
if (tok.IsQuote())
{
tok = tok.RealCompleteText();
}
else if (tok.IsCharDelimiter())
{
tok = tok.RealCompleteChar();
}
else if (tok.IsBlockCommentStart())
{
tok = tok.RealCompleteBlockComment();
}
else if (tok.IsLineCommentStart())
{
tok = tok.RealCompleteLineComment();
}
else if (tok.IsDirectiveStart())
{
tok = tok.RealCompleteDirective();
}
else
{
prev = tok;
tok++;
}
}
// add semicolons to class definitions
tok = root;
for (tok = root; !tok.IsEnd(); tok = tok.Next())
{
if (tok.IsEnum() || tok.IsClass() || tok.IsInterface())
{
tok = tok.PreCompleteClass();
if (tok.IsEnd())
{
Token semi = new Token();
semi.Type = Token.Typeenum.Symbol;
semi.Value = ";";
Token tail = tok.Prev();
tail.InsertAfter(semi);
tail.Whitespaces = "";
tok = semi;
}
else if (tok.Next().IsEnd() || !tok.Next().IsSemi())
{
Token semi = new Token();
semi.Type = Token.Typeenum.Symbol;
semi.Value = ";";
semi.Whitespaces = tok.Whitespaces;
tok.Whitespaces = "";
tok.InsertAfter(semi);
tok = semi;
}
}
}
// expand arrays
enumstate state = enumstate.Clear;
tok = root;
while (!tok.Next().IsEnd())
{
tok = tok.Next();
}
for (; !tok.IsBegin(); tok = tok.Prev())
{
if (tok.IsLeftBracket())
{
state = enumstate.FoundBracket;
continue;
}
else if (tok.IsAssign())
{
if (state == enumstate.FoundBracket)
{
state = enumstate.FoundAssign;
}
else
{
state = enumstate.Clear;
}
continue;
}
else if (tok.IsIdentifier())
{
if (state == enumstate.FoundAssign)
{
state = enumstate.FoundIdentifier;
}
else
{
state = enumstate.Clear;
}
continue;
}
if (tok.IsArrayEnd())
{
if (state == enumstate.FoundIdentifier)
{
// go back until identifier
while (!tok.IsEnd() && !tok.IsIdentifier())
{
tok = tok.Prev();
}
if (tok.IsEnd())
{
Program.Log("Assignment in beginning of the file?");
continue;
}
continue;
}
// make an array
tok.RemoveSelf();
tok = tok.Prev();
tok.RemoveSelf();
tok = tok.Prev();
Token ident = tok;
while (!ident.IsBegin() && !ident.IsIdentifier())
{
ident = ident.Prev();
}
if (ident.IsBegin())
{
Program.Log("Array without a type, am I parsing something inlined?");
continue;
}
replaceWithArray(ident, tok);
tok = tok.Next();
}
state = enumstate.Clear;
}
tok = root;
for (tok = root; !tok.IsEnd(); tok = tok.Next())
{
if (tok.IsNot())
{
tok.Value = "!";
tok.Whitespaces = "";
}
}
if (Program.Fixargs)
{
fixAllArguments();
}
}
private void replaceWithArray(Token from, Token to)
{
Token arr = new Token();
arr.Type = Token.Typeenum.Identifier;
arr.Value = "array";
Token left = new Token();
left.Type = Token.Typeenum.Symbol;
left.Value = "<";
Token right = new Token();
right.Type = Token.Typeenum.TemplateClosure;
right.Value = ">";
linkTokens(arr, left);
linkTokens(from.RealPrev(), arr);
linkTokens(left,from);
linkTokens(right,to.RealNext());
if (right.Next().IsIdentifier() || right.Next().Type == Token.Typeenum.TemplateClosure) right.Whitespaces = " ";
linkTokens(to, right);
}
