public Token RealCompleteText()
{
Token tok = this.RealNext();
while (!tok.IsEnd() && !tok.IsQuote())
{
if (tok.IsControl() && tok.Whitespaces.Length == 0)
{
tok++;
}
else if (tok.IsQuote() && tok.Whitespaces.Length == 0 && tok.RealNext().IsQuote())
{
tok++;
}
tok++;
}
if (tok.IsEnd())
{
Program.Log("Text block not finished?");
return(tok);
}
Tokenizer.SetUnimportantRange(this, tok);
return(tok.RealNext());
}
public Token RealCompleteDirective()
{
Token tok = this;
bool no_control = true;
while (!tok.IsEnd() && !tok.MakesNewLine() && no_control)
{
if (tok.IsQuote())
{
tok = tok.RealCompleteText();
if (tok.RealPrev().MakesNewLine())
{
tok = tok.RealPrev();
break;
}
continue;
}
if (tok.IsControl())
{
no_control = false;
}
if (tok.MakesNewLine())
{
no_control = true;
}
tok = tok.RealNext();
}
Tokenizer.SetUnimportantRange(this, tok);
return(tok.RealNext());
}
private void process()
{
// real tokens, cut out comments and such
Token prev = root;
Token tok = root.RealNext();
while (!tok.IsEnd())
{
if (tok.IsQuote())
{
tok = tok.RealCompleteText();
}
else if (tok.IsCharDelimiter())
{
tok = tok.RealCompleteChar();
}
else if (tok.IsBlockCommentStart())
{
tok = tok.RealCompleteBlockComment();
}
else if (tok.IsLineCommentStart())
{
tok = tok.RealCompleteLineComment();
}
else if (tok.IsDirectiveStart())
{
tok = tok.RealCompleteDirective();
}
else
{
prev = tok;
tok++;
}
}
// add semicolons to class definitions
tok = root;
for (tok = root; !tok.IsEnd(); tok = tok.Next())
{
if (tok.IsEnum() || tok.IsClass() || tok.IsInterface())
{
tok = tok.PreCompleteClass();
if (tok.IsEnd())
{
Token semi = new Token();
semi.Type = Token.Typeenum.Symbol;
semi.Value = ";";
Token tail = tok.Prev();
tail.InsertAfter(semi);
tail.Whitespaces = "";
tok = semi;
}
else if (tok.Next().IsEnd() || !tok.Next().IsSemi())
{
Token semi = new Token();
semi.Type = Token.Typeenum.Symbol;
semi.Value = ";";
semi.Whitespaces = tok.Whitespaces;
tok.Whitespaces = "";
tok.InsertAfter(semi);
tok = semi;
}
}
}
// expand arrays
enumstate state = enumstate.Clear;
tok = root;
while (!tok.Next().IsEnd())
{
tok = tok.Next();
}
for (; !tok.IsBegin(); tok = tok.Prev())
{
if (tok.IsLeftBracket())
{
state = enumstate.FoundBracket;
continue;
}
else if (tok.IsAssign())
{
if (state == enumstate.FoundBracket)
{
state = enumstate.FoundAssign;
}
else
{
state = enumstate.Clear;
}
continue;
}
else if (tok.IsIdentifier())
{
if (state == enumstate.FoundAssign)
{
state = enumstate.FoundIdentifier;
}
else
{
state = enumstate.Clear;
}
continue;
}
if (tok.IsArrayEnd())
{
if (state == enumstate.FoundIdentifier)
{
// go back until identifier
while (!tok.IsEnd() && !tok.IsIdentifier())
{
tok = tok.Prev();
}
if (tok.IsEnd())
{
Program.Log("Assignment in beginning of the file?");
continue;
}
continue;
}
// make an array
tok.RemoveSelf();
tok = tok.Prev();
tok.RemoveSelf();
tok = tok.Prev();
Token ident = tok;
while (!ident.IsBegin() && !ident.IsIdentifier())
{
ident = ident.Prev();
}
if (ident.IsBegin())
{
Program.Log("Array without a type, am I parsing something inlined?");
continue;
}
replaceWithArray(ident, tok);
tok = tok.Next();
}
state = enumstate.Clear;
}
tok = root;
for (tok = root; !tok.IsEnd(); tok = tok.Next())
{
if (tok.IsNot())
{
tok.Value = "!";
tok.Whitespaces = "";
}
}
if (Program.Fixargs)
{
fixAllArguments();
}
}