public Token Tokenize(String stream, String filePath)
{
int Pos = 0;
Token Result;
Token FileNode = new Token();
Rule Root = new RuleName(null);
FileNode.SetValue("", 0, Root, Root);
DateTime _start = DateTime.Now;
Log.getInstance().Add("Tokenize " + filePath, Log.EnuSeverity.Info, "");
while (Pos < stream.Length)
{
Result = Rules.Evaluate(stream, ref Pos);
if (Result != null && Result.IsValid())
{
FileNode.AddLast(Result);
}
else
{
RuleAnyLine x = new RuleAnyLine(null);
Result = x.Evaluate(stream, ref Pos);
Result.SetError("invalid:" + Rules.GetError() + " at " + Pos.ToString());
FileNode.AddLast(Result);
Log.getInstance().Add("invalid:" + Rules.GetError() + " at " + Pos.ToString(),
Log.EnuSeverity.Warn,
filePath + "@" + Pos.ToString());
break;
}
}
DateTime _end = DateTime.Now;
TimeSpan dt = _end - _start;
Log.getInstance().Add("Tokenized " + filePath + " in " + dt.TotalMilliseconds.ToString() + " ms", Log.EnuSeverity.Info, "");
return(FileNode);
}