public void Next(LexerMode mode)
{
switch (mode)
{
case LexerMode.Block:
this.current = this.NextBlock ();
break;
case LexerMode.Raw:
this.current = this.NextRaw ();
break;
default:
throw new ParseException (this.column, this.line, "<?>", "block or raw text");
}
}
public bool Reset(TextReader reader)
{
this.column = 1;
this.current = new Lexem ();
this.eof = false;
this.last = '\0';
this.line = 1;
this.pending.Clear ();
this.reader = reader;
return this.Read ();
}
private Lexem NextRaw()
{
StringBuilder buffer;
bool cancel;
Lexem lexem;
string text;
StringBuilder token;
int trail;
LexemType type;
buffer = new StringBuilder ();
for (; !this.eof; this.Read ())
{
cancel = this.last == this.escape && this.Read ();
trail = 0;
this.cursors.Enqueue (new LexemCursor (this.last, this.root));
foreach (LexemCursor cursor in this.cursors)
{
if (cancel)
cursor.Cancel ();
else if (cursor.Move (this.last, out type))
{
while (trail-- > 0)
buffer.Append (this.cursors.Dequeue ().Character);
token = new StringBuilder ();
while (this.cursors.Count > 0)
token.Append (this.cursors.Dequeue ().Character);
text = buffer.ToString ();
if (string.IsNullOrEmpty (text))
lexem = new Lexem (type, token.ToString ());
else
{
for (int i = 0; i < token.Length; ++i)
this.pending.Enqueue (token[i]);
lexem = new Lexem (LexemType.Text, text);
}
this.Read ();
return lexem;
}
++trail;
}
while (this.cursors.Count > 0 && this.cursors.Peek ().State == null)
buffer.Append (this.cursors.Dequeue ().Character);
}
while (this.cursors.Count > 0)
buffer.Append (this.cursors.Dequeue ().Character);
text = buffer.ToString ();
if (!string.IsNullOrEmpty (text))
return new Lexem (LexemType.Text, text);
return new Lexem (LexemType.EndOfFile, "<EOF>");
}
private Lexem NextChar(LexemType type)
{
Lexem lexem;
lexem = new Lexem (type, this.last.ToString ());
this.Read ();
return lexem;
}
private Lexem NextRaw()
{
StringBuilder buffer;
bool cancel;
Lexem lexem;
string text;
StringBuilder token;
int trail;
LexemType type;
buffer = new StringBuilder();
for (; !this.eof; this.Read())
{
cancel = this.last == this.escape && this.Read();
trail = 0;
this.cursors.Enqueue(new LexemCursor(this.last, this.root));
foreach (LexemCursor cursor in this.cursors)
{
if (cancel)
{
cursor.Cancel();
}
else if (cursor.Move(this.last, out type))
{
while (trail-- > 0)
{
buffer.Append(this.cursors.Dequeue().Character);
}
token = new StringBuilder();
while (this.cursors.Count > 0)
{
token.Append(this.cursors.Dequeue().Character);
}
text = buffer.ToString();
if (string.IsNullOrEmpty(text))
{
lexem = new Lexem(type, token.ToString());
}
else
{
for (int i = 0; i < token.Length; ++i)
{
this.pending.Enqueue(token[i]);
}
lexem = new Lexem(LexemType.Text, text);
}
this.Read();
return(lexem);
}
++trail;
}
while (this.cursors.Count > 0 && this.cursors.Peek().State == null)
{
buffer.Append(this.cursors.Dequeue().Character);
}
}
while (this.cursors.Count > 0)
{
buffer.Append(this.cursors.Dequeue().Character);
}
text = buffer.ToString();
if (!string.IsNullOrEmpty(text))
{
return(new Lexem(LexemType.Text, text));
}
return(new Lexem(LexemType.EndOfFile, "<EOF>"));
}
private Lexem NextRaw()
{
StringBuilder buffer;
int copy;
int first;
Lexem lexem;
LexemCursor next;
string text;
StringBuilder token;
buffer = new StringBuilder();
for (; !this.eof; this.Read())
{
// Escape sequence found, cancel all pending cursors
if (this.last == this.escape && this.Read())
{
foreach (LexemCursor cursor in this.cursors)
{
buffer.Append(cursor.Character);
}
this.cursors.Clear();
buffer.Append(this.last);
}
// Not an escape sequence, move all cursors
else
{
this.cursors.Add(new LexemCursor(this.last, this.root));
for (int candidate = 0; candidate < this.cursors.Count; ++candidate)
{
next = this.cursors[candidate].Move(this.last);
this.cursors[candidate] = next;
// No lexem matched for this cursor (and no risk of LOH allocation), continue loop
if ((next.State == null || next.State.Type == LexemType.None) && (buffer.Length < MaxBufferSize || this.pending.Count > 0))
{
continue;
}
// Lexem matched, flush characters located before it
// Or text too long, risk for LOH
for (int i = 0; i < candidate; ++i)
{
buffer.Append(this.cursors[i].Character);
}
text = buffer.ToString();
// We were in the case of No lexem matched for this cursor
if (next.State == null || next.State.Type == LexemType.None)
{
this.Read();
return(new Lexem(LexemType.Text, text));
}
// Concatenate matched characters to build matched lexem string
token = new StringBuilder();
while (candidate < this.cursors.Count)
{
token.Append(this.cursors[candidate++].Character);
}
// Return lexem if no text was located before or enqueue otherwise
if (string.IsNullOrEmpty(text))
{
lexem = new Lexem(next.State.Type, token.ToString());
}
else
{
for (int i = 0; i < token.Length; ++i)
{
this.pending.Enqueue(token[i]);
}
lexem = new Lexem(LexemType.Text, text);
}
this.Read();
this.cursors.Clear();
return(lexem);
}
// Remove dead cursors and shift next ones
first = 0;
while (first < this.cursors.Count && this.cursors[first].State == null)
{
buffer.Append(this.cursors[first++].Character);
}
if (first > 0)
{
copy = 0;
while (first < this.cursors.Count)
{
this.cursors[copy++] = this.cursors[first++];
}
this.cursors.RemoveRange(copy, this.cursors.Count - copy);
}
}
}
foreach (LexemCursor cursor in this.cursors)
{
buffer.Append(cursor.Character);
}
this.cursors.Clear();
text = buffer.ToString();
if (!string.IsNullOrEmpty(text))
{
return(new Lexem(LexemType.Text, text));
}
return(new Lexem(LexemType.EndOfFile, "<EOF>"));
}
