public void Next(LexerMode mode) { switch (mode) { case LexerMode.Block: this.current = this.NextBlock (); break; case LexerMode.Raw: this.current = this.NextRaw (); break; default: throw new ParseException (this.column, this.line, "<?>", "block or raw text"); } }
public bool Reset(TextReader reader) { this.column = 1; this.current = new Lexem (); this.eof = false; this.last = '\0'; this.line = 1; this.pending.Clear (); this.reader = reader; return this.Read (); }
private Lexem NextRaw() { StringBuilder buffer; bool cancel; Lexem lexem; string text; StringBuilder token; int trail; LexemType type; buffer = new StringBuilder (); for (; !this.eof; this.Read ()) { cancel = this.last == this.escape && this.Read (); trail = 0; this.cursors.Enqueue (new LexemCursor (this.last, this.root)); foreach (LexemCursor cursor in this.cursors) { if (cancel) cursor.Cancel (); else if (cursor.Move (this.last, out type)) { while (trail-- > 0) buffer.Append (this.cursors.Dequeue ().Character); token = new StringBuilder (); while (this.cursors.Count > 0) token.Append (this.cursors.Dequeue ().Character); text = buffer.ToString (); if (string.IsNullOrEmpty (text)) lexem = new Lexem (type, token.ToString ()); else { for (int i = 0; i < token.Length; ++i) this.pending.Enqueue (token[i]); lexem = new Lexem (LexemType.Text, text); } this.Read (); return lexem; } ++trail; } while (this.cursors.Count > 0 && this.cursors.Peek ().State == null) buffer.Append (this.cursors.Dequeue ().Character); } while (this.cursors.Count > 0) buffer.Append (this.cursors.Dequeue ().Character); text = buffer.ToString (); if (!string.IsNullOrEmpty (text)) return new Lexem (LexemType.Text, text); return new Lexem (LexemType.EndOfFile, "<EOF>"); }
private Lexem NextChar(LexemType type) { Lexem lexem; lexem = new Lexem (type, this.last.ToString ()); this.Read (); return lexem; }
private Lexem NextRaw() { StringBuilder buffer; bool cancel; Lexem lexem; string text; StringBuilder token; int trail; LexemType type; buffer = new StringBuilder(); for (; !this.eof; this.Read()) { cancel = this.last == this.escape && this.Read(); trail = 0; this.cursors.Enqueue(new LexemCursor(this.last, this.root)); foreach (LexemCursor cursor in this.cursors) { if (cancel) { cursor.Cancel(); } else if (cursor.Move(this.last, out type)) { while (trail-- > 0) { buffer.Append(this.cursors.Dequeue().Character); } token = new StringBuilder(); while (this.cursors.Count > 0) { token.Append(this.cursors.Dequeue().Character); } text = buffer.ToString(); if (string.IsNullOrEmpty(text)) { lexem = new Lexem(type, token.ToString()); } else { for (int i = 0; i < token.Length; ++i) { this.pending.Enqueue(token[i]); } lexem = new Lexem(LexemType.Text, text); } this.Read(); return(lexem); } ++trail; } while (this.cursors.Count > 0 && this.cursors.Peek().State == null) { buffer.Append(this.cursors.Dequeue().Character); } } while (this.cursors.Count > 0) { buffer.Append(this.cursors.Dequeue().Character); } text = buffer.ToString(); if (!string.IsNullOrEmpty(text)) { return(new Lexem(LexemType.Text, text)); } return(new Lexem(LexemType.EndOfFile, "<EOF>")); }
private Lexem NextRaw() { StringBuilder buffer; int copy; int first; Lexem lexem; LexemCursor next; string text; StringBuilder token; buffer = new StringBuilder(); for (; !this.eof; this.Read()) { // Escape sequence found, cancel all pending cursors if (this.last == this.escape && this.Read()) { foreach (LexemCursor cursor in this.cursors) { buffer.Append(cursor.Character); } this.cursors.Clear(); buffer.Append(this.last); } // Not an escape sequence, move all cursors else { this.cursors.Add(new LexemCursor(this.last, this.root)); for (int candidate = 0; candidate < this.cursors.Count; ++candidate) { next = this.cursors[candidate].Move(this.last); this.cursors[candidate] = next; // No lexem matched for this cursor (and no risk of LOH allocation), continue loop if ((next.State == null || next.State.Type == LexemType.None) && (buffer.Length < MaxBufferSize || this.pending.Count > 0)) { continue; } // Lexem matched, flush characters located before it // Or text too long, risk for LOH for (int i = 0; i < candidate; ++i) { buffer.Append(this.cursors[i].Character); } text = buffer.ToString(); // We were in the case of No lexem matched for this cursor if (next.State == null || next.State.Type == LexemType.None) { this.Read(); return(new Lexem(LexemType.Text, text)); } // Concatenate matched characters to build matched lexem string token = new StringBuilder(); while (candidate < this.cursors.Count) { token.Append(this.cursors[candidate++].Character); } // Return lexem if no text was located before or enqueue otherwise if (string.IsNullOrEmpty(text)) { lexem = new Lexem(next.State.Type, token.ToString()); } else { for (int i = 0; i < token.Length; ++i) { this.pending.Enqueue(token[i]); } lexem = new Lexem(LexemType.Text, text); } this.Read(); this.cursors.Clear(); return(lexem); } // Remove dead cursors and shift next ones first = 0; while (first < this.cursors.Count && this.cursors[first].State == null) { buffer.Append(this.cursors[first++].Character); } if (first > 0) { copy = 0; while (first < this.cursors.Count) { this.cursors[copy++] = this.cursors[first++]; } this.cursors.RemoveRange(copy, this.cursors.Count - copy); } } } foreach (LexemCursor cursor in this.cursors) { buffer.Append(cursor.Character); } this.cursors.Clear(); text = buffer.ToString(); if (!string.IsNullOrEmpty(text)) { return(new Lexem(LexemType.Text, text)); } return(new Lexem(LexemType.EndOfFile, "<EOF>")); }