protected int RunLexerUntil(int stopAt) { if (_lexer.InputPosition < stopAt) { int startAt = _lexer.InputPosition; _tokens.ClearSpace(startAt, stopAt - startAt); _nestedTokens.ClearSpace(startAt, stopAt - startAt); for (Maybe <Token> t_; _lexer.InputPosition < stopAt && (t_ = _lexer.NextToken()).HasValue;) { Token t = t_.Value; if (t.EndIndex > stopAt) { _tokens.ClearSpace(t.StartIndex, t.Length); _nestedTokens.ClearSpace(t.StartIndex, t.Length); } if (t.Children != null) { foreach (var ct in t.Children) { _nestedTokens[ct.StartIndex] = new EditorToken(ct.TypeInt, ct.Length, ct.Value); } } if (!IsWhitespace(t.TypeInt)) { var et = new EditorToken(t.TypeInt, t.Length, t.Value); _tokens[t.StartIndex] = StoreLexerError(et); } } if (_lexerError != null && _tokens.Count != 0) { _tokens.Last = StoreLexerError(_tokens.Last); } } return(_lexer.InputPosition); }
EditorToken StoreLexerError(EditorToken token) { if (_lexerError != null) { _haveLexerErrors = true; _lexerError.OriginalValue = token.Value; token.Value = _lexerError; _lexerError = null; } return(token); }
public static DList <Token> ToNormalTokens(SparseAList <EditorToken> eTokens) { var output = new DList <Token>(); int?index = null; for (;;) { EditorToken eTok = eTokens.NextHigherItem(ref index); if (index == null) { break; } output.Add(eTok.ToToken(index.Value)); } return(output); }
void AddLexerErrors(NormalizedSnapshotSpanCollection spans, List <ITagSpan <ErrorTag> > errors) { if (_haveLexerErrors) { foreach (var span in spans) { int? i = span.Start.Position + 1; EditorToken t = _tokens.NextLowerItem(ref i); while (i != null) { do { if (t.Value is LexerMessage) { errors.Add(LexerMessageToErrorTagSpan(span.Snapshot, t.ToToken(i.Value, false), t.Value as LexerMessage)); } t = _tokens.NextHigherItem(ref i); }while (i < span.End.Position); } } } }
EditorToken StoreLexerError(EditorToken token) { if (_lexerError != null) { _haveLexerErrors = true; _lexerError.OriginalValue = token.Value; token.Value = _lexerError; _lexerError = null; } return token; }
protected int RunLexerUntil(int stopAt) { if (_lexer.InputPosition < stopAt) { int startAt = _lexer.InputPosition; _tokens.ClearSpace(startAt, stopAt - startAt); _nestedTokens.ClearSpace(startAt, stopAt - startAt); for (Maybe<Token> t_; _lexer.InputPosition < stopAt && (t_ = _lexer.NextToken()).HasValue; ) { Token t = t_.Value; if (t.EndIndex > stopAt) { _tokens.ClearSpace(t.StartIndex, t.Length); _nestedTokens.ClearSpace(t.StartIndex, t.Length); } if (t.Children != null) foreach (var ct in t.Children) _nestedTokens[ct.StartIndex] = new EditorToken(ct.TypeInt, ct.Length, ct.Value); if (!IsWhitespace(t.TypeInt)) { var et = new EditorToken(t.TypeInt, t.Length, t.Value); _tokens[t.StartIndex] = StoreLexerError(et); } } if (_lexerError != null && _tokens.Count != 0) _tokens.Last = StoreLexerError(_tokens.Last); } return _lexer.InputPosition; }