/// <summary> /// Tokenizes the input. /// </summary> public void Tokenize(IReader reader) { this.reader = reader; try { CurrentLine = 1; PositionOnLine = 0; LastToken = null; LastTokenPosition = 0; Tokens.Clear(); CurrentTokenChars.Clear(); TokenizeCore(); } finally { reader.Dispose(); } }
protected bool TokenizeInternal(string sourceText, Func <bool> readFunction) { this.sourceText = sourceText; try { position = 0; CurrentLine = 1; PositionOnLine = 0; LastToken = null; LastTokenPosition = 0; Tokens.Clear(); CurrentTokenChars.Clear(); return(readFunction()); } catch (Exception ex) when(ex.Message == "Assertion failed!") { return(false); } }