public static IEnumerable <LineToken> GetTokens(string input, bool treatWhitespaceOnlyLinesAsBlankLines) { if (input is null) { throw new ArgumentNullException(nameof(input)); } var currentLine = new StringBuilder(); var currentLineIsWhitespace = true; var lineNumber = 1; LineToken GetCurrentToken() { // In "Loose Mode", lines that consist only of whitespace characters are considered blank lines if (currentLine.Length == 0 || (treatWhitespaceOnlyLinesAsBlankLines && currentLineIsWhitespace)) { // clear current value of line currentLine.Clear(); return(LineToken.Blank(lineNumber++)); } else { // get current value and reset state var value = currentLine.GetValueAndClear(); currentLineIsWhitespace = true; return(LineToken.Line(value, lineNumber++)); } } for (var i = 0; i < input.Length; i++) { var currentChar = input[i]; var nextChar = i + 1 < input.Length ? (char?)input[i + 1] : null; switch ((currentChar, nextChar)) { case ('\r', '\n'): yield return(GetCurrentToken()); i += 1; break; case ('\n', _): yield return(GetCurrentToken()); break; default: currentLineIsWhitespace &= char.IsWhiteSpace(currentChar); currentLine.Append(currentChar); break; } } if (currentLine.Length > 0) { yield return(GetCurrentToken()); } yield return(LineToken.Eof(lineNumber)); }
static object[] TestCase(string input, params FooterToken[] tokens) => new object[] { new XunitSerializableLineToken(LineToken.Line(input, 1)), tokens.Select(t => new XunitSerializableFooterToken(t)).ToArray() };