public void IsEndOfLine_LineFeed() { const char c = '\n'; var result = ReadHelper.IsEndOfLine(c); Assert.True(result); }
public void IsEndOfLine_CarriageReturn() { const char c = '\r'; var result = ReadHelper.IsEndOfLine(c); Assert.True(result); }
private static string ReadLine(IInputBytes input, StringBuilder stringBuilder) { stringBuilder.Clear(); while (ReadHelper.IsWhitespace(input.CurrentByte) && input.MoveNext()) { } stringBuilder.Append((char)input.CurrentByte); while (input.MoveNext() && !ReadHelper.IsEndOfLine(input.CurrentByte)) { stringBuilder.Append((char)input.CurrentByte); } return(stringBuilder.ToString()); }
private string ReadComment() { commentBuffer.Clear(); while (bytes.MoveNext()) { var c = (char)bytes.CurrentByte; if (ReadHelper.IsEndOfLine(c)) { break; } commentBuffer.Append(c); } return(commentBuffer.ToString()); }
public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token) { token = null; if (currentByte != '%') { return(false); } var builder = new StringBuilder(); while (inputBytes.MoveNext() && !ReadHelper.IsEndOfLine(inputBytes.CurrentByte)) { builder.Append((char)inputBytes.CurrentByte); } token = new CommentToken(builder.ToString()); return(true); }
public void IsEndOfLine_OtherCharacters(char c) { var result = ReadHelper.IsEndOfLine(c); Assert.False(result); }
public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token) { token = null; if (inputBytes == null) { return(false); } if (currentByte != '(') { return(false); } var builder = stringBuilder; var numberOfBrackets = 1; var isEscapeActive = false; var isLineBreaking = false; var octalModeActive = false; short[] octal = { 0, 0, 0 }; var octalsRead = 0; while (inputBytes.MoveNext()) { var b = inputBytes.CurrentByte; var c = (char)b; if (octalModeActive) { var nextCharacterOctal = c >= '0' && c <= '7'; if (nextCharacterOctal) { // left shift the octals. LeftShiftOctal(c, octalsRead, octal); octalsRead++; } if (octalsRead == 3 || !nextCharacterOctal) { var characterCode = OctalHelpers.FromOctalDigits(octal); // For now :( // TODO: I have a sneaking suspicion this is wrong, not sure what behaviour is for large octal numbers builder.Append((char)characterCode); octal[0] = 0; octal[1] = 0; octal[2] = 0; octalsRead = 0; octalModeActive = false; } if (nextCharacterOctal) { continue; } } switch (c) { case ')': isLineBreaking = false; if (!isEscapeActive) { numberOfBrackets--; } isEscapeActive = false; if (numberOfBrackets > 0) { builder.Append(c); } // TODO: Check for other ends of string where the string is improperly formatted. See commented method numberOfBrackets = CheckForEndOfString(numberOfBrackets, inputBytes); break; case '(': isLineBreaking = false; if (!isEscapeActive) { numberOfBrackets++; } isEscapeActive = false; builder.Append(c); break; // Escape case '\\': isLineBreaking = false; // Escaped backslash if (isEscapeActive) { builder.Append(c); isEscapeActive = false; } else { isEscapeActive = true; } break; default: if (isLineBreaking) { if (ReadHelper.IsEndOfLine(c)) { continue; } isLineBreaking = false; builder.Append(c); } else if (isEscapeActive) { ProcessEscapedCharacter(c, builder, octal, ref octalModeActive, ref octalsRead, ref isLineBreaking); isEscapeActive = false; } else { builder.Append(c); } break; } if (numberOfBrackets <= 0) { break; } } StringToken.Encoding encodedWith; string tokenStr; if (builder.Length >= 2) { if (builder[0] == 0xFE && builder[1] == 0xFF) { var rawBytes = OtherEncodings.StringAsLatin1Bytes(builder.ToString()); tokenStr = Encoding.BigEndianUnicode.GetString(rawBytes).Substring(1); encodedWith = StringToken.Encoding.Utf16BE; } else if (builder[0] == 0xFF && builder[1] == 0xFE) { var rawBytes = OtherEncodings.StringAsLatin1Bytes(builder.ToString()); tokenStr = Encoding.Unicode.GetString(rawBytes).Substring(1); encodedWith = StringToken.Encoding.Utf16; } else { tokenStr = builder.ToString(); encodedWith = StringToken.Encoding.Iso88591; } } else { tokenStr = builder.ToString(); encodedWith = StringToken.Encoding.Iso88591; } builder.Clear(); token = new StringToken(tokenStr, encodedWith); return(true); }
public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token) { var builder = new StringBuilder(); token = null; if (inputBytes == null) { return(false); } if (currentByte != '(') { return(false); } int numberOfBrackets = 1; bool isEscapeActive = false; bool isLineBreaking = false; bool octalModeActive = false; short[] octal = { 0, 0, 0 }; int octalsRead = 0; while (inputBytes.MoveNext()) { var b = inputBytes.CurrentByte; var c = (char)b; if (octalModeActive) { var nextCharacterOctal = c >= '0' && c <= '7'; if (nextCharacterOctal) { // left shift the octals. LeftShiftOctal(c, octalsRead, octal); octalsRead++; } if (octalsRead == 3 || !nextCharacterOctal) { var characterCode = OctalHelpers.FromOctalDigits(octal); // For now :( // TODO: I have a sneaking suspicion this is wrong, not sure what behaviour is for large octal numbers builder.Append((char)characterCode); octal[0] = 0; octal[1] = 0; octal[2] = 0; octalsRead = 0; octalModeActive = false; } if (nextCharacterOctal) { continue; } } switch (c) { case ')': isLineBreaking = false; if (!isEscapeActive) { numberOfBrackets--; } isEscapeActive = false; if (numberOfBrackets > 0) { builder.Append(c); } // TODO: Check for other ends of string where the string is improperly formatted. See commented method // numberOfBrackets = CheckForEndOfString(inputBytes, numberOfBrackets); break; case '(': isLineBreaking = false; if (!isEscapeActive) { numberOfBrackets++; } isEscapeActive = false; builder.Append(c); break; // Escape case '\\': isLineBreaking = false; // Escaped backslash if (isEscapeActive) { builder.Append(c); } else { isEscapeActive = true; } break; default: if (isLineBreaking) { if (ReadHelper.IsEndOfLine(c)) { continue; } isLineBreaking = false; builder.Append(c); } else if (isEscapeActive) { ProcessEscapedCharacter(c, builder, octal, ref octalModeActive, ref octalsRead, ref isLineBreaking); isEscapeActive = false; } else { builder.Append(c); } break; } if (numberOfBrackets <= 0) { break; } } token = new StringToken(builder.ToString()); return(true); }