private static bool SkipComment(StringStream stream)
{
// Skip single line comments
if (stream.Peek() == '/' && stream.Peek(1) == '/')
{
while (!stream.IsEndOfStream && stream.Peek() != StringStream.NewLine)
{
stream.Read();
}
return(true);
}
// Skip multiline comments
if (stream.Peek() == '/' && stream.Peek(1) == '*')
{
stream.Position += 2;
int nestCount = 1;
while (nestCount > 0)
{
if (stream.IsEndOfStream)
{
throw new LexerException("Unterminated multi-line comment ran past end of file.");
}
if (stream.Peek() == '/' && stream.Peek(1) == '*')
{
++nestCount;
}
else if (stream.Peek() == '*' && stream.Peek(1) == '/')
{
--nestCount;
}
else
{
++stream.Position;
continue;
}
stream.Position += 2;
if (stream.Read() == StringStream.NewLine)
{
++Line;
}
}
return(true);
}
return(false);
}
private static bool ParseString(StringStream stream, ICollection <Token> tokens)
{
if (!(stream.Peek() == '@' && stream.Peek(1) == '\"') && stream.Peek() != '\"')
{
return(false);
}
uint startLine = Line;
string value = "";
bool escaped = true;
if (stream.Peek() == '@')
{
escaped = false;
stream.Read();
}
while (stream.Peek() == '\"')
{
char ch;
++stream.Position;
while ((ch = stream.Read()) != '\"')
{
if (stream.IsEndOfStream)
{
throw new LexerException("Unterminated string on line {0}.", startLine);
}
if (ch == StringStream.NewLine)
{
++Line;
}
value += ch;
if (ch == '\\')
{
value += stream.Read();
}
}
SkipWhitespace(stream);
}
value = escaped ? ParseEscapeSequence(value) : value;
tokens.Add(new Token(TokenType.String, value, Filename, startLine));
return(true);
}
private static bool ParseWord(StringStream stream, ICollection <Token> tokens)
{
if (char.IsLetter(stream.Peek()) || stream.Peek() == '_')
{
string word = "";
while (!stream.IsEndOfStream && (char.IsLetterOrDigit(stream.Peek()) || stream.Peek() == '_'))
{
word += stream.Read();
}
switch (word)
{
case "true":
tokens.Add(new Token(TokenType.Number, "1", Filename, Line));
break;
case "false":
tokens.Add(new Token(TokenType.Number, "0", Filename, Line));
break;
default:
tokens.Add(new Token(TokenType.Word, word, Filename, Line));
break;
}
return(true);
}
return(false);
}
private static void SkipWhitespace(StringStream stream)
{
while (!stream.IsEndOfStream && char.IsWhiteSpace(stream.Peek()))
{
// Read new lines
if (stream.Read() == StringStream.NewLine)
{
++Line;
}
}
}
public void NotExpandable_WithInitialValue_SpecificEncoding(string data, Encoding encoding)
{
using (var stream = new StringStream(data, encoding))
{
Assert.That(stream.Encoding, Is.EqualTo(encoding));
Assert.That(stream.ToString(), Is.EqualTo(data));
var bytes = new byte[stream.Length];
stream.Read(bytes, 0, bytes.Length);
Assert.That(stream.Encoding.GetString(bytes), Is.EqualTo(data));
Assert.Throws <NotSupportedException>(() => stream.Capacity += 100);
}
}
private static List <Token> ParseStream(StringStream stream)
{
List <Token> tokens = new List <Token>();
Line = 1;
while (!stream.IsEndOfStream)
{
SkipWhitespace(stream);
if (SkipComment(stream))
{
continue;
}
if (ParseString(stream, tokens))
{
continue;
}
if (ParseWord(stream, tokens))
{
continue;
}
if (ParseNumber(stream, tokens))
{
continue;
}
if (ParseDelimiter(stream, tokens))
{
continue;
}
if (stream.IsEndOfStream)
{
break;
}
throw new LexerException("Unexpected token '{0}' on line {1}.", stream.Read(), Line);
}
tokens.Add(new Token(TokenType.EndOfStream, Filename, Line));
return(tokens);
}
public void Expandable_WithoutInitialValue_SpecificEncoding(string data, Encoding encoding)
{
using (var stream = new StringStream(encoding))
{
Assert.That(stream.Encoding, Is.EqualTo(encoding));
var bytes = stream.Encoding.GetBytes(data);
stream.Write(bytes, 0, bytes.Length);
Assert.That(stream.ToString(), Is.EqualTo(data));
bytes = new byte[stream.Position];
stream.Position = 0;
stream.Read(bytes, 0, bytes.Length);
Assert.That(stream.Encoding.GetString(bytes), Is.EqualTo(data));
var oldCapacity = stream.Capacity;
stream.Capacity += 100;
Assert.That(stream.Capacity, Is.EqualTo(oldCapacity + 100));
}
}
private static string ParseEscapeSequences(string value)
{
StringStream ss = new StringStream(value);
string parsedValue = "";
while (!ss.IsEndOfStream)
{
// Read non-escaping characters
if (ss.Peek() != '\\')
{
parsedValue += ss.Read();
continue;
}
// Skip backslash
ss.Read();
switch (ss.Read())
{
case 'a':
parsedValue += '\a';
continue;
case 'b':
parsedValue += '\b';
continue;
case 'f':
parsedValue += '\f';
continue;
case 'n':
parsedValue += '\n';
continue;
case 'r':
parsedValue += '\r';
continue;
case 't':
parsedValue += '\t';
continue;
case 'v':
parsedValue += '\v';
continue;
case '\'':
parsedValue += '\'';
continue;
case '\"':
parsedValue += '\"';
continue;
case '\\':
parsedValue += '\\';
continue;
case 'x':
{
string hexString = new string(new[] { ss.Read(), ss.Read() });
parsedValue += (char)Convert.ToInt32(hexString, 16);
continue;
}
case 'u':
case 'U':
throw new NotImplementedException();
}
throw new Exception("Bad escape sequence.");
}
return parsedValue;
}
private static void SkipWhitespace(StringStream stream)
{
while (!stream.IsEndOfStream && char.IsWhiteSpace(stream.Peek()))
{
// Read new lines
if (stream.Read() == StringStream.NewLine)
++line;
}
}
private static bool SkipComment(StringStream stream)
{
// Skip single line comments
if (stream.Peek() == '/' && stream.PeekAhead(1) == '/')
{
while (!stream.IsEndOfStream && stream.Peek() != StringStream.NewLine)
stream.Read();
return true;
}
// Skip multi line comments
if (stream.Peek() == '/' && stream.PeekAhead(1) == '*')
{
while (!stream.IsEndOfStream && !(stream.Peek() == '*' && stream.PeekAhead(1) == '/'))
{
if (stream.Read() == StringStream.NewLine)
++line;
}
stream.Position += 2;
return true;
}
return false;
}
private static bool ParseWord(StringStream stream, List<Token> tokens)
{
if (char.IsLetter(stream.Peek()) || stream.Peek() == '_')
{
string word = "";
while (!stream.IsEndOfStream && (char.IsLetterOrDigit(stream.Peek()) || stream.Peek() == '_'))
word += stream.Read();
switch (word)
{
case "true":
tokens.Add(new Token(TokenType.Number, "1", filename, line));
break;
case "false":
tokens.Add(new Token(TokenType.Number, "0", filename, line));
break;
default:
tokens.Add(new Token(TokenType.Word, word, filename, line));
break;
}
return true;
}
return false;
}
private static bool ParseString(StringStream stream, List<Token> tokens)
{
if (stream.Peek() != '\"')
return false;
uint startLine = line;
string value = "";
while (stream.Peek() == '\"')
{
char ch;
++stream.Position;
while ((ch = stream.Read()) != '\"')
{
if (stream.IsEndOfStream)
throw new Exception("Unterminated string on line " + startLine);
if (ch == StringStream.NewLine)
++line;
value += ch;
}
SkipWhitespace(stream);
}
tokens.Add(new Token(TokenType.String, ParseEscapeSequences(value), filename, startLine));
return true;
}
private static List<Token> ParseStream(StringStream stream)
{
List<Token> tokens = new List<Token>();
line = 1;
while (!stream.IsEndOfStream)
{
SkipWhitespace(stream);
if (SkipComment(stream))
continue;
if (ParseString(stream, tokens))
continue;
if (ParseWord(stream, tokens))
continue;
if (ParseNumber(stream, tokens))
continue;
if (ParseDelimiter(stream, tokens))
continue;
if (stream.IsEndOfStream)
break;
throw new Exception("Unexpected token \"" + stream.Read() + "\" on line " + line);
}
tokens.Add(new Token(TokenType.EndOfStream, filename, line));
return tokens;
}
private static string ParseEscapeSequence(string value)
{
StringStream stringStream = new StringStream(value);
string parsedValue = "";
while (!stringStream.IsEndOfStream)
{
// Read non-escaping characters
if (stringStream.Peek() != '\\')
{
parsedValue += stringStream.Read();
continue;
}
// Skip backslash
stringStream.Read();
switch (stringStream.Read())
{
case '\'':
parsedValue += '\'';
continue;
case '\"':
parsedValue += '\"';
continue;
case '\\':
parsedValue += '\\';
continue;
case '%':
parsedValue += '%';
continue;
case ':':
parsedValue += ':';
continue;
case '\0':
parsedValue += '\0';
continue;
case 'a':
parsedValue += '\a';
continue;
case 'b':
parsedValue += '\b';
continue;
case 'f':
parsedValue += '\f';
continue;
case 'n':
parsedValue += '\n';
continue;
case 'r':
parsedValue += '\r';
continue;
case 't':
parsedValue += '\t';
continue;
case 'u':
case 'U':
case 'x':
{
string unicodeValue = "";
for (int i = 0; i < 8; ++i)
{
char read = stringStream.Read();
unicodeValue += read;
if (!stringStream.Peek(1).IsHex() && (i == 1 || i == 3 || i == 7))
{
break;
}
}
uint unicodeIntValue;
if (!uint.TryParse(unicodeValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out unicodeIntValue))
{
throw new LexerException("Malformed escape code \\x, \\u or \\U on line {0}.", Line);
}
parsedValue += Convert.ToChar(unicodeIntValue);
continue;
}
case 'v':
parsedValue += '\v';
continue;
}
throw new LexerException("Bad escape sequence.");
}
return(parsedValue);
}
private static bool ParseNumber(StringStream stream, ICollection <Token> tokens)
{
if (char.IsDigit(stream.Peek()))
{
string number = "";
if (stream.Peek() == '0' && stream.Peek(1) == 'x')
{
stream.Position += 2;
while (!stream.IsEndOfStream && stream.Peek().IsHex())
{
number += stream.Read();
}
ulong result;
try
{
result = ulong.Parse(number, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
catch (OverflowException)
{
throw new LexerException("Number is larger than 64bits on line {0}.", Line);
}
catch (Exception)
{
throw new LexerException("Invalid number on line {0}.", Line);
}
number = result.ToString("G");
}
else if (stream.Peek() == '0' && stream.Peek(1) == 'b')
{
stream.Position += 2;
while (!stream.IsEndOfStream && char.IsDigit(stream.Peek()))
{
number += stream.Read();
}
long result;
try
{
result = Convert.ToInt64(number, 2);
}
catch (OverflowException)
{
throw new LexerException("Number is larger than 64bits on line {0}.", Line);
}
catch (Exception)
{
throw new LexerException("Invalid number on line {0}.", Line);
}
number = result.ToString("G");
}
else
{
while (!stream.IsEndOfStream && (char.IsDigit(stream.Peek()) || (stream.Peek() == '.' && stream.Peek(1) != '.')))
{
number += stream.Read();
}
}
tokens.Add(new Token(TokenType.Number, number, Filename, Line));
return(true);
}
return(false);
}
private void ParseSectionHeader(ref StringStream stream,
out string ref_sectionName)
{
// Skip prefix character
stream.Read();
ref_sectionName = string.Empty;
while (true)
{
if (stream.IsAtEndOfStream)
{
break;
}
if (stream.PeekUnchecked() == SectionSuffix)
{
// Read the suffix character
stream.Read();
break;
}
ref_sectionName += (char)stream.Read();
}
}
private void ParseComment(ref StringStream stream,
out string out_comment)
{
// Skip initial prefix character
stream.Read();
out_comment = stream.ReadLine();
}
private static bool ParseNumber(StringStream stream, List<Token> tokens)
{
if (char.IsDigit(stream.Peek()) || (stream.Peek() == '-' && char.IsDigit(stream.PeekAhead(1))))
{
string number = "";
if (stream.Peek() == '-')
number += stream.Read();
if (stream.Peek() == '0' && stream.PeekAhead(1) == 'x')
{
stream.Position += 2;
while (!stream.IsEndOfStream && char.IsLetterOrDigit(stream.Peek()))
number += stream.Read();
ulong result;
try
{
result = ulong.Parse(number, NumberStyles.HexNumber, CultureInfo.InvariantCulture);
}
catch (OverflowException)
{
throw new Exception("Number is larger than 64bits on line " + line);
}
catch (Exception)
{
throw new Exception("Invalid number on line " + line);
}
number = result.ToString("G");
}
else
{
while (!stream.IsEndOfStream && (char.IsDigit(stream.Peek()) || stream.Peek() == '.'))
number += stream.Read();
}
tokens.Add(new Token(TokenType.Number, number, filename, line));
return true;
}
return false;
}
