public bool GotoNextLine()
{
_position = Reader.Position;
TokenByteSplice splice = Reader.ReadLine();
_line = splice.Bytes;
if (_line != null)
{
_length = splice.Start + splice.Length;
_index = splice.Start;
return(true);
}
else
{
return(false);
}
}
private void SkipWhitespace()
{
while (true)
{
// Do we need to fetch the next line of characters?
if ((_line == null) || (_index == _length))
{
_position = Reader.Position;
TokenByteSplice splice = Reader.ReadLine();
_line = splice.Bytes;
if (_line != null)
{
_length = splice.Start + splice.Length;
_index = splice.Start;
}
else
{
// No more lines, finished
break;
}
}
// Skip all whitespace characters
while (_index < _length)
{
if (_lookupWhitespace[_line[_index]])
{
_index++;
}
else
{
return;
}
}
}
}
private TokenObject GetStringLiteral()
{
long position = _position + _index;
// Move past the '(' start literal string marker
_index++;
int nesting = 0;
int first = _index;
int scanned = 0;
bool continuation = false;
bool checkedBOM = false;
StringBuilder sb = new StringBuilder();
// Keep scanning until we get to the end of literal string ')' marker
while (true)
{
// Scan rest of the current line
while (_index < _length)
{
// Is this the start of an escape sequence
if (_line[_index] == 92) // '\'
{
// If the last character, then '\' indicates that no newline should be appended into the literal string
if (_index >= (_length - 1))
{
continuation = true;
}
else
{
// Skip over the following escaped character for first digit of escaped number
_index++;
}
}
else if (_line[_index] == 41) // ')'
{
// If the balancing end marker then we are finished
if (nesting == 0)
{
sb.Append(Encoding.ASCII.GetString(_line, first, _index - first));
// Move past the ')' marker
_index++;
return(new TokenStringLiteral(sb.ToString()));
}
else
{
nesting--;
}
}
else if (_line[_index] == 40) // '('
{
nesting++;
}
_index++;
scanned++;
if (!checkedBOM && (scanned == 2))
{
// Check for the UTF16 Byte Order Mark (little endian or big endian versions)
if ((_line[_index - 2] == 0xFE) && (_line[_index - 1] == 0xFF))
{
return(GetStringLiteralUTF16(position, true));
}
else if ((_line[_index - 2] == 0xFF) && (_line[_index - 1] == 0xFE))
{
return(GetStringLiteralUTF16(position, false));
}
}
}
checkedBOM = true;
if (continuation)
{
// Append everything from the first character
sb.Append(Encoding.ASCII.GetString(_line, first, _index - first - 1));
}
else
{
// Append everything from the first character but excluding the continuation marker
sb.Append(Encoding.ASCII.GetString(_line, first, _index - first));
sb.Append("\n");
}
TokenByteSplice splice = Reader.ReadLine();
_line = splice.Bytes;
if (_line != null)
{
_length = splice.Start + splice.Length;
_index = splice.Start;
first = _index;
continuation = false;
}
else
{
// End of content before end of string literal
return(new TokenError(position, $"End of content before end of literal string character ')'."));
}
}
}
