public bool GotoNextLine() { _position = Reader.Position; TokenByteSplice splice = Reader.ReadLine(); _line = splice.Bytes; if (_line != null) { _length = splice.Start + splice.Length; _index = splice.Start; return(true); } else { return(false); } }
private void SkipWhitespace() { while (true) { // Do we need to fetch the next line of characters? if ((_line == null) || (_index == _length)) { _position = Reader.Position; TokenByteSplice splice = Reader.ReadLine(); _line = splice.Bytes; if (_line != null) { _length = splice.Start + splice.Length; _index = splice.Start; } else { // No more lines, finished break; } } // Skip all whitespace characters while (_index < _length) { if (_lookupWhitespace[_line[_index]]) { _index++; } else { return; } } } }
private TokenObject GetStringLiteral() { long position = _position + _index; // Move past the '(' start literal string marker _index++; int nesting = 0; int first = _index; int scanned = 0; bool continuation = false; bool checkedBOM = false; StringBuilder sb = new StringBuilder(); // Keep scanning until we get to the end of literal string ')' marker while (true) { // Scan rest of the current line while (_index < _length) { // Is this the start of an escape sequence if (_line[_index] == 92) // '\' { // If the last character, then '\' indicates that no newline should be appended into the literal string if (_index >= (_length - 1)) { continuation = true; } else { // Skip over the following escaped character for first digit of escaped number _index++; } } else if (_line[_index] == 41) // ')' { // If the balancing end marker then we are finished if (nesting == 0) { sb.Append(Encoding.ASCII.GetString(_line, first, _index - first)); // Move past the ')' marker _index++; return(new TokenStringLiteral(sb.ToString())); } else { nesting--; } } else if (_line[_index] == 40) // '(' { nesting++; } _index++; scanned++; if (!checkedBOM && (scanned == 2)) { // Check for the UTF16 Byte Order Mark (little endian or big endian versions) if ((_line[_index - 2] == 0xFE) && (_line[_index - 1] == 0xFF)) { return(GetStringLiteralUTF16(position, true)); } else if ((_line[_index - 2] == 0xFF) && (_line[_index - 1] == 0xFE)) { return(GetStringLiteralUTF16(position, false)); } } } checkedBOM = true; if (continuation) { // Append everything from the first character sb.Append(Encoding.ASCII.GetString(_line, first, _index - first - 1)); } else { // Append everything from the first character but excluding the continuation marker sb.Append(Encoding.ASCII.GetString(_line, first, _index - first)); sb.Append("\n"); } TokenByteSplice splice = Reader.ReadLine(); _line = splice.Bytes; if (_line != null) { _length = splice.Start + splice.Length; _index = splice.Start; first = _index; continuation = false; } else { // End of content before end of string literal return(new TokenError(position, $"End of content before end of literal string character ')'.")); } } }