private string _tokenWhiteSpace, _lookaheadWhiteSpace; // the whitespace for the current and lookahead tokens as provided from the parser
#endregion Fields
#region Constructors
private Parser(Tokenizer tokenizer, ErrorSink errorSink, JLanguageVersion langVersion, bool verbatim, bool bindRefs, string privatePrefix)
{
Contract.Assert(tokenizer != null);
Contract.Assert(errorSink != null);
tokenizer.ErrorSink = new TokenizerErrorSink(this);
_tokenizer = tokenizer;
_errors = errorSink;
_langVersion = langVersion;
_verbatim = verbatim;
_bindReferences = bindRefs;
if (langVersion.Is7x()) {
// 3.x always does true division
_languageFeatures |= FutureOptions.TrueDivision;
}
Reset(FutureOptions.None);
_privatePrefix = privatePrefix;
}
/*!*/
private static StreamReader GetStreamReaderWithEncoding(Stream/*!*/ stream, Encoding/*!*/ defaultEncoding, ErrorSink errors)
{
// we choose ASCII by default, if the file has a Unicode pheader though
// we'll automatically get it as unicode.
Encoding encoding = JAsciiEncoding.SourceEncoding;
byte[] bomBuffer = new byte[3];
int bomRead = stream.Read(bomBuffer, 0, 3);
int bytesRead = 0;
bool isUtf8 = false;
List<byte> readBytes = new List<byte>();
if (bomRead == 3 && (bomBuffer[0] == 0xef && bomBuffer[1] == 0xbb && bomBuffer[2] == 0xbf)) {
isUtf8 = true;
bytesRead = 3;
readBytes.AddRange(bomBuffer);
} else {
for (int i = 0; i < bomRead; i++) {
readBytes.Add(bomBuffer[i]);
}
}
int lineLength;
string line = ReadOneLine(readBytes, ref bytesRead, stream, out lineLength);
bool? gotEncoding = false;
string encodingName = null;
// magic encoding must be on line 1 or 2
int lineNo = 1;
int encodingIndex = 0;
if ((gotEncoding = TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName, out encodingIndex)) == false) {
var prevLineLength = lineLength;
line = ReadOneLine(readBytes, ref bytesRead, stream, out lineLength);
lineNo = 2;
gotEncoding = TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName, out encodingIndex);
encodingIndex += prevLineLength;
}
if ((gotEncoding == null || gotEncoding == true) && isUtf8 && encodingName != "utf-8") {
// we have both a BOM & an encoding type, throw an error
errors.Add("file has both Unicode marker and PEP-263 file encoding. You must use \"utf-8\" as the encoding name when a BOM is present.",
GetEncodingLineNumbers(readBytes),
encodingIndex,
encodingIndex + encodingName.Length,
ErrorCodes.SyntaxError,
Severity.FatalError
);
encoding = Encoding.UTF8;
} else if (isUtf8) {
return new StreamReader(new PartiallyReadStream(readBytes, stream), UTF8Throwing);
} else if (encoding == null) {
if (gotEncoding == null) {
// get line number information for the bytes we've read...
errors.Add(
String.Format("encoding problem: unknown encoding (line {0})", lineNo),
GetEncodingLineNumbers(readBytes),
encodingIndex,
encodingIndex + encodingName.Length,
ErrorCodes.SyntaxError,
Severity.Error
);
}
return new StreamReader(new PartiallyReadStream(readBytes, stream), defaultEncoding);
}
// re-read w/ the correct encoding type...
return new StreamReader(new PartiallyReadStream(readBytes, stream), encoding);
}