Beispiel #1
0
        private string _tokenWhiteSpace, _lookaheadWhiteSpace; // the whitespace for the current and lookahead tokens as provided from the parser

        #endregion Fields

        #region Constructors

        private Parser(Tokenizer tokenizer, ErrorSink errorSink, JLanguageVersion langVersion, bool verbatim, bool bindRefs, string privatePrefix)
        {
            Contract.Assert(tokenizer != null);
            Contract.Assert(errorSink != null);

            tokenizer.ErrorSink = new TokenizerErrorSink(this);

            _tokenizer = tokenizer;
            _errors = errorSink;
            _langVersion = langVersion;
            _verbatim = verbatim;
            _bindReferences = bindRefs;

            if (langVersion.Is7x()) {
                // 3.x always does true division
                _languageFeatures |= FutureOptions.TrueDivision;
            }

            Reset(FutureOptions.None);

            _privatePrefix = privatePrefix;
        }
Beispiel #2
0
        /*!*/
        private static StreamReader GetStreamReaderWithEncoding(Stream/*!*/ stream, Encoding/*!*/ defaultEncoding, ErrorSink errors)
        {
            // we choose ASCII by default, if the file has a Unicode pheader though
            // we'll automatically get it as unicode.
            Encoding encoding = JAsciiEncoding.SourceEncoding;

            byte[] bomBuffer = new byte[3];
            int bomRead = stream.Read(bomBuffer, 0, 3);
            int bytesRead = 0;
            bool isUtf8 = false;
            List<byte> readBytes = new List<byte>();
            if (bomRead == 3 && (bomBuffer[0] == 0xef && bomBuffer[1] == 0xbb && bomBuffer[2] == 0xbf)) {
                isUtf8 = true;
                bytesRead = 3;
                readBytes.AddRange(bomBuffer);
            } else {
                for (int i = 0; i < bomRead; i++) {
                    readBytes.Add(bomBuffer[i]);
                }
            }

            int lineLength;
            string line = ReadOneLine(readBytes, ref bytesRead, stream, out lineLength);

            bool? gotEncoding = false;
            string encodingName = null;
            // magic encoding must be on line 1 or 2
            int lineNo = 1;
            int encodingIndex = 0;
            if ((gotEncoding = TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName, out encodingIndex)) == false) {
                var prevLineLength = lineLength;
                line = ReadOneLine(readBytes, ref bytesRead, stream, out lineLength);
                lineNo = 2;
                gotEncoding = TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName, out encodingIndex);
                encodingIndex += prevLineLength;
            }

            if ((gotEncoding == null || gotEncoding == true) && isUtf8 && encodingName != "utf-8") {
                // we have both a BOM & an encoding type, throw an error
                errors.Add("file has both Unicode marker and PEP-263 file encoding.  You must use \"utf-8\" as the encoding name when a BOM is present.",
                    GetEncodingLineNumbers(readBytes),
                    encodingIndex,
                    encodingIndex + encodingName.Length,
                    ErrorCodes.SyntaxError,
                    Severity.FatalError
                );
                encoding = Encoding.UTF8;
            } else if (isUtf8) {
                return new StreamReader(new PartiallyReadStream(readBytes, stream), UTF8Throwing);
            } else if (encoding == null) {
                if (gotEncoding == null) {
                    // get line number information for the bytes we've read...
                    errors.Add(
                        String.Format("encoding problem: unknown encoding (line {0})", lineNo),
                        GetEncodingLineNumbers(readBytes),
                        encodingIndex,
                        encodingIndex + encodingName.Length,
                        ErrorCodes.SyntaxError,
                        Severity.Error
                    );
                }
                return new StreamReader(new PartiallyReadStream(readBytes, stream), defaultEncoding);
            }

            // re-read w/ the correct encoding type...
            return new StreamReader(new PartiallyReadStream(readBytes, stream), encoding);
        }