Ejemplo n.º 1
0
        private Parser(Tokenizer tokenizer, ErrorSink errorSink, PythonLanguageVersion langVersion, bool verbatim, bool bindRefs, string privatePrefix) {
            Contract.Assert(tokenizer != null);
            Contract.Assert(errorSink != null);

            tokenizer.ErrorSink = new TokenizerErrorSink(this);

            _tokenizer = tokenizer;
            _errors = errorSink;
            _langVersion = langVersion;
            _verbatim = verbatim;
            _bindReferences = bindRefs;
            
            if (langVersion.Is3x()) {
                // 3.x always does true division and absolute import
                _languageFeatures |= FutureOptions.TrueDivision | FutureOptions.AbsoluteImports;
            }

            Reset(FutureOptions.None);

            _privatePrefix = privatePrefix;
        }
Ejemplo n.º 2
0
        private static StreamReader/*!*/ GetStreamReaderWithEncoding(Stream/*!*/ stream, Encoding/*!*/ defaultEncoding, ErrorSink errors) {
            // Python 2.x should pass ASCII as the default
            // Python 3.x should pass UTF-8
            // A BOM or encoding comment can override the default
            Encoding encoding = defaultEncoding;

            List<byte> readBytes = new List<byte>();
            try {
                byte[] bomBuffer = new byte[3];
                int bomRead = stream.Read(bomBuffer, 0, 3);
                int bytesRead = 0;
                bool isUtf8 = false;
                if (bomRead == 3 && (bomBuffer[0] == 0xef && bomBuffer[1] == 0xbb && bomBuffer[2] == 0xbf)) {
                    isUtf8 = true;
                    bytesRead = 3;
                    readBytes.AddRange(bomBuffer);
                } else {
                    for (int i = 0; i < bomRead; i++) {
                        readBytes.Add(bomBuffer[i]);
                    }
                }

                int lineLength;
                string line = ReadOneLine(readBytes, ref bytesRead, stream, out lineLength);

                bool? gotEncoding = false;
                string encodingName = null;
                // magic encoding must be on line 1 or 2
                int lineNo = 1;
                int encodingIndex = 0;
                if ((gotEncoding = TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName, out encodingIndex)) == false) {
                    var prevLineLength = lineLength;
                    line = ReadOneLine(readBytes, ref bytesRead, stream, out lineLength);
                    lineNo = 2;
                    gotEncoding = TryGetEncoding(defaultEncoding, line, ref encoding, out encodingName, out encodingIndex);
                    encodingIndex += prevLineLength;
                }

                if ((gotEncoding == null || gotEncoding == true) && isUtf8 && encodingName != "utf-8") {
                    // we have both a BOM & an encoding type, throw an error
                    errors.Add("file has both Unicode marker and PEP-263 file encoding.  You must use \"utf-8\" as the encoding name when a BOM is present.",
                        GetEncodingLineNumbers(readBytes),
                        encodingIndex,
                        encodingIndex + encodingName.Length,
                        ErrorCodes.SyntaxError,
                        Severity.FatalError
                    );
                    encoding = Encoding.UTF8;
                } else if (isUtf8) {
                    return new StreamReader(new PartiallyReadStream(readBytes, stream), UTF8Throwing);
                } else if (encoding == null) {
                    if (gotEncoding == null) {
                        // get line number information for the bytes we've read...
                        errors.Add(
                            String.Format("encoding problem: unknown encoding (line {0})", lineNo),
                            GetEncodingLineNumbers(readBytes),
                            encodingIndex,
                            encodingIndex + encodingName.Length,
                            ErrorCodes.SyntaxError,
                            Severity.Error
                        );
                    }
                    return new StreamReader(new PartiallyReadStream(readBytes, stream), defaultEncoding);
                }

                // re-read w/ the correct encoding type...
                return new StreamReader(new PartiallyReadStream(readBytes, stream), encoding);
            } catch (EncoderFallbackException ex) {
                errors.Add(
                    ex.Message,
                    new int[0],
                    ex.Index,
                    ex.Index + 1,
                    ErrorCodes.SyntaxError,
                    Severity.FatalError
                );
                return new StreamReader(new PartiallyReadStream(readBytes, stream), encoding);
            }
        }
Ejemplo n.º 3
0
 private static PythonAst ParseFile(string filename, ErrorSink errorSink, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Ignore) {
     var parser = Parser.CreateParser(TestData.Read(Path.Combine("TestData\\Grammar", filename)), version, new ParserOptions() { ErrorSink = errorSink, IndentationInconsistencySeverity = indentationInconsistencySeverity });
     var ast = parser.ParseFile();
     return ast;
 }