/// <summary> /// Creates source unit and parses given <paramref name="code"/>. /// </summary> /// <param name="code">Source code to be parsed.</param> /// <param name="filePath">Source file used for error reporting.</param> /// <param name="factory">Nodes factory and error sink.</param> /// <param name="errors">Error sink. Can be <c>null</c>.</param> /// <param name="recovery">Error recovery. Can be <c>null</c>.</param> /// <param name="features">Optional. Language features.</param> /// <param name="initialState"> /// Optional. Initial parser state. /// This allows e.g. to parse PHP code without encapsulating the code into opening and closing tags.</param> /// <returns>New <see cref="CodeSourceUnit"/> object.</returns> public static SourceUnit /*!*/ ParseCode(string code, string filePath, INodesFactory <LangElement, Span> factory = null, Errors.IErrorSink <Span> errors = null, Errors.IErrorRecovery recovery = null, LanguageFeatures features = LanguageFeatures.Basic, Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL) { var unit = new CodeSourceUnit(code, filePath, Encoding.UTF8, initialState, features); if (factory == null) { factory = new BasicNodesFactory(unit); } if (errors == null) { errors = (factory as Errors.IErrorSink <Span>) ?? new EmptyErrorSink <Span>(); } //var lexer = new Lexer(new StringReader(code), Encoding.UTF8, errors, features, 0, initialState); unit.Parse(factory, errors, recovery); unit.Close(); // return(unit); }
public Scanner(Parsers.Position initialPosition, TextReader /*!*/ reader, SourceUnit /*!*/ sourceUnit, ErrorSink /*!*/ errors, ICommentsSink commentsSink, LanguageFeatures features) : base(reader) { if (reader == null) { throw new ArgumentNullException("reader"); } if (sourceUnit == null) { throw new ArgumentNullException("sourceUnit"); } if (errors == null) { throw new ArgumentNullException("errors"); } this.lineShift = initialPosition.FirstLine; this.columnShift = initialPosition.FirstColumn; this.offsetShift = initialPosition.FirstOffset; this.errors = errors; this.commentsSink = commentsSink ?? new _NullCommentsSink(); this.features = features; this.sourceUnit = sourceUnit; this.streamOffset = 0; this.pure = sourceUnit.CompilationUnit.IsPure; this.encoding = sourceUnit.Encoding; AllowAspTags = (features & LanguageFeatures.AspTags) != 0; AllowShortTags = (features & LanguageFeatures.ShortOpenTags) != 0; }
public GlobalCode Parse(SourceUnit /*!*/ sourceUnit, TextReader /*!*/ reader, ErrorSink /*!*/ errors, IReductionsSink reductionsSink, Parsers.Position initialPosition, Lexer.LexicalStates initialLexicalState, LanguageFeatures features) { Debug.Assert(reader != null && errors != null); // initialization: this.sourceUnit = sourceUnit; this.errors = errors; this.features = features; this.reader = reader; this.reductionsSink = reductionsSink ?? NullReductionSink; InitializeFields(); this.scanner = new Scanner(initialPosition, reader, sourceUnit, errors, reductionsSink as ICommentsSink, features); this.scanner.CurrentLexicalState = initialLexicalState; this.currentScope = new Scope(1); // starts assigning scopes from 2 (1 is reserved for prepended inclusion) this.unicodeSemantics = (features & LanguageFeatures.UnicodeSemantics) != 0; base.Scanner = this.scanner; base.Parse(); GlobalCode result = astRoot; // clean and let GC collect unused AST and other stuff: ClearFields(); return(result); }
/// <summary> /// Keeps stream open. /// </summary> /// <exception cref="InvalidSourceException">Source file cannot be opened for reading.</exception> public override void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink, Parsers.Position initialPosition, LanguageFeatures features) { Parser parser = new Parser(); StreamReader source_reader; try { stream = new FileStream(sourceFile.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read); source_reader = new StreamReader(stream, encoding, true); // if the file contains a byte-order-mark, we must advance the offset in order to // get correct token positions in lexer (unfortunately, StreamReader skips BOM without // a possibility to detect it from outside, so we have to do that manually): // TODO: // initialPosition.FirstOffset += FileSystemUtils.GetByteOrderMarkLength(stream); } catch (Exception e) { throw new InvalidSourceException(sourceFile.FullPath, e); } ast = parser.Parse(this, source_reader, errors, reductionsSink, initialPosition, Lexer.LexicalStates.INITIAL, features); }
public Scanner(TextReader /*!*/ reader, SourceUnit /*!*/ sourceUnit, ErrorSink /*!*/ errors, ICommentsSink commentsSink, IScannerHandler scannerHandler, LanguageFeatures features, int positionShift) : base(reader) { if (reader == null) { throw new ArgumentNullException("reader"); } if (sourceUnit == null) { throw new ArgumentNullException("sourceUnit"); } if (errors == null) { throw new ArgumentNullException("errors"); } this.errors = errors; this.commentsSink = commentsSink ?? new NullCommentsSink(); this.scannerHandler = scannerHandler ?? new NullScannerHandler(); this.features = features; this.sourceUnit = sourceUnit; this.charOffset = positionShift; this.AllowAspTags = (features & LanguageFeatures.AspTags) != 0; this.AllowShortTags = (features & LanguageFeatures.ShortOpenTags) != 0; }
public TestSourceUnit(string /*!*/ code, string /*!*/ filePath, Encoding /*!*/ encoding, Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL, LanguageFeatures features = LanguageFeatures.Basic) : base(code, filePath, encoding, initialState, features) { _features = features; }
public CodeSourceUnit(string /*!*/ code, string /*!*/ filePath, Encoding /*!*/ encoding, Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL, LanguageFeatures features = LanguageFeatures.Basic) : base(filePath, encoding, Text.LineBreaks.Create(code)) { this.code = code; this.initialState = initialState; this.features = features; }
private void ClearFields() { scanner = null; features = 0; errors = null; sourceUnit = null; reductionsSink = null; astRoot = null; reader = null; }
public override void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink, Position initialPosition, LanguageFeatures features) { Parser parser = new Parser(); using (StringReader source_reader = new StringReader(code)) { ast = parser.Parse(this, source_reader, errors, reductionsSink, initialPosition, initialState, features); } }
public override void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink, Position initialPosition, LanguageFeatures features) { Parser parser = new Parser(); parser.AllowGlobalCode = this.allowGlobalCode; using (StringReader reader = new StringReader(code)) { ast = parser.Parse(this, reader, errors, reductionsSink, initialPosition, Lexer.LexicalStates.INITIAL, features); } }
/// <summary> /// Creates source unit and parses given <paramref name="code"/>. /// </summary> /// <param name="code">Source code to be parsed.</param> /// <param name="sourceFile">Source file used for error reporting.</param> /// <param name="errors">Errors sink. Can be <c>null</c>.</param> /// <param name="reductionsSink">Reduction sink. Can be <c>null</c>.</param> /// <param name="features">Optional. Language features.</param> /// <param name="initialState"> /// Optional. Initial parser state. /// This allows e.g. to parse PHP code without encapsulating the code into opening and closing tags.</param> /// <returns></returns> public static SourceUnit /*!*/ ParseCode(string code, PhpSourceFile sourceFile, ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink = null, LanguageFeatures features = LanguageFeatures.Php5, Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL) { var /*!*/ unit = new CodeSourceUnit(code, sourceFile, Encoding.UTF8, initialState); unit.Parse(errors, reductionsSink, features); unit.Close(); // return(unit); }
/// <summary> /// Lexer constructor that initializes all the necessary members /// </summary> /// <param name="reader">Text reader containing the source code.</param> /// <param name="encoding">Source file encoding to convert UTF characters.</param> /// <param name="errors">Error sink used to report lexical error.</param> /// <param name="features">Allow or disable short oppening tags for PHP.</param> /// <param name="positionShift">Starting position of the first token, used during custom restart.</param> /// <param name="initialState">Initial state of the lexer, used during custom restart.</param> public Lexer( System.IO.TextReader reader, Encoding encoding, IErrorSink <Span> errors = null, LanguageFeatures features = LanguageFeatures.Basic, int positionShift = 0, LexicalStates initialState = LexicalStates.INITIAL) { _encoding = encoding ?? Encoding.UTF8; _errors = errors ?? new EmptyErrorSink <Span>(); _charOffset = positionShift; _allowShortTags = (features & LanguageFeatures.ShortOpenTags) != 0; Initialize(reader, initialState); }
/// <summary> /// Lexer constructor that initializes all the necessary members /// </summary> /// <param name="reader">Text reader containing the source code.</param> /// <param name="encoding">Source file encoding to convert UTF characters.</param> /// <param name="errors">Error sink used to report lexical error.</param> /// <param name="features">Allow or disable short oppening tags for PHP.</param> /// <param name="positionShift">Starting position of the first token, used during custom restart.</param> /// <param name="initialState">Initial state of the lexer, used during custom restart.</param> public Lexer( System.IO.TextReader reader, Encoding encoding, IErrorSink <Span> errors = null, LanguageFeatures features = LanguageFeatures.Basic, int positionShift = 0, LexicalStates initialState = LexicalStates.INITIAL) { _encoding = encoding ?? Encoding.UTF8; _errors = errors ?? new EmptyErrorSink <Span>(); _charOffset = positionShift; _features = features; _strings = StringTable.GetInstance(); Initialize(reader, initialState); }
/// <summary> /// Keeps stream open. /// </summary> /// <exception cref="InvalidSourceException">Source file cannot be opened for reading.</exception> public override void Parse(ErrorSink errors, IReductionsSink reductionsSink, LanguageFeatures features) { Parser parser = new Parser(); StreamReader source_reader; try { stream = new FileStream(sourceFile.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read); source_reader = new StreamReader(stream, encoding, true); } catch (Exception e) { throw new InvalidSourceException(sourceFile.FullPath, e); } ast = parser.Parse(this, source_reader, errors, reductionsSink, Lexer.LexicalStates.INITIAL, features); }
public LangElement Parse( ITokenProvider <SemanticValueType, Span> lexer, INodesFactory <LangElement, Span> astFactory, LanguageFeatures language, IErrorSink <Span> errors = null, IErrorRecovery errorRecovery = null, int positionShift = 0) { if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } if (astFactory == null) { throw new ArgumentNullException(nameof(astFactory)); } // initialization: _languageFeatures = language; if (errorRecovery != null) { _lexer = new BufferedLexer(new CompliantLexer(lexer)); } else { _lexer = new CompliantLexer(lexer); } _astFactory = astFactory; _errors = errors ?? new EmptyErrorSink <Span>(); _errorRecovery = errorRecovery ?? new EmptyErrorRecovery(); //InitializeFields(); _currentScope = new Scope(0); base.Scanner = _lexer; bool accept = base.Parse(); LangElement result = _astRoot; // clean and let GC collect unused AST and other stuff: //ClearFields(); return(result); }
/// <summary> /// Lexer constructor that initializes all the necessary members /// </summary> /// <param name="reader">Text reader containing the source code.</param> /// <param name="encoding">Source file encoding to convert UTF characters.</param> /// <param name="errors">Error sink used to report lexical error.</param> /// <param name="features">Allow or disable short oppening tags for PHP.</param> /// <param name="docBlockFactory">Factory for documentary comment semantic elements.</param> /// <param name="positionShift">Starting position of the first token, used during custom restart.</param> /// <param name="initialState">Initial state of the lexer, used during custom restart.</param> public Lexer( System.IO.TextReader reader, Encoding encoding, IErrorSink <Span> errors = null, LanguageFeatures features = LanguageFeatures.Basic, IDocBlockFactory docBlockFactory = null, int positionShift = 0, LexicalStates initialState = LexicalStates.INITIAL) { _encoding = encoding ?? Encoding.UTF8; _errors = errors ?? new EmptyErrorSink <Span>(); _docblockFactory = docBlockFactory ?? DefaultDocBlockFactory.Instance; _charOffset = positionShift; _features = features; _strings = StringTable.GetInstance(); _processDoubleQuotedString = ProcessDoubleQuotedString; Initialize(reader, initialState); }
public void Parse(NodesFactory factory, IErrorSink <Span> errors, IErrorRecovery recovery = null, LanguageFeatures features = LanguageFeatures.Basic, Lexer.LexicalStates state = Lexer.LexicalStates.INITIAL) { var parser = new Parser(); using (var source = new StringReader(SourceText.ToString())) { using (var provider = new AdditionalSyntaxProvider( new PhpTokenProvider( new Lexer(source, Encoding.UTF8, errors, features, 0, state), this), factory, parser.CreateTypeRef)) { ast = parser.Parse(provider, factory, features, errors, recovery); } } }
public LangElement Parse( ITokenProvider <SemanticValueType, Span> lexer, INodesFactory <LangElement, Span> astFactory, LanguageFeatures language, IErrorSink <Span> errors = null, IErrorRecovery errorRecovery = null, int positionShift = 0) { if (lexer == null) { throw new ArgumentNullException(nameof(lexer)); } // initialization: _languageFeatures = language; _lexer = new CompliantLexer(lexer, language); _astFactory = astFactory ?? throw new ArgumentNullException(nameof(astFactory)); _errors = errors ?? new EmptyErrorSink <Span>(); if (errorRecovery != null) { _lexer = new BufferedLexer(_lexer); _errorRecovery = errorRecovery; } else { _errorRecovery = EmptyErrorRecovery.Instance; } //InitializeFields(); _currentScope = new Scope(0); base.Scanner = _lexer; bool accept = base.Parse(); // return(_astRoot); }
public abstract void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink, Parsers.Position initialPosition, LanguageFeatures features);
public static bool HasFeature(this LanguageFeatures value, LanguageFeatures feature) => (value & feature) == feature;
/// <summary> /// Initializes a new instance of the LanguageDefinition class. /// </summary> /// <param name="name">The name of the language.</param> /// <param name="fileExtensions">The file extensions for script files of this language.</param> /// <param name="features">Supported features of the language.</param> public LanguageDefinition(string name, string[] fileExtensions, LanguageFeatures features) { this.languageName = name; this.supportedFileExtensions = fileExtensions.Clone() as string[]; this.features = features; }
bool HasFeatureSet(LanguageFeatures fset) => (_features & fset) == fset;
public abstract void Parse( ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink, LanguageFeatures features);
/// <summary> /// Lexer constructor that initializes all the necessary members /// </summary> /// <param name="provider">Underlaying tokens provider.</param> /// <param name="language">Language features.</param> public CompliantLexer(ITokenProvider <SemanticValueType, Span> provider, LanguageFeatures language = LanguageFeatures.Basic) { _provider = provider; _features = language; }
/// <summary> /// Parse PHP code and return its AST. /// </summary> /// <param name="code">The PHP code to be parsed.</param> /// <param name="encoding">Encoding of the source code.</param> /// <param name="lang">Language features that may change parser behavior.</param> /// <param name="file">PHP Source file with the file name & location</param> /// <returns>Returns the parsed AST node.</returns> public AST.GlobalCode ParseString(string code, Encoding encoding, PhpSourceFile file, LanguageFeatures lang) { PhpScriptSourceUnit srcUnit = new PhpScriptSourceUnit(this, code, file, encoding, 0, 0); using (StringReader reader = new StringReader(code)) { Parser parser = new Parser(); return(parser.Parse(srcUnit, reader, new ParserErrorSink(), this, Lexer.LexicalStates.INITIAL, lang)); } }
/// <summary> /// Parse PHP code and return its AST. /// </summary> /// <param name="code">The PHP code to be parsed.</param> /// <param name="encoding">Encoding of the source code.</param> /// <param name="lang">Language features that may change parser behavior.</param> /// <param name="file">PHP Source file with the file name & location</param> /// <returns>Returns the parsed AST node.</returns> public AST.GlobalCode ParseString(string code, Encoding encoding, PhpSourceFile file, LanguageFeatures lang) { PhpScriptSourceUnit srcUnit = new PhpScriptSourceUnit (this, code, file, encoding, 0, 0); using(StringReader reader = new StringReader(code)) { Parser parser = new Parser(); return parser.Parse(srcUnit, reader, new ParserErrorSink(), this, Position.Initial, Lexer.LexicalStates.INITIAL, lang); } }
public abstract void Parse(ErrorSink/*!*/ errors, IReductionsSink/*!*/ reductionsSink, Parsers.Position initialPosition, LanguageFeatures features);