/// <summary>
/// Creates source unit and parses given <paramref name="code"/>.
/// </summary>
/// <param name="code">Source code to be parsed.</param>
/// <param name="filePath">Source file used for error reporting.</param>
/// <param name="factory">Nodes factory and error sink.</param>
/// <param name="errors">Error sink. Can be <c>null</c>.</param>
/// <param name="recovery">Error recovery. Can be <c>null</c>.</param>
/// <param name="features">Optional. Language features.</param>
/// <param name="initialState">
/// Optional. Initial parser state.
/// This allows e.g. to parse PHP code without encapsulating the code into opening and closing tags.</param>
/// <returns>New <see cref="CodeSourceUnit"/> object.</returns>
public static SourceUnit /*!*/ ParseCode(string code, string filePath,
INodesFactory <LangElement, Span> factory = null,
Errors.IErrorSink <Span> errors = null,
Errors.IErrorRecovery recovery = null,
LanguageFeatures features = LanguageFeatures.Basic,
Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL)
{
var unit = new CodeSourceUnit(code, filePath, Encoding.UTF8, initialState, features);
if (factory == null)
{
factory = new BasicNodesFactory(unit);
}
if (errors == null)
{
errors = (factory as Errors.IErrorSink <Span>) ?? new EmptyErrorSink <Span>();
}
//var lexer = new Lexer(new StringReader(code), Encoding.UTF8, errors, features, 0, initialState);
unit.Parse(factory, errors, recovery);
unit.Close();
//
return(unit);
}
public Scanner(Parsers.Position initialPosition, TextReader /*!*/ reader, SourceUnit /*!*/ sourceUnit,
ErrorSink /*!*/ errors, ICommentsSink commentsSink, LanguageFeatures features)
: base(reader)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (sourceUnit == null)
{
throw new ArgumentNullException("sourceUnit");
}
if (errors == null)
{
throw new ArgumentNullException("errors");
}
this.lineShift = initialPosition.FirstLine;
this.columnShift = initialPosition.FirstColumn;
this.offsetShift = initialPosition.FirstOffset;
this.errors = errors;
this.commentsSink = commentsSink ?? new _NullCommentsSink();
this.features = features;
this.sourceUnit = sourceUnit;
this.streamOffset = 0;
this.pure = sourceUnit.CompilationUnit.IsPure;
this.encoding = sourceUnit.Encoding;
AllowAspTags = (features & LanguageFeatures.AspTags) != 0;
AllowShortTags = (features & LanguageFeatures.ShortOpenTags) != 0;
}
public GlobalCode Parse(SourceUnit /*!*/ sourceUnit, TextReader /*!*/ reader, ErrorSink /*!*/ errors,
IReductionsSink reductionsSink, Parsers.Position initialPosition, Lexer.LexicalStates initialLexicalState,
LanguageFeatures features)
{
Debug.Assert(reader != null && errors != null);
// initialization:
this.sourceUnit = sourceUnit;
this.errors = errors;
this.features = features;
this.reader = reader;
this.reductionsSink = reductionsSink ?? NullReductionSink;
InitializeFields();
this.scanner = new Scanner(initialPosition, reader, sourceUnit, errors, reductionsSink as ICommentsSink, features);
this.scanner.CurrentLexicalState = initialLexicalState;
this.currentScope = new Scope(1); // starts assigning scopes from 2 (1 is reserved for prepended inclusion)
this.unicodeSemantics = (features & LanguageFeatures.UnicodeSemantics) != 0;
base.Scanner = this.scanner;
base.Parse();
GlobalCode result = astRoot;
// clean and let GC collect unused AST and other stuff:
ClearFields();
return(result);
}
/// <summary>
/// Keeps stream open.
/// </summary>
/// <exception cref="InvalidSourceException">Source file cannot be opened for reading.</exception>
public override void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink,
Parsers.Position initialPosition, LanguageFeatures features)
{
Parser parser = new Parser();
StreamReader source_reader;
try
{
stream = new FileStream(sourceFile.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
source_reader = new StreamReader(stream, encoding, true);
// if the file contains a byte-order-mark, we must advance the offset in order to
// get correct token positions in lexer (unfortunately, StreamReader skips BOM without
// a possibility to detect it from outside, so we have to do that manually):
// TODO:
// initialPosition.FirstOffset += FileSystemUtils.GetByteOrderMarkLength(stream);
}
catch (Exception e)
{
throw new InvalidSourceException(sourceFile.FullPath, e);
}
ast = parser.Parse(this, source_reader, errors, reductionsSink, initialPosition,
Lexer.LexicalStates.INITIAL, features);
}
public Scanner(TextReader /*!*/ reader, SourceUnit /*!*/ sourceUnit,
ErrorSink /*!*/ errors, ICommentsSink commentsSink, IScannerHandler scannerHandler,
LanguageFeatures features, int positionShift)
: base(reader)
{
if (reader == null)
{
throw new ArgumentNullException("reader");
}
if (sourceUnit == null)
{
throw new ArgumentNullException("sourceUnit");
}
if (errors == null)
{
throw new ArgumentNullException("errors");
}
this.errors = errors;
this.commentsSink = commentsSink ?? new NullCommentsSink();
this.scannerHandler = scannerHandler ?? new NullScannerHandler();
this.features = features;
this.sourceUnit = sourceUnit;
this.charOffset = positionShift;
this.AllowAspTags = (features & LanguageFeatures.AspTags) != 0;
this.AllowShortTags = (features & LanguageFeatures.ShortOpenTags) != 0;
}
public TestSourceUnit(string /*!*/ code, string /*!*/ filePath,
Encoding /*!*/ encoding,
Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL,
LanguageFeatures features = LanguageFeatures.Basic)
: base(code, filePath, encoding, initialState, features)
{
_features = features;
}
public CodeSourceUnit(string /*!*/ code, string /*!*/ filePath,
Encoding /*!*/ encoding,
Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL,
LanguageFeatures features = LanguageFeatures.Basic)
: base(filePath, encoding, Text.LineBreaks.Create(code))
{
this.code = code;
this.initialState = initialState;
this.features = features;
}
private void ClearFields()
{
scanner = null;
features = 0;
errors = null;
sourceUnit = null;
reductionsSink = null;
astRoot = null;
reader = null;
}
public override void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink, Position initialPosition,
LanguageFeatures features)
{
Parser parser = new Parser();
using (StringReader source_reader = new StringReader(code))
{
ast = parser.Parse(this, source_reader, errors, reductionsSink, initialPosition,
initialState, features);
}
}
public override void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink, Position initialPosition,
LanguageFeatures features)
{
Parser parser = new Parser();
parser.AllowGlobalCode = this.allowGlobalCode;
using (StringReader reader = new StringReader(code))
{
ast = parser.Parse(this, reader, errors, reductionsSink, initialPosition,
Lexer.LexicalStates.INITIAL, features);
}
}
/// <summary>
/// Creates source unit and parses given <paramref name="code"/>.
/// </summary>
/// <param name="code">Source code to be parsed.</param>
/// <param name="sourceFile">Source file used for error reporting.</param>
/// <param name="errors">Errors sink. Can be <c>null</c>.</param>
/// <param name="reductionsSink">Reduction sink. Can be <c>null</c>.</param>
/// <param name="features">Optional. Language features.</param>
/// <param name="initialState">
/// Optional. Initial parser state.
/// This allows e.g. to parse PHP code without encapsulating the code into opening and closing tags.</param>
/// <returns></returns>
public static SourceUnit /*!*/ ParseCode(string code, PhpSourceFile sourceFile,
ErrorSink /*!*/ errors,
IReductionsSink /*!*/ reductionsSink = null,
LanguageFeatures features = LanguageFeatures.Php5,
Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL)
{
var /*!*/ unit = new CodeSourceUnit(code, sourceFile, Encoding.UTF8, initialState);
unit.Parse(errors, reductionsSink, features);
unit.Close();
//
return(unit);
}
/// <summary>
/// Lexer constructor that initializes all the necessary members
/// </summary>
/// <param name="reader">Text reader containing the source code.</param>
/// <param name="encoding">Source file encoding to convert UTF characters.</param>
/// <param name="errors">Error sink used to report lexical error.</param>
/// <param name="features">Allow or disable short oppening tags for PHP.</param>
/// <param name="positionShift">Starting position of the first token, used during custom restart.</param>
/// <param name="initialState">Initial state of the lexer, used during custom restart.</param>
public Lexer(
System.IO.TextReader reader,
Encoding encoding,
IErrorSink <Span> errors = null,
LanguageFeatures features = LanguageFeatures.Basic,
int positionShift = 0,
LexicalStates initialState = LexicalStates.INITIAL)
{
_encoding = encoding ?? Encoding.UTF8;
_errors = errors ?? new EmptyErrorSink <Span>();
_charOffset = positionShift;
_allowShortTags = (features & LanguageFeatures.ShortOpenTags) != 0;
Initialize(reader, initialState);
}
/// <summary>
/// Lexer constructor that initializes all the necessary members
/// </summary>
/// <param name="reader">Text reader containing the source code.</param>
/// <param name="encoding">Source file encoding to convert UTF characters.</param>
/// <param name="errors">Error sink used to report lexical error.</param>
/// <param name="features">Allow or disable short oppening tags for PHP.</param>
/// <param name="positionShift">Starting position of the first token, used during custom restart.</param>
/// <param name="initialState">Initial state of the lexer, used during custom restart.</param>
public Lexer(
System.IO.TextReader reader,
Encoding encoding,
IErrorSink <Span> errors = null,
LanguageFeatures features = LanguageFeatures.Basic,
int positionShift = 0,
LexicalStates initialState = LexicalStates.INITIAL)
{
_encoding = encoding ?? Encoding.UTF8;
_errors = errors ?? new EmptyErrorSink <Span>();
_charOffset = positionShift;
_features = features;
_strings = StringTable.GetInstance();
Initialize(reader, initialState);
}
/// <summary>
/// Keeps stream open.
/// </summary>
/// <exception cref="InvalidSourceException">Source file cannot be opened for reading.</exception>
public override void Parse(ErrorSink errors, IReductionsSink reductionsSink, LanguageFeatures features)
{
Parser parser = new Parser();
StreamReader source_reader;
try
{
stream = new FileStream(sourceFile.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read);
source_reader = new StreamReader(stream, encoding, true);
}
catch (Exception e)
{
throw new InvalidSourceException(sourceFile.FullPath, e);
}
ast = parser.Parse(this, source_reader, errors, reductionsSink, Lexer.LexicalStates.INITIAL, features);
}
public LangElement Parse(
ITokenProvider <SemanticValueType, Span> lexer,
INodesFactory <LangElement, Span> astFactory,
LanguageFeatures language,
IErrorSink <Span> errors = null,
IErrorRecovery errorRecovery = null,
int positionShift = 0)
{
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
if (astFactory == null)
{
throw new ArgumentNullException(nameof(astFactory));
}
// initialization:
_languageFeatures = language;
if (errorRecovery != null)
{
_lexer = new BufferedLexer(new CompliantLexer(lexer));
}
else
{
_lexer = new CompliantLexer(lexer);
}
_astFactory = astFactory;
_errors = errors ?? new EmptyErrorSink <Span>();
_errorRecovery = errorRecovery ?? new EmptyErrorRecovery();
//InitializeFields();
_currentScope = new Scope(0);
base.Scanner = _lexer;
bool accept = base.Parse();
LangElement result = _astRoot;
// clean and let GC collect unused AST and other stuff:
//ClearFields();
return(result);
}
/// <summary>
/// Lexer constructor that initializes all the necessary members
/// </summary>
/// <param name="reader">Text reader containing the source code.</param>
/// <param name="encoding">Source file encoding to convert UTF characters.</param>
/// <param name="errors">Error sink used to report lexical error.</param>
/// <param name="features">Allow or disable short oppening tags for PHP.</param>
/// <param name="docBlockFactory">Factory for documentary comment semantic elements.</param>
/// <param name="positionShift">Starting position of the first token, used during custom restart.</param>
/// <param name="initialState">Initial state of the lexer, used during custom restart.</param>
public Lexer(
System.IO.TextReader reader,
Encoding encoding,
IErrorSink <Span> errors = null,
LanguageFeatures features = LanguageFeatures.Basic,
IDocBlockFactory docBlockFactory = null,
int positionShift = 0,
LexicalStates initialState = LexicalStates.INITIAL)
{
_encoding = encoding ?? Encoding.UTF8;
_errors = errors ?? new EmptyErrorSink <Span>();
_docblockFactory = docBlockFactory ?? DefaultDocBlockFactory.Instance;
_charOffset = positionShift;
_features = features;
_strings = StringTable.GetInstance();
_processDoubleQuotedString = ProcessDoubleQuotedString;
Initialize(reader, initialState);
}
public void Parse(NodesFactory factory, IErrorSink <Span> errors,
IErrorRecovery recovery = null,
LanguageFeatures features = LanguageFeatures.Basic,
Lexer.LexicalStates state = Lexer.LexicalStates.INITIAL)
{
var parser = new Parser();
using (var source = new StringReader(SourceText.ToString()))
{
using (var provider = new AdditionalSyntaxProvider(
new PhpTokenProvider(
new Lexer(source, Encoding.UTF8, errors, features, 0, state),
this),
factory,
parser.CreateTypeRef))
{
ast = parser.Parse(provider, factory, features, errors, recovery);
}
}
}
public LangElement Parse(
ITokenProvider <SemanticValueType, Span> lexer,
INodesFactory <LangElement, Span> astFactory,
LanguageFeatures language,
IErrorSink <Span> errors = null,
IErrorRecovery errorRecovery = null,
int positionShift = 0)
{
if (lexer == null)
{
throw new ArgumentNullException(nameof(lexer));
}
// initialization:
_languageFeatures = language;
_lexer = new CompliantLexer(lexer, language);
_astFactory = astFactory ?? throw new ArgumentNullException(nameof(astFactory));
_errors = errors ?? new EmptyErrorSink <Span>();
if (errorRecovery != null)
{
_lexer = new BufferedLexer(_lexer);
_errorRecovery = errorRecovery;
}
else
{
_errorRecovery = EmptyErrorRecovery.Instance;
}
//InitializeFields();
_currentScope = new Scope(0);
base.Scanner = _lexer;
bool accept = base.Parse();
//
return(_astRoot);
}
public abstract void Parse(ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink,
Parsers.Position initialPosition, LanguageFeatures features);
public static bool HasFeature(this LanguageFeatures value, LanguageFeatures feature) => (value & feature) == feature;
/// <summary>
/// Initializes a new instance of the LanguageDefinition class.
/// </summary>
/// <param name="name">The name of the language.</param>
/// <param name="fileExtensions">The file extensions for script files of this language.</param>
/// <param name="features">Supported features of the language.</param>
public LanguageDefinition(string name, string[] fileExtensions, LanguageFeatures features)
{
this.languageName = name;
this.supportedFileExtensions = fileExtensions.Clone() as string[];
this.features = features;
}
bool HasFeatureSet(LanguageFeatures fset) => (_features & fset) == fset;
public abstract void Parse(
ErrorSink /*!*/ errors, IReductionsSink /*!*/ reductionsSink,
LanguageFeatures features);
/// <summary>
/// Lexer constructor that initializes all the necessary members
/// </summary>
/// <param name="provider">Underlaying tokens provider.</param>
/// <param name="language">Language features.</param>
public CompliantLexer(ITokenProvider <SemanticValueType, Span> provider, LanguageFeatures language = LanguageFeatures.Basic)
{
_provider = provider;
_features = language;
}
/// <summary>
/// Parse PHP code and return its AST.
/// </summary>
/// <param name="code">The PHP code to be parsed.</param>
/// <param name="encoding">Encoding of the source code.</param>
/// <param name="lang">Language features that may change parser behavior.</param>
/// <param name="file">PHP Source file with the file name & location</param>
/// <returns>Returns the parsed AST node.</returns>
public AST.GlobalCode ParseString(string code, Encoding encoding, PhpSourceFile file, LanguageFeatures lang)
{
PhpScriptSourceUnit srcUnit = new PhpScriptSourceUnit(this, code, file, encoding, 0, 0);
using (StringReader reader = new StringReader(code))
{
Parser parser = new Parser();
return(parser.Parse(srcUnit, reader, new ParserErrorSink(), this, Lexer.LexicalStates.INITIAL, lang));
}
}
/// <summary>
/// Parse PHP code and return its AST.
/// </summary>
/// <param name="code">The PHP code to be parsed.</param>
/// <param name="encoding">Encoding of the source code.</param>
/// <param name="lang">Language features that may change parser behavior.</param>
/// <param name="file">PHP Source file with the file name & location</param>
/// <returns>Returns the parsed AST node.</returns>
public AST.GlobalCode ParseString(string code, Encoding encoding, PhpSourceFile file, LanguageFeatures lang) {
PhpScriptSourceUnit srcUnit = new PhpScriptSourceUnit
(this, code, file, encoding, 0, 0);
using(StringReader reader = new StringReader(code)) {
Parser parser = new Parser();
return parser.Parse(srcUnit, reader, new ParserErrorSink(), this,
Position.Initial, Lexer.LexicalStates.INITIAL, lang);
}
}
public abstract void Parse(ErrorSink/*!*/ errors, IReductionsSink/*!*/ reductionsSink,
Parsers.Position initialPosition, LanguageFeatures features);
