private void checkMissingKeyword(object endToken, ParserRuleContext context, string msg)
{
if (endToken == null)
{
// if there is an error inside the start .. end then there is no need to report
// a missing end
int start = context.Start.StartIndex;
int end = context.Stop.StopIndex;
bool haserrorinblock = false;
foreach (var error in _parseErrors)
{
if (error.Node.Position > start && error.Node.Position < end)
{
haserrorinblock = true;
break;
}
}
if (!haserrorinblock)
{
var err = ErrorCode.ERR_SyntaxError;
IToken anchor = context.Stop;
if (anchor == null)
{
anchor = context.Start;
}
var errdata = new ParseErrorData(anchor, err, msg);
_parseErrors.Add(errdata);
}
}
return;
}
public override void ExitLiteralValue([NotNull] XSharpParser.LiteralValueContext context)
{
if (context.Token.Type == XSharpLexer.INCOMPLETE_STRING_CONST)
{
var err = ErrorCode.ERR_UnterminatedStringLit;
IToken anchor = context.Stop;
if (anchor == null)
{
anchor = context.Start;
}
var errdata = new ParseErrorData(anchor, err);
_parseErrors.Add(errdata);
}
else if (context.Token.Type == XSharpLexer.INVALID_NUMBER)
{
var err = ErrorCode.ERR_InvalidNumber;
IToken anchor = context.Stop;
if (anchor == null)
{
anchor = context.Start;
}
var errdata = new ParseErrorData(anchor, err);
_parseErrors.Add(errdata);
}
}
internal void AddError(ParseErrorData e)
{
if (ErrorData == null)
{
ErrorData = new List <ParseErrorData>();
}
ErrorData.Add(e);
}
public override void ExitBinaryExpression([NotNull] XSharpParser.BinaryExpressionContext context)
{
if (context.Left != null && context.Right == null)
{
var err = ErrorCode.ERR_SyntaxError;
IToken anchor = context.Stop;
if (anchor == null)
{
anchor = context.Start;
}
var errdata = new ParseErrorData(anchor, err, "Expression after '" + context.Op.Text + "' operator");
_parseErrors.Add(errdata);
}
}
public static bool Lex(string sourceText, string fileName, CSharpParseOptions options, IErrorListener listener,
out ITokenStream tokens)
{
tokens = null;
var parseErrors = ParseErrorData.NewBag();
try
{
var lexer = XSharpLexer.Create(sourceText, fileName, options);
lexer.Options = options;
var tokenStream = lexer.GetTokenStream();
tokenStream.Fill();
tokens = tokenStream;
GetLexerErrors(lexer, tokenStream, parseErrors);
#region Determine if we need to preprocess
bool mustPreprocess = true;
if (options.NoStdDef)
{
mustPreprocess = lexer.MustBeProcessed || lexer.HasPreprocessorTokens;
}
#endregion
XSharpPreprocessor pp = null;
BufferedTokenStream ppStream = null;
pp = new XSharpPreprocessor(lexer, tokenStream, options, fileName, Encoding.Unicode, SourceHashAlgorithm.None, parseErrors);
if (mustPreprocess)
{
var ppTokens = pp.PreProcess();
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens));
}
else
{
// No Standard Defs and no preprocessor tokens in the lexer
// so we bypass the preprocessor and use the lexer token stream
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokenStream.GetTokens()));
}
ppStream.Fill();
}
catch (Exception)
{
}
ReportErrors(parseErrors, listener);
return(tokens != null);
}
public override void VisitErrorNode([NotNull] IErrorNode node)
{
if (node.Symbol.Type == XSharpLexer.INCOMPLETE_STRING_CONST)
{
var err = ErrorCode.ERR_UnterminatedStringLit;
IToken anchor = node.Symbol;
var errdata = new ParseErrorData(anchor, err);
_parseErrors.Add(errdata);
}
else if (node.Symbol.Type == XSharpLexer.INVALID_NUMBER)
{
var err = ErrorCode.ERR_InvalidNumber;
IToken anchor = node.Symbol;
var errdata = new ParseErrorData(anchor, err);
_parseErrors.Add(errdata);
}
//else
//{
// _parseErrors.Add(new ParseErrorData(node, ErrorCode.ERR_SyntaxError, node));
//}
}
private void checkMissingToken(IToken l, IToken r, ParserRuleContext context)
{
if (l != null && r == null)
{
ErrorCode err = ErrorCode.ERR_SyntaxError;
object par = null;
switch (l.Type)
{
case XSharpLexer.LPAREN:
err = ErrorCode.ERR_CloseParenExpected;
break;
case XSharpLexer.LCURLY:
err = ErrorCode.ERR_RbraceExpected;
break;
case XSharpLexer.LBRKT:
err = ErrorCode.ERR_SyntaxError;
par = ']';
break;
}
IToken anchor = context.Stop;
if (anchor == null)
{
anchor = l;
}
ParseErrorData errdata;
if (par != null)
{
errdata = new ParseErrorData(anchor, err, par);
}
else
{
errdata = new ParseErrorData(anchor, err);
}
_parseErrors.Add(errdata);
}
}
private static void Parse(string fileName)
{
ITokenStream stream;
IList <ParseErrorData> parseErrors = ParseErrorData.NewBag();
var filestream = new AntlrFileStream(fileName);
var lexer = new XSharpLexer(filestream);
lexer.TokenFactory = XSharpTokenFactory.Default;
stream = new CommonTokenStream(lexer, Lexer.DefaultTokenChannel);
var parser = new XSharpParser(stream);
parser.IsScript = false;
parser.AllowFunctionInsideClass = false;
parser.AllowNamedArgs = false;
parser.AllowXBaseVariables = false;
parser.RemoveErrorListeners();
parser.Interpreter.PredictionMode = PredictionMode.Sll;
parser.Interpreter.reportAmbiguities = true;
parser.Interpreter.enable_global_context_dfa = true; // default false
parser.Interpreter.optimize_tail_calls = true;
parser.Interpreter.tail_call_preserves_sll = true;
//parser.Interpreter.userWantsCtxSensitive = true; // default true
parser.ErrorHandler = new XSharpErrorStrategy();
parser.AddErrorListener(new XSharpErrorListener(fileName, parseErrors, true));
XSharpParserRuleContext tree;
try
{
tree = parser.source();
}
catch (ParseCanceledException)
{
Console.WriteLine("Parse error, Errors from SLL mode");
showErrors(parseErrors);
parseErrors.Clear();
parser.ErrorHandler = new XSharpErrorStrategy();
parser.AddErrorListener(new XSharpErrorListener(fileName, parseErrors, true));
parser.Interpreter.PredictionMode = PredictionMode.Ll;
parser.Interpreter.force_global_context = true;
parser.Interpreter.optimize_ll1 = false;
parser.Interpreter.reportAmbiguities = true;
parser.Reset();
try
{
tree = parser.source();
}
catch (Exception e)
{
tree = null;
Console.WriteLine(e.Message);
}
}
// find parser errors (missing tokens etc)
foreach (var e in lexer.LexErrors)
{
parseErrors.Add(e);
}
var walker = new ParseTreeWalker();
var errchecker = new XSharpParseErrorAnalysis(parser, parseErrors);
if (tree != null)
{
walker.Walk(errchecker, tree);
}
Console.WriteLine("Parse error, Errors:");
showErrors(parseErrors);
}
internal CompilationUnitSyntax ParseCompilationUnitCore()
{
#if DEBUG && DUMP_TIMES
DateTime t = DateTime.Now;
#endif
if (_options.ShowIncludes)
{
_options.ConsoleOutput.WriteLine("Compiling {0}", _fileName);
}
var sourceText = _text.ToString();
XSharpLexer lexer = null;
XSharpPreprocessor pp = null;
XSharpParserRuleContext tree = new XSharpParserRuleContext();
XSharpParser parser = null;
var parseErrors = ParseErrorData.NewBag();
try
{
lexer = XSharpLexer.Create(sourceText, _fileName, _options);
lexer.Options = _options;
_lexerTokenStream = lexer.GetTokenStream();
}
catch (Exception e)
{
// Exception during Lexing
parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace));
// create empty token stream so we can continue the rest of the code
_lexerTokenStream = new BufferedTokenStream(new XSharpListTokenSource(lexer, new List <IToken>()));
}
#if DEBUG && DUMP_TIMES
{
var ts = DateTime.Now - t;
t += ts;
Debug.WriteLine("Lexing completed in {0}", ts);
}
#endif
// do not pre-process when there were lexer exceptions
if (lexer != null && parseErrors.Count == 0)
{
foreach (var e in lexer.LexErrors)
{
parseErrors.Add(e);
}
BufferedTokenStream ppStream = null;
try
{
// Check for #pragma in the lexerTokenStream
_lexerTokenStream.Fill();
if (!_options.MacroScript)
{
pp = new XSharpPreprocessor(lexer, _lexerTokenStream, _options, _fileName, _text.Encoding, _text.ChecksumAlgorithm, parseErrors);
}
var mustPreprocess = !_options.MacroScript && (lexer.HasPreprocessorTokens || !_options.NoStdDef);
if (mustPreprocess)
{
var ppTokens = pp.PreProcess();
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens));
}
else
{
// No Standard Defs and no preprocessor tokens in the lexer
// so we bypass the preprocessor and use the lexer tokenstream
// but if a .ppo is required we must use the preprocessor to
// write the source text to the .ppo file
if (_options.PreprocessorOutput && pp != null)
{
pp.writeToPPO(sourceText, false);
}
BufferedTokenStream ts = (BufferedTokenStream)_lexerTokenStream;
var tokens = ts.GetTokens();
// commontokenstream filters on tokens on the default channel. All other tokens are ignored
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokens));
}
ppStream.Fill();
_preprocessorTokenStream = ppStream;
}
catch (Exception e)
{
// Exception during Preprocessing
parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace));
// create empty token stream so we can continue the rest of the code
_preprocessorTokenStream = new BufferedTokenStream(new XSharpListTokenSource(lexer, new List <IToken>()));
}
}
#if DEBUG && DUMP_TIMES
{
var ts = DateTime.Now - t;
t += ts;
Debug.WriteLine("Preprocessing completed in {0}", ts);
}
#endif
parser = new XSharpParser(_preprocessorTokenStream)
{
Options = _options
};
tree = new XSharpParserRuleContext();
if (_options.ParseLevel != ParseLevel.Lex)
{
// When parsing in Sll mode we do not record any parser errors.
// When this fails, then we try again with LL mode and then we record errors
parser.RemoveErrorListeners();
parser.Interpreter.PredictionMode = PredictionMode.Sll;
// some options to have FAST parsing
parser.Interpreter.tail_call_preserves_sll = false;
parser.Interpreter.treat_sllk1_conflict_as_ambiguity = true;
parser.ErrorHandler = new BailErrorStrategy();
try
{
tree = buildTree(parser);
}
catch (ParseCanceledException e)
{
if (_options.Verbose)
{
string msg = _GetInnerExceptionMessage(e);
_options.ConsoleOutput.WriteLine("Antlr: SLL parsing failed with failure: " + msg + ". Trying again in LL mode.");
}
var errorListener = new XSharpErrorListener(_fileName, parseErrors);
parser.AddErrorListener(errorListener);
parser.ErrorHandler = new XSharpErrorStrategy();
// we need to set force_global_context to get proper error messages. This makes parsing slower
// but gives better messages
parser.Interpreter.treat_sllk1_conflict_as_ambiguity = false;
parser.Interpreter.force_global_context = true;
parser.Interpreter.enable_global_context_dfa = true;
parser.Interpreter.PredictionMode = PredictionMode.Ll;
_preprocessorTokenStream.Reset();
if (_options.Verbose && pp != null)
{
pp.DumpStats();
}
if (pp != null)
{
pp.Close();
}
parser.Reset();
try
{
tree = buildTree(parser);
}
catch (Exception e1)
{
// Cannot parse again. Must be a syntax error.
if (_options.Verbose)
{
string msg = _GetInnerExceptionMessage(e1);
_options.ConsoleOutput.WriteLine("Antlr: LL parsing also failed with failure: " + msg);
}
}
}
}// _options.ParseLevel < Complete
#if DEBUG && DUMP_TIMES
{
var ts = DateTime.Now - t;
t += ts;
Debug.WriteLine("Parsing completed in {0}", ts);
}
#endif
if (_options.DumpAST && tree != null)
{
string strTree = tree.ToStringTree();
string file = System.IO.Path.ChangeExtension(_fileName, "ast");
strTree = strTree.Replace(@"\r\n)))))", @"\r\n*)))))" + "\r\n");
strTree = strTree.Replace(@"\r\n))))", @"\r\n*)))" + "\r\n");
strTree = strTree.Replace(@"\r\n)))", @"\r\n*)))" + "\r\n");
strTree = strTree.Replace(@"\r\n))", @"\r\n*))" + "\r\n");
strTree = strTree.Replace(@"\r\n)", @"\r\n*)" + "\r\n");
strTree = strTree.Replace(@"\r\n*)", @"\r\n)");
System.IO.File.WriteAllText(file, strTree);
}
var walker = new ParseTreeWalker();
if (_options.ParseLevel == ParseLevel.Complete)
{
// check for parser errors, such as missing tokens
// This adds items to the parseErrors list for missing
// tokens and missing keywords
try
{
var errchecker = new XSharpParseErrorAnalysis(parser, parseErrors, _options);
walker.Walk(errchecker, tree);
}
catch (Exception e)
{
parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace));
}
}
var treeTransform = CreateTransform(parser, _options, _pool, _syntaxFactory, _fileName);
bool hasErrors = false;
SyntaxToken eof = null;
try
{
if (_options.ParseLevel < ParseLevel.Complete || parser.NumberOfSyntaxErrors != 0 ||
(parseErrors.Count != 0 && parseErrors.Contains(p => !ErrorFacts.IsWarning(p.Code))))
{
eof = SyntaxFactory.Token(SyntaxKind.EndOfFileToken);
eof = AddLeadingSkippedSyntax(eof, ParserErrorsAsTrivia(parseErrors, pp.IncludedFiles));
if (tree != null)
{
eof.XNode = new XTerminalNodeImpl(tree.Stop);
}
else
{
eof.XNode = new XTerminalNodeImpl(_lexerTokenStream.Get(_lexerTokenStream.Size - 1));
}
hasErrors = true;
}
if (!hasErrors)
{
try
{
walker.Walk(treeTransform, tree);
}
catch (Exception e)
{
parseErrors.Add(new ParseErrorData(_fileName, ErrorCode.ERR_Internal, e.Message, e.StackTrace));
}
eof = SyntaxFactory.Token(SyntaxKind.EndOfFileToken);
if (!parseErrors.IsEmpty())
{
eof = AddLeadingSkippedSyntax(eof, ParserErrorsAsTrivia(parseErrors, pp.IncludedFiles));
}
}
var result = _syntaxFactory.CompilationUnit(
treeTransform.GlobalEntities.Externs,
treeTransform.GlobalEntities.Usings,
treeTransform.GlobalEntities.Attributes,
treeTransform.GlobalEntities.Members, eof);
result.XNode = tree;
tree.CsNode = result;
result.XTokens = _lexerTokenStream;
result.XPPTokens = _preprocessorTokenStream;
result.HasDocComments = lexer.HasDocComments;
if (!_options.MacroScript && !hasErrors)
{
result.InitProcedures = treeTransform.GlobalEntities.InitProcedures;
result.Globals = treeTransform.GlobalEntities.Globals;
result.PragmaWarnings = treeTransform.GlobalEntities.PragmaWarnings;
result.PragmaOptions = treeTransform.GlobalEntities.PragmaOptions;
result.IncludedFiles = pp?.IncludedFiles;
result.FileWidePublics = treeTransform.GlobalEntities.FileWidePublics;
result.HasPCall = treeTransform.GlobalEntities.HasPCall;
result.NeedsProcessing = treeTransform.GlobalEntities.NeedsProcessing;
if (_options.HasRuntime)
{
result.LiteralSymbols = ((XSharpTreeTransformationRT)treeTransform).LiteralSymbols;
result.LiteralPSZs = ((XSharpTreeTransformationRT)treeTransform).LiteralPSZs;
}
}
return(result);
}
finally
{
#if DEBUG && DUMP_TIMES
{
var ts = DateTime.Now - t;
t += ts;
Debug.WriteLine("Tree transform completed in {0}", ts);
}
#endif
treeTransform.Free();
if (pp != null)
{
pp.Close();
}
}
}
public override void ExitPragma([NotNull] XSharpParser.PragmaContext context)
{
ErrorCode error = ErrorCode.Unknown;
var tokens = context._Tokens;
IToken i1 = null;
IToken i2 = null;
var numbers = new List <IToken>();
Pragmastate state = Pragmastate.Default;
IToken errortoken = null;
bool isWarning = false;
if (tokens.Count > 0)
{
i1 = tokens[0];
if (tokens.Count > 1)
{
i2 = tokens[1];
// 0 1 2 3 4 5
// #pragma warnings ( 123 , off )
// #pragma warnings ( pop )
if (i2.Type == XSharpParser.LPAREN)
{
if (tokens.Count >= 6 && tokens[3].Type == XSharpParser.COMMA && tokens[5].Type == XSharpParser.RPAREN)
{
i2 = tokens[4];
numbers.Add(tokens[2]);
}
else if (tokens.Count >= 4 && tokens[3].Type == XSharpParser.RPAREN)
{
i2 = tokens[2];
}
}
else
{
// 0 1 2 3 4 5 6
// #pragma warnings disable 1 , 2 , 3
if (tokens.Count > 2)
{
for (int i = 2; i < context._Tokens.Count; i++)
{
if (tokens[i].Type != XSharpParser.COMMA)
{
numbers.Add(tokens[i]);
}
}
}
}
}
}
if (i1 == null)
{
error = ErrorCode.WRN_IllegalPragma;
errortoken = context.P;
}
if (error == ErrorCode.Unknown)
{
switch (i1.Text.ToLower())
{
case "options":
isWarning = false;
break;
case "warning":
case "warnings":
isWarning = true;
break;
default:
error = ErrorCode.WRN_IllegalPragma;
errortoken = i1;
break;
}
}
if (error == ErrorCode.Unknown)
{
if (i2 != null)
{
switch (i2.Text.ToLower())
{
case "on":
state = Pragmastate.On;
break;
case "disable":
case "off":
state = Pragmastate.Off;
break;
case "restore":
case "default":
case "pop":
state = Pragmastate.Default;
break;
default:
error = ErrorCode.WRN_IllegalPPWarning;
errortoken = i2;
break;
}
}
else
{
error = ErrorCode.WRN_IllegalPPWarning;
errortoken = i1;
}
}
if (error == ErrorCode.Unknown)
{
if (isWarning)
{
context.Pragma = new PragmaWarning(context, state, numbers, i1, i2);
}
else if (numbers.Count == 0)
{
// options pop
if (i2.Text.ToLower() == "pop")
{
context.Pragma = new PragmaOption(context, Pragmastate.Default, CompilerOption.All);
}
else
{
error = ErrorCode.WRN_IllegalPPWarning;
errortoken = i2;
}
}
else
{
var token = numbers[0];
var opt = token.Text.ToLower();
if (token.Type == XSharpParser.STRING_CONST && opt.StartsWith("\"") && opt.EndsWith("\"") && opt.Length > 2)
{
opt = opt.Substring(1, opt.Length - 2);
switch (opt)
{
case "az":
case "initlocals":
case "lb":
case "memvar":
case "memvars":
case "undeclared":
case "vo2": // Initialize string variables with empty strings
case "vo4": // SignedUnsignedConversion
case "vo5": // Implicit Clipper Calling convention
case "vo6": // ResolveTypedFunctionPointersToPtr
case "vo7": // Implicit Casts and Conversions
case "vo9": // Allow missing return statements or missing return values
case "vo10": // Compatible IIF
case "vo11": // ArithmeticConversions
case "vo12": // Clipper Integer divisions
case "vo13": // StringComparisons
case "vo14": // Embed real constants as float
context.Pragma = new PragmaOption(context, state, CompilerOptionDecoder.Decode(opt));
break;
default:
error = ErrorCode.WRN_IllegalPPOption;
errortoken = numbers[0];
break;
}
}
}
}
// C# does not validate the error codes, so we will not do that either.
if (error != ErrorCode.Unknown)
{
var errdata = new ParseErrorData(errortoken, error, errortoken.Text);
_parseErrors.Add(errdata);
}
}
public static bool Parse(string sourceText, string fileName, CSharpParseOptions options, IErrorListener listener,
out ITokenStream tokens, out XSharpParser.SourceContext tree)
{
tree = null;
tokens = null;
var parseErrors = ParseErrorData.NewBag();
try
{
var lexer = XSharpLexer.Create(sourceText, fileName, options);
lexer.Options = options;
BufferedTokenStream tokenStream = lexer.GetTokenStream();
tokenStream.Fill();
tokens = (ITokenStream)tokenStream;
GetLexerErrors(lexer, tokenStream, parseErrors);
// do we need to preprocess
#region Determine if we really need the preprocessor
bool mustPreprocess = true;
if (lexer.HasPreprocessorTokens || !options.NoStdDef)
{
// no need to pre process in partial compilation
// if lexer does not contain UDCs, Messages or Includes
mustPreprocess = lexer.MustBeProcessed;
}
else
{
mustPreprocess = false;
}
#endregion
XSharpPreprocessor pp = null;
BufferedTokenStream ppStream = null;
pp = new XSharpPreprocessor(lexer, tokenStream, options, fileName, Encoding.Unicode, SourceHashAlgorithm.None, parseErrors);
if (mustPreprocess)
{
var ppTokens = pp.PreProcess();
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens));
}
else
{
// No Standard Defs and no preprocessor tokens in the lexer
// so we bypass the preprocessor and use the lexer token stream
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokenStream.GetTokens()));
}
ppStream.Fill();
var parser = new XSharpParser(ppStream);
parser.Interpreter.tail_call_preserves_sll = false; // default = true Setting to FALSE will reduce memory used by parser
parser.Options = options;
tree = null;
parser.RemoveErrorListeners();
parser.Interpreter.PredictionMode = PredictionMode.Sll;
parser.ErrorHandler = new BailErrorStrategy();
try
{
tree = parser.source();
}
catch (Exception)
{
var errorListener = new XSharpErrorListener(fileName, parseErrors);
parser.AddErrorListener(errorListener);
parser.ErrorHandler = new XSharpErrorStrategy();
parser.Interpreter.PredictionMode = PredictionMode.Ll;
ppStream.Reset();
parser.Reset();
try
{
tree = parser.source();
}
catch (Exception)
{
tree = null;
}
}
}
catch (Exception)
{
tree = null;
}
ReportErrors(parseErrors, listener);
return(tree != null);
}
