private string GetGrammarFromCodeFileName(RuntimeInfo runtimeInfo, string codeFileName)
{
string result = _grammar.Files.FirstOrDefault(file => file.EndsWith(codeFileName));
if (result != null)
{
return(result);
}
result = Path.GetFileNameWithoutExtension(codeFileName);
if (_grammar.Type == GrammarType.Combined)
{
if (result.EndsWith(runtimeInfo.LexerPostfix))
{
result = result.Remove(result.Length - runtimeInfo.LexerPostfix.Length);
}
else if (result.EndsWith(runtimeInfo.ParserPostfix))
{
result = result.Remove(result.Length - runtimeInfo.ParserPostfix.Length);
}
}
result = result + Grammar.AntlrDotExt;
return(_grammar.Files.FirstOrDefault(file => file.EndsWith(result)));
}
private void CopyCompiledSources(Runtime runtime, string workingDirectory)
{
RuntimeInfo runtimeInfo = RuntimeInfo.InitOrGetRuntimeInfo(runtime);
string extension = runtimeInfo.Extensions[0];
foreach (string fileName in _grammar.Files)
{
if (Path.GetExtension(fileName).Equals("." + extension, StringComparison.OrdinalIgnoreCase))
{
string sourceFileName = Path.Combine(_grammar.Directory, fileName);
string shortFileName = Path.GetFileName(fileName);
if ((runtimeInfo.Runtime == Runtime.Java || runtimeInfo.Runtime == Runtime.Go) &&
!string.IsNullOrWhiteSpace(_result.ParserGeneratedState.PackageName))
{
shortFileName = Path.Combine(_result.ParserGeneratedState.PackageName, shortFileName);
}
string destFileName = Path.Combine(workingDirectory, shortFileName);
File.Copy(sourceFileName, destFileName, true);
}
}
}
public static RuntimeInfo InitOrGetRuntimeInfo(Runtime runtime)
{
RuntimeInfo runtimeInfo = Runtimes[runtime];
if (!runtimeInfo.Initialized)
{
try
{
var processor = new Processor(runtimeInfo.RuntimeToolName, runtimeInfo.VersionArg);
string version = "";
if (runtimeInfo.ErrorVersionStream)
{
processor.ErrorDataReceived += VersionCollectFunc;
}
else
{
processor.OutputDataReceived += VersionCollectFunc;
}
void VersionCollectFunc(object sender, DataReceivedEventArgs e)
{
if (!(runtime == Runtime.Java && e.IsIgnoreJavaError()))
{
version += e.Data + Environment.NewLine;
}
}
processor.Start();
runtimeInfo.Version = version.Trim();
}
catch
{
runtimeInfo.Version = null;
}
runtimeInfo.Initialized = true;
}
return(runtimeInfo);
}
public ParserCompiliedState Compile(ParserGeneratedState state,
CancellationToken cancellationToken = default)
{
_grammar = state.GrammarCheckedState.InputState.Grammar;
Runtime runtime = state.Runtime;
_result = new ParserCompiliedState(state);
_currentRuntimeInfo = RuntimeInfo.InitOrGetRuntimeInfo(runtime);
Processor processor = null;
try
{
string runtimeSource = runtime.GetGeneralRuntimeName();
string runtimeDir = Path.Combine(RuntimesDirName, runtimeSource);
string runtimeLibraryPath = RuntimeLibrary ?? Path.Combine(runtimeDir, _currentRuntimeInfo.RuntimeLibrary);
string workingDirectory = Path.Combine(ParserGenerator.HelperDirectoryName, _grammar.Name, runtime.ToString());
var generatedFiles = new List <string>();
_grammarCodeMapping = new Dictionary <string, List <TextSpanMapping> >();
string generatedGrammarName =
runtime != Runtime.Go ? _grammar.Name : _grammar.Name.ToLowerInvariant();
if (_grammar.Type != GrammarType.Lexer)
{
GetGeneratedFileNames(state.GrammarCheckedState, generatedGrammarName, workingDirectory, generatedFiles, false);
}
GetGeneratedFileNames(state.GrammarCheckedState, generatedGrammarName, workingDirectory, generatedFiles, true);
CopyCompiledSources(runtime, workingDirectory);
if (_grammar.Type != GrammarType.Lexer)
{
if (state.IncludeListener)
{
GetGeneratedListenerOrVisitorFiles(generatedGrammarName, workingDirectory, generatedFiles, false);
}
if (state.IncludeVisitor)
{
GetGeneratedListenerOrVisitorFiles(generatedGrammarName, workingDirectory, generatedFiles, true);
}
}
string arguments = "";
switch (runtime)
{
case Runtime.CSharpOptimized:
case Runtime.CSharpStandard:
arguments = PrepareCSharpFiles(workingDirectory, runtimeDir);
break;
case Runtime.Java:
arguments = PrepareJavaFiles(generatedFiles, runtimeDir, workingDirectory, runtimeLibraryPath);
break;
case Runtime.Python2:
case Runtime.Python3:
arguments = PreparePythonFiles(generatedFiles, runtimeDir, workingDirectory);
break;
case Runtime.JavaScript:
arguments = PrepareJavaScriptFiles(generatedFiles, workingDirectory, runtimeDir);
break;
case Runtime.Go:
arguments = PrepareGoFiles(generatedFiles, runtimeDir, workingDirectory);
break;
}
PrepareParserCode(workingDirectory, runtimeDir);
_buffer = new List <string>();
_processedMessages = new HashSet <string>();
_result.Command = _currentRuntimeInfo.RuntimeToolName + " " + arguments;
processor = new Processor(_currentRuntimeInfo.RuntimeToolName, arguments, workingDirectory);
processor.CancellationToken = cancellationToken;
processor.ErrorDataReceived += ParserCompilation_ErrorDataReceived;
processor.OutputDataReceived += ParserCompilation_OutputDataReceived;
processor.Start();
if (_buffer.Count > 0)
{
if (runtime.IsPythonRuntime())
{
AddPythonError();
}
else if (runtime == Runtime.JavaScript)
{
AddJavaScriptError();
}
}
cancellationToken.ThrowIfCancellationRequested();
}
catch (Exception ex)
{
_result.Exception = ex;
if (!(ex is OperationCanceledException))
{
AddError(new ParsingError(ex, WorkflowStage.ParserCompilied));
}
}
finally
{
processor?.Dispose();
}
return(_result);
}
private void PrepareParserCode(string workingDirectory, string runtimeDir)
{
string templateFile = Path.Combine(workingDirectory, _currentRuntimeInfo.MainFile);
Runtime runtime = _currentRuntimeInfo.Runtime;
string code = File.ReadAllText(Path.Combine(runtimeDir, _currentRuntimeInfo.MainFile));
string packageName = _result.ParserGeneratedState.PackageName;
code = code.Replace(TemplateGrammarName, _grammar.Name);
if (!string.IsNullOrWhiteSpace(packageName))
{
if (runtime.IsCSharpRuntime())
{
code = code.Replace(PackageNamePart, "using " + packageName + ";");
}
else if (runtime == Runtime.Java)
{
code = code.Replace(PackageNamePart, "import " + packageName + ".*;");
}
else if (runtime == Runtime.Go)
{
code = code.Replace(PackageNamePart, "\"./" + packageName + "\"")
.Replace("/*$PackageName2$*/", packageName + ".");
}
}
else
{
code = code.Replace(PackageNamePart, "");
if (runtime == Runtime.Go)
{
code = code.Replace("/*$PackageName2$*/", "");
}
}
if (_grammar.CaseInsensitiveType != CaseInsensitiveType.None)
{
string antlrInputStream = RuntimeInfo.InitOrGetRuntimeInfo(runtime).AntlrInputStream;
string caseInsensitiveStream = "AntlrCaseInsensitiveInputStream";
if (runtime == Runtime.Java)
{
caseInsensitiveStream = "new " + caseInsensitiveStream;
}
else if (runtime == Runtime.Go)
{
caseInsensitiveStream = "New" + caseInsensitiveStream;
}
var antlrInputStreamRegex = new Regex($@"{antlrInputStream}\(([^\)]+)\)");
string isLowerBool = (_grammar.CaseInsensitiveType == CaseInsensitiveType.lower).ToString();
if (!runtime.IsPythonRuntime())
{
isLowerBool = isLowerBool.ToLowerInvariant();
}
code = antlrInputStreamRegex.Replace(code,
m => $"{caseInsensitiveStream}({m.Groups[1].Value}, {isLowerBool})");
if (runtime.IsPythonRuntime())
{
code = code.Replace("from antlr4.InputStream import InputStream", "")
.Replace("'''AntlrCaseInsensitive'''",
"from AntlrCaseInsensitiveInputStream import AntlrCaseInsensitiveInputStream");
}
else if (runtime == Runtime.JavaScript)
{
code = code.Replace("/*AntlrCaseInsensitive*/",
"var AntlrCaseInsensitiveInputStream = require('./AntlrCaseInsensitiveInputStream').AntlrCaseInsensitiveInputStream;");
}
}
else
{
if (runtime.IsPythonRuntime())
{
code = code.Replace("'''AntlrCaseInsensitive'''", "");
}
else if (runtime == Runtime.JavaScript)
{
code = code.Replace("/*AntlrCaseInsensitive*/", "");
}
}
Regex parserPartStart, parserPartEnd;
if (runtime.IsPythonRuntime())
{
string newValue = runtime == Runtime.Python2
? "print \"Tree \" + tree.toStringTree(recog=parser);"
: "print(\"Tree \", tree.toStringTree(recog=parser));";
code = code.Replace("'''PrintTree'''", newValue);
parserPartStart = ParserPartStartPython;
parserPartEnd = ParserPartEndPython;
code = RemoveCodeOrClearMarkers(code, ParserIncludeStartPython, ParserIncludeEndPython);
}
else
{
parserPartStart = ParserPartStart;
parserPartEnd = ParserPartEnd;
if (runtime == Runtime.JavaScript || runtime == Runtime.Go)
{
code = RemoveCodeOrClearMarkers(code, ParserIncludeStartJavaScript, ParserIncludeEndJavaScript);
}
}
code = RemoveCodeOrClearMarkers(code, parserPartStart, parserPartEnd);
File.WriteAllText(templateFile, code);
}
private void Generate(Grammar grammar, GrammarCheckedState state, CancellationToken cancellationToken)
{
Processor processor = null;
try
{
string runtimeDirectoryName = Path.Combine(HelperDirectoryName, grammar.Name, Runtime.ToString());
if ((Runtime == Runtime.Java || Runtime == Runtime.Go) && !string.IsNullOrWhiteSpace(PackageName))
{
runtimeDirectoryName = Path.Combine(runtimeDirectoryName, PackageName);
}
if (Directory.Exists(runtimeDirectoryName))
{
Directory.Delete(runtimeDirectoryName, true);
}
Directory.CreateDirectory(runtimeDirectoryName);
cancellationToken.ThrowIfCancellationRequested();
RuntimeInfo runtimeInfo = RuntimeInfo.InitOrGetRuntimeInfo(Runtime);
var jarGenerator = GeneratorTool ?? Path.Combine("Generators", runtimeInfo.JarGenerator);
foreach (string grammarFileName in state.InputState.Grammar.Files)
{
string extension = Path.GetExtension(grammarFileName);
if (extension != Grammar.AntlrDotExt)
{
continue;
}
_currentGrammarSource = state.GrammarFilesData[grammarFileName];
var arguments =
$@"-jar ""{jarGenerator}"" ""{Path.Combine(grammar.Directory, grammarFileName)}"" " +
$@"-o ""{runtimeDirectoryName}"" " +
$"-Dlanguage={runtimeInfo.DLanguage} " +
$"{(GenerateVisitor ? "-visitor" : "-no-visitor")} " +
$"{(GenerateListener ? "-listener" : "-no-listener")}";
if (!string.IsNullOrWhiteSpace(PackageName))
{
arguments += " -package " + PackageName;
}
else if (Runtime == Runtime.Go)
{
arguments += " -package main";
}
if (grammarFileName.Contains(Grammar.LexerPostfix) && state.LexerSuperClass != null)
{
arguments += " -DsuperClass=" + state.LexerSuperClass;
}
if (grammarFileName.Contains(Grammar.ParserPostfix) && state.ParserSuperClass != null)
{
arguments += " -DsuperClass=" + state.ParserSuperClass;
}
_result.Command = "java " + arguments;
processor = new Processor("java", arguments, ".");
processor.CancellationToken = cancellationToken;
processor.ErrorDataReceived += ParserGeneration_ErrorDataReceived;
processor.OutputDataReceived += ParserGeneration_OutputDataReceived;
processor.Start();
cancellationToken.ThrowIfCancellationRequested();
}
}
catch (Exception ex)
{
_result.Exception = ex;
if (!(ex is OperationCanceledException))
{
ErrorEvent?.Invoke(this, new ParsingError(ex, WorkflowStage.ParserGenerated));
}
}
finally
{
processor?.Dispose();
}
}
