internal static StepBroTypeScanListener.FileContent TypeScanFile(string content) { ITokenSource lexer = new Grammar.StepBroLexer(new AntlrInputStream(content)); ITokenStream tokens = new CommonTokenStream(lexer); var parser = new SBP(tokens); var errors = new ErrorCollector(null); parser.AddErrorListener(errors); parser.BuildParseTree = true; var listener = new StepBroTypeScanListener("Angus"); // Some random default namespace var context = parser.compilationUnit(); var walker = new ParseTreeWalker(); walker.Walk(listener, context); if (errors.ErrorCount > 0) { throw new Exception("PARSING ERRORS"); } return(listener.GetContent()); }
internal static int ParseFiles(ServiceManager services, ILogger logger, ScriptFile topfile = null) { var addons = services.Get <IAddonManager>(); var filesManager = services.Get <ILoadedFilesManager>(); var filesToParse = new List <ScriptFile>(); if (topfile != null) { filesToParse.Add(topfile); } else { filesToParse = filesManager.ListFiles <ScriptFile>().ToList(); } var fileListeners = new Dictionary <ScriptFile, StepBroListener>(); var fileContexts = new Dictionary <ScriptFile, SBP.CompilationUnitContext>(); var namespaceFiles = new Dictionary <string, IdentifierInfo>(); //==============================================================// #region STEP 1: PRE-SCAN ALL FILES TO OPEN ALL DEPENDENCY FILES // //==============================================================// var fileParsingStack = new Queue <ScriptFile>(filesToParse); while (fileParsingStack.Count > 0) { var file = fileParsingStack.Dequeue(); file.MarkForTypeScanning(); ITokenSource lexer = new Grammar.StepBroLexer(file.GetParserFileStream()); ITokenStream tokens = new CommonTokenStream(lexer); var parser = new SBP(tokens); parser.RemoveErrorListeners(); (file.Errors as ErrorCollector).Clear(); parser.AddErrorListener(file.Errors as ErrorCollector); parser.BuildParseTree = true; var context = parser.compilationUnit(); fileContexts.Add(file, context); if (String.IsNullOrEmpty(file.Namespace)) { file.SetNamespace(System.IO.Path.GetFileNameWithoutExtension(file.FileName)); } var visitor = new StepBroTypeVisitor(); visitor.Visit(context); var scanListener = new StepBroTypeScanListener(file.Namespace); var walker = new ParseTreeWalker(); walker.Walk(scanListener, context); if (file.Errors.ErrorCount > 0) { continue; // Stop parsing this file } var parserListener = new StepBroListener(file.Errors as ErrorCollector, addons, file); fileListeners.Add(file, parserListener); var fileScanData = scanListener.GetContent(); file.PreScanFileContent = fileScanData; System.Diagnostics.Debug.Assert(!String.IsNullOrEmpty(fileScanData.TopElement.Name)); file.SetNamespace(fileScanData.TopElement.Name); // Update the namespace from the scanning. foreach (var @using in fileScanData.ListUsings()) { var line = @using.Line; var type = @using.Type; var name = @using.Name; if (type == "i" || type == "I") { if (!file.AddNamespaceUsing(line, name)) { throw new Exception($"Namespace using already added ({name})."); } } else if (type == "p" || type == "P") { if (!file.AddFileUsing(line, name)) { throw new Exception($"File using already added ({name})."); } } else { throw new Exception($"Unhandled using type: {type}"); } } file.ResolveFileUsings( fu => { string basefolder = Path.GetDirectoryName(file.GetFullPath()); var fuName = Path.GetFileName(fu); // Check the already parsed or loaded files first. foreach (var f in filesToParse) { if (!Object.ReferenceEquals(file, f) && String.Equals(fuName, f.FileName, StringComparison.InvariantCulture)) { return(f); } } foreach (var f in filesManager.ListFiles <ScriptFile>()) { if (!Object.ReferenceEquals(file, f) && String.Equals(fuName, f.FileName, StringComparison.InvariantCulture)) { fileParsingStack.Enqueue(f); filesToParse.Insert(0, f); // Put in front return(f); } } // File was not found among the already loaded files. Try loading the file. string foundMatchingFile = null; if (fu.Contains("[")) // It's a path using a folder shortcut. { throw new NotImplementedException(); } else if (fu.Contains("\\")) // It's a relative or absolute path { string path = Path.GetFullPath(Path.Combine(basefolder, fu)); if (System.IO.File.Exists(path)) { foundMatchingFile = path; } } else { // Start at the location of this file. while (basefolder != Path.GetPathRoot(basefolder)) { string path = Path.GetFullPath(Path.Combine(basefolder, fu)); if (System.IO.File.Exists(path)) { foundMatchingFile = path; break; } // Not found yet; try the parent folder. basefolder = Path.GetDirectoryName(basefolder); } } if (foundMatchingFile != null) { // Load and add file to fileParsingStack object dummyUser = new object(); // The parser will set the current scriptfile as a dependant. if (Main.LoadScriptFile(user: dummyUser, filepath: foundMatchingFile) is ScriptFile loadedFile) { loadedFile.UnregisterDependant(dummyUser); fileParsingStack.Enqueue(loadedFile); filesToParse.Add(loadedFile); return(loadedFile); } } return(null); // Not found! } ); file.ResolveNamespaceUsings( id => { var foundIdentifier = addons.Lookup(null, id); if (foundIdentifier != null) { return(foundIdentifier); } var scriptFilesInNamespace = new List <ScriptFile>(); foreach (var f in filesToParse) { if (f.Namespace.Equals(id, StringComparison.InvariantCulture)) { if (foundIdentifier == null) { foundIdentifier = new IdentifierInfo(id, id, IdentifierType.FileNamespace, null, scriptFilesInNamespace); } scriptFilesInNamespace.Add(f); // More files can have same namespace. } } if (foundIdentifier != null) { return(foundIdentifier); } foreach (var f in filesManager.ListFiles <ScriptFile>()) { if (f.Namespace.Equals(id, StringComparison.InvariantCulture)) { if (foundIdentifier == null) { foundIdentifier = new IdentifierInfo(id, id, IdentifierType.FileNamespace, null, scriptFilesInNamespace); } scriptFilesInNamespace.Add(f); // Different files can have the same namespace. fileParsingStack.Enqueue(f); filesToParse.Insert(0, f); // Put in front } } return(foundIdentifier); // Is null if not found } ); } #endregion //===================================================// #region STEP 2: COLLECT ALL THE PROCEDURE SIGNATURES // //===================================================// // TODO: Sort files after dependencies (usings) var beforeSorting = filesToParse; List <ScriptFile> sortedAfterDependencies = new List <ScriptFile>(); var filesToCheck = new Queue <ScriptFile>(filesToParse); while (filesToCheck.Count > 0) { var file = filesToCheck.Dequeue(); bool addNow = true; foreach (var fu in file.ListReferencedScriptFiles()) { if (!sortedAfterDependencies.Contains(fu)) { filesToCheck.Enqueue(file); // Put back in queue. addNow = false; break; } } if (addNow) { sortedAfterDependencies.Add(file); } } filesToParse = sortedAfterDependencies; foreach (var file in filesToParse) { var fileScanData = file.PreScanFileContent; if (fileScanData != null) { foreach (var element in fileScanData.TopElement.Childs) { var firstPropFlag = (element.PropertyFlags != null) ? element.PropertyFlags[0] : null; var accessModifier = (element.Modifiers != null && element.Modifiers.Count > 0) ? (AccessModifier)Enum.Parse(typeof(AccessModifier), element.Modifiers[0], true) : DefaultAccess; switch (element.Type) { case FileElementType.Using: break; case FileElementType.Namespace: file.SetNamespace(element.Name); throw new Exception(); //break; case FileElementType.EnumDeclaration: break; case FileElementType.ProcedureDeclaration: { var procedure = new FileProcedure(file, accessModifier, element.Line, null, file.Namespace, element.Name) { Flags = (element.IsFunction ? ProcedureFlags.IsFunction : ProcedureFlags.None), HasBody = element.HasBody, BaseElementName = firstPropFlag }; file.AddProcedure(procedure); procedure.CheckForPrototypeChange(element.Parameters, element.ReturnTypeData); } break; case FileElementType.FileVariable: break; case FileElementType.TestList: { var testlist = new FileTestList(file, accessModifier, element.Line, null, file.Namespace, element.Name) { BaseElementName = firstPropFlag }; file.AddTestList(testlist); } break; case FileElementType.Datatable: break; default: break; } } file.LastTypeScan = DateTime.Now; } } #endregion //=================================================// #region STEP 3: PARSE ALL THE PROCEDURE SIGNATURES // //=================================================// var signaturesToParseNow = new List <Tuple <FileElement, StepBroListener> >(); // Collect file elements to parse from _all_ files foreach (var file in filesToParse) { StepBroListener listener; if (fileListeners.TryGetValue(file, out listener)) { signaturesToParseNow.AddRange( file.ListElements().Cast <FileElement>().Select(e => new Tuple <FileElement, StepBroListener>(e, listener))); } } // Update the lookup tables before further parsing. foreach (var file in filesToParse) { file.UpdateRootIdentifiers(); } var numberToParse = signaturesToParseNow.Count; var numberParsedLast = int.MaxValue; var signaturesToParseAgain = new List <Tuple <FileElement, StepBroListener> >(); // Continue parsing signatures until no more elements can be resolved while (numberToParse > 0 && numberToParse < numberParsedLast) { foreach (var d in signaturesToParseNow) { d.Item1.ParseBaseElement(); if (d.Item1.ParseSignature(d.Item2, false) > 0) { signaturesToParseAgain.Add(d); } } numberParsedLast = signaturesToParseNow.Count; signaturesToParseNow = signaturesToParseAgain; numberToParse = signaturesToParseNow.Count; signaturesToParseAgain = new List <Tuple <FileElement, StepBroListener> >(); } if (numberToParse > 0) { foreach (var d in signaturesToParseNow) { d.Item1.ParseSignature(d.Item2, true); // Parse again and report the errors } //throw new Exception("Not all signatures could be parsed."); // TBD return(signaturesToParseNow.Count); } #endregion //=================================// #region STEP 4: PARSE ALL THE FILES # //=================================// int totalErrors = 0; try { foreach (var file in filesToParse) { file.SaveCurrentFileVariables(); StepBroListener listener; if (file.Errors.ErrorCount == 0) { if (fileListeners.TryGetValue(file, out listener)) { var context = fileContexts[file]; try { var walker = new ParseTreeWalker(); walker.Walk(listener, context); } finally { } } else { throw new NotImplementedException(); } } totalErrors += file.Errors.ErrorCount; if (file.Errors.ErrorCount == 0) { file.InitializeFileVariables(logger); file.LastParsing = DateTime.Now; file.DisposeUnusedFileVariables(logger); } } } finally { foreach (var file in filesToParse) { file.DisposeFileStream(); } } #endregion return(totalErrors); }