/// <summary> /// Given a request, run it. No need to clean up afterwards as we are already there. /// </summary> /// <returns></returns> /// <remarks> /// We can't run more than one of these at a time as root is not able to /// run more than a thread at a time without significant work. Other /// executors get to take care of this, sadly. /// </remarks> public async Task <IDictionary <string, ROOTNET.Interface.NTObject> > Execute( Uri[] files, FileInfo templateFile, DirectoryInfo queryDirectory, IEnumerable <KeyValuePair <string, object> > varsToTransfer) { using (var waiter = await _globalLock.LockAsync()) { // Remove flag characters in teh source file - parse for clean up subsitutions we can do // only when we know how we are going to do the execution. ReWritePathsInQuery(templateFile); // Get the environment setup for this call await ExecutionUtilities.Init(); PreExecutionInit(Environment.ClassesToDictify); TraceHelpers.TraceInfo(12, "ExecuteQueuedQueries: Loading all extra objects"); AssembleAndLoadExtraObjects(Environment.ExtraComponentFiles); // // Load the query up // if (Environment.BreakToDebugger) { System.Diagnostics.Debugger.Break(); } CompileAndLoad(templateFile); try { // To help with possible debugging and other things, if a pdb was generated, then copy it over and rename it // correctly. if (File.Exists("vc100.pdb")) { File.Copy("vc100.pdb", Path.Combine(queryDirectory.FullName, Path.GetFileNameWithoutExtension(templateFile.Name) + ".pdb")); } // Get the file name of the selector. TraceHelpers.TraceInfo(14, "ExecuteQueuedQueries: Startup - Running the code"); var localFiles = files.Select(u => new FileInfo(u.LocalPath)).ToArray(); return(await RunNtupleQuery(Path.GetFileNameWithoutExtension(templateFile.Name), varsToTransfer, Environment.TreeName, localFiles)); } finally { // And cleanup (even if something goes wrong during the run). TraceHelpers.TraceInfo(16, "ExecuteQueuedQueries: unloading all results"); ExecutionUtilities.UnloadAllModules(_loadedModuleNames); if (Environment.CleanupQuery) { queryDirectory.Delete(true); } } } }
/// <summary> /// Global init for an executor that needs to do local compiles on this ROOT machine. /// </summary> public static async Task Init() { using (var holder = await _initLock.LockAsync()) { if (gGlobalInit) { return; } gGlobalInit = true; /// /// A directory where we can store all of the temp files we need to create /// TempDirectory = new DirectoryInfo(Path.GetTempPath() + "\\LINQToROOT"); if (!TempDirectory.Exists) { TempDirectory.Create(); TempDirectory.Refresh(); } if (!DictDirectory.Exists) { DictDirectory.Create(); } /// /// Next the common source files. Make sure that the include files passed to the old compiler has /// this common file directory in there! /// var cf = CommonSourceDirectory(); if (!cf.Exists) { cf.Create(); } if (!ROOTNET.NTSystem.gSystem.IncludePath.Contains(cf.FullName)) { ROOTNET.NTSystem.gSystem.AddIncludePath("-I\"" + cf.FullName + "\""); } // // Make sure the environment is setup correctly! // ExecutionUtilities.SetupENV(); } }
/// <summary> /// Compile/Load in extra C++ files and move the include files locally so they can be easily /// referenced by the query code. /// </summary> /// <param name="queryDirectory">Location where we move the files to compiling</param> private async Task LoadExtraCPPFiles(DirectoryInfo queryDirectory, StringBuilder cmds) { // Move everything over, and then compile! if (Environment.ExtraComponentFiles != null) { foreach (var fd in Environment.ExtraComponentFiles) { var output = ExecutionUtilities.CopyToDirectory(fd, queryDirectory); try { await CompileAndLoad(output, cmds, queryDirectory); } catch (Exception) { Console.WriteLine("Failed to build {0}. Ignoring and crossing fingers.", output.Name); } } } }
/// <summary> /// If there are some extra files we need to be loading, go after them here. /// </summary> private void AssembleAndLoadExtraObjects(FileInfo[] extraComponentFiles) { /// /// First, do the files that are part of our infrastructure. /// We currently have no infrastructure files needed. /// /// /// Next, build any files that are required to build run this ntuple /// foreach (var fd in extraComponentFiles) { var output = ExecutionUtilities.CopyToCommonDirectory(fd); try { CompileAndLoad(output); } catch (Exception) { Console.WriteLine("Failed to build {0}. Ignoring and crossing fingers.", output.Name); } } }
/// <summary> /// Execute a TSelector script on a group of files. /// </summary> /// <param name="queryFile">The C++ file we are going to run the query against</param> /// <param name="queryDirectory">Directory where we run the query</param> /// <param name="varsToTransfer">Variables we need to move over to the query</param> /// <returns>The results from running the TSelector</returns> /// <remarks> /// Some exception conditions are handled: /// - Once in a while while executing we have a corrupt datafile that is corrupt due to networking, not because it is a bad /// file. This happens often when things are over shares. We will retry three time if we detect root is having trouble with /// bad data. /// </remarks> public virtual async Task <IDictionary <string, NTObject> > Execute(Uri[] files, FileInfo queryFile, DirectoryInfo queryDirectory, IEnumerable <KeyValuePair <string, object> > varsToTransfer) { // Record our passing for testing, etc. Interlocked.Increment(ref NumberOfExecutesCalled); // Setup for building a command await ExecutionUtilities.Init(); var cmds = new StringBuilder(); cmds.AppendLine("{"); // Rewrite the query if it contains special file paths ReWritePathsInQuery(queryFile, queryDirectory); // Put our run-directory in the list of includes. var includePath = await NormalizeFileForTarget(new DirectoryInfo(System.Environment.CurrentDirectory), queryDirectory); cmds.AppendLine($"gSystem->AddIncludePath(\"-I\\\"{includePath}\\\"\");"); // Load up extra objects & dictionaries await LoadExtraCPPFiles(queryDirectory, cmds); LoadExtraDictionaries(Environment.ClassesToDictify, cmds); // Compile the macro await CompileAndLoad(queryFile, cmds, queryDirectory); // Run the query in a second file. var subfileCommands = new StringBuilder(); var resultsFile = new FileInfo(Path.Combine(queryDirectory.FullName, "selector_results.root")); await RunNtupleQuery(subfileCommands, resultsFile, Path.GetFileNameWithoutExtension(queryFile.Name), varsToTransfer, Environment.TreeName, files, queryDirectory); // Write out the temp file. using (var secondFile = File.CreateText(Path.Combine(queryDirectory.FullName, "RunTSelector1.C"))) { secondFile.WriteLine("void RunTSelector1() {"); secondFile.Write(" " + subfileCommands.ToString()); secondFile.WriteLine("}"); secondFile.Close(); } cmds.AppendLine("gROOT->ProcessLine(\".X RunTSelector1.C\");"); // Run the root script. Retry if we detect an understood error condition. cmds.AppendLine("exit(0);"); cmds.AppendLine("}"); await NormalizeFileForTarget(queryDirectory, queryDirectory); await Policy .Handle <CommandLineExecutionException>(ex => ex.Message.Contains("error reading from file")) .RetryAsync(3) .ExecuteAsync(async() => { await ExecuteRootScript("Query", cmds.ToString(), queryDirectory, fetchFiles: new[] { new Uri(resultsFile.FullName) }, timeout: TimeSpan.FromHours(4)); }); // Get back results var results = await LoadSelectorResults(resultsFile); return(results); }
/// <summary> /// Copy this source file (along with any includes in it) to /// our common area. /// </summary> /// <param name="sourceFile"></param> public static FileInfo CopyToCommonDirectory(FileInfo sourceFile) { return(ExecutionUtilities.CopyToDirectory(sourceFile, CommonSourceDirectory())); }