/// <summary>
/// Given a request, run it. No need to clean up afterwards as we are already there.
/// </summary>
/// <returns></returns>
/// <remarks>
/// We can't run more than one of these at a time as root is not able to
/// run more than a thread at a time without significant work. Other
/// executors get to take care of this, sadly.
/// </remarks>
public async Task <IDictionary <string, ROOTNET.Interface.NTObject> > Execute(
Uri[] files,
FileInfo templateFile,
DirectoryInfo queryDirectory,
IEnumerable <KeyValuePair <string, object> > varsToTransfer)
{
using (var waiter = await _globalLock.LockAsync())
{
// Remove flag characters in teh source file - parse for clean up subsitutions we can do
// only when we know how we are going to do the execution.
ReWritePathsInQuery(templateFile);
// Get the environment setup for this call
await ExecutionUtilities.Init();
PreExecutionInit(Environment.ClassesToDictify);
TraceHelpers.TraceInfo(12, "ExecuteQueuedQueries: Loading all extra objects");
AssembleAndLoadExtraObjects(Environment.ExtraComponentFiles);
//
// Load the query up
//
if (Environment.BreakToDebugger)
{
System.Diagnostics.Debugger.Break();
}
CompileAndLoad(templateFile);
try
{
// To help with possible debugging and other things, if a pdb was generated, then copy it over and rename it
// correctly.
if (File.Exists("vc100.pdb"))
{
File.Copy("vc100.pdb", Path.Combine(queryDirectory.FullName, Path.GetFileNameWithoutExtension(templateFile.Name) + ".pdb"));
}
// Get the file name of the selector.
TraceHelpers.TraceInfo(14, "ExecuteQueuedQueries: Startup - Running the code");
var localFiles = files.Select(u => new FileInfo(u.LocalPath)).ToArray();
return(await RunNtupleQuery(Path.GetFileNameWithoutExtension(templateFile.Name), varsToTransfer, Environment.TreeName, localFiles));
}
finally
{
// And cleanup (even if something goes wrong during the run).
TraceHelpers.TraceInfo(16, "ExecuteQueuedQueries: unloading all results");
ExecutionUtilities.UnloadAllModules(_loadedModuleNames);
if (Environment.CleanupQuery)
{
queryDirectory.Delete(true);
}
}
}
}
/// <summary>
/// Global init for an executor that needs to do local compiles on this ROOT machine.
/// </summary>
public static async Task Init()
{
using (var holder = await _initLock.LockAsync())
{
if (gGlobalInit)
{
return;
}
gGlobalInit = true;
///
/// A directory where we can store all of the temp files we need to create
///
TempDirectory = new DirectoryInfo(Path.GetTempPath() + "\\LINQToROOT");
if (!TempDirectory.Exists)
{
TempDirectory.Create();
TempDirectory.Refresh();
}
if (!DictDirectory.Exists)
{
DictDirectory.Create();
}
///
/// Next the common source files. Make sure that the include files passed to the old compiler has
/// this common file directory in there!
///
var cf = CommonSourceDirectory();
if (!cf.Exists)
{
cf.Create();
}
if (!ROOTNET.NTSystem.gSystem.IncludePath.Contains(cf.FullName))
{
ROOTNET.NTSystem.gSystem.AddIncludePath("-I\"" + cf.FullName + "\"");
}
//
// Make sure the environment is setup correctly!
//
ExecutionUtilities.SetupENV();
}
}
/// <summary>
/// Compile/Load in extra C++ files and move the include files locally so they can be easily
/// referenced by the query code.
/// </summary>
/// <param name="queryDirectory">Location where we move the files to compiling</param>
private async Task LoadExtraCPPFiles(DirectoryInfo queryDirectory, StringBuilder cmds)
{
// Move everything over, and then compile!
if (Environment.ExtraComponentFiles != null)
{
foreach (var fd in Environment.ExtraComponentFiles)
{
var output = ExecutionUtilities.CopyToDirectory(fd, queryDirectory);
try
{
await CompileAndLoad(output, cmds, queryDirectory);
}
catch (Exception)
{
Console.WriteLine("Failed to build {0}. Ignoring and crossing fingers.", output.Name);
}
}
}
}
/// <summary>
/// If there are some extra files we need to be loading, go after them here.
/// </summary>
private void AssembleAndLoadExtraObjects(FileInfo[] extraComponentFiles)
{
///
/// First, do the files that are part of our infrastructure.
/// We currently have no infrastructure files needed.
///
///
/// Next, build any files that are required to build run this ntuple
///
foreach (var fd in extraComponentFiles)
{
var output = ExecutionUtilities.CopyToCommonDirectory(fd);
try
{
CompileAndLoad(output);
}
catch (Exception)
{
Console.WriteLine("Failed to build {0}. Ignoring and crossing fingers.", output.Name);
}
}
}
/// <summary>
/// Execute a TSelector script on a group of files.
/// </summary>
/// <param name="queryFile">The C++ file we are going to run the query against</param>
/// <param name="queryDirectory">Directory where we run the query</param>
/// <param name="varsToTransfer">Variables we need to move over to the query</param>
/// <returns>The results from running the TSelector</returns>
/// <remarks>
/// Some exception conditions are handled:
/// - Once in a while while executing we have a corrupt datafile that is corrupt due to networking, not because it is a bad
/// file. This happens often when things are over shares. We will retry three time if we detect root is having trouble with
/// bad data.
/// </remarks>
public virtual async Task <IDictionary <string, NTObject> > Execute(Uri[] files, FileInfo queryFile, DirectoryInfo queryDirectory, IEnumerable <KeyValuePair <string, object> > varsToTransfer)
{
// Record our passing for testing, etc.
Interlocked.Increment(ref NumberOfExecutesCalled);
// Setup for building a command
await ExecutionUtilities.Init();
var cmds = new StringBuilder();
cmds.AppendLine("{");
// Rewrite the query if it contains special file paths
ReWritePathsInQuery(queryFile, queryDirectory);
// Put our run-directory in the list of includes.
var includePath = await NormalizeFileForTarget(new DirectoryInfo(System.Environment.CurrentDirectory), queryDirectory);
cmds.AppendLine($"gSystem->AddIncludePath(\"-I\\\"{includePath}\\\"\");");
// Load up extra objects & dictionaries
await LoadExtraCPPFiles(queryDirectory, cmds);
LoadExtraDictionaries(Environment.ClassesToDictify, cmds);
// Compile the macro
await CompileAndLoad(queryFile, cmds, queryDirectory);
// Run the query in a second file.
var subfileCommands = new StringBuilder();
var resultsFile = new FileInfo(Path.Combine(queryDirectory.FullName, "selector_results.root"));
await RunNtupleQuery(subfileCommands, resultsFile, Path.GetFileNameWithoutExtension(queryFile.Name), varsToTransfer,
Environment.TreeName, files, queryDirectory);
// Write out the temp file.
using (var secondFile = File.CreateText(Path.Combine(queryDirectory.FullName, "RunTSelector1.C")))
{
secondFile.WriteLine("void RunTSelector1() {");
secondFile.Write(" " + subfileCommands.ToString());
secondFile.WriteLine("}");
secondFile.Close();
}
cmds.AppendLine("gROOT->ProcessLine(\".X RunTSelector1.C\");");
// Run the root script. Retry if we detect an understood error condition.
cmds.AppendLine("exit(0);");
cmds.AppendLine("}");
await NormalizeFileForTarget(queryDirectory, queryDirectory);
await Policy
.Handle <CommandLineExecutionException>(ex => ex.Message.Contains("error reading from file"))
.RetryAsync(3)
.ExecuteAsync(async() =>
{
await ExecuteRootScript("Query", cmds.ToString(), queryDirectory,
fetchFiles: new[] { new Uri(resultsFile.FullName) },
timeout: TimeSpan.FromHours(4));
});
// Get back results
var results = await LoadSelectorResults(resultsFile);
return(results);
}
/// <summary>
/// Copy this source file (along with any includes in it) to
/// our common area.
/// </summary>
/// <param name="sourceFile"></param>
public static FileInfo CopyToCommonDirectory(FileInfo sourceFile)
{
return(ExecutionUtilities.CopyToDirectory(sourceFile, CommonSourceDirectory()));
}