public void BuildJobFunctionally()
{
var job = new AtlasJob()
.NameVersionRelease("DiVertAnalysis", 3, "2.3.32")
.Package("JetSelectorTools")
.Package("atlasphys-exo/Physics/Exotic/UEH/DisplacedJets/Run2/AnalysisCode/trunk/DiVertAnalysis", "248132")
.Command("grep -v \"emf < 0.05\" JetSelectorTools/Root/JetCleaningTool.cxx > JetSelectorTools/Root/JetCleaningTool-New.cxx")
.Command("mv JetSelectorTools/Root/JetCleaningTool-New.cxx JetSelectorTools/Root/JetCleaningTool.cxx")
.SubmitCommand("DiVertAnalysisRunner -EventLoopDriver GRID *INPUTDS* -ELGRIDOutputSampleName *OUTPUTDS* -WaitTillDone FALSE -isLLPMC true");
Assert.AreEqual("DiVertAnalysis", job.Name);
Assert.AreEqual(3, job.Version);
Assert.AreEqual("2.3.32", job.Release.Name);
Assert.AreEqual(2, job.Packages.Length);
Assert.AreEqual("JetSelectorTools", job.Packages[0].Name);
Assert.AreEqual("", job.Packages[0].SCTag);
Assert.AreEqual("atlasphys-exo/Physics/Exotic/UEH/DisplacedJets/Run2/AnalysisCode/trunk/DiVertAnalysis", job.Packages[1].Name);
Assert.AreEqual("248132", job.Packages[1].SCTag);
Assert.AreEqual(2, job.Commands.Length);
Assert.AreEqual("grep -v \"emf < 0.05\" JetSelectorTools/Root/JetCleaningTool.cxx > JetSelectorTools/Root/JetCleaningTool-New.cxx", job.Commands[0].CommandLine);
Assert.AreEqual("mv JetSelectorTools/Root/JetCleaningTool-New.cxx JetSelectorTools/Root/JetCleaningTool.cxx", job.Commands[1].CommandLine);
Assert.AreEqual("DiVertAnalysisRunner -EventLoopDriver GRID *INPUTDS* -ELGRIDOutputSampleName *OUTPUTDS* -WaitTillDone FALSE -isLLPMC true", job.SubmitCommand.SubmitCommand.CommandLine);
}
public void DxAODDatasetSlightlyDifferent()
{
// Seen in the wild. Need to take into account the fact that there might be subtle differences in the dataset names.
var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var ds1 = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3668_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus");
var ds2 = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3669_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus");
Assert.AreNotEqual(ds1, ds2);
}
public void HashIsDifferent()
{
var j = new AtlasJob() { Name = "Hi there", Version = 12 };
var h1 = j.Hash();
j.Name = "not there";
var h2 = j.Hash();
Assert.AreNotEqual(h1, h2);
}
/// <summary>
/// Set the name and version of the job.
/// </summary>
/// <param name="job"></param>
/// <param name="name"></param>
/// <param name="version"></param>
/// <returns></returns>
public static AtlasJob NameVersionRelease(this AtlasJob job, string name, int version, string release)
{
job = job == null ? new AtlasJob() : job;
job.Name = name;
job.Version = version;
job.Release = new Release()
{
Name = release
};
return(job);
}
/// <summary>
/// Fetch from GRID the credential name.
/// </summary>
/// <param name="job"></param>
/// <param name="origDSName"></param>
/// <returns></returns>
public static string ResultingDataSetName(this AtlasJob job, string origDSName, int jobIteration = 0)
{
using (var credList = new CredentialSet("GRID"))
{
var gridCredentials = credList.Load().FirstOrDefault();
if (gridCredentials == null)
{
throw new ArgumentException("Please create a generic windows credential with the target 'GRID' with the user name as the rucio grid user name and the password to be used with voms proxy init");
}
return(job.ResultingDataSetName(origDSName, gridCredentials, jobIteration));
}
}
/// <summary>
/// Add the submit command to the atlas job.
/// </summary>
/// <param name="job"></param>
/// <param name="commandLine"></param>
/// <returns></returns>
public static AtlasJob SubmitCommand(this AtlasJob job, string commandLine)
{
job = job == null ? new AtlasJob() : job;
job.SubmitCommand = new Submit()
{
SubmitCommand = new Jobs.Command()
{
CommandLine = commandLine
}
};
return(job);
}
/// <summary>
/// Do a deep copy of the job.
/// </summary>
/// <returns></returns>
internal AtlasJob Clone()
{
var r = new AtlasJob ();
r.Name = Name;
r.Version = Version;
r.Release = Release?.Clone();
r.Packages = Packages?.Select(p => p.Clone()).ToArray();
r.Commands = Commands?.Select(c => c.Clone()).ToArray();
r.SubmitCommand = SubmitCommand?.Clone();
r.SubmitPatternCommands = SubmitPatternCommands?.Select(sp => sp.Clone()).ToArray();
return r;
}
/// <summary>
/// Do a deep copy of the job.
/// </summary>
/// <returns></returns>
internal AtlasJob Clone()
{
var r = new AtlasJob();
r.Name = Name;
r.Version = Version;
r.Release = Release?.Clone();
r.Packages = Packages?.Select(p => p.Clone()).ToArray();
r.Commands = Commands?.Select(c => c.Clone()).ToArray();
r.SubmitCommand = SubmitCommand?.Clone();
r.SubmitPatternCommands = SubmitPatternCommands?.Select(sp => sp.Clone()).ToArray();
return(r);
}
public void JobOutput()
{
var j = new AtlasJob()
{
Commands = new Command[] { new Command() { CommandLine = "ls" } },
Name = "MyJob",
Version = 1234,
Release = new Release() { Name = "notmyrelease" },
SubmitCommand = new Submit() { SubmitCommand = new Command() { CommandLine = "submit" } },
Packages = new Package[] { new Package() { Name = "hithere", SCTag = "tag" } }
};
Assert.AreEqual("job(MyJob,1234){release(notmyrelease)package(hithere,tag)submit(submit)}", j.Print());
}
/// <summary>
/// Add a new submit command pattern to the job.
/// </summary>
/// <param name="job"></param>
/// <param name="commandLine"></param>
/// <param name="pattern"></param>
/// <returns></returns>
public static AtlasJob SubmitCommandPattern(this AtlasJob job, string commandLine, string pattern)
{
job = job ?? new AtlasJob();
job.SubmitPatternCommands = job.SubmitPatternCommands
.Append(new SubmitPattern()
{
Regex = pattern, SubmitCommand = new Submit()
{
SubmitCommand = new Jobs.Command()
{
CommandLine = commandLine
}
}
});
return(job);
}
private static StringBuilder Print(this AtlasJob r, StringBuilder bld, bool prettyPrint = false)
{
if (prettyPrint && bld.Length > 0)
{
bld.AppendLine();
}
bld.AppendFormat("job({0},{1}){{", r.Name, r.Version);
if (prettyPrint)
{
bld.AppendLine();
}
if (r.Release != null)
{
r.Release.Print(bld, prettyPrint: prettyPrint);
}
if (r.Packages != null)
{
foreach (var p in r.Packages.OrderBy(p => $"{p.Name}-{p.SCTag}"))
{
p.Print(bld, prettyPrint: prettyPrint);
}
}
if (r.SubmitPatternCommands != null && r.SubmitPatternCommands.Length > 0)
{
foreach (var sc in r.SubmitPatternCommands.OrderBy(p => $"{p.Regex}-{p.SubmitCommand.SubmitCommand}"))
{
sc.Print(bld, prettyPrint: prettyPrint);
}
}
else
{
if (r.SubmitCommand != null)
{
r.SubmitCommand.Print(bld, prettyPrint: prettyPrint);
}
}
bld.Append("}");
return(bld);
}
public void HashSameForReorderedPackages()
{
var j1 = new AtlasJob() { Name = "DiVertAnalysis", Version = 22,
Packages = new Package[]
{
new Package() { Name = "pkg1", SCTag = "v00-00-00" },
new Package() { Name = "pkg2", SCTag = "v00-01-00" }
},
};
var j2 = new AtlasJob()
{
Name = "DiVertAnalysis",
Version = 22,
Packages = new Package[]
{
new Package() { Name = "pkg2", SCTag = "v00-01-00" },
new Package() { Name = "pkg1", SCTag = "v00-00-00" },
},
};
Assert.AreEqual(j1.Hash(), j2.Hash());
}
public void HashTheSame()
{
var j1 = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var j2 = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
Assert.AreEqual(j1.Hash(), j2.Hash());
}
public void DxNoDash()
{
var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var ds = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3668_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus");
Assert.IsFalse(ds.Contains("-"), ds);
}
public void DxAODDatasetFromEmma()
{
var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var ds = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3668_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus");
Assert.IsTrue(ds.StartsWith("user.bogus.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.r6282__EXOT15_EXT0.DiVertAnalysis_v22_3B233454"), ds);
}
public void DataDerivationxAOD()
{
var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var ds = j.ResultingDatasetName("data15_13TeV:data15_13TeV.00280500.physics_Main.merge.DAOD_EXOT15.f631_m1504_p2425_tid06603342_00", "user.bogus");
Assert.IsTrue(ds.StartsWith("user.bogus.00280500.physics_Main.DAOD_EXOT15.f631_m1504_p2425.DiVertAnalysis_v22_3B233454"));
}
public static string Print(this AtlasJob r, bool prettyPrint = false)
{
return(r.Print(new StringBuilder(), prettyPrint: prettyPrint).ToString());
}
public void StableHash()
{
var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var h = j.Hash();
Assert.AreEqual("3B233454", h);
}
public void PrintEmptyJob()
{
var j = new AtlasJob() { Name = "hihere", Version = 10 };
var s = j.Print();
Assert.AreEqual("job(hihere,10){}", s);
}
/// <summary>
/// We will run the job submission. We assume that any checking has already gone one
/// and we are just going to execute all the commands required of this job request.
/// Assume setupATLAS and rucio and voms proxy init have all been done before
/// this guy is called!
/// </summary>
/// <param name="connection"></param>
/// <param name="job"></param>
/// <param name="datasetToStartWith"></param>
/// <param name="credSet">Set of credentials to load. Default to CERN</param>
/// <returns></returns>
public static async Task <ISSHConnection> SubmitJobAsync(this ISSHConnection connection, AtlasJob job, string inputDataSet, string resultingDataSet, Action <string> statusUpdate = null, Func <bool> failNow = null, bool sameJobAsLastTime = false, string credSet = "CERN", bool dumpOnly = false)
{
// Get the status update protected.
Action <string> update = statusUpdate != null ?
statusUpdate
: s => { };
// Figure out the proper submit command.
string submitCmd = (job.SubmitPatternCommands.Length > 0
? MatchSubmitPattern(job.SubmitPatternCommands, inputDataSet)
: job.SubmitCommand.SubmitCommand.CommandLine)
.Replace("*INPUTDS*", "{0}")
.Replace("*OUTPUTDS*", "{1}");
var cernCred = new CredentialSet(credSet)
.Load()
.FirstOrDefault()
.ThrowIfNull(() => new GRIDSubmitException($"Please create a windows generic credential with a target of '{credSet}' to allow access to kinit"));
// If this is the first time through with a single job, then setup a directory we can use.
if (!sameJobAsLastTime)
{
var linuxLocation = string.Format("/tmp/{0}", resultingDataSet);
await connection.Apply(() => update("Removing old build directory"))
.ExecuteLinuxCommandAsync("rm -rf " + linuxLocation, dumpOnly: dumpOnly);
await connection
.Apply(() => update("Setting up panda"))
.ExecuteLinuxCommandAsync("lsetup panda", dumpOnly: dumpOnly);
await connection.Apply(() => update("Setting up release"))
.SetupRcReleaseAsync(linuxLocation, job.Release.Name, dumpOnly: dumpOnly);
await connection.Apply(() => update("Getting CERN credentials"))
.KinitAsync(cernCred.Username, cernCred.Password, dumpOnly: dumpOnly);
await connection
.ApplyAsync(job.Packages, (c, j) => c.Apply(() => update("Checking out package " + j.Name)).CheckoutPackageAsync(j.Name, j.SCTag, failNow: failNow, dumpOnly: dumpOnly));
await connection
.ApplyAsync(job.Commands, (co, cm) => co.Apply(() => update("Running command " + cm.CommandLine)).ExecuteLinuxCommandAsync(cm.CommandLine, failNow: failNow, dumpOnly: dumpOnly));
await connection
.Apply(() => update("Compiling release"))
.BuildWorkAreaAsync(failNow: failNow, dumpOnly: dumpOnly);
}
// We should now be in the directory where everything is - so submit!
return(await connection
.Apply(() => update($"Running submit command ({inputDataSet})"))
.ExecuteLinuxCommandAsync(string.Format(submitCmd, inputDataSet, resultingDataSet), failNow: failNow, dumpOnly: dumpOnly));
}
/// <summary>
/// Return the hash code for this job definition.
/// </summary>
/// <param name="J"></param>
/// <returns></returns>
public static string Hash(this AtlasJob J)
{
return(J.Print().ComputeMD5Hash());
}
public void LatestreamDSNoScope()
{
var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var ds = j.ResultingDatasetName("data15_13TeV.00277025.physics_Late.merge.DAOD_EXOT15.f622_m1486_p2425_tid06570038_00", "user.bogus");
Assert.IsTrue(ds.StartsWith("user.bogus.00277025.physics_Late.DAOD_EXOT15.f622_m1486_p2425.DiVertAnalysis_v22_3B233454"));
}
public void xAODMCDataset()
{
var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 };
var ds = j.ResultingDatasetName("mc15_13TeV.361020.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ0W.merge.AOD.e3569_s2576_s2132_r6765_r6282", "user.bogus");
Assert.IsTrue(ds.StartsWith("user.bogus.361020.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ0W.s2132_r6765_r6282.DiVertAnalysis_v22_3B233454"), ds);
}
/// <summary>
/// Returns a dataset name that this job should produce given the original dataset name.
/// </summary>
/// <param name="job"></param>
/// <param name="originalDSName"></param>
/// <returns>
/// Use the name of the job and the version number as part of the name
/// Use a hash of the commands incase the user changes the commands but forgets to change the version number
/// Truncate the ds name if it gets too long (rucio has some limits)
/// Make sure that the uniqueness of the lost dataset name is not lost.
/// </returns>
public static string ResultingDataSetName(this AtlasJob job, string originalDSName, string scopeDSName, int jobIteration = 0)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
// Remove the scope if it is there.
var sanitizedDSName = originalDSName.RemoveBefore(":");
// Grab a hash to deal with bits we are dropping (for uniqueness)
var originalHash = sanitizedDSName.ComputeMD5Hash();
// Split the dataset into its constituent parts.
var dsParts = sanitizedDSName.Split('.').ToList();
// If the first item in the list is a user, remove that and the next item.
if (dsParts[0] == "user")
{
dsParts.RemoveAt(0);
dsParts.RemoveAt(0);
}
// Remove a few key words
dsParts = dsParts
.Where(i => i != "merge")
.Where(i => i != "AOD")
.ToList();
// Look for the tid job spec. Remove it if it is there.
var containsTID = dsParts.Where(i => i.Contains("tid")).FirstOrDefault();
if (containsTID != null)
{
var minusTID = containsTID.RemoveTIDString();
dsParts.Replace(containsTID, minusTID);
}
// Ok, if the second guy is a number, then it is a run number. That means the first one is almost certainly a scope. So remove it.
int runNumber = 0;
if (dsParts.Count > 1 && int.TryParse(dsParts[1], out runNumber))
{
dsParts.RemoveAt(0);
}
// Done. Combine into a single string and check length!
bool shortEnough = false;
string dsNew = "";
while (!shortEnough)
{
var jobVersionString = jobIteration == 0
? job.Version.ToString()
: $"{job.Version}.{jobIteration}";
dsNew = $"{scopeDSName}.{dsParts.AsSingleString(".")}.{job.Name}_v{jobVersionString}_{job.Hash()}_{originalHash}";
shortEnough = (dsNew + "_hist-output.root/").Length <= 133;
if (!shortEnough)
{
dsParts = dsParts.RemoveATag();
if (dsParts.Count == 0)
{
StringBuilder bld = new StringBuilder();
bld.AppendFormat("Dataset '{0}' is more than 133 characters after accounting for EventLoop naming (from ds '{1}'). It needs to be made shorter", dsNew, originalDSName);
throw new ArgumentException(bld.ToString());
}
}
}
Trace.WriteLine($"Job {job.Name} v{job.Version} (iteration {jobIteration} run on dataset {sanitizedDSName} will be sent to the dataset {dsNew}.");
return(dsNew);
}
public static Dictionary<string, string> AddSubmitInfo(this Dictionary<string,string> source, AtlasJob j, string submitCommand = "submit")
{
var d = new Dictionary<string, string>()
{
{ "rm -rf /tmp/ds1-out", "" },
{ "lsetup panda", "" },
{ "mkdir /tmp/ds1-out", "" },
{ "cd /tmp/ds1-out", "" },
{ "rcSetup notmyrelease", "Found ASG release with" },
{ "echo fubar | kinit [email protected]", "Password for [email protected]: " },
{ $"rc checkout_pkg {j.Packages[0].Name}/trunk@tag", "Checked out revision" },
{ "mv trunk@tag hithere", "" },
{ "rc find_packages", "" },
{ "rc compile", "" },
{ "ls", "" },
{ "echo $?", "0" },
{ submitCommand, "dude" }
};
return source.AddEntry(d);
}
