public void BuildJobFunctionally() { var job = new AtlasJob() .NameVersionRelease("DiVertAnalysis", 3, "2.3.32") .Package("JetSelectorTools") .Package("atlasphys-exo/Physics/Exotic/UEH/DisplacedJets/Run2/AnalysisCode/trunk/DiVertAnalysis", "248132") .Command("grep -v \"emf < 0.05\" JetSelectorTools/Root/JetCleaningTool.cxx > JetSelectorTools/Root/JetCleaningTool-New.cxx") .Command("mv JetSelectorTools/Root/JetCleaningTool-New.cxx JetSelectorTools/Root/JetCleaningTool.cxx") .SubmitCommand("DiVertAnalysisRunner -EventLoopDriver GRID *INPUTDS* -ELGRIDOutputSampleName *OUTPUTDS* -WaitTillDone FALSE -isLLPMC true"); Assert.AreEqual("DiVertAnalysis", job.Name); Assert.AreEqual(3, job.Version); Assert.AreEqual("2.3.32", job.Release.Name); Assert.AreEqual(2, job.Packages.Length); Assert.AreEqual("JetSelectorTools", job.Packages[0].Name); Assert.AreEqual("", job.Packages[0].SCTag); Assert.AreEqual("atlasphys-exo/Physics/Exotic/UEH/DisplacedJets/Run2/AnalysisCode/trunk/DiVertAnalysis", job.Packages[1].Name); Assert.AreEqual("248132", job.Packages[1].SCTag); Assert.AreEqual(2, job.Commands.Length); Assert.AreEqual("grep -v \"emf < 0.05\" JetSelectorTools/Root/JetCleaningTool.cxx > JetSelectorTools/Root/JetCleaningTool-New.cxx", job.Commands[0].CommandLine); Assert.AreEqual("mv JetSelectorTools/Root/JetCleaningTool-New.cxx JetSelectorTools/Root/JetCleaningTool.cxx", job.Commands[1].CommandLine); Assert.AreEqual("DiVertAnalysisRunner -EventLoopDriver GRID *INPUTDS* -ELGRIDOutputSampleName *OUTPUTDS* -WaitTillDone FALSE -isLLPMC true", job.SubmitCommand.SubmitCommand.CommandLine); }
public void DxAODDatasetSlightlyDifferent() { // Seen in the wild. Need to take into account the fact that there might be subtle differences in the dataset names. var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var ds1 = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3668_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus"); var ds2 = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3669_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus"); Assert.AreNotEqual(ds1, ds2); }
public void HashIsDifferent() { var j = new AtlasJob() { Name = "Hi there", Version = 12 }; var h1 = j.Hash(); j.Name = "not there"; var h2 = j.Hash(); Assert.AreNotEqual(h1, h2); }
/// <summary> /// Set the name and version of the job. /// </summary> /// <param name="job"></param> /// <param name="name"></param> /// <param name="version"></param> /// <returns></returns> public static AtlasJob NameVersionRelease(this AtlasJob job, string name, int version, string release) { job = job == null ? new AtlasJob() : job; job.Name = name; job.Version = version; job.Release = new Release() { Name = release }; return(job); }
/// <summary> /// Fetch from GRID the credential name. /// </summary> /// <param name="job"></param> /// <param name="origDSName"></param> /// <returns></returns> public static string ResultingDataSetName(this AtlasJob job, string origDSName, int jobIteration = 0) { using (var credList = new CredentialSet("GRID")) { var gridCredentials = credList.Load().FirstOrDefault(); if (gridCredentials == null) { throw new ArgumentException("Please create a generic windows credential with the target 'GRID' with the user name as the rucio grid user name and the password to be used with voms proxy init"); } return(job.ResultingDataSetName(origDSName, gridCredentials, jobIteration)); } }
/// <summary> /// Add the submit command to the atlas job. /// </summary> /// <param name="job"></param> /// <param name="commandLine"></param> /// <returns></returns> public static AtlasJob SubmitCommand(this AtlasJob job, string commandLine) { job = job == null ? new AtlasJob() : job; job.SubmitCommand = new Submit() { SubmitCommand = new Jobs.Command() { CommandLine = commandLine } }; return(job); }
/// <summary> /// Do a deep copy of the job. /// </summary> /// <returns></returns> internal AtlasJob Clone() { var r = new AtlasJob (); r.Name = Name; r.Version = Version; r.Release = Release?.Clone(); r.Packages = Packages?.Select(p => p.Clone()).ToArray(); r.Commands = Commands?.Select(c => c.Clone()).ToArray(); r.SubmitCommand = SubmitCommand?.Clone(); r.SubmitPatternCommands = SubmitPatternCommands?.Select(sp => sp.Clone()).ToArray(); return r; }
/// <summary> /// Do a deep copy of the job. /// </summary> /// <returns></returns> internal AtlasJob Clone() { var r = new AtlasJob(); r.Name = Name; r.Version = Version; r.Release = Release?.Clone(); r.Packages = Packages?.Select(p => p.Clone()).ToArray(); r.Commands = Commands?.Select(c => c.Clone()).ToArray(); r.SubmitCommand = SubmitCommand?.Clone(); r.SubmitPatternCommands = SubmitPatternCommands?.Select(sp => sp.Clone()).ToArray(); return(r); }
public void JobOutput() { var j = new AtlasJob() { Commands = new Command[] { new Command() { CommandLine = "ls" } }, Name = "MyJob", Version = 1234, Release = new Release() { Name = "notmyrelease" }, SubmitCommand = new Submit() { SubmitCommand = new Command() { CommandLine = "submit" } }, Packages = new Package[] { new Package() { Name = "hithere", SCTag = "tag" } } }; Assert.AreEqual("job(MyJob,1234){release(notmyrelease)package(hithere,tag)submit(submit)}", j.Print()); }
/// <summary> /// Add a new submit command pattern to the job. /// </summary> /// <param name="job"></param> /// <param name="commandLine"></param> /// <param name="pattern"></param> /// <returns></returns> public static AtlasJob SubmitCommandPattern(this AtlasJob job, string commandLine, string pattern) { job = job ?? new AtlasJob(); job.SubmitPatternCommands = job.SubmitPatternCommands .Append(new SubmitPattern() { Regex = pattern, SubmitCommand = new Submit() { SubmitCommand = new Jobs.Command() { CommandLine = commandLine } } }); return(job); }
private static StringBuilder Print(this AtlasJob r, StringBuilder bld, bool prettyPrint = false) { if (prettyPrint && bld.Length > 0) { bld.AppendLine(); } bld.AppendFormat("job({0},{1}){{", r.Name, r.Version); if (prettyPrint) { bld.AppendLine(); } if (r.Release != null) { r.Release.Print(bld, prettyPrint: prettyPrint); } if (r.Packages != null) { foreach (var p in r.Packages.OrderBy(p => $"{p.Name}-{p.SCTag}")) { p.Print(bld, prettyPrint: prettyPrint); } } if (r.SubmitPatternCommands != null && r.SubmitPatternCommands.Length > 0) { foreach (var sc in r.SubmitPatternCommands.OrderBy(p => $"{p.Regex}-{p.SubmitCommand.SubmitCommand}")) { sc.Print(bld, prettyPrint: prettyPrint); } } else { if (r.SubmitCommand != null) { r.SubmitCommand.Print(bld, prettyPrint: prettyPrint); } } bld.Append("}"); return(bld); }
public void HashSameForReorderedPackages() { var j1 = new AtlasJob() { Name = "DiVertAnalysis", Version = 22, Packages = new Package[] { new Package() { Name = "pkg1", SCTag = "v00-00-00" }, new Package() { Name = "pkg2", SCTag = "v00-01-00" } }, }; var j2 = new AtlasJob() { Name = "DiVertAnalysis", Version = 22, Packages = new Package[] { new Package() { Name = "pkg2", SCTag = "v00-01-00" }, new Package() { Name = "pkg1", SCTag = "v00-00-00" }, }, }; Assert.AreEqual(j1.Hash(), j2.Hash()); }
public void HashTheSame() { var j1 = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var j2 = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; Assert.AreEqual(j1.Hash(), j2.Hash()); }
public void DxNoDash() { var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var ds = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3668_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus"); Assert.IsFalse(ds.Contains("-"), ds); }
public void DxAODDatasetFromEmma() { var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var ds = j.ResultingDatasetName("user.emmat.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.merge.AOD.e3668_s2576_s2132_r6765_r6282__EXOT15_EXT0", "user.bogus"); Assert.IsTrue(ds.StartsWith("user.bogus.361023.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ3W.r6282__EXOT15_EXT0.DiVertAnalysis_v22_3B233454"), ds); }
public void DataDerivationxAOD() { var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var ds = j.ResultingDatasetName("data15_13TeV:data15_13TeV.00280500.physics_Main.merge.DAOD_EXOT15.f631_m1504_p2425_tid06603342_00", "user.bogus"); Assert.IsTrue(ds.StartsWith("user.bogus.00280500.physics_Main.DAOD_EXOT15.f631_m1504_p2425.DiVertAnalysis_v22_3B233454")); }
public static string Print(this AtlasJob r, bool prettyPrint = false) { return(r.Print(new StringBuilder(), prettyPrint: prettyPrint).ToString()); }
public void StableHash() { var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var h = j.Hash(); Assert.AreEqual("3B233454", h); }
public void PrintEmptyJob() { var j = new AtlasJob() { Name = "hihere", Version = 10 }; var s = j.Print(); Assert.AreEqual("job(hihere,10){}", s); }
/// <summary> /// We will run the job submission. We assume that any checking has already gone one /// and we are just going to execute all the commands required of this job request. /// Assume setupATLAS and rucio and voms proxy init have all been done before /// this guy is called! /// </summary> /// <param name="connection"></param> /// <param name="job"></param> /// <param name="datasetToStartWith"></param> /// <param name="credSet">Set of credentials to load. Default to CERN</param> /// <returns></returns> public static async Task <ISSHConnection> SubmitJobAsync(this ISSHConnection connection, AtlasJob job, string inputDataSet, string resultingDataSet, Action <string> statusUpdate = null, Func <bool> failNow = null, bool sameJobAsLastTime = false, string credSet = "CERN", bool dumpOnly = false) { // Get the status update protected. Action <string> update = statusUpdate != null ? statusUpdate : s => { }; // Figure out the proper submit command. string submitCmd = (job.SubmitPatternCommands.Length > 0 ? MatchSubmitPattern(job.SubmitPatternCommands, inputDataSet) : job.SubmitCommand.SubmitCommand.CommandLine) .Replace("*INPUTDS*", "{0}") .Replace("*OUTPUTDS*", "{1}"); var cernCred = new CredentialSet(credSet) .Load() .FirstOrDefault() .ThrowIfNull(() => new GRIDSubmitException($"Please create a windows generic credential with a target of '{credSet}' to allow access to kinit")); // If this is the first time through with a single job, then setup a directory we can use. if (!sameJobAsLastTime) { var linuxLocation = string.Format("/tmp/{0}", resultingDataSet); await connection.Apply(() => update("Removing old build directory")) .ExecuteLinuxCommandAsync("rm -rf " + linuxLocation, dumpOnly: dumpOnly); await connection .Apply(() => update("Setting up panda")) .ExecuteLinuxCommandAsync("lsetup panda", dumpOnly: dumpOnly); await connection.Apply(() => update("Setting up release")) .SetupRcReleaseAsync(linuxLocation, job.Release.Name, dumpOnly: dumpOnly); await connection.Apply(() => update("Getting CERN credentials")) .KinitAsync(cernCred.Username, cernCred.Password, dumpOnly: dumpOnly); await connection .ApplyAsync(job.Packages, (c, j) => c.Apply(() => update("Checking out package " + j.Name)).CheckoutPackageAsync(j.Name, j.SCTag, failNow: failNow, dumpOnly: dumpOnly)); await connection .ApplyAsync(job.Commands, (co, cm) => co.Apply(() => update("Running command " + cm.CommandLine)).ExecuteLinuxCommandAsync(cm.CommandLine, failNow: failNow, dumpOnly: dumpOnly)); await connection .Apply(() => update("Compiling release")) .BuildWorkAreaAsync(failNow: failNow, dumpOnly: dumpOnly); } // We should now be in the directory where everything is - so submit! return(await connection .Apply(() => update($"Running submit command ({inputDataSet})")) .ExecuteLinuxCommandAsync(string.Format(submitCmd, inputDataSet, resultingDataSet), failNow: failNow, dumpOnly: dumpOnly)); }
/// <summary> /// Return the hash code for this job definition. /// </summary> /// <param name="J"></param> /// <returns></returns> public static string Hash(this AtlasJob J) { return(J.Print().ComputeMD5Hash()); }
public void LatestreamDSNoScope() { var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var ds = j.ResultingDatasetName("data15_13TeV.00277025.physics_Late.merge.DAOD_EXOT15.f622_m1486_p2425_tid06570038_00", "user.bogus"); Assert.IsTrue(ds.StartsWith("user.bogus.00277025.physics_Late.DAOD_EXOT15.f622_m1486_p2425.DiVertAnalysis_v22_3B233454")); }
public void xAODMCDataset() { var j = new AtlasJob() { Name = "DiVertAnalysis", Version = 22 }; var ds = j.ResultingDatasetName("mc15_13TeV.361020.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ0W.merge.AOD.e3569_s2576_s2132_r6765_r6282", "user.bogus"); Assert.IsTrue(ds.StartsWith("user.bogus.361020.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ0W.s2132_r6765_r6282.DiVertAnalysis_v22_3B233454"), ds); }
/// <summary> /// Returns a dataset name that this job should produce given the original dataset name. /// </summary> /// <param name="job"></param> /// <param name="originalDSName"></param> /// <returns> /// Use the name of the job and the version number as part of the name /// Use a hash of the commands incase the user changes the commands but forgets to change the version number /// Truncate the ds name if it gets too long (rucio has some limits) /// Make sure that the uniqueness of the lost dataset name is not lost. /// </returns> public static string ResultingDataSetName(this AtlasJob job, string originalDSName, string scopeDSName, int jobIteration = 0) { if (job == null) { throw new ArgumentNullException(nameof(job)); } // Remove the scope if it is there. var sanitizedDSName = originalDSName.RemoveBefore(":"); // Grab a hash to deal with bits we are dropping (for uniqueness) var originalHash = sanitizedDSName.ComputeMD5Hash(); // Split the dataset into its constituent parts. var dsParts = sanitizedDSName.Split('.').ToList(); // If the first item in the list is a user, remove that and the next item. if (dsParts[0] == "user") { dsParts.RemoveAt(0); dsParts.RemoveAt(0); } // Remove a few key words dsParts = dsParts .Where(i => i != "merge") .Where(i => i != "AOD") .ToList(); // Look for the tid job spec. Remove it if it is there. var containsTID = dsParts.Where(i => i.Contains("tid")).FirstOrDefault(); if (containsTID != null) { var minusTID = containsTID.RemoveTIDString(); dsParts.Replace(containsTID, minusTID); } // Ok, if the second guy is a number, then it is a run number. That means the first one is almost certainly a scope. So remove it. int runNumber = 0; if (dsParts.Count > 1 && int.TryParse(dsParts[1], out runNumber)) { dsParts.RemoveAt(0); } // Done. Combine into a single string and check length! bool shortEnough = false; string dsNew = ""; while (!shortEnough) { var jobVersionString = jobIteration == 0 ? job.Version.ToString() : $"{job.Version}.{jobIteration}"; dsNew = $"{scopeDSName}.{dsParts.AsSingleString(".")}.{job.Name}_v{jobVersionString}_{job.Hash()}_{originalHash}"; shortEnough = (dsNew + "_hist-output.root/").Length <= 133; if (!shortEnough) { dsParts = dsParts.RemoveATag(); if (dsParts.Count == 0) { StringBuilder bld = new StringBuilder(); bld.AppendFormat("Dataset '{0}' is more than 133 characters after accounting for EventLoop naming (from ds '{1}'). It needs to be made shorter", dsNew, originalDSName); throw new ArgumentException(bld.ToString()); } } } Trace.WriteLine($"Job {job.Name} v{job.Version} (iteration {jobIteration} run on dataset {sanitizedDSName} will be sent to the dataset {dsNew}."); return(dsNew); }
public static Dictionary<string, string> AddSubmitInfo(this Dictionary<string,string> source, AtlasJob j, string submitCommand = "submit") { var d = new Dictionary<string, string>() { { "rm -rf /tmp/ds1-out", "" }, { "lsetup panda", "" }, { "mkdir /tmp/ds1-out", "" }, { "cd /tmp/ds1-out", "" }, { "rcSetup notmyrelease", "Found ASG release with" }, { "echo fubar | kinit [email protected]", "Password for [email protected]: " }, { $"rc checkout_pkg {j.Packages[0].Name}/trunk@tag", "Checked out revision" }, { "mv trunk@tag hithere", "" }, { "rc find_packages", "" }, { "rc compile", "" }, { "ls", "" }, { "echo $?", "0" }, { submitCommand, "dude" } }; return source.AddEntry(d); }