/// <summary>
/// Get the full listing from svn of all datasets. This isn't fast (hence the updates).
/// </summary>
private void RefreshDatabase()
{
var pr = new ProgressRecord(1, "ATLAS MC Dataset Listing", "Fetching from CERN");
pr.PercentComplete = 0;
WriteProgress(pr);
var svnRootDir = MCJobSVNHelpers.BuildTarget(".", MCCampaign);
var listing = MCJobSVNHelpers.FetchListing(svnRootDir, new SvnListArgs()
{
Depth = SvnDepth.Infinity
});
pr.PercentComplete = 50;
pr.StatusDescription = $"Saving to local datafile ({listing.Count} datasets found)";
WriteProgress(pr);
// Write it all out to the database file.
var dbFileInfo = GetDatabaseFilename();
if (!dbFileInfo.Directory.Exists)
{
dbFileInfo.Directory.Create();
}
using (var writer = dbFileInfo.CreateText())
{
foreach (var svnFile in listing)
{
writer.WriteLine(svnFile.Name);
}
}
pr.PercentComplete = 100;
WriteProgress(pr);
}
protected override void ProcessRecord()
{
// Make sure this is a 6 digit number as a string.
var runID = MCJobNumber.ToString("D6");
// First, for this run number, see if we can't figure out what DSID it is going to be
// cached under.
var DSIDDirectory = $"DSID{runID.Substring(0, 3)}xxx";
WriteVerbose($"Fetching svn listing from {DSIDDirectory}");
var dsidListTarget = MCJobSVNHelpers.FetchListing(MCJobSVNHelpers.BuildTarget($"share/{DSIDDirectory}", MCCampaign));
var myMCFile = dsidListTarget
.Where(ads => ads.Name.Contains(runID))
.ToArray();
if (myMCFile.Length == 0)
{
var err = new ArgumentException($"Unable to find dataset run number for {runID}.");
WriteError(new ErrorRecord(err, "NoSuchDataset", ErrorCategory.InvalidArgument, null));
}
if (myMCFile.Length != 1)
{
var err = new ArgumentException($"Found multiple dataset files for {runID} - giving up.");
WriteError(new ErrorRecord(err, "MoreThanOneDataset", ErrorCategory.InvalidArgument, null));
}
var ds = myMCFile[0];
// Write out the raw lines, or the files to the local directory, depending.
if (ExtractionPath == null)
{
IEnumerable <string> lines = GetSvnFileLines(BuildTarget(ds.Uri));
foreach (var line in lines)
{
WriteObject(line);
}
}
else
{
var files = GetSvnFiles(BuildTarget(ds.Uri), ExtractionPath);
foreach (var f in files)
{
WriteObject(f);
}
}
base.ProcessRecord();
}
/// <summary>
/// Given a listing, return an item.
/// </summary>
/// <param name="ds"></param>
/// <returns></returns>
private IEnumerable <string> GetSvnFileLines(SvnTarget ds)
{
// Next, fetch the file down.
var targetTempPath = Path.GetTempFileName();
WriteVerbose($"Downloading svn file {ds.TargetName}");
MCJobSVNHelpers.ExtractFile(ds, new FileInfo(targetTempPath));
// Transfer process the lines
var lines = new FileInfo(targetTempPath)
.ReadLines()
.SelectMany(l => ReplaceIncludeFiles(l, ExpandIncludeFiles));
return(lines);
}
/// <summary>
/// Fetch the files, and if requested, all the include files as well.
/// </summary>
/// <param name="ds"></param>
/// <param name="extractionPath"></param>
/// <returns></returns>
private IEnumerable <FileInfo> GetSvnFiles(SvnTarget ds, PathInfo extractionPath)
{
// Build the location of this file to write out.
var outfile = new FileInfo(Path.Combine(extractionPath.Path, ds.FileName));
WriteVerbose($"Downloading svn file {ds.TargetName}");
MCJobSVNHelpers.ExtractFile(ds, outfile);
yield return(outfile);
// Next, we need to dip into all the levels down to see if we can't
// figure out if there are includes.
var includeFiles = outfile
.ReadLines()
.SelectMany(l => ExtractIncludedFiles(l, extractionPath));
foreach (var l in includeFiles)
{
yield return(l);
}
}
/// <summary>
/// Update the cache with all the files.
/// </summary>
private void ReloadCache()
{
var subdirs = new string[] { "common", "nonStandard", "higgscontrol", "susycontrol", "topcontrol" };
// A list fo the files to access, one after the other
var opt = new SvnListArgs()
{
Depth = SvnDepth.Infinity
};
var linesCommon = subdirs
.Select(d => MCJobSVNHelpers.BuildTarget(d, MCCampaign))
.SelectMany(dl => MCJobSVNHelpers.FetchListing(dl, opt));
// Now, write it out.
var cacheFile = GetCacheFileInfo();
using (var writer = cacheFile.CreateText())
{
foreach (var item in linesCommon)
{
writer.WriteLine(item.Uri.OriginalString);
}
}
}
