public void NormalCall()
{
// Make sure it doesn't break.
var location = new Locator();
var ls = location.FindBestLocations();
Assert.IsNotNull(ls);
}
public void GetCERNLocation()
{
AtlasWorkFlows.Utils.IPLocationTests.SetIpName("bogus.nytimes.com");
var locator = new Locator();
var lst = locator.FindLocation("MyTestLocation");
Assert.IsNotNull(lst);
}
public void ScreenLocations()
{
AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch");
Locator.SetLocationFilter(loc => false);
var locator = new Locator();
var lst = locator.FindBestLocations();
Assert.IsNotNull(lst);
Assert.AreEqual(0, lst.Length);
}
public void CERNNotAtCERN()
{
AtlasWorkFlows.Utils.IPLocationTests.SetIpName("bogus.nytimes.com");
var locator = new Locator();
var lst = locator.FindBestLocations();
Assert.IsNotNull(lst);
var cern = lst.Where(l => l.Name == "MyTestLocation").FirstOrDefault();
Assert.IsNull(cern);
}
/// <summary>
/// Find the DSInfo with the best possible location provider (local, remote, etc.).
/// </summary>
/// <param name="datasetname">The GRID dataset name to find pointers to.</param>
/// <param name="fileFilter">Filter the potential list of files to be returned or downloaded (can really speed up things if you want one file in a large dataset).</param>
/// <param name="locationFilter">Filter out the locations that we are allowed to fetch the file from.</param>
/// <returns></returns>
public static DSInfo FetchDSInfo(string datasetname, Func<string[], string[]> fileFilter, Func<string, bool> locationFilter)
{
// Basic parameter checks
if (string.IsNullOrWhiteSpace(datasetname))
{
throw new ArgumentException("Dataset name is empty");
}
// Get a location so we can see if we can fetch the dataset
var locator = new Locator();
var locationList = locator.FindBestLocations();
if (locationFilter != null)
{
locationList = locationList.Where(loc => locationFilter(loc.Name)).ToArray();
}
if (locationList.Length == 0)
{
throw new InvalidOperationException(string.Format("There are no valid ways to download dataset '{0}' from your current location.", datasetname));
}
// Get the info for each dataset.
var allDSInfo = (from loc in locationList
let dsi = loc.GetDSInfo(datasetname)
select new
{
dsinfo = dsi,
islocal = dsi.IsLocal(fileFilter)
}).ToArray();
// First, if anyone has something local, then we should grab that.
var localDS = allDSInfo.Where(d => d.islocal).OrderByDescending(d => d.dsinfo.LocationProvider.Priority).ToArray();
// Ok, now we need the ones that can generate it.
if (localDS.Length == 0)
{
localDS = allDSInfo.Where(d => d.dsinfo.CanBeGeneratedAutomatically).OrderByDescending(d => d.dsinfo.LocationProvider.Priority).ToArray();
}
// Did we strike out?
var locationInfo = localDS.FirstOrDefault();
if (locationInfo == null)
{
var locationListText = "";
foreach (var l in locationList)
{
locationListText += l.Name + " ";
}
throw new ArgumentException(string.Format("Do not know how to generate the dataset '{0}' at any of the locations {1} (which are the only ones working where the computer is located)", datasetname, locationListText));
}
var dsinfo = locationInfo.dsinfo;
return dsinfo;
}
public void FailBecauseLocalRepoNotCreated()
{
AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch");
var dsname = "ds1.1.1";
var d1 = utils.BuildSampleDirectoryBeforeBuild("LoadNewFilesToLocalWhenMissingRemote", dsname);
var d2 = new DirectoryInfo("FailBecauseLocalRepoNotCreated");
if (d2.Exists)
{
d2.Delete(true);
}
d2.Refresh();
Locator._getLocations = () => utils.GetLocal(d1, d2);
var locator = new Locator();
var l = locator.FindLocation("MyTestLocalLocation");
var r = l.GetDSInfo("ds1.1.1");
Assert.IsFalse(r.IsLocal(null));
var files = l.GetDS(r, null, null, null, 0);
}
/// <summary>
/// Main routine to find all the URI's that point to a dataset.
/// </summary>
/// <param name="datasetname">The GRID dataset name to find pointers to.</param>
/// <param name="fileFilter">Filter the potential list of files to be returned or downloaded (can really speed up things if you want one file in a large dataset).</param>
/// <param name="locationFilter">Filter out the locations that we are allowed to fetch the file from.</param>
/// <param name="statusUpdate">Downloads from the grid can take a long time. Status updates will be posted here if not null.</param>
/// <returns></returns>
public static Uri[] FetchDatasetUrisAtLocation(string locationName, string datasetname,
Action<string> statusUpdate = null,
Func<string[], string[]> fileFilter = null,
Func<bool> failNow = null,
int timeoutDownloadSecs = 3600)
{
var locator = new Locator();
var location = locator.FindLocation(locationName);
if (location == null) {
throw new InvalidOperationException(string.Format("Location {0} isn't known.", locationName));
}
if (!location.LocationIsGood()) {
throw new InvalidOperationException(string.Format("Location {0} isn't active and can't be used", locationName));
}
var dsinfo = location.GetDSInfo(datasetname);
// And delegate all the rest of our work to fetching.
return dsinfo.LocationProvider.GetDS(dsinfo, statusUpdate, fileFilter, failNow, timeoutDownloadSecs);
}
public void FetchOneMissingFileFromBackupSource()
{
// Dataset isn't here and there is no remote... Backup to the rescue!
var dsname = "ds1.1.1";
var d2 = new DirectoryInfo("FetchOneMissingFileFromBackupSourceLocal");
if (d2.Exists)
{
d2.Delete(true);
}
d2.Create();
d2.Refresh();
Locator._getLocations = () => utils.GetLocal(null, d2);
// Configure a dummy fetcher that will get the datasets (they will get pulled when things are created).
// This isn't a real Linux downloader, so we need to put the files in some other place
var dtemp = new DirectoryInfo("FetchOneMissingFileFromBackupSourceRemoteLinuxDownload");
if (dtemp.Exists)
{
dtemp.Delete(true);
}
dtemp.Create();
FetchToRemoteLinuxDirInstance._test = new LinuxMirrorDownloaderPretend(dtemp, dsname);
var locator = new Locator();
var l = locator.FindLocation("MyTestLocalLocation");
var r = l.GetDSInfo("ds1.1.1");
Assert.IsFalse(r.IsLocal(null));
var files = l.GetDS(r, null, null, null, 0);
Assert.AreEqual(5, files.Length);
Assert.IsTrue(files[0].LocalPath.Contains("FetchOneMissingFileFromBackupSourceLocal"));
Assert.AreEqual(5, d2.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Where(f => !f.Name.EndsWith(".part")).Count());
}
public void LoadTwoFilesToLocalWhenMissing()
{
// Setup the infrastructure. For Local to fetch files, it will callback into the global
// infrastructure - so that needs to be setup for the test.
AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch");
var dsname = "ds1.1.1";
var d1 = utils.BuildSampleDirectoryBeforeBuild("LoadNewFilesToLocalWhenMissingRemote", dsname);
var d2 = new DirectoryInfo("LoadNewFilesToLocalWhenMissingLocal");
if (d2.Exists)
{
d2.Delete(true);
}
d2.Create();
d2.Refresh();
Locator._getLocations = () => utils.GetLocal(d1, d2);
var locator = new Locator();
var l = locator.FindLocation("MyTestLocalLocation");
var r = l.GetDSInfo("ds1.1.1");
Assert.IsFalse(r.IsLocal(null));
var files = l.GetDS(r, null, fslist => fslist.Take(2).ToArray(), null, 0);
Assert.AreEqual(2, files.Length);
Assert.IsTrue(files[0].LocalPath.Contains("LoadNewFilesToLocalWhenMissingLocal"));
Assert.AreEqual(2, d2.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Count());
}
public void LoadNewFilesToLocalWhenEmptyDirectory()
{
// Seen in the wild. Local directory is actually there for the dataset, but
// empty due to an earlier crash. Make sure the copy still occurs.
AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch");
var dsname = "ds1.1.1";
var d1 = utils.BuildSampleDirectoryBeforeBuild("LoadNewFilesToLocalWhenMissingRemote", dsname);
var d2 = new DirectoryInfo("LoadNewFilesToLocalWhenMissingLocal");
if (d2.Exists)
{
d2.Delete(true);
}
d2.Create();
d2.Refresh();
d2.SubDir("ds1.1.1").Create();
Locator._getLocations = () => utils.GetLocal(d1, d2);
var locator = new Locator();
var l = locator.FindLocation("MyTestLocalLocation");
var r = l.GetDSInfo("ds1.1.1");
Assert.IsFalse(r.IsLocal(null));
var files = l.GetDS(r, null, null, null, 0);
Assert.AreEqual(5, files.Length);
Assert.IsTrue(files[0].LocalPath.Contains("LoadNewFilesToLocalWhenMissingLocal"));
Assert.AreEqual("ds1.1.1", d2.EnumerateDirectories().First().Name);
}
public void FetchOneMissingFileFromRemote()
{
// Local is complete in all but one file. Force a fetch of just that one file.
// Setup the infrastructure. For Local to fetch files, it will callback into the global
// infrastructure - so that needs to be setup for the test.
AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch");
var dsname = "ds1.1.1";
var d1 = utils.BuildSampleDirectoryBeforeBuild("FetchOneMissingFileFromRemoteRemote", dsname);
var d2 = utils.BuildSampleDirectoryBeforeBuild("FetchOneMissingFileFromRemoteLocal", dsname);
Locator._getLocations = () => utils.GetLocal(d1, d2);
var fbad = new FileInfo(Path.Combine(d2.FullName, "ds1.1.1", "sub2", "file.root.5"));
Assert.IsTrue(fbad.Exists);
fbad.Delete();
utils.MakePartial(d2, "ds1.1.1");
var locator = new Locator();
var l = locator.FindLocation("MyTestLocalLocation");
var r = l.GetDSInfo("ds1.1.1");
Assert.IsFalse(r.IsLocal(null));
var files = l.GetDS(r, null, null, null, 0);
Assert.AreEqual(5, files.Length);
Assert.IsTrue(files[0].LocalPath.Contains("FetchOneMissingFileFromRemoteLocal"));
Assert.AreEqual(5, d2.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Where(f => !f.Name.EndsWith(".part")).Count());
}