/// <summary>
/// Return the location for CERN
/// </summary>
/// <param name="props">Property bag to configure the object</param>
/// <returns></returns>
public static Location GetLocation(Dictionary<string, string> props)
{
var l = new Location();
l.Name = props["Name"];
var dnsString = props["DNSEndString"];
Func<bool> test = () => dnsString.Split(',').Select(s => s.Trim()).Select(en => IPLocationTests.FindLocalIpName().EndsWith(en)).Where(t => t).Any();
l.LocationTests.Add(() => test());
var fetcher = FetchToRemoteLinuxDirInstance.FetchRemoteLinuxInstance(props);
var dsfinder = new GRIDFetchToLinuxVisibleOnWindows(new DirectoryInfo(props["WindowsPath"]), fetcher, props["LinuxPath"]);
l.GetDSInfo = name =>
{
var nfiles = dsfinder.CheckNumberOfFiles(name);
return new DSInfo()
{
Name = name,
IsLocal = filter => dsfinder.CheckIfLocal(name, filter),
CanBeGeneratedAutomatically = true,
ListOfFiles = () => dsfinder.ListOfFiles(name),
LocationProvider = l,
};
};
l.GetDS = dsfinder.GetDS;
return l;
}
public void AttemptToDownloadBadDS()
{
// Seen in wild: try to download a bad dataset, and it creates a directory
// anyway. Ops!
var dsinfo = MakeDSInfo("ds1.1.1");
var d = new DirectoryInfo("AttemptToDownloadBadDS");
if (d.Exists)
{
d.Delete(true);
}
var ld = new LinuxMirrorDownloaderPretend(d, "forkitover");
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store");
try {
var r = gf.GetDS(dsinfo);
} catch (ArgumentException)
{
// Expecting it to throw here - no dataset should exist by that name!
}
// Did a local directory get created?
d.Refresh();
Assert.IsFalse(d.Exists);
}
public void CountFilesDownloaded()
{
var dsinfo = MakeDSInfo("ds1.1.1");
var d = utils.BuildSampleDirectoryBeforeBuild("CountFilesDownloaded", dsinfo.Name);
utils.MakePartial(d, dsinfo.Name);
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null);
Assert.AreEqual(5, gf.CheckNumberOfFiles(dsinfo.Name));
}
public void DownloadAllFilesSecondTime()
{
var dsinfo = MakeDSInfo("ds1.1.1");
var d = new DirectoryInfo("DownloadAllFilesSecondTime");
if (d.Exists)
{
d.Delete(true);
}
var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name);
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store");
var r = gf.GetDS(dsinfo, fileFilter: flist => flist.OrderBy(f => f).Take(1).ToArray());
Assert.AreEqual(1, r.Length);
r = gf.GetDS(dsinfo);
Assert.AreEqual(5, r.Length);
Assert.AreEqual(5, d.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Where(f => !f.FullName.EndsWith(".part")).Count());
}
public void MakeSureContentsFileIsUpdated()
{
var dsinfo = MakeDSInfo("ds1.1.1");
var d = new DirectoryInfo("MakeSureContentsFileIsUpdated");
if (d.Exists)
{
d.Delete(true);
}
d.Create();
var f = new FileInfo(Path.Combine(d.FullName, "contents.txt"));
using (var wr = f.CreateText())
{
wr.WriteLine("otherdataset.data.set gwatts");
}
var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name);
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store");
var r = gf.GetDS(dsinfo);
Assert.IsTrue(f.Exists);
Assert.AreEqual("otherdataset.data.set gwatts", f.ReadLine(1));
Assert.IsTrue(f.ReadLine(2).StartsWith(dsinfo.Name));
}
public void MakeSureContentsFileIsCreated()
{
var dsinfo = MakeDSInfo("ds1.1.1");
var d = new DirectoryInfo("MakeSureContentsFileIsCreated");
if (d.Exists)
{
d.Delete(true);
}
var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name);
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store");
var r = gf.GetDS(dsinfo);
var f = new FileInfo(Path.Combine(d.FullName, "contents.txt"));
Assert.IsTrue(f.Exists);
Assert.IsTrue(f.ReadLine().StartsWith(dsinfo.Name));
}
public void GetFileListForLocalFilesWithNoWork()
{
var dsinfo = MakeDSInfo("ds1.1.1.1");
var d = utils.BuildSampleDirectoryBeforeBuild("GetFileListForLocalFilesWithNoWork", dsinfo.Name);
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null);
Assert.AreEqual(5, gf.ListOfFiles("ds1.1.1.1").Length);
}
public void FindSomeFilesWithScopedDataset()
{
var dsinfo = MakeDSInfo("user.norm:ds1.1.1");
var d = utils.BuildSampleDirectoryBeforeBuild("FindLocationWithScopedDataset", "ds1.1.1");
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null);
var r = gf.GetDS(dsinfo, fileFilter: fs => fs.Where(fname => fname.Contains("root.1")).ToArray());
Assert.IsNotNull(r);
Assert.AreEqual(1, r.Length);
}
public void FindLocationWithScopedDataset()
{
var dsinfo = MakeDSInfo("user.norm:ds1.1.1");
var d = utils.BuildSampleDirectoryBeforeBuild("FindLocationWithScopedDataset", "ds1.1.1");
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null);
var r = gf.GetDS(dsinfo);
Assert.IsNotNull(r);
Assert.AreEqual(5, r.Length);
}
public void FindLocalFilesWithNoWork()
{
var dsinfo = MakeDSInfo("ds1.1.1.1");
var d = utils.BuildSampleDirectoryBeforeBuild("FindLocalFilesWithNoWork", dsinfo.Name);
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null);
var r = gf.GetDS(dsinfo);
Assert.IsNotNull(r);
Assert.AreEqual(5, r.Length);
}
public void DownloadToLinuxDirectoryThatIsAWindowsDirectoryWithScoppedDS()
{
var dsinfo = MakeDSInfo("user.norm:ds1.1.1");
var d = new DirectoryInfo("DownloadToLinuxDirectoryThatIsAWindowsDirectory");
if (d.Exists)
{
d.Delete(true);
}
var ld = new LinuxMirrorDownloaderPretend(d, "ds1.1.1");
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store");
var r = gf.GetDS(dsinfo);
Assert.IsNotNull(r);
Assert.AreEqual(5, r.Length);
Assert.AreEqual("/bogus/files/store/ds1.1.1", ld.LinuxDest);
}
public void DownloadToLinuxDirectoryThatIsAWindowsDirectoryAndAreadyCreated()
{
// Seen in the wild. A crash (or other interruption) means the dataset directory has
// been created, but is empty (for whatever reason). In that particular case, we should
// treat it as not there.
var dsinfo = MakeDSInfo("ds1.1.1");
var d = new DirectoryInfo("DownloadToLinuxDirectoryThatIsAWindowsDirectory");
if (d.Exists)
{
d.Delete(true);
}
d.Create();
var dsdir = d.SubDir(dsinfo.Name);
dsdir.Create();
var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name);
var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store");
var r = gf.GetDS(dsinfo);
Assert.IsNotNull(r);
Assert.AreEqual(5, r.Length);
Assert.AreEqual("/bogus/files/store/ds1.1.1", ld.LinuxDest);
}