/// <summary> /// Return the location for CERN /// </summary> /// <param name="props">Property bag to configure the object</param> /// <returns></returns> public static Location GetLocation(Dictionary<string, string> props) { var l = new Location(); l.Name = props["Name"]; var dnsString = props["DNSEndString"]; Func<bool> test = () => dnsString.Split(',').Select(s => s.Trim()).Select(en => IPLocationTests.FindLocalIpName().EndsWith(en)).Where(t => t).Any(); l.LocationTests.Add(() => test()); var fetcher = FetchToRemoteLinuxDirInstance.FetchRemoteLinuxInstance(props); var dsfinder = new GRIDFetchToLinuxVisibleOnWindows(new DirectoryInfo(props["WindowsPath"]), fetcher, props["LinuxPath"]); l.GetDSInfo = name => { var nfiles = dsfinder.CheckNumberOfFiles(name); return new DSInfo() { Name = name, IsLocal = filter => dsfinder.CheckIfLocal(name, filter), CanBeGeneratedAutomatically = true, ListOfFiles = () => dsfinder.ListOfFiles(name), LocationProvider = l, }; }; l.GetDS = dsfinder.GetDS; return l; }
public void AttemptToDownloadBadDS() { // Seen in wild: try to download a bad dataset, and it creates a directory // anyway. Ops! var dsinfo = MakeDSInfo("ds1.1.1"); var d = new DirectoryInfo("AttemptToDownloadBadDS"); if (d.Exists) { d.Delete(true); } var ld = new LinuxMirrorDownloaderPretend(d, "forkitover"); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store"); try { var r = gf.GetDS(dsinfo); } catch (ArgumentException) { // Expecting it to throw here - no dataset should exist by that name! } // Did a local directory get created? d.Refresh(); Assert.IsFalse(d.Exists); }
public void CountFilesDownloaded() { var dsinfo = MakeDSInfo("ds1.1.1"); var d = utils.BuildSampleDirectoryBeforeBuild("CountFilesDownloaded", dsinfo.Name); utils.MakePartial(d, dsinfo.Name); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null); Assert.AreEqual(5, gf.CheckNumberOfFiles(dsinfo.Name)); }
public void DownloadAllFilesSecondTime() { var dsinfo = MakeDSInfo("ds1.1.1"); var d = new DirectoryInfo("DownloadAllFilesSecondTime"); if (d.Exists) { d.Delete(true); } var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store"); var r = gf.GetDS(dsinfo, fileFilter: flist => flist.OrderBy(f => f).Take(1).ToArray()); Assert.AreEqual(1, r.Length); r = gf.GetDS(dsinfo); Assert.AreEqual(5, r.Length); Assert.AreEqual(5, d.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Where(f => !f.FullName.EndsWith(".part")).Count()); }
public void MakeSureContentsFileIsUpdated() { var dsinfo = MakeDSInfo("ds1.1.1"); var d = new DirectoryInfo("MakeSureContentsFileIsUpdated"); if (d.Exists) { d.Delete(true); } d.Create(); var f = new FileInfo(Path.Combine(d.FullName, "contents.txt")); using (var wr = f.CreateText()) { wr.WriteLine("otherdataset.data.set gwatts"); } var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store"); var r = gf.GetDS(dsinfo); Assert.IsTrue(f.Exists); Assert.AreEqual("otherdataset.data.set gwatts", f.ReadLine(1)); Assert.IsTrue(f.ReadLine(2).StartsWith(dsinfo.Name)); }
public void MakeSureContentsFileIsCreated() { var dsinfo = MakeDSInfo("ds1.1.1"); var d = new DirectoryInfo("MakeSureContentsFileIsCreated"); if (d.Exists) { d.Delete(true); } var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store"); var r = gf.GetDS(dsinfo); var f = new FileInfo(Path.Combine(d.FullName, "contents.txt")); Assert.IsTrue(f.Exists); Assert.IsTrue(f.ReadLine().StartsWith(dsinfo.Name)); }
public void GetFileListForLocalFilesWithNoWork() { var dsinfo = MakeDSInfo("ds1.1.1.1"); var d = utils.BuildSampleDirectoryBeforeBuild("GetFileListForLocalFilesWithNoWork", dsinfo.Name); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null); Assert.AreEqual(5, gf.ListOfFiles("ds1.1.1.1").Length); }
public void FindSomeFilesWithScopedDataset() { var dsinfo = MakeDSInfo("user.norm:ds1.1.1"); var d = utils.BuildSampleDirectoryBeforeBuild("FindLocationWithScopedDataset", "ds1.1.1"); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null); var r = gf.GetDS(dsinfo, fileFilter: fs => fs.Where(fname => fname.Contains("root.1")).ToArray()); Assert.IsNotNull(r); Assert.AreEqual(1, r.Length); }
public void FindLocationWithScopedDataset() { var dsinfo = MakeDSInfo("user.norm:ds1.1.1"); var d = utils.BuildSampleDirectoryBeforeBuild("FindLocationWithScopedDataset", "ds1.1.1"); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null); var r = gf.GetDS(dsinfo); Assert.IsNotNull(r); Assert.AreEqual(5, r.Length); }
public void FindLocalFilesWithNoWork() { var dsinfo = MakeDSInfo("ds1.1.1.1"); var d = utils.BuildSampleDirectoryBeforeBuild("FindLocalFilesWithNoWork", dsinfo.Name); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, null, null); var r = gf.GetDS(dsinfo); Assert.IsNotNull(r); Assert.AreEqual(5, r.Length); }
public void DownloadToLinuxDirectoryThatIsAWindowsDirectoryWithScoppedDS() { var dsinfo = MakeDSInfo("user.norm:ds1.1.1"); var d = new DirectoryInfo("DownloadToLinuxDirectoryThatIsAWindowsDirectory"); if (d.Exists) { d.Delete(true); } var ld = new LinuxMirrorDownloaderPretend(d, "ds1.1.1"); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store"); var r = gf.GetDS(dsinfo); Assert.IsNotNull(r); Assert.AreEqual(5, r.Length); Assert.AreEqual("/bogus/files/store/ds1.1.1", ld.LinuxDest); }
public void DownloadToLinuxDirectoryThatIsAWindowsDirectoryAndAreadyCreated() { // Seen in the wild. A crash (or other interruption) means the dataset directory has // been created, but is empty (for whatever reason). In that particular case, we should // treat it as not there. var dsinfo = MakeDSInfo("ds1.1.1"); var d = new DirectoryInfo("DownloadToLinuxDirectoryThatIsAWindowsDirectory"); if (d.Exists) { d.Delete(true); } d.Create(); var dsdir = d.SubDir(dsinfo.Name); dsdir.Create(); var ld = new LinuxMirrorDownloaderPretend(d, dsinfo.Name); var gf = new GRIDFetchToLinuxVisibleOnWindows(d, ld, "/bogus/files/store"); var r = gf.GetDS(dsinfo); Assert.IsNotNull(r); Assert.AreEqual(5, r.Length); Assert.AreEqual("/bogus/files/store/ds1.1.1", ld.LinuxDest); }