public void NormalCall() { // Make sure it doesn't break. var location = new Locator(); var ls = location.FindBestLocations(); Assert.IsNotNull(ls); }
public void GetCERNLocation() { AtlasWorkFlows.Utils.IPLocationTests.SetIpName("bogus.nytimes.com"); var locator = new Locator(); var lst = locator.FindLocation("MyTestLocation"); Assert.IsNotNull(lst); }
public void ScreenLocations() { AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch"); Locator.SetLocationFilter(loc => false); var locator = new Locator(); var lst = locator.FindBestLocations(); Assert.IsNotNull(lst); Assert.AreEqual(0, lst.Length); }
public void CERNNotAtCERN() { AtlasWorkFlows.Utils.IPLocationTests.SetIpName("bogus.nytimes.com"); var locator = new Locator(); var lst = locator.FindBestLocations(); Assert.IsNotNull(lst); var cern = lst.Where(l => l.Name == "MyTestLocation").FirstOrDefault(); Assert.IsNull(cern); }
/// <summary> /// Find the DSInfo with the best possible location provider (local, remote, etc.). /// </summary> /// <param name="datasetname">The GRID dataset name to find pointers to.</param> /// <param name="fileFilter">Filter the potential list of files to be returned or downloaded (can really speed up things if you want one file in a large dataset).</param> /// <param name="locationFilter">Filter out the locations that we are allowed to fetch the file from.</param> /// <returns></returns> public static DSInfo FetchDSInfo(string datasetname, Func<string[], string[]> fileFilter, Func<string, bool> locationFilter) { // Basic parameter checks if (string.IsNullOrWhiteSpace(datasetname)) { throw new ArgumentException("Dataset name is empty"); } // Get a location so we can see if we can fetch the dataset var locator = new Locator(); var locationList = locator.FindBestLocations(); if (locationFilter != null) { locationList = locationList.Where(loc => locationFilter(loc.Name)).ToArray(); } if (locationList.Length == 0) { throw new InvalidOperationException(string.Format("There are no valid ways to download dataset '{0}' from your current location.", datasetname)); } // Get the info for each dataset. var allDSInfo = (from loc in locationList let dsi = loc.GetDSInfo(datasetname) select new { dsinfo = dsi, islocal = dsi.IsLocal(fileFilter) }).ToArray(); // First, if anyone has something local, then we should grab that. var localDS = allDSInfo.Where(d => d.islocal).OrderByDescending(d => d.dsinfo.LocationProvider.Priority).ToArray(); // Ok, now we need the ones that can generate it. if (localDS.Length == 0) { localDS = allDSInfo.Where(d => d.dsinfo.CanBeGeneratedAutomatically).OrderByDescending(d => d.dsinfo.LocationProvider.Priority).ToArray(); } // Did we strike out? var locationInfo = localDS.FirstOrDefault(); if (locationInfo == null) { var locationListText = ""; foreach (var l in locationList) { locationListText += l.Name + " "; } throw new ArgumentException(string.Format("Do not know how to generate the dataset '{0}' at any of the locations {1} (which are the only ones working where the computer is located)", datasetname, locationListText)); } var dsinfo = locationInfo.dsinfo; return dsinfo; }
public void FailBecauseLocalRepoNotCreated() { AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch"); var dsname = "ds1.1.1"; var d1 = utils.BuildSampleDirectoryBeforeBuild("LoadNewFilesToLocalWhenMissingRemote", dsname); var d2 = new DirectoryInfo("FailBecauseLocalRepoNotCreated"); if (d2.Exists) { d2.Delete(true); } d2.Refresh(); Locator._getLocations = () => utils.GetLocal(d1, d2); var locator = new Locator(); var l = locator.FindLocation("MyTestLocalLocation"); var r = l.GetDSInfo("ds1.1.1"); Assert.IsFalse(r.IsLocal(null)); var files = l.GetDS(r, null, null, null, 0); }
/// <summary> /// Main routine to find all the URI's that point to a dataset. /// </summary> /// <param name="datasetname">The GRID dataset name to find pointers to.</param> /// <param name="fileFilter">Filter the potential list of files to be returned or downloaded (can really speed up things if you want one file in a large dataset).</param> /// <param name="locationFilter">Filter out the locations that we are allowed to fetch the file from.</param> /// <param name="statusUpdate">Downloads from the grid can take a long time. Status updates will be posted here if not null.</param> /// <returns></returns> public static Uri[] FetchDatasetUrisAtLocation(string locationName, string datasetname, Action<string> statusUpdate = null, Func<string[], string[]> fileFilter = null, Func<bool> failNow = null, int timeoutDownloadSecs = 3600) { var locator = new Locator(); var location = locator.FindLocation(locationName); if (location == null) { throw new InvalidOperationException(string.Format("Location {0} isn't known.", locationName)); } if (!location.LocationIsGood()) { throw new InvalidOperationException(string.Format("Location {0} isn't active and can't be used", locationName)); } var dsinfo = location.GetDSInfo(datasetname); // And delegate all the rest of our work to fetching. return dsinfo.LocationProvider.GetDS(dsinfo, statusUpdate, fileFilter, failNow, timeoutDownloadSecs); }
public void FetchOneMissingFileFromBackupSource() { // Dataset isn't here and there is no remote... Backup to the rescue! var dsname = "ds1.1.1"; var d2 = new DirectoryInfo("FetchOneMissingFileFromBackupSourceLocal"); if (d2.Exists) { d2.Delete(true); } d2.Create(); d2.Refresh(); Locator._getLocations = () => utils.GetLocal(null, d2); // Configure a dummy fetcher that will get the datasets (they will get pulled when things are created). // This isn't a real Linux downloader, so we need to put the files in some other place var dtemp = new DirectoryInfo("FetchOneMissingFileFromBackupSourceRemoteLinuxDownload"); if (dtemp.Exists) { dtemp.Delete(true); } dtemp.Create(); FetchToRemoteLinuxDirInstance._test = new LinuxMirrorDownloaderPretend(dtemp, dsname); var locator = new Locator(); var l = locator.FindLocation("MyTestLocalLocation"); var r = l.GetDSInfo("ds1.1.1"); Assert.IsFalse(r.IsLocal(null)); var files = l.GetDS(r, null, null, null, 0); Assert.AreEqual(5, files.Length); Assert.IsTrue(files[0].LocalPath.Contains("FetchOneMissingFileFromBackupSourceLocal")); Assert.AreEqual(5, d2.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Where(f => !f.Name.EndsWith(".part")).Count()); }
public void LoadTwoFilesToLocalWhenMissing() { // Setup the infrastructure. For Local to fetch files, it will callback into the global // infrastructure - so that needs to be setup for the test. AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch"); var dsname = "ds1.1.1"; var d1 = utils.BuildSampleDirectoryBeforeBuild("LoadNewFilesToLocalWhenMissingRemote", dsname); var d2 = new DirectoryInfo("LoadNewFilesToLocalWhenMissingLocal"); if (d2.Exists) { d2.Delete(true); } d2.Create(); d2.Refresh(); Locator._getLocations = () => utils.GetLocal(d1, d2); var locator = new Locator(); var l = locator.FindLocation("MyTestLocalLocation"); var r = l.GetDSInfo("ds1.1.1"); Assert.IsFalse(r.IsLocal(null)); var files = l.GetDS(r, null, fslist => fslist.Take(2).ToArray(), null, 0); Assert.AreEqual(2, files.Length); Assert.IsTrue(files[0].LocalPath.Contains("LoadNewFilesToLocalWhenMissingLocal")); Assert.AreEqual(2, d2.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Count()); }
public void LoadNewFilesToLocalWhenEmptyDirectory() { // Seen in the wild. Local directory is actually there for the dataset, but // empty due to an earlier crash. Make sure the copy still occurs. AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch"); var dsname = "ds1.1.1"; var d1 = utils.BuildSampleDirectoryBeforeBuild("LoadNewFilesToLocalWhenMissingRemote", dsname); var d2 = new DirectoryInfo("LoadNewFilesToLocalWhenMissingLocal"); if (d2.Exists) { d2.Delete(true); } d2.Create(); d2.Refresh(); d2.SubDir("ds1.1.1").Create(); Locator._getLocations = () => utils.GetLocal(d1, d2); var locator = new Locator(); var l = locator.FindLocation("MyTestLocalLocation"); var r = l.GetDSInfo("ds1.1.1"); Assert.IsFalse(r.IsLocal(null)); var files = l.GetDS(r, null, null, null, 0); Assert.AreEqual(5, files.Length); Assert.IsTrue(files[0].LocalPath.Contains("LoadNewFilesToLocalWhenMissingLocal")); Assert.AreEqual("ds1.1.1", d2.EnumerateDirectories().First().Name); }
public void FetchOneMissingFileFromRemote() { // Local is complete in all but one file. Force a fetch of just that one file. // Setup the infrastructure. For Local to fetch files, it will callback into the global // infrastructure - so that needs to be setup for the test. AtlasWorkFlows.Utils.IPLocationTests.SetIpName("pc.cern.ch"); var dsname = "ds1.1.1"; var d1 = utils.BuildSampleDirectoryBeforeBuild("FetchOneMissingFileFromRemoteRemote", dsname); var d2 = utils.BuildSampleDirectoryBeforeBuild("FetchOneMissingFileFromRemoteLocal", dsname); Locator._getLocations = () => utils.GetLocal(d1, d2); var fbad = new FileInfo(Path.Combine(d2.FullName, "ds1.1.1", "sub2", "file.root.5")); Assert.IsTrue(fbad.Exists); fbad.Delete(); utils.MakePartial(d2, "ds1.1.1"); var locator = new Locator(); var l = locator.FindLocation("MyTestLocalLocation"); var r = l.GetDSInfo("ds1.1.1"); Assert.IsFalse(r.IsLocal(null)); var files = l.GetDS(r, null, null, null, 0); Assert.AreEqual(5, files.Length); Assert.IsTrue(files[0].LocalPath.Contains("FetchOneMissingFileFromRemoteLocal")); Assert.AreEqual(5, d2.EnumerateFiles("*.root.*", SearchOption.AllDirectories).Where(f => !f.Name.EndsWith(".part")).Count()); }