/// <summary> /// Given the dictionary of parameters, create a place. /// </summary> /// <param name="info"></param> /// <returns></returns> private static async Task <IEnumerable <IPlace> > ParseSingleConfig(Dictionary <string, string> info) { // Check to see if there are any restrictions on using these guys bool isGoodConfig = true; if (info.ContainsKey("UseOnlyWhenDNSEndStringIs")) { // We will use this iff the DNS string ends with the argument. isGoodConfig = (await CheckDNS(info["UseOnlyWhenDNSEndStringIs"])).Any(t => t); } if (info.ContainsKey("UseWhenDNSEndStringIsnt")) { // We will use this config only if the IP address doesn't end in one of the given ones here. if (isGoodConfig) { isGoodConfig = (await CheckDNS(info["UseWhenDNSEndStringIsnt"])).All(t => !t); } } var result = new List <IPlace>(); if (isGoodConfig) { // Now, create the end points as needed. var type = info["LocationType"]; if (type == "LocalWindowsFilesystem") { result.Add(CreateWindowsFilesystemPlace(info)); } else if (type == "LinuxWithLocalAndGRID") { // First, do the linux remote var name = info["Name"]; var p_linux = CreateLinuxFilesystemPlace($"{name}-linux", info); result.Add(p_linux); // Next, the grid var p_grid = new PlaceGRID($"{name}-GRID", p_linux); result.Add(p_grid); // Next, the local if that is "ok" to create. var localVisible = info.ContainsKey("WindowsAccessibleDNSEndString") ? (await CheckDNS(info["WindowsAccessibleDNSEndString"])).Any(t => t) : true; if (localVisible) { Trace.WriteLine($"Linux-GRID-Local {name} is locally visible.", "ParseSingleConfig"); // Create the local disk - but this is a big server, so a copy confirmation isn't needed var ip = CreateWindowsFilesystemPlace(info, needsCopyConfirmation: false); result.Add(ip); } } else { throw new UnknownLocationTypeException($"Location type {type} is not known as read from the configuration file."); } } return(result); }
public async Task HasExpectedFile() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); Assert.IsTrue(await grid_p.HasFileAsync(new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"))); }
public async Task DoesNotHaveWeirdFile() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); Assert.IsFalse(await grid_p.HasFileAsync(new Uri($"gridds://{_good_dsname}/myfile.root"))); }
public void CanCopyToProperLocation() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); Assert.IsTrue(grid_p.CanSourceCopy(local_p)); }
public async Task GetNonDatasetFileList() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var files = await grid_p.GetListOfFilesForDataSetAsync("mc15_13TeV.361032.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ12W.merge.DAOD_EXOT15.bogus"); Assert.IsNull(files); }
public async Task GetExistingDatasetFileList() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var files = await grid_p.GetListOfFilesForDataSetAsync(_good_dsname); Assert.AreEqual(197, files.Length); // This might not be safe b.c. file order might not be idempotent, but try this for now. Assert.AreEqual(_good_dsfile_1, files[0]); }
public async Task CopyOneFilesTunnel() { // Should be setup to deal with about 4 GB of data in one file. _ssh = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnelBigData"); _ssh.CreateRepro(); var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var uris = new Uri[] { new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"), }; await grid_p.CopyToAsync(local_p, uris); var files = await local_p.GetListOfFilesForDataSetAsync(_good_dsname); Assert.AreEqual(197, files.Length); Assert.IsTrue(await local_p.HasFileAsync(uris[0])); }
public async Task CopyTwoFiles() { _ssh.CreateRepro(); var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var uris = new Uri[] { new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"), new Uri($"gridds://{_good_dsname}/{_good_dsfile_2}"), }; await grid_p.CopyToAsync(local_p, uris); var files = await local_p.GetListOfFilesForDataSetAsync(_good_dsname); Assert.AreEqual(197, files.Length); Assert.IsTrue(await local_p.HasFileAsync(uris[0])); Assert.IsTrue(await local_p.HasFileAsync(uris[1])); }