public void CopyWhenAlreadyThere() { // The whole thing is already there. _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var ssh2 = new UtilsForBuildingLinuxDatasets(); ssh2.RemotePath += "2"; ssh2.CreateRepro(); ssh2.CreateDS("ds1", "f1.root", "f2.root"); var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var p2 = new PlaceLinuxRemote("test", ssh2.RemotePath, _ssh.RemoteHostInfo); var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") }; p1.CopyTo(p2, fileList); Assert.IsTrue(p2.HasFile(fileList[0])); var allfiles = ssh2.GetAllFilesInRepro("ds1"); foreach (var f in allfiles) { Console.WriteLine(f); } Assert.AreEqual(1, allfiles.Where(l => l.Contains("f1.root")).Count()); }
public async Task CopyWhenOneAlreadyThereAsPart() { // The whole thing is already there. _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var ssh2 = new UtilsForBuildingLinuxDatasets(); ssh2.RemotePath += "2"; ssh2.CreateRepro(); ssh2.CreateDS("ds1", "f1.root", "f2.root"); ssh2.RemoveFileInDS("ds1", "f1.root"); ssh2.AddFileToDS("ds1", "f1.root.part"); var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var p2 = new PlaceLinuxRemote("test", ssh2.RemotePath, _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p1, p2); var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") }; await p1.CopyToAsync(p2, fileList); Assert.IsTrue(await p2.HasFileAsync(fileList[0])); Assert.IsTrue(await p2.HasFileAsync(fileList[1])); }
public void CanSourceCopyFromSelf() { _ssh = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnel"); var p1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo); var p2 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo); Assert.IsTrue(p1.CanSourceCopy(p2)); }
public void CanNotSourceCopyToNonISCPTarget() { var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var other = new DummyPlace("testmeout"); DataSetManager.ResetDSM(p, other); Assert.IsFalse(p.CanSourceCopy(other)); }
public void CanCopyToProperLocation() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); Assert.IsTrue(grid_p.CanSourceCopy(local_p)); }
public async Task HasExpectedFile() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); Assert.IsTrue(await grid_p.HasFileAsync(new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"))); }
public async Task DoesNotHaveWeirdFile() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); Assert.IsFalse(await grid_p.HasFileAsync(new Uri($"gridds://{_good_dsname}/myfile.root"))); }
public async Task HasFileGoodFile() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p); Assert.IsTrue(await p.HasFileAsync(new Uri("gridds://ds1/f1.root"))); }
public void CanNotSourceCopyFromSomeoneSomewhereElse() { var _ssh1 = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnel"); var p1 = new PlaceLinuxRemote("test1", _ssh1.RemotePath, _ssh1.RemoteHostInfo); var _ssh2 = new UtilsForBuildingLinuxDatasets(); var p2 = new PlaceLinuxRemote("test1", _ssh2.RemotePath, _ssh2.RemoteHostInfo); DataSetManager.ResetDSM(p1, p2); Assert.IsFalse(p2.CanSourceCopy(p1)); }
public async Task GetReproDatasetFileListForBadDS() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p); var files = await p.GetListOfFilesForDataSetAsync("ds2"); Assert.IsNull(files); }
public async Task GetNonDatasetFileList() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var files = await grid_p.GetListOfFilesForDataSetAsync("mc15_13TeV.361032.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ12W.merge.DAOD_EXOT15.bogus"); Assert.IsNull(files); }
public async Task HasPartFileInDataset() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); _ssh.RemoveFileInDS("ds1", "f1.root"); _ssh.AddFileToDS("ds1", "f1.root.part"); var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p); Assert.IsFalse(await p.HasFileAsync(new Uri("gridds://ds1/f1.root"))); }
public void CanNotSourceCopyToISCPTarget() { var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var other = new ScpTargetDummy() { DoVisibility = false }; DataSetManager.ResetDSM(p, other); Assert.IsFalse(p.CanSourceCopy(other)); }
public async Task GetExistingDatasetFileList() { var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var files = await grid_p.GetListOfFilesForDataSetAsync(_good_dsname); Assert.AreEqual(197, files.Length); // This might not be safe b.c. file order might not be idempotent, but try this for now. Assert.AreEqual(_good_dsfile_1, files[0]); }
public async Task GetLocalFileURIsOnLinux() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p); var files = await p.GetLocalFileLocationsAsync(new [] { new Uri("gridds://ds1/f1.root") }); Assert.IsNotNull(files); Assert.AreEqual(1, files.Count()); Assert.AreEqual("file://test/phys/users/gwatts/bogus_ds_repro/ds1/files/f1.root", files.First().ToString()); }
public async Task LinuxRemoteCopyToFromBadDS() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var p2 = new PlaceLinuxRemote("test", _ssh.RemotePath + "2", _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p1, p2); var fileList = new Uri[] { new Uri("gridds://ds2/f1.root"), new Uri("gridds://ds2/f2.root") }; await p1.CopyToAsync(p2, fileList); Assert.IsTrue(await p2.HasFileAsync(fileList[0])); }
public async Task CopyFrom() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var p1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo); var p2 = new PlaceLinuxRemote("test2", _ssh.RemotePath + "2", _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p1, p2); var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") }; await p2.CopyFromAsync(p1, fileList); Assert.IsTrue(await p2.HasFileAsync(fileList[0])); Assert.AreEqual(2, (await p2.GetListOfFilesForDataSetAsync("ds1")).Length); }
public async Task CopyToWithMissingFile() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); _ssh.RemoveFileInDS("ds1", "f1.root"); var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var p2 = new PlaceLinuxRemote("test", _ssh.RemotePath + "2", _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p1, p2); var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") }; // This should fail because there is a missing file and we've explicitly requested both files to be copied. await p1.CopyToAsync(p2, fileList); Assert.IsTrue(await p2.HasFileAsync(fileList[0])); }
public async Task CopyFromViaTunnelWithPassword() { _ssh = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnelBigData"); _ssh.CreateRepro(); var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var sshRemote = new UtilsForBuildingLinuxDatasets(); sshRemote.CreateRepro(); sshRemote.CreateDS("ds1", "f1.root", "f2.root"); var p2 = new PlaceLinuxRemote("test", sshRemote.RemotePath, sshRemote.RemoteHostInfo); DataSetManager.ResetDSM(p1, p2); var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") }; await p1.CopyFromAsync(p2, fileList); Assert.IsTrue(await p1.HasFileAsync(fileList[0])); }
public async Task CopyFromSCPTarget() { // Build remote dataset up on linux _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var place1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo); var repro1 = BuildRepro("repro2"); var place2 = new PlaceLocalWindowsDisk("test1", repro1); DataSetManager.ResetDSM(place1, place2); var uris = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") }; await place2.CopyFromAsync(place1, uris); var files = await place2.GetListOfFilesForDataSetAsync("ds1"); Assert.AreEqual(2, files.Length); Assert.IsTrue(await place2.HasFileAsync(uris[0])); Assert.IsTrue(await place2.HasFileAsync(uris[1])); }
public async Task CopyToTwoStep() { _ssh.CreateRepro(); _ssh.CreateDS("ds1", "f1.root", "f2.root"); var p1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo); _ssh.CreateRepro(_ssh.RemotePath + "2"); var p2 = new PlaceLinuxRemote("test2", _ssh.RemotePath + "2", _ssh.RemoteHostInfo); DataSetManager.ResetDSM(p1, p2); var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") }; await p1.CopyToAsync(p2, fileList.Take(1).ToArray()); Assert.IsTrue(await p2.HasFileAsync(fileList[0])); Assert.IsFalse(await p2.HasFileAsync(fileList[1])); await p1.CopyToAsync(p2, fileList.Skip(1).ToArray()); Assert.IsTrue(await p2.HasFileAsync(fileList[1])); }
public async Task CopyOneFilesTunnel() { // Should be setup to deal with about 4 GB of data in one file. _ssh = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnelBigData"); _ssh.CreateRepro(); var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var uris = new Uri[] { new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"), }; await grid_p.CopyToAsync(local_p, uris); var files = await local_p.GetListOfFilesForDataSetAsync(_good_dsname); Assert.AreEqual(197, files.Length); Assert.IsTrue(await local_p.HasFileAsync(uris[0])); }
public async Task CopyTwoFiles() { _ssh.CreateRepro(); var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo); var grid_p = new PlaceGRID("test-GRID", local_p); DataSetManager.ResetDSM(local_p, grid_p); var uris = new Uri[] { new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"), new Uri($"gridds://{_good_dsname}/{_good_dsfile_2}"), }; await grid_p.CopyToAsync(local_p, uris); var files = await local_p.GetListOfFilesForDataSetAsync(_good_dsname); Assert.AreEqual(197, files.Length); Assert.IsTrue(await local_p.HasFileAsync(uris[0])); Assert.IsTrue(await local_p.HasFileAsync(uris[1])); }