public void CopyWhenAlreadyThere()
{
// The whole thing is already there.
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var ssh2 = new UtilsForBuildingLinuxDatasets();
ssh2.RemotePath += "2";
ssh2.CreateRepro();
ssh2.CreateDS("ds1", "f1.root", "f2.root");
var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var p2 = new PlaceLinuxRemote("test", ssh2.RemotePath, _ssh.RemoteHostInfo);
var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") };
p1.CopyTo(p2, fileList);
Assert.IsTrue(p2.HasFile(fileList[0]));
var allfiles = ssh2.GetAllFilesInRepro("ds1");
foreach (var f in allfiles)
{
Console.WriteLine(f);
}
Assert.AreEqual(1, allfiles.Where(l => l.Contains("f1.root")).Count());
}
public async Task CopyWhenOneAlreadyThereAsPart()
{
// The whole thing is already there.
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var ssh2 = new UtilsForBuildingLinuxDatasets();
ssh2.RemotePath += "2";
ssh2.CreateRepro();
ssh2.CreateDS("ds1", "f1.root", "f2.root");
ssh2.RemoveFileInDS("ds1", "f1.root");
ssh2.AddFileToDS("ds1", "f1.root.part");
var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var p2 = new PlaceLinuxRemote("test", ssh2.RemotePath, _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p1, p2);
var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") };
await p1.CopyToAsync(p2, fileList);
Assert.IsTrue(await p2.HasFileAsync(fileList[0]));
Assert.IsTrue(await p2.HasFileAsync(fileList[1]));
}
public void CanSourceCopyFromSelf()
{
_ssh = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnel");
var p1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo);
var p2 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo);
Assert.IsTrue(p1.CanSourceCopy(p2));
}
public void CanNotSourceCopyToNonISCPTarget()
{
var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var other = new DummyPlace("testmeout");
DataSetManager.ResetDSM(p, other);
Assert.IsFalse(p.CanSourceCopy(other));
}
public void CanCopyToProperLocation()
{
var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var grid_p = new PlaceGRID("test-GRID", local_p);
DataSetManager.ResetDSM(local_p, grid_p);
Assert.IsTrue(grid_p.CanSourceCopy(local_p));
}
public async Task HasExpectedFile()
{
var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var grid_p = new PlaceGRID("test-GRID", local_p);
DataSetManager.ResetDSM(local_p, grid_p);
Assert.IsTrue(await grid_p.HasFileAsync(new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}")));
}
public async Task DoesNotHaveWeirdFile()
{
var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var grid_p = new PlaceGRID("test-GRID", local_p);
DataSetManager.ResetDSM(local_p, grid_p);
Assert.IsFalse(await grid_p.HasFileAsync(new Uri($"gridds://{_good_dsname}/myfile.root")));
}
public async Task HasFileGoodFile()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p);
Assert.IsTrue(await p.HasFileAsync(new Uri("gridds://ds1/f1.root")));
}
public void CanNotSourceCopyFromSomeoneSomewhereElse()
{
var _ssh1 = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnel");
var p1 = new PlaceLinuxRemote("test1", _ssh1.RemotePath, _ssh1.RemoteHostInfo);
var _ssh2 = new UtilsForBuildingLinuxDatasets();
var p2 = new PlaceLinuxRemote("test1", _ssh2.RemotePath, _ssh2.RemoteHostInfo);
DataSetManager.ResetDSM(p1, p2);
Assert.IsFalse(p2.CanSourceCopy(p1));
}
public async Task GetReproDatasetFileListForBadDS()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p);
var files = await p.GetListOfFilesForDataSetAsync("ds2");
Assert.IsNull(files);
}
public async Task GetNonDatasetFileList()
{
var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var grid_p = new PlaceGRID("test-GRID", local_p);
DataSetManager.ResetDSM(local_p, grid_p);
var files = await grid_p.GetListOfFilesForDataSetAsync("mc15_13TeV.361032.Pythia8EvtGen_A14NNPDF23LO_jetjet_JZ12W.merge.DAOD_EXOT15.bogus");
Assert.IsNull(files);
}
public async Task HasPartFileInDataset()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
_ssh.RemoveFileInDS("ds1", "f1.root");
_ssh.AddFileToDS("ds1", "f1.root.part");
var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p);
Assert.IsFalse(await p.HasFileAsync(new Uri("gridds://ds1/f1.root")));
}
public void CanNotSourceCopyToISCPTarget()
{
var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var other = new ScpTargetDummy()
{
DoVisibility = false
};
DataSetManager.ResetDSM(p, other);
Assert.IsFalse(p.CanSourceCopy(other));
}
public async Task GetExistingDatasetFileList()
{
var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var grid_p = new PlaceGRID("test-GRID", local_p);
DataSetManager.ResetDSM(local_p, grid_p);
var files = await grid_p.GetListOfFilesForDataSetAsync(_good_dsname);
Assert.AreEqual(197, files.Length);
// This might not be safe b.c. file order might not be idempotent, but try this for now.
Assert.AreEqual(_good_dsfile_1, files[0]);
}
public async Task GetLocalFileURIsOnLinux()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p);
var files = await p.GetLocalFileLocationsAsync(new [] { new Uri("gridds://ds1/f1.root") });
Assert.IsNotNull(files);
Assert.AreEqual(1, files.Count());
Assert.AreEqual("file://test/phys/users/gwatts/bogus_ds_repro/ds1/files/f1.root", files.First().ToString());
}
public async Task LinuxRemoteCopyToFromBadDS()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var p2 = new PlaceLinuxRemote("test", _ssh.RemotePath + "2", _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p1, p2);
var fileList = new Uri[] { new Uri("gridds://ds2/f1.root"), new Uri("gridds://ds2/f2.root") };
await p1.CopyToAsync(p2, fileList);
Assert.IsTrue(await p2.HasFileAsync(fileList[0]));
}
public async Task CopyFrom()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var p1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo);
var p2 = new PlaceLinuxRemote("test2", _ssh.RemotePath + "2", _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p1, p2);
var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") };
await p2.CopyFromAsync(p1, fileList);
Assert.IsTrue(await p2.HasFileAsync(fileList[0]));
Assert.AreEqual(2, (await p2.GetListOfFilesForDataSetAsync("ds1")).Length);
}
public async Task CopyToWithMissingFile()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
_ssh.RemoveFileInDS("ds1", "f1.root");
var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var p2 = new PlaceLinuxRemote("test", _ssh.RemotePath + "2", _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p1, p2);
var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") };
// This should fail because there is a missing file and we've explicitly requested both files to be copied.
await p1.CopyToAsync(p2, fileList);
Assert.IsTrue(await p2.HasFileAsync(fileList[0]));
}
public async Task CopyFromViaTunnelWithPassword()
{
_ssh = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnelBigData");
_ssh.CreateRepro();
var p1 = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var sshRemote = new UtilsForBuildingLinuxDatasets();
sshRemote.CreateRepro();
sshRemote.CreateDS("ds1", "f1.root", "f2.root");
var p2 = new PlaceLinuxRemote("test", sshRemote.RemotePath, sshRemote.RemoteHostInfo);
DataSetManager.ResetDSM(p1, p2);
var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") };
await p1.CopyFromAsync(p2, fileList);
Assert.IsTrue(await p1.HasFileAsync(fileList[0]));
}
public async Task CopyFromSCPTarget()
{
// Build remote dataset up on linux
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var place1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo);
var repro1 = BuildRepro("repro2");
var place2 = new PlaceLocalWindowsDisk("test1", repro1);
DataSetManager.ResetDSM(place1, place2);
var uris = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") };
await place2.CopyFromAsync(place1, uris);
var files = await place2.GetListOfFilesForDataSetAsync("ds1");
Assert.AreEqual(2, files.Length);
Assert.IsTrue(await place2.HasFileAsync(uris[0]));
Assert.IsTrue(await place2.HasFileAsync(uris[1]));
}
public async Task CopyToTwoStep()
{
_ssh.CreateRepro();
_ssh.CreateDS("ds1", "f1.root", "f2.root");
var p1 = new PlaceLinuxRemote("test1", _ssh.RemotePath, _ssh.RemoteHostInfo);
_ssh.CreateRepro(_ssh.RemotePath + "2");
var p2 = new PlaceLinuxRemote("test2", _ssh.RemotePath + "2", _ssh.RemoteHostInfo);
DataSetManager.ResetDSM(p1, p2);
var fileList = new Uri[] { new Uri("gridds://ds1/f1.root"), new Uri("gridds://ds1/f2.root") };
await p1.CopyToAsync(p2, fileList.Take(1).ToArray());
Assert.IsTrue(await p2.HasFileAsync(fileList[0]));
Assert.IsFalse(await p2.HasFileAsync(fileList[1]));
await p1.CopyToAsync(p2, fileList.Skip(1).ToArray());
Assert.IsTrue(await p2.HasFileAsync(fileList[1]));
}
public async Task CopyOneFilesTunnel()
{
// Should be setup to deal with about 4 GB of data in one file.
_ssh = new UtilsForBuildingLinuxDatasets("LinuxRemoteTestTunnelBigData");
_ssh.CreateRepro();
var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var grid_p = new PlaceGRID("test-GRID", local_p);
DataSetManager.ResetDSM(local_p, grid_p);
var uris = new Uri[]
{
new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"),
};
await grid_p.CopyToAsync(local_p, uris);
var files = await local_p.GetListOfFilesForDataSetAsync(_good_dsname);
Assert.AreEqual(197, files.Length);
Assert.IsTrue(await local_p.HasFileAsync(uris[0]));
}
public async Task CopyTwoFiles()
{
_ssh.CreateRepro();
var local_p = new PlaceLinuxRemote("test", _ssh.RemotePath, _ssh.RemoteHostInfo);
var grid_p = new PlaceGRID("test-GRID", local_p);
DataSetManager.ResetDSM(local_p, grid_p);
var uris = new Uri[]
{
new Uri($"gridds://{_good_dsname}/{_good_dsfile_1}"),
new Uri($"gridds://{_good_dsname}/{_good_dsfile_2}"),
};
await grid_p.CopyToAsync(local_p, uris);
var files = await local_p.GetListOfFilesForDataSetAsync(_good_dsname);
Assert.AreEqual(197, files.Length);
Assert.IsTrue(await local_p.HasFileAsync(uris[0]));
Assert.IsTrue(await local_p.HasFileAsync(uris[1]));
}
