public void HadoopFileSystemGetDirectoryTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
// valid parameters
fileSystem.CreateDirectory(_directoryPath);
fileSystem.CreateDirectory(_directoryPath + "/InternalDirectory");
Assert.AreEqual("/", fileSystem.GetDirectory("/"));
Assert.AreEqual(_directoryPath, fileSystem.GetDirectory(_directoryPath));
Assert.AreEqual(_directoryPath, fileSystem.GetDirectory(_directoryPath + "/"));
Assert.AreEqual(_directoryPath + "/InternalDirectory", fileSystem.GetDirectory(_directoryPath + "/InternalDirectory"));
fileSystem.Delete(_directoryPath);
// exceptions
Assert.Throws <ArgumentNullException>(() => fileSystem.GetDirectory(null));
Assert.Throws <ArgumentException>(() => fileSystem.GetDirectory(String.Empty));
Assert.Throws <ArgumentException>(() => fileSystem.GetDirectory(":"));
Assert.Throws <ArgumentException>(() => fileSystem.GetDirectory("/NotExistingPath/"));
}
public void HadoopFileSystemGetRootDirectoriesTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
Assert.AreEqual(1, fileSystem.GetRootDirectories().Length);
Assert.AreEqual("/", fileSystem.GetRootDirectories()[0]);
}
public void HadoopFileSystemDeleteTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
// existing path
fileSystem.CreateDirectory(_directoryPath);
Assert.IsTrue(fileSystem.Exists(_directoryPath));
Assert.IsTrue(fileSystem.IsDirectory(_directoryPath));
fileSystem.Delete(_directoryPath);
Assert.IsFalse(fileSystem.Exists(_directoryPath));
Assert.IsFalse(fileSystem.IsDirectory(_directoryPath));
// exceptions
Assert.Throws <ArgumentNullException>(() => fileSystem.Delete(null));
Assert.Throws <ArgumentException>(() => fileSystem.Delete(String.Empty));
Assert.Throws <ArgumentException>(() => fileSystem.Delete("InvalidPath"));
Assert.Throws <ArgumentException>(() => fileSystem.Delete("/NotExistingPath"));
}
public void HadoopFileSystemGetDirectoryTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(this.hostname, this.portNumber, this.authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
// valid parameters
fileSystem.CreateDirectory(this.directoryPath);
fileSystem.CreateDirectory(this.directoryPath + "/InternalDirectory");
fileSystem.GetDirectory("/").ShouldBe("/");
fileSystem.GetDirectory(this.directoryPath).ShouldBe(this.directoryPath);
fileSystem.GetDirectory(this.directoryPath + "/").ShouldBe(this.directoryPath);
fileSystem.GetDirectory(this.directoryPath + "/InternalDirectory").ShouldBe(this.directoryPath + "/InternalDirectory");
fileSystem.Delete(this.directoryPath);
// exceptions
Should.Throw <ArgumentNullException>(() => fileSystem.GetDirectory(null));
Should.Throw <ArgumentException>(() => fileSystem.GetDirectory(String.Empty));
Should.Throw <ArgumentException>(() => fileSystem.GetDirectory(":"));
Should.Throw <ArgumentException>(() => fileSystem.GetDirectory("/NotExistingPath/"));
}
/// <summary>
/// Sets up the file system instance to be used for the tests.
/// </summary>
public TestHadoopFileSystem()
{
_fileSystem =
TangFactory.GetTang()
.NewInjector(HadoopFileSystemConfiguration.ConfigurationModule.Build())
.GetInstance <HadoopFileSystem>();
}
public void HadoopFileSystemDeleteTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(this.hostname, this.portNumber, this.authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
// existing path
fileSystem.CreateDirectory(this.directoryPath);
fileSystem.Exists(this.directoryPath).ShouldBeTrue();
fileSystem.IsDirectory(this.directoryPath).ShouldBeTrue();
fileSystem.Delete(this.directoryPath);
fileSystem.Exists(this.directoryPath).ShouldBeFalse();
fileSystem.IsDirectory(this.directoryPath).ShouldBeFalse();
// exceptions
Should.Throw <ArgumentNullException>(() => fileSystem.Delete(null));
Should.Throw <ArgumentException>(() => fileSystem.Delete(String.Empty));
Should.Throw <ArgumentException>(() => fileSystem.Delete("InvalidPath"));
Should.Throw <ArgumentException>(() => fileSystem.Delete("/NotExistingPath"));
}
public void TearDown()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
// remove the test directory
if (fileSystem.IsConnected && fileSystem.Exists(_directoryPath))
{
fileSystem.Delete(_directoryPath);
}
}
public void HadoopFileSystemGetFileSystemEntriesTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(this.hostname, this.portNumber, this.authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
// empty directory
fileSystem.CreateDirectory(this.directoryPath);
fileSystem.GetFileSystemEntries(this.directoryPath).ShouldBeEmpty();
// only directories
fileSystem.CreateDirectory(this.directoryPath + "/InnerDirectory1");
fileSystem.CreateDirectory(this.directoryPath + "/InnerDirectory2");
FileSystemEntry[] directories = fileSystem.GetFileSystemEntries(this.directoryPath);
directories.Length.ShouldBe(2);
directories[0].Path.ShouldBe(this.directoryPath + "/InnerDirectory1");
directories[1].Path.ShouldBe(this.directoryPath + "/InnerDirectory2");
// with search pattern
directories = fileSystem.GetFileSystemEntries(this.directoryPath, "*1", false);
directories.Length.ShouldBe(1);
directories[0].Path.ShouldBe(this.directoryPath + "/InnerDirectory1");
// recursive
fileSystem.CreateDirectory(this.directoryPath + "/InnerDirectory1/DeepInnerDirectory1");
fileSystem.CreateDirectory(this.directoryPath + "/InnerDirectory1/DeepInnerDirectory2");
directories = fileSystem.GetFileSystemEntries(this.directoryPath, "*", true);
directories.Length.ShouldBe(4);
directories[0].Path.ShouldBe(this.directoryPath + "/InnerDirectory1");
directories[1].Path.ShouldBe(this.directoryPath + "/InnerDirectory1/DeepInnerDirectory1");
directories[2].Path.ShouldBe(this.directoryPath + "/InnerDirectory1/DeepInnerDirectory2");
directories[3].Path.ShouldBe(this.directoryPath + "/InnerDirectory2");
fileSystem.Delete(this.directoryPath);
// exceptions
Should.Throw <ArgumentNullException>(() => fileSystem.GetFileSystemEntries(null));
Should.Throw <ArgumentException>(() => fileSystem.GetFileSystemEntries(String.Empty));
Should.Throw <ArgumentException>(() => fileSystem.GetFileSystemEntries(":"));
Should.Throw <ArgumentException>(() => fileSystem.GetFileSystemEntries("/NotExistingPath/"));
}
public void HadoopFileSystemCreateDirectoryTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
// new directory
Assert.IsFalse(fileSystem.Exists(_directoryPath));
fileSystem.CreateDirectory(_directoryPath);
Assert.IsTrue(fileSystem.Exists(_directoryPath));
Assert.IsTrue(fileSystem.IsDirectory(_directoryPath));
fileSystem.CreateDirectory(_directoryPath + "/InternalDirectory");
Assert.IsTrue(fileSystem.Exists(_directoryPath + "/InternalDirectory"));
Assert.IsTrue(fileSystem.IsDirectory(_directoryPath + "/InternalDirectory"));
// existing directory
fileSystem.CreateDirectory("/");
fileSystem.CreateDirectory(_directoryPath);
Assert.IsTrue(fileSystem.Exists(_directoryPath));
fileSystem.Delete(_directoryPath);
// multiple new directories
fileSystem.CreateDirectory(_directoryPath + "/InternalDirectory");
Assert.IsTrue(fileSystem.Exists(_directoryPath));
Assert.IsTrue(fileSystem.Exists(_directoryPath + "/InternalDirectory"));
fileSystem.Delete(_directoryPath);
// exceptions
Assert.Throws <ArgumentNullException>(() => fileSystem.CreateDirectory(null));
Assert.Throws <ArgumentException>(() => fileSystem.CreateDirectory(String.Empty));
Assert.Throws <ArgumentException>(() => fileSystem.CreateDirectory("InvalidPath"));
}
public void HadoopFileSystemGetDirectoryRootTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(this.hostname, this.portNumber, this.authentication);
// valid parameters
fileSystem.GetDirectoryRoot("/").ShouldBe("/");
fileSystem.GetDirectoryRoot(this.directoryPath).ShouldBe("/");
// exceptions
Should.Throw <ArgumentNullException>(() => fileSystem.GetDirectoryRoot(null));
Should.Throw <ArgumentException>(() => fileSystem.GetDirectoryRoot(String.Empty));
}
public void HadoopFileSystemGetDirectoryRootTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
// valid parameters
Assert.AreEqual("/", fileSystem.GetDirectoryRoot("/"));
Assert.AreEqual("/", fileSystem.GetDirectoryRoot(_directoryPath));
// exceptions
Assert.Throws <ArgumentNullException>(() => fileSystem.GetDirectoryRoot(null));
Assert.Throws <ArgumentException>(() => fileSystem.GetDirectoryRoot(String.Empty));
}
public void HadoopFileSystemConstructorTest()
{
HadoopFileSystem fileSystem;
Uri location = new Uri(String.Format("http://{0}:{1}", _hostname, _portNumber));
// hostname and port parameters
fileSystem = new HadoopFileSystem(_hostname, _portNumber);
Assert.IsFalse(fileSystem.IsConnected);
Assert.AreEqual('/', fileSystem.DirectorySeparator);
Assert.AreEqual('/', fileSystem.VolumeSeparator);
Assert.AreEqual(';', fileSystem.PathSeparator);
Assert.AreEqual('/', fileSystem.VolumeSeparator);
Assert.AreEqual("hdfs", fileSystem.UriScheme);
Assert.IsTrue(fileSystem.IsContentBrowsingSupported);
Assert.IsTrue(fileSystem.IsContentWritingSupported);
Assert.IsTrue(fileSystem.IsStreamingSupported);
Assert.AreEqual(location, fileSystem.Location);
Assert.AreEqual(FileSystemAuthenticationType.Anonymous, fileSystem.Authentication.AutenticationType);
// host, port and authentication parameters
fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
Assert.AreEqual(_authentication, fileSystem.Authentication);
// exceptions
Assert.Throws <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(null, 1));
Assert.Throws <ArgumentException>(() => fileSystem = new HadoopFileSystem(String.Empty, 1));
Assert.Throws <ArgumentException>(() => fileSystem = new HadoopFileSystem(":", 1));
Assert.Throws <ArgumentOutOfRangeException>(() => fileSystem = new HadoopFileSystem(_hostname, 0));
Assert.Throws <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(null));
Assert.Throws <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(null, _authentication));
Assert.Throws <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(location, (IHadoopFileSystemAuthentication)null));
Assert.Throws <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(location, _authentication, (HttpClient)null));
Assert.Throws <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(location, (HttpClient)null));
}
public void HadoopFileSystemConstructorTest()
{
HadoopFileSystem fileSystem;
Uri location = new Uri(String.Format("http://{0}:{1}", this.hostname, this.portNumber));
// host name and port parameters
fileSystem = new HadoopFileSystem(this.hostname, this.portNumber);
fileSystem.IsConnected.ShouldBeFalse();
fileSystem.DirectorySeparator.ShouldBe('/');
fileSystem.VolumeSeparator.ShouldBe('/');
fileSystem.PathSeparator.ShouldBe(';');
fileSystem.VolumeSeparator.ShouldBe('/');
fileSystem.UriScheme.ShouldBe("hdfs");
fileSystem.IsContentBrowsingSupported.ShouldBeTrue();
fileSystem.IsContentWritingSupported.ShouldBeTrue();
fileSystem.IsStreamingSupported.ShouldBeTrue();
fileSystem.Location.ShouldBe(location);
fileSystem.Authentication.AutenticationType.ShouldBe(StorageAuthenticationType.Anonymous);
// host, port and authentication parameters
fileSystem = new HadoopFileSystem(this.hostname, this.portNumber, this.authentication);
fileSystem.Authentication.ShouldBe(this.authentication);
// exceptions
Should.Throw <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(null, 1));
Should.Throw <ArgumentException>(() => fileSystem = new HadoopFileSystem(String.Empty, 1));
Should.Throw <ArgumentException>(() => fileSystem = new HadoopFileSystem(":", 1));
Should.Throw <ArgumentOutOfRangeException>(() => fileSystem = new HadoopFileSystem(this.hostname, 0));
Should.Throw <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(null));
Should.Throw <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(null, this.authentication));
Should.Throw <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(location, this.authentication, null));
Should.Throw <ArgumentNullException>(() => fileSystem = new HadoopFileSystem(location, (HttpClient)null));
}
public void HadoopFileSystemOpenFileTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(this.hostname, this.portNumber, this.authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
if (!fileSystem.Exists(this.directoryPath + "TestFile.txt"))
{
Assert.Inconclusive();
}
// read existing file
Stream fileStream = fileSystem.OpenFile(this.directoryPath + "TestFile.txt", FileMode.Open, FileAccess.Read);
fileSystem.ShouldNotBeNull();
StreamReader reader = new StreamReader(fileStream);
reader.ReadToEnd().Length.ShouldBeGreaterThan(0);
}
public void HadoopFileSystemOpenFileTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
if (!fileSystem.Exists(_directoryPath + "TestFile.txt"))
{
Assert.Inconclusive();
}
// read existing file
Stream fileStream = fileSystem.OpenFile(_directoryPath + "TestFile.txt", FileMode.Open, FileAccess.Read);
Assert.IsNotNull(fileSystem);
StreamReader reader = new StreamReader(fileStream);
Assert.IsTrue(reader.ReadToEnd().Length > 0);
}
public void HadoopFileSystemGetFileSystemEntriesTest()
{
HadoopFileSystem fileSystem = new HadoopFileSystem(_hostname, _portNumber, _authentication);
if (!fileSystem.IsConnected)
{
Assert.Inconclusive();
}
// empty directory
fileSystem.CreateDirectory(_directoryPath);
Assert.AreEqual(0, fileSystem.GetFileSystemEntries(_directoryPath).Length);
// only directories
fileSystem.CreateDirectory(_directoryPath + "/InnerDirectory1");
fileSystem.CreateDirectory(_directoryPath + "/InnerDirectory2");
FileSystemEntry[] directories = fileSystem.GetFileSystemEntries(_directoryPath);
Assert.AreEqual(2, directories.Length);
Assert.AreEqual(_directoryPath + "/InnerDirectory1", directories[0].Path);
Assert.AreEqual(_directoryPath + "/InnerDirectory2", directories[1].Path);
// with search pattern
directories = fileSystem.GetFileSystemEntries(_directoryPath, "*1", false);
Assert.AreEqual(1, directories.Length);
Assert.AreEqual(_directoryPath + "/InnerDirectory1", directories[0].Path);
// recursive
fileSystem.CreateDirectory(_directoryPath + "/InnerDirectory1/DeepInnerDirectory1");
fileSystem.CreateDirectory(_directoryPath + "/InnerDirectory1/DeepInnerDirectory2");
directories = fileSystem.GetFileSystemEntries(_directoryPath, "*", true);
Assert.AreEqual(4, directories.Length);
Assert.AreEqual(_directoryPath + "/InnerDirectory1", directories[0].Path);
Assert.AreEqual(_directoryPath + "/InnerDirectory1/DeepInnerDirectory1", directories[1].Path);
Assert.AreEqual(_directoryPath + "/InnerDirectory1/DeepInnerDirectory2", directories[2].Path);
Assert.AreEqual(_directoryPath + "/InnerDirectory2", directories[3].Path);
fileSystem.Delete(_directoryPath);
// exceptions
Assert.Throws <ArgumentNullException>(() => fileSystem.GetFileSystemEntries(null));
Assert.Throws <ArgumentException>(() => fileSystem.GetFileSystemEntries(String.Empty));
Assert.Throws <ArgumentException>(() => fileSystem.GetFileSystemEntries(":"));
Assert.Throws <ArgumentException>(() => fileSystem.GetFileSystemEntries("/NotExistingPath/"));
}
