/// <summary>
/// Test that
/// <see cref="LocalDirAllocator.GetAllLocalPathsToRead(string, Org.Apache.Hadoop.Conf.Configuration)
/// "/>
///
/// returns correct filenames and "file" schema.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestGetAllLocalPathsToRead()
{
Assume.AssumeTrue(!isWindows);
string dir0 = BuildBufferDir(Root, 0);
string dir1 = BuildBufferDir(Root, 1);
try
{
conf.Set(Context, dir0 + "," + dir1);
Assert.True(localFs.Mkdirs(new Path(dir0)));
Assert.True(localFs.Mkdirs(new Path(dir1)));
localFs.Create(new Path(dir0 + Path.Separator + Filename));
localFs.Create(new Path(dir1 + Path.Separator + Filename));
// check both the paths are returned as paths to read:
IEnumerable <Path> pathIterable = dirAllocator.GetAllLocalPathsToRead(Filename, conf
);
int count = 0;
foreach (Path p in pathIterable)
{
count++;
Assert.Equal(Filename, p.GetName());
Assert.Equal("file", p.GetFileSystem(conf).GetUri().GetScheme(
));
}
Assert.Equal(2, count);
// test #next() while no element to iterate any more:
try
{
Path p_1 = pathIterable.GetEnumerator().Next();
NUnit.Framework.Assert.IsFalse("NoSuchElementException must be thrown, but returned ["
+ p_1 + "] instead.", true);
}
catch (NoSuchElementException)
{
}
// exception expected
// okay
// test modification not allowed:
IEnumerable <Path> pathIterable2 = dirAllocator.GetAllLocalPathsToRead(Filename, conf
);
IEnumerator <Path> it = pathIterable2.GetEnumerator();
try
{
it.Remove();
NUnit.Framework.Assert.IsFalse(true);
}
catch (NotSupportedException)
{
}
}
finally
{
// exception expected
// okay
Shell.ExecCommand(new string[] { "chmod", "u+w", BufferDirRoot });
RmBufferDirs();
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCloseForErroneousSequenceFile()
{
Configuration conf = new Configuration();
LocalFileSystem fs = FileSystem.GetLocal(conf);
// create an empty file (which is not a valid sequence file)
Path path = new Path(Runtime.GetProperty("test.build.data", ".") + "/broken.seq");
fs.Create(path).Close();
// try to create SequenceFile.Reader
TestSequenceFile.TestFSDataInputStream[] openedFile = new TestSequenceFile.TestFSDataInputStream
[1];
try
{
new _Reader_509(openedFile, fs, path, conf);
// this method is called by the SequenceFile.Reader constructor, overwritten, so we can access the opened file
Fail("IOException expected.");
}
catch (IOException)
{
}
NUnit.Framework.Assert.IsNotNull(path + " should have been opened.", openedFile[0
]);
Assert.True("InputStream for " + path + " should have been closed."
, openedFile[0].IsClosed());
}
public virtual void PrepFiles()
{
lfs.SetVerifyChecksum(true);
lfs.SetWriteChecksum(true);
lfs.Delete(srcPath, true);
lfs.Delete(dstPath, true);
FSDataOutputStream @out = lfs.Create(srcPath);
@out.WriteChars("hi");
@out.Close();
Assert.True(lfs.Exists(lfs.GetChecksumFile(srcPath)));
}
/// <summary>
/// Создание папки
/// </summary>
private static void CreateFolder()
{
if (_commandArray.Length < 2)
{
_lastMessage = "Possible command: create file | folder name";
return;
}
switch (_commandArray[1])
{
case "file":
_lfs.Create(new File(Name(2), _currentPath, null), _currentPath);
_lastMessage = "";
return;
case "folder":
_lfs.Create(new Folder(Name(2), _currentPath, null), _currentPath);
_lastMessage = "";
return;
default:
_lastMessage = "Possible command: create file | folder name";
return;
}
}
public void Create()
{
string path = Path.GetTempFileName();
string testData = Guid.NewGuid().ToString();
LocalFileSystem fs = new LocalFileSystem();
using (StreamWriter r = new StreamWriter(fs.Create(path)))
{
r.Write(testData);
}
string data = File.ReadAllText(path);
Assert.Equal(testData, data);
File.Delete(path);
}
public virtual void TestVerifyChecksum()
{
Path testPath = new Path(TestRootDir, "testPath");
Path testPath11 = new Path(TestRootDir, "testPath11");
FSDataOutputStream fout = localFs.Create(testPath);
fout.Write(Runtime.GetBytesForString("testing"));
fout.Close();
fout = localFs.Create(testPath11);
fout.Write(Runtime.GetBytesForString("testing you"));
fout.Close();
// Exercise some boundary cases - a divisor of the chunk size
// the chunk size, 2x chunk size, and +/-1 around these.
FileSystemTestHelper.ReadFile(localFs, testPath, 128);
FileSystemTestHelper.ReadFile(localFs, testPath, 511);
FileSystemTestHelper.ReadFile(localFs, testPath, 512);
FileSystemTestHelper.ReadFile(localFs, testPath, 513);
FileSystemTestHelper.ReadFile(localFs, testPath, 1023);
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
FileSystemTestHelper.ReadFile(localFs, testPath, 1025);
localFs.Delete(localFs.GetChecksumFile(testPath), true);
Assert.True("checksum deleted", !localFs.Exists(localFs.GetChecksumFile
(testPath)));
//copying the wrong checksum file
FileUtil.Copy(localFs, localFs.GetChecksumFile(testPath11), localFs, localFs.GetChecksumFile
(testPath), false, true, localFs.GetConf());
Assert.True("checksum exists", localFs.Exists(localFs.GetChecksumFile
(testPath)));
bool errorRead = false;
try
{
FileSystemTestHelper.ReadFile(localFs, testPath, 1024);
}
catch (ChecksumException)
{
errorRead = true;
}
Assert.True("error reading", errorRead);
//now setting verify false, the read should succeed
localFs.SetVerifyChecksum(false);
string str = FileSystemTestHelper.ReadFile(localFs, testPath, 1024).ToString();
Assert.True("read", "testing".Equals(str));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestReportChecksumFailure()
{
@base.Mkdirs();
Assert.True(@base.Exists() && @base.IsDirectory());
FilePath dir1 = new FilePath(@base, "dir1");
FilePath dir2 = new FilePath(dir1, "dir2");
dir2.Mkdirs();
Assert.True(dir2.Exists() && FileUtil.CanWrite(dir2));
string dataFileName = "corruptedData";
Path dataPath = new Path(new FilePath(dir2, dataFileName).ToURI());
Path checksumPath = fileSys.GetChecksumFile(dataPath);
FSDataOutputStream fsdos = fileSys.Create(dataPath);
try
{
fsdos.WriteUTF("foo");
}
finally
{
fsdos.Close();
}
Assert.True(fileSys.PathToFile(dataPath).Exists());
long dataFileLength = fileSys.GetFileStatus(dataPath).GetLen();
Assert.True(dataFileLength > 0);
// check the the checksum file is created and not empty:
Assert.True(fileSys.PathToFile(checksumPath).Exists());
long checksumFileLength = fileSys.GetFileStatus(checksumPath).GetLen();
Assert.True(checksumFileLength > 0);
// this is a hack to force the #reportChecksumFailure() method to stop
// climbing up at the 'base' directory and use 'dir1/bad_files' as the
// corrupted files storage:
FileUtil.SetWritable(@base, false);
FSDataInputStream dataFsdis = fileSys.Open(dataPath);
FSDataInputStream checksumFsdis = fileSys.Open(checksumPath);
bool retryIsNecessary = fileSys.ReportChecksumFailure(dataPath, dataFsdis, 0, checksumFsdis
, 0);
Assert.True(!retryIsNecessary);
// the data file should be moved:
Assert.True(!fileSys.PathToFile(dataPath).Exists());
// the checksum file should be moved:
Assert.True(!fileSys.PathToFile(checksumPath).Exists());
// check that the files exist in the new location where they were moved:
FilePath[] dir1files = dir1.ListFiles(new _FileFilter_352());
Assert.True(dir1files != null);
Assert.True(dir1files.Length == 1);
FilePath badFilesDir = dir1files[0];
FilePath[] badFiles = badFilesDir.ListFiles();
Assert.True(badFiles != null);
Assert.True(badFiles.Length == 2);
bool dataFileFound = false;
bool checksumFileFound = false;
foreach (FilePath badFile in badFiles)
{
if (badFile.GetName().StartsWith(dataFileName))
{
Assert.True(dataFileLength == badFile.Length());
dataFileFound = true;
}
else
{
if (badFile.GetName().Contains(dataFileName + ".crc"))
{
Assert.True(checksumFileLength == badFile.Length());
checksumFileFound = true;
}
}
}
Assert.True(dataFileFound);
Assert.True(checksumFileFound);
}
/// <exception cref="System.IO.IOException"/>
protected internal override OutputStream GetOutputStream(int bufferSize, byte[] key
, byte[] iv)
{
return(new CryptoOutputStream(fileSys.Create(file), codec, bufferSize, key, iv));
}
