public virtual void TestReadClosedStream()
{
Path testFile = new Path("/testfile+2");
FSDataOutputStream os = hdfs.Create(testFile, true);
os.WriteBytes("0123456789");
os.Close();
// ByteRangeInputStream delays opens until reads. Make sure it doesn't
// open a closed stream that has never been opened
FSDataInputStream @in = hftpFs.Open(testFile);
@in.Close();
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
// force the stream to connect and then close it
@in = hftpFs.Open(testFile);
int ch = @in.Read();
NUnit.Framework.Assert.AreEqual('0', ch);
@in.Close();
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
// make sure seeking doesn't automagically reopen the stream
@in.Seek(4);
CheckClosedStream(@in);
CheckClosedStream(@in.GetWrappedStream());
}
public virtual void TestOpenFileTwice()
{
Describe("verify that two opened file streams are independent");
Path path = Path("testopenfiletwice.txt");
byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255);
//this file now has a simple rule: offset => value
ContractTestUtils.CreateFile(GetFileSystem(), path, false, block);
//open first
FSDataInputStream instream1 = GetFileSystem().Open(path);
int c = instream1.Read();
Assert.Equal(0, c);
FSDataInputStream instream2 = null;
try
{
instream2 = GetFileSystem().Open(path);
Assert.Equal("first read of instream 2", 0, instream2.Read());
Assert.Equal("second read of instream 1", 1, instream1.Read());
instream1.Close();
Assert.Equal("second read of instream 2", 1, instream2.Read());
//close instream1 again
instream1.Close();
}
finally
{
IOUtils.CloseStream(instream1);
IOUtils.CloseStream(instream2);
}
}
/// <summary>
/// Adding as part of jira HDFS-5343
/// Test for checking the cat command on snapshot path it
/// cannot read a file beyond snapshot file length
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestSnapshotFileLengthWithCatCommand()
{
FSDataInputStream fis = null;
FileStatus fileStatus = null;
int bytesRead;
byte[] buffer = new byte[Blocksize * 8];
hdfs.Mkdirs(sub);
Path file1 = new Path(sub, file1Name);
DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed);
hdfs.AllowSnapshot(sub);
hdfs.CreateSnapshot(sub, snapshot1);
DFSTestUtil.AppendFile(hdfs, file1, Blocksize);
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.GetFileStatus(file1);
NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus
.GetLen());
fis = hdfs.Open(file1);
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead
);
fis.Close();
Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name);
fis = hdfs.Open(file1snap1);
fileStatus = hdfs.GetFileStatus(file1snap1);
NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize);
// Make sure we can only read up to the snapshot length.
bytesRead = fis.Read(buffer, 0, buffer.Length);
NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead);
fis.Close();
TextWriter outBackup = System.Console.Out;
TextWriter errBackup = System.Console.Error;
ByteArrayOutputStream bao = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bao));
Runtime.SetErr(new TextWriter(bao));
// Make sure we can cat the file upto to snapshot length
FsShell shell = new FsShell();
try
{
ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size
());
}
finally
{
Runtime.SetOut(outBackup);
Runtime.SetErr(errBackup);
}
}
/// <exception cref="System.IO.IOException"/>
public static void Check(FileSystem fs, Path p, long length)
{
int i = -1;
try
{
FileStatus status = fs.GetFileStatus(p);
FSDataInputStream @in = fs.Open(p);
if (@in.GetWrappedStream() is DFSInputStream)
{
long len = ((DFSInputStream)@in.GetWrappedStream()).GetFileLength();
NUnit.Framework.Assert.AreEqual(length, len);
}
else
{
NUnit.Framework.Assert.AreEqual(length, status.GetLen());
}
for (i++; i < length; i++)
{
NUnit.Framework.Assert.AreEqual(unchecked ((byte)i), unchecked ((byte)@in.Read()));
}
i = -(int)length;
NUnit.Framework.Assert.AreEqual(-1, @in.Read());
//EOF
@in.Close();
}
catch (IOException ioe)
{
throw new IOException("p=" + p + ", length=" + length + ", i=" + i, ioe);
}
}
/// <exception cref="System.Exception"/>
private void ReadTestFile(string testFileName)
{
Path filePath = new Path(testFileName);
FSDataInputStream istream = dfs.Open(filePath, 10240);
ByteBuffer buf = ByteBuffer.Allocate(10240);
int count = 0;
try
{
while (istream.Read(buf) > 0)
{
count += 1;
buf.Clear();
istream.Seek(istream.GetPos() + 5);
}
}
catch (IOException)
{
}
finally
{
// Ignore this it's probably a seek after eof.
istream.Close();
}
}
//
// validates that file encounters BlockMissingException
//
/// <exception cref="System.IO.IOException"/>
private void ValidateFile(FileSystem fileSys, Path name)
{
FSDataInputStream stm = fileSys.Open(name);
byte[] b = new byte[4192];
int num = 0;
bool gotException = false;
try
{
while (num >= 0)
{
num = stm.Read(b);
if (num < 0)
{
break;
}
}
}
catch (BlockMissingException)
{
gotException = true;
}
stm.Close();
NUnit.Framework.Assert.IsTrue("Expected BlockMissingException ", gotException);
}
/// <summary>test seek</summary>
/// <exception cref="System.IO.IOException"/>
internal static void VerifySeek(FileSystem fs, Path p, long offset, long length,
byte[] buf, byte[] expected)
{
long remaining = length - offset;
long @checked = 0;
Log.Info("XXX SEEK: offset=" + offset + ", remaining=" + remaining);
TestWebHDFS.Ticker t = new TestWebHDFS.Ticker("SEEK", "offset=%d, remaining=%d",
offset, remaining);
FSDataInputStream @in = fs.Open(p, 64 << 10);
@in.Seek(offset);
for (; remaining > 0;)
{
t.Tick(@checked, "offset=%d, remaining=%d", offset, remaining);
int n = (int)Math.Min(remaining, buf.Length);
@in.ReadFully(buf, 0, n);
CheckData(offset, remaining, n, buf, expected);
offset += n;
remaining -= n;
@checked += n;
}
@in.Close();
t.End(@checked);
}
public virtual void TestBothOldAndNewShortCircuitConfigured()
{
short ReplFactor = 1;
int FileLength = 512;
Assume.AssumeTrue(null == DomainSocket.GetLoadingFailureReason());
TemporarySocketDirectory socketDir = new TemporarySocketDirectory();
HdfsConfiguration conf = GetConfiguration(socketDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
socketDir.Close();
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("/foo");
byte[] orig = new byte[FileLength];
for (int i = 0; i < orig.Length; i++)
{
orig[i] = unchecked ((byte)(i % 10));
}
FSDataOutputStream fos = fs.Create(path, (short)1);
fos.Write(orig);
fos.Close();
DFSTestUtil.WaitReplication(fs, path, ReplFactor);
FSDataInputStream fis = cluster.GetFileSystem().Open(path);
byte[] buf = new byte[FileLength];
IOUtils.ReadFully(fis, buf, 0, FileLength);
fis.Close();
Assert.AssertArrayEquals(orig, buf);
Arrays.Equals(orig, buf);
cluster.Shutdown();
}
public virtual void TestGetPos()
{
Path testFile = new Path("/testfile+1");
// Write a test file.
FSDataOutputStream @out = hdfs.Create(testFile, true);
@out.WriteBytes("0123456789");
@out.Close();
FSDataInputStream @in = hftpFs.Open(testFile);
// Test read().
for (int i = 0; i < 5; ++i)
{
NUnit.Framework.Assert.AreEqual(i, @in.GetPos());
@in.Read();
}
// Test read(b, off, len).
NUnit.Framework.Assert.AreEqual(5, @in.GetPos());
byte[] buffer = new byte[10];
NUnit.Framework.Assert.AreEqual(2, @in.Read(buffer, 0, 2));
NUnit.Framework.Assert.AreEqual(7, @in.GetPos());
// Test read(b).
int bytesRead = @in.Read(buffer);
NUnit.Framework.Assert.AreEqual(7 + bytesRead, @in.GetPos());
// Test EOF.
for (int i_1 = 0; i_1 < 100; ++i_1)
{
@in.Read();
}
NUnit.Framework.Assert.AreEqual(10, @in.GetPos());
@in.Close();
}
/// <summary>Tests read/seek/getPos/skipped opeation for input stream.</summary>
/// <exception cref="System.Exception"/>
private void TestChecker(FileSystem fileSys, bool readCS)
{
Path file = new Path("try.dat");
WriteFile(fileSys, file);
try
{
if (!readCS)
{
fileSys.SetVerifyChecksum(false);
}
stm = fileSys.Open(file);
CheckReadAndGetPos();
CheckSeek();
CheckSkip();
//checkMark
NUnit.Framework.Assert.IsFalse(stm.MarkSupported());
stm.Close();
}
finally
{
if (!readCS)
{
fileSys.SetVerifyChecksum(true);
}
CleanupFile(fileSys, file);
}
}
/// <exception cref="System.IO.IOException"/>
private long DumpFromOffset(PathData item, long offset)
{
long fileSize = item.RefreshStatus().GetLen();
if (offset > fileSize)
{
return(fileSize);
}
// treat a negative offset as relative to end of the file, floor of 0
if (offset < 0)
{
offset = Math.Max(fileSize + offset, 0);
}
FSDataInputStream @in = item.fs.Open(item.path);
try
{
@in.Seek(offset);
// use conf so the system configured io block size is used
IOUtils.CopyBytes(@in, System.Console.Out, GetConf(), false);
offset = @in.GetPos();
}
finally
{
@in.Close();
}
return(offset);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestVLongRandom()
{
int count = 1024 * 1024;
long[] data = new long[count];
Random rng = new Random();
for (int i = 0; i < data.Length; ++i)
{
int shift = rng.Next(long.Size) + 1;
long mask = (1L << shift) - 1;
long a = ((long)rng.Next()) << 32;
long b = ((long)rng.Next()) & unchecked ((long)(0xffffffffL));
data[i] = (a + b) & mask;
}
FSDataOutputStream @out = fs.Create(path);
for (int i_1 = 0; i_1 < data.Length; ++i_1)
{
Utils.WriteVLong(@out, data[i_1]);
}
@out.Close();
FSDataInputStream @in = fs.Open(path);
for (int i_2 = 0; i_2 < data.Length; ++i_2)
{
Assert.Equal(Utils.ReadVLong(@in), data[i_2]);
}
@in.Close();
fs.Delete(path, false);
}
public virtual void TestShortCircuitTraceHooks()
{
Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows);
conf = new Configuration();
conf.Set(DFSConfigKeys.DfsClientHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix
, typeof(TestTracing.SetSpanReceiver).FullName);
conf.SetLong("dfs.blocksize", 100 * 1024);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "testShortCircuitTraceHooks._PORT"
);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
dfs = cluster.GetFileSystem();
try
{
DFSTestUtil.CreateFile(dfs, TestPath, TestLength, (short)1, 5678L);
TraceScope ts = Trace.StartSpan("testShortCircuitTraceHooks", Sampler.Always);
FSDataInputStream stream = dfs.Open(TestPath);
byte[] buf = new byte[TestLength];
IOUtils.ReadFully(stream, buf, 0, TestLength);
stream.Close();
ts.Close();
string[] expectedSpanNames = new string[] { "OpRequestShortCircuitAccessProto", "ShortCircuitShmRequestProto" };
TestTracing.AssertSpanNamesFound(expectedSpanNames);
}
finally
{
dfs.Close();
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
public SpillRecord(Path indexFileName, JobConf job, Checksum crc, string expectedIndexOwner
)
{
FileSystem rfs = FileSystem.GetLocal(job).GetRaw();
FSDataInputStream @in = SecureIOUtils.OpenFSDataInputStream(new FilePath(indexFileName
.ToUri().GetRawPath()), expectedIndexOwner, null);
try
{
long length = rfs.GetFileStatus(indexFileName).GetLen();
int partitions = (int)length / MapTask.MapOutputIndexRecordLength;
int size = partitions * MapTask.MapOutputIndexRecordLength;
buf = ByteBuffer.Allocate(size);
if (crc != null)
{
crc.Reset();
CheckedInputStream chk = new CheckedInputStream(@in, crc);
IOUtils.ReadFully(chk, ((byte[])buf.Array()), 0, size);
if (chk.GetChecksum().GetValue() != @in.ReadLong())
{
throw new ChecksumException("Checksum error reading spill index: " + indexFileName
, -1);
}
}
else
{
IOUtils.ReadFully(@in, ((byte[])buf.Array()), 0, size);
}
entries = buf.AsLongBuffer();
}
finally
{
@in.Close();
}
}
public virtual void TestRamDiskShortCircuitRead()
{
StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, 2 * BlockSize - 1, true);
// 1 replica + delta, SCR read
string MethodName = GenericTestUtils.GetMethodName();
int Seed = unchecked ((int)(0xFADED));
Path path = new Path("/" + MethodName + ".dat");
MakeRandomTestFile(path, BlockSize, true, Seed);
EnsureFileReplicasOnStorageType(path, StorageType.RamDisk);
// Sleep for a short time to allow the lazy writer thread to do its job
Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000);
//assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
FSDataInputStream fis = fs.Open(path);
// Verify SCR read counters
try
{
fis = fs.Open(path);
byte[] buf = new byte[BufferLength];
fis.Read(0, buf, 0, BufferLength);
HdfsDataInputStream dfsis = (HdfsDataInputStream)fis;
NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead
());
NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead
());
}
finally
{
fis.Close();
fis = null;
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override void ProcessArguments(List <PathData> items)
{
base.ProcessArguments(items);
if (exitCode != 0)
{
// check for error collecting paths
return;
}
FSDataOutputStream @out = dst.fs.Create(dst.path);
try
{
foreach (PathData src in srcs)
{
FSDataInputStream @in = src.fs.Open(src.path);
try
{
IOUtils.CopyBytes(@in, @out, GetConf(), false);
if (delimiter != null)
{
@out.Write(Runtime.GetBytesForString(delimiter, "UTF-8"));
}
}
finally
{
@in.Close();
}
}
}
finally
{
@out.Close();
}
}
/*
* Read some data, skip a few bytes and read more. HADOOP-922.
*/
/// <exception cref="System.IO.IOException"/>
private void SmallReadSeek(FileSystem fileSys, Path name)
{
if (fileSys is ChecksumFileSystem)
{
fileSys = ((ChecksumFileSystem)fileSys).GetRawFileSystem();
}
// Make the buffer size small to trigger code for HADOOP-922
FSDataInputStream stmRaw = fileSys.Open(name, 1);
byte[] expected = new byte[Onemb];
Random rand = new Random(seed);
rand.NextBytes(expected);
// Issue a simple read first.
byte[] actual = new byte[128];
stmRaw.Seek(100000);
stmRaw.Read(actual, 0, actual.Length);
CheckAndEraseData(actual, 100000, expected, "First Small Read Test");
// now do a small seek of 4 bytes, within the same block.
int newpos1 = 100000 + 128 + 4;
stmRaw.Seek(newpos1);
stmRaw.Read(actual, 0, actual.Length);
CheckAndEraseData(actual, newpos1, expected, "Small Seek Bug 1");
// seek another 256 bytes this time
int newpos2 = newpos1 + 256;
stmRaw.Seek(newpos2);
stmRaw.Read(actual, 0, actual.Length);
CheckAndEraseData(actual, newpos2, expected, "Small Seek Bug 2");
// all done
stmRaw.Close();
}
/// <summary>Read the file and convert to a byte dataset.</summary>
/// <remarks>
/// Read the file and convert to a byte dataset.
/// This implements readfully internally, so that it will read
/// in the file without ever having to seek()
/// </remarks>
/// <param name="fs">filesystem</param>
/// <param name="path">path to read from</param>
/// <param name="len">length of data to read</param>
/// <returns>the bytes</returns>
/// <exception cref="System.IO.IOException">IO problems</exception>
public static byte[] ReadDataset(FileSystem fs, Path path, int len)
{
FSDataInputStream @in = fs.Open(path);
byte[] dest = new byte[len];
int offset = 0;
int nread = 0;
try
{
while (nread < len)
{
int nbytes = @in.Read(dest, offset + nread, len - nread);
if (nbytes < 0)
{
throw new EOFException("End of file reached before reading fully.");
}
nread += nbytes;
}
}
finally
{
@in.Close();
}
return(dest);
}
/// <exception cref="System.IO.IOException"/>
private void CheckFile(FileSystem fileSys, Path name)
{
BlockLocation[] locations = fileSys.GetFileBlockLocations(fileSys.GetFileStatus(name
), 0, fileSize);
NUnit.Framework.Assert.AreEqual("Number of blocks", fileSize, locations.Length);
FSDataInputStream stm = fileSys.Open(name);
byte[] expected = new byte[fileSize];
if (simulatedStorage)
{
for (int i = 0; i < expected.Length; ++i)
{
expected[i] = SimulatedFSDataset.DefaultDatabyte;
}
}
else
{
Random rand = new Random(seed);
rand.NextBytes(expected);
}
// do a sanity check. Read the file
byte[] actual = new byte[fileSize];
stm.ReadFully(0, actual);
CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
stm.Close();
}
public virtual void TestAppendLessThanChecksumChunk()
{
byte[] buf = new byte[1024];
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes
(1).Build();
cluster.WaitActive();
try
{
using (DistributedFileSystem fs = cluster.GetFileSystem())
{
int len1 = 200;
int len2 = 300;
Path p = new Path("/foo");
FSDataOutputStream @out = fs.Create(p);
@out.Write(buf, 0, len1);
@out.Close();
@out = fs.Append(p);
@out.Write(buf, 0, len2);
// flush but leave open
@out.Hflush();
// read data to verify the replica's content and checksum are correct
FSDataInputStream @in = fs.Open(p);
int length = @in.Read(0, buf, 0, len1 + len2);
NUnit.Framework.Assert.IsTrue(length > 0);
@in.Close();
@out.Close();
}
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Check file content, reading as user
/// <paramref name="readingUser"/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal static void CheckFileContent(URI uri, Path name, byte[] expected, int readOffset
, string readingUser, Configuration conf, bool legacyShortCircuitFails)
{
// Ensure short circuit is enabled
DistributedFileSystem fs = GetFileSystem(readingUser, uri, conf);
ClientContext getClientContext = ClientContext.GetFromConf(conf);
if (legacyShortCircuitFails)
{
NUnit.Framework.Assert.IsFalse(getClientContext.GetDisableLegacyBlockReaderLocal(
));
}
FSDataInputStream stm = fs.Open(name);
byte[] actual = new byte[expected.Length - readOffset];
stm.ReadFully(readOffset, actual);
CheckData(actual, readOffset, expected, "Read 2");
stm.Close();
// Now read using a different API.
actual = new byte[expected.Length - readOffset];
stm = fs.Open(name);
IOUtils.SkipFully(stm, readOffset);
//Read a small number of bytes first.
int nread = stm.Read(actual, 0, 3);
nread += stm.Read(actual, nread, 2);
//Read across chunk boundary
nread += stm.Read(actual, nread, 517);
CheckData(actual, readOffset, expected, nread, "A few bytes");
//Now read rest of it
while (nread < actual.Length)
{
int nbytes = stm.Read(actual, nread, actual.Length - nread);
if (nbytes < 0)
{
throw new EOFException("End of file reached before reading fully.");
}
nread += nbytes;
}
CheckData(actual, readOffset, expected, "Read 3");
if (legacyShortCircuitFails)
{
NUnit.Framework.Assert.IsTrue(getClientContext.GetDisableLegacyBlockReaderLocal()
);
}
stm.Close();
}
public virtual void TestRedirect()
{
string dir = "/testRedirect/";
string filename = "file";
Path p = new Path(dir, filename);
string[] writeStrings = CreateStrings("write to webhdfs ", "write");
string[] appendStrings = CreateStrings("append to webhdfs ", "append");
//test create: create a file for each namenode
for (int i = 0; i < webhdfs.Length; i++)
{
FSDataOutputStream @out = webhdfs[i].Create(p);
@out.Write(Sharpen.Runtime.GetBytesForString(writeStrings[i]));
@out.Close();
}
for (int i_1 = 0; i_1 < webhdfs.Length; i_1++)
{
//check file length
long expected = writeStrings[i_1].Length;
NUnit.Framework.Assert.AreEqual(expected, webhdfs[i_1].GetFileStatus(p).GetLen());
}
//test read: check file content for each namenode
for (int i_2 = 0; i_2 < webhdfs.Length; i_2++)
{
FSDataInputStream @in = webhdfs[i_2].Open(p);
for (int c; (c = @in.Read()) != -1; j++)
{
NUnit.Framework.Assert.AreEqual(writeStrings[i_2][j], c);
}
@in.Close();
}
//test append: append to the file for each namenode
for (int i_3 = 0; i_3 < webhdfs.Length; i_3++)
{
FSDataOutputStream @out = webhdfs[i_3].Append(p);
@out.Write(Sharpen.Runtime.GetBytesForString(appendStrings[i_3]));
@out.Close();
}
for (int i_4 = 0; i_4 < webhdfs.Length; i_4++)
{
//check file length
long expected = writeStrings[i_4].Length + appendStrings[i_4].Length;
NUnit.Framework.Assert.AreEqual(expected, webhdfs[i_4].GetFileStatus(p).GetLen());
}
//test read: check file content for each namenode
for (int i_5 = 0; i_5 < webhdfs.Length; i_5++)
{
StringBuilder b = new StringBuilder();
FSDataInputStream @in = webhdfs[i_5].Open(p);
for (int c; (c = @in.Read()) != -1;)
{
b.Append((char)c);
}
int wlen = writeStrings[i_5].Length;
NUnit.Framework.Assert.AreEqual(writeStrings[i_5], b.Substring(0, wlen));
NUnit.Framework.Assert.AreEqual(appendStrings[i_5], b.Substring(wlen));
@in.Close();
}
}
/// <exception cref="System.IO.IOException"/>
public static JobSplit.TaskSplitMetaInfo[] ReadSplitMetaInfo(JobID jobId, FileSystem
fs, Configuration conf, Path jobSubmitDir)
{
long maxMetaInfoSize = conf.GetLong(MRJobConfig.SplitMetainfoMaxsize, MRJobConfig
.DefaultSplitMetainfoMaxsize);
Path metaSplitFile = JobSubmissionFiles.GetJobSplitMetaFile(jobSubmitDir);
string jobSplitFile = JobSubmissionFiles.GetJobSplitFile(jobSubmitDir).ToString();
FileStatus fStatus = fs.GetFileStatus(metaSplitFile);
if (maxMetaInfoSize > 0 && fStatus.GetLen() > maxMetaInfoSize)
{
throw new IOException("Split metadata size exceeded " + maxMetaInfoSize + ". Aborting job "
+ jobId);
}
FSDataInputStream @in = fs.Open(metaSplitFile);
byte[] header = new byte[JobSplit.MetaSplitFileHeader.Length];
@in.ReadFully(header);
if (!Arrays.Equals(JobSplit.MetaSplitFileHeader, header))
{
throw new IOException("Invalid header on split file");
}
int vers = WritableUtils.ReadVInt(@in);
if (vers != JobSplit.MetaSplitVersion)
{
@in.Close();
throw new IOException("Unsupported split version " + vers);
}
int numSplits = WritableUtils.ReadVInt(@in);
//TODO: check for insane values
JobSplit.TaskSplitMetaInfo[] allSplitMetaInfo = new JobSplit.TaskSplitMetaInfo[numSplits
];
for (int i = 0; i < numSplits; i++)
{
JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo();
splitMetaInfo.ReadFields(@in);
JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(jobSplitFile, splitMetaInfo
.GetStartOffset());
allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex, splitMetaInfo.GetLocations
(), splitMetaInfo.GetInputDataLength());
}
@in.Close();
return(allSplitMetaInfo);
}
/// <exception cref="System.IO.IOException"/>
public override void Close()
{
datas.Close();
if (sums != null)
{
sums.Close();
}
Set(fs.verifyChecksum, null, 1, 0);
}
/// <exception cref="System.IO.IOException"/>
private static string GetJobSummary(FileContext fc, Path path)
{
Path qPath = fc.MakeQualified(path);
FSDataInputStream @in = fc.Open(qPath);
string jobSummaryString = @in.ReadUTF();
@in.Close();
return(jobSummaryString);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSeek()
{
Path dir = new Path("/test/testSeek");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
{
//test zero file size
Path zero = new Path(dir, "zero");
fs.Create(zero).Close();
int count = 0;
FSDataInputStream @in = fs.Open(zero);
for (; @in.Read() != -1; count++)
{
}
@in.Close();
NUnit.Framework.Assert.AreEqual(0, count);
}
byte[] mydata = new byte[1 << 20];
new Random().NextBytes(mydata);
Path p = new Path(dir, "file");
FSDataOutputStream @out = fs.Create(p, false, 4096, (short)3, 1L << 17);
@out.Write(mydata, 0, mydata.Length);
@out.Close();
int one_third = mydata.Length / 3;
int two_third = one_third * 2;
{
//test seek
int offset = one_third;
int len = mydata.Length - offset;
byte[] buf = new byte[len];
FSDataInputStream @in = fs.Open(p);
@in.Seek(offset);
//read all remaining data
@in.ReadFully(buf);
@in.Close();
for (int i = 0; i < buf.Length; i++)
{
NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length="
+ len, mydata[i + offset], buf[i]);
}
}
{
//test position read (read the data after the two_third location)
int offset = two_third;
int len = mydata.Length - offset;
byte[] buf = new byte[len];
FSDataInputStream @in = fs.Open(p);
@in.ReadFully(offset, buf);
@in.Close();
for (int i = 0; i < buf.Length; i++)
{
NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length="
+ len, mydata[i + offset], buf[i]);
}
}
}
/// <summary>
/// Moves files to a bad file directory on the same device, so that their
/// storage will not be reused.
/// </summary>
public override bool ReportChecksumFailure(Path p, FSDataInputStream @in, long inPos
, FSDataInputStream sums, long sumsPos)
{
try
{
// canonicalize f
FilePath f = ((RawLocalFileSystem)fs).PathToFile(p).GetCanonicalFile();
// find highest writable parent dir of f on the same device
string device = new DF(f, GetConf()).GetMount();
FilePath parent = f.GetParentFile();
FilePath dir = null;
while (parent != null && FileUtil.CanWrite(parent) && parent.ToString().StartsWith
(device))
{
dir = parent;
parent = parent.GetParentFile();
}
if (dir == null)
{
throw new IOException("not able to find the highest writable parent dir");
}
// move the file there
FilePath badDir = new FilePath(dir, "bad_files");
if (!badDir.Mkdirs())
{
if (!badDir.IsDirectory())
{
throw new IOException("Mkdirs failed to create " + badDir.ToString());
}
}
string suffix = "." + rand.Next();
FilePath badFile = new FilePath(badDir, f.GetName() + suffix);
Log.Warn("Moving bad file " + f + " to " + badFile);
@in.Close();
// close it first
bool b = f.RenameTo(badFile);
// rename it
if (!b)
{
Log.Warn("Ignoring failure of renameTo");
}
// move checksum file too
FilePath checkFile = ((RawLocalFileSystem)fs).PathToFile(GetChecksumFile(p));
// close the stream before rename to release the file handle
sums.Close();
b = checkFile.RenameTo(new FilePath(badDir, checkFile.GetName() + suffix));
if (!b)
{
Log.Warn("Ignoring failure of renameTo");
}
}
catch (IOException e)
{
Log.Warn("Error moving bad file " + p + ": " + e);
}
return(false);
}
// make sure bytes are available and match expected
/// <exception cref="System.IO.IOException"/>
private void AssertBytesAvailable(FileSystem fileSystem, Path path, int numBytes)
{
byte[] buffer = new byte[numBytes];
FSDataInputStream inputStream = fileSystem.Open(path);
IOUtils.ReadFully(inputStream, buffer, 0, numBytes);
inputStream.Close();
NUnit.Framework.Assert.IsTrue("unable to validate bytes", ValidateSequentialBytes
(buffer, 0, numBytes));
}
/// <exception cref="System.Exception"/>
private void CheckFile(Path name)
{
FSDataInputStream stm = fileSys.Open(name);
// do a sanity check. Read the file
stm.ReadFully(0, actual);
CheckAndEraseData(actual, 0, expected, "Read Sanity Test");
stm.Close();
// do a sanity check. Get the file checksum
fileSys.GetFileChecksum(name);
}
/// <exception cref="System.IO.IOException"/>
private void TestSeekAndRead(FileSystem fileSys)
{
Path file = new Path("try.dat");
WriteFile(fileSys, file);
stm = fileSys.Open(file, fileSys.GetConf().GetInt(CommonConfigurationKeys.IoFileBufferSizeKey
, 4096));
CheckSeekAndRead();
stm.Close();
CleanupFile(fileSys, file);
}
