public virtual void TestReadClosedStream() { Path testFile = new Path("/testfile+2"); FSDataOutputStream os = hdfs.Create(testFile, true); os.WriteBytes("0123456789"); os.Close(); // ByteRangeInputStream delays opens until reads. Make sure it doesn't // open a closed stream that has never been opened FSDataInputStream @in = hftpFs.Open(testFile); @in.Close(); CheckClosedStream(@in); CheckClosedStream(@in.GetWrappedStream()); // force the stream to connect and then close it @in = hftpFs.Open(testFile); int ch = @in.Read(); NUnit.Framework.Assert.AreEqual('0', ch); @in.Close(); CheckClosedStream(@in); CheckClosedStream(@in.GetWrappedStream()); // make sure seeking doesn't automagically reopen the stream @in.Seek(4); CheckClosedStream(@in); CheckClosedStream(@in.GetWrappedStream()); }
public virtual void TestOpenFileTwice() { Describe("verify that two opened file streams are independent"); Path path = Path("testopenfiletwice.txt"); byte[] block = ContractTestUtils.Dataset(TestFileLen, 0, 255); //this file now has a simple rule: offset => value ContractTestUtils.CreateFile(GetFileSystem(), path, false, block); //open first FSDataInputStream instream1 = GetFileSystem().Open(path); int c = instream1.Read(); Assert.Equal(0, c); FSDataInputStream instream2 = null; try { instream2 = GetFileSystem().Open(path); Assert.Equal("first read of instream 2", 0, instream2.Read()); Assert.Equal("second read of instream 1", 1, instream1.Read()); instream1.Close(); Assert.Equal("second read of instream 2", 1, instream2.Read()); //close instream1 again instream1.Close(); } finally { IOUtils.CloseStream(instream1); IOUtils.CloseStream(instream2); } }
/// <summary> /// Adding as part of jira HDFS-5343 /// Test for checking the cat command on snapshot path it /// cannot read a file beyond snapshot file length /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotFileLengthWithCatCommand() { FSDataInputStream fis = null; FileStatus fileStatus = null; int bytesRead; byte[] buffer = new byte[Blocksize * 8]; hdfs.Mkdirs(sub); Path file1 = new Path(sub, file1Name); DFSTestUtil.CreateFile(hdfs, file1, Blocksize, Replication, Seed); hdfs.AllowSnapshot(sub); hdfs.CreateSnapshot(sub, snapshot1); DFSTestUtil.AppendFile(hdfs, file1, Blocksize); // Make sure we can read the entire file via its non-snapshot path. fileStatus = hdfs.GetFileStatus(file1); NUnit.Framework.Assert.AreEqual("Unexpected file length", Blocksize * 2, fileStatus .GetLen()); fis = hdfs.Open(file1); bytesRead = fis.Read(buffer, 0, buffer.Length); NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize * 2, bytesRead ); fis.Close(); Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name); fis = hdfs.Open(file1snap1); fileStatus = hdfs.GetFileStatus(file1snap1); NUnit.Framework.Assert.AreEqual(fileStatus.GetLen(), Blocksize); // Make sure we can only read up to the snapshot length. bytesRead = fis.Read(buffer, 0, buffer.Length); NUnit.Framework.Assert.AreEqual("Unexpected # bytes read", Blocksize, bytesRead); fis.Close(); TextWriter outBackup = System.Console.Out; TextWriter errBackup = System.Console.Error; ByteArrayOutputStream bao = new ByteArrayOutputStream(); Runtime.SetOut(new TextWriter(bao)); Runtime.SetErr(new TextWriter(bao)); // Make sure we can cat the file upto to snapshot length FsShell shell = new FsShell(); try { ToolRunner.Run(conf, shell, new string[] { "-cat", "/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" }); NUnit.Framework.Assert.AreEqual("Unexpected # bytes from -cat", Blocksize, bao.Size ()); } finally { Runtime.SetOut(outBackup); Runtime.SetErr(errBackup); } }
/// <exception cref="System.IO.IOException"/> public static void Check(FileSystem fs, Path p, long length) { int i = -1; try { FileStatus status = fs.GetFileStatus(p); FSDataInputStream @in = fs.Open(p); if (@in.GetWrappedStream() is DFSInputStream) { long len = ((DFSInputStream)@in.GetWrappedStream()).GetFileLength(); NUnit.Framework.Assert.AreEqual(length, len); } else { NUnit.Framework.Assert.AreEqual(length, status.GetLen()); } for (i++; i < length; i++) { NUnit.Framework.Assert.AreEqual(unchecked ((byte)i), unchecked ((byte)@in.Read())); } i = -(int)length; NUnit.Framework.Assert.AreEqual(-1, @in.Read()); //EOF @in.Close(); } catch (IOException ioe) { throw new IOException("p=" + p + ", length=" + length + ", i=" + i, ioe); } }
/// <exception cref="System.Exception"/> private void ReadTestFile(string testFileName) { Path filePath = new Path(testFileName); FSDataInputStream istream = dfs.Open(filePath, 10240); ByteBuffer buf = ByteBuffer.Allocate(10240); int count = 0; try { while (istream.Read(buf) > 0) { count += 1; buf.Clear(); istream.Seek(istream.GetPos() + 5); } } catch (IOException) { } finally { // Ignore this it's probably a seek after eof. istream.Close(); } }
// // validates that file encounters BlockMissingException // /// <exception cref="System.IO.IOException"/> private void ValidateFile(FileSystem fileSys, Path name) { FSDataInputStream stm = fileSys.Open(name); byte[] b = new byte[4192]; int num = 0; bool gotException = false; try { while (num >= 0) { num = stm.Read(b); if (num < 0) { break; } } } catch (BlockMissingException) { gotException = true; } stm.Close(); NUnit.Framework.Assert.IsTrue("Expected BlockMissingException ", gotException); }
/// <summary>test seek</summary> /// <exception cref="System.IO.IOException"/> internal static void VerifySeek(FileSystem fs, Path p, long offset, long length, byte[] buf, byte[] expected) { long remaining = length - offset; long @checked = 0; Log.Info("XXX SEEK: offset=" + offset + ", remaining=" + remaining); TestWebHDFS.Ticker t = new TestWebHDFS.Ticker("SEEK", "offset=%d, remaining=%d", offset, remaining); FSDataInputStream @in = fs.Open(p, 64 << 10); @in.Seek(offset); for (; remaining > 0;) { t.Tick(@checked, "offset=%d, remaining=%d", offset, remaining); int n = (int)Math.Min(remaining, buf.Length); @in.ReadFully(buf, 0, n); CheckData(offset, remaining, n, buf, expected); offset += n; remaining -= n; @checked += n; } @in.Close(); t.End(@checked); }
public virtual void TestBothOldAndNewShortCircuitConfigured() { short ReplFactor = 1; int FileLength = 512; Assume.AssumeTrue(null == DomainSocket.GetLoadingFailureReason()); TemporarySocketDirectory socketDir = new TemporarySocketDirectory(); HdfsConfiguration conf = GetConfiguration(socketDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); socketDir.Close(); FileSystem fs = cluster.GetFileSystem(); Path path = new Path("/foo"); byte[] orig = new byte[FileLength]; for (int i = 0; i < orig.Length; i++) { orig[i] = unchecked ((byte)(i % 10)); } FSDataOutputStream fos = fs.Create(path, (short)1); fos.Write(orig); fos.Close(); DFSTestUtil.WaitReplication(fs, path, ReplFactor); FSDataInputStream fis = cluster.GetFileSystem().Open(path); byte[] buf = new byte[FileLength]; IOUtils.ReadFully(fis, buf, 0, FileLength); fis.Close(); Assert.AssertArrayEquals(orig, buf); Arrays.Equals(orig, buf); cluster.Shutdown(); }
public virtual void TestGetPos() { Path testFile = new Path("/testfile+1"); // Write a test file. FSDataOutputStream @out = hdfs.Create(testFile, true); @out.WriteBytes("0123456789"); @out.Close(); FSDataInputStream @in = hftpFs.Open(testFile); // Test read(). for (int i = 0; i < 5; ++i) { NUnit.Framework.Assert.AreEqual(i, @in.GetPos()); @in.Read(); } // Test read(b, off, len). NUnit.Framework.Assert.AreEqual(5, @in.GetPos()); byte[] buffer = new byte[10]; NUnit.Framework.Assert.AreEqual(2, @in.Read(buffer, 0, 2)); NUnit.Framework.Assert.AreEqual(7, @in.GetPos()); // Test read(b). int bytesRead = @in.Read(buffer); NUnit.Framework.Assert.AreEqual(7 + bytesRead, @in.GetPos()); // Test EOF. for (int i_1 = 0; i_1 < 100; ++i_1) { @in.Read(); } NUnit.Framework.Assert.AreEqual(10, @in.GetPos()); @in.Close(); }
/// <summary>Tests read/seek/getPos/skipped opeation for input stream.</summary> /// <exception cref="System.Exception"/> private void TestChecker(FileSystem fileSys, bool readCS) { Path file = new Path("try.dat"); WriteFile(fileSys, file); try { if (!readCS) { fileSys.SetVerifyChecksum(false); } stm = fileSys.Open(file); CheckReadAndGetPos(); CheckSeek(); CheckSkip(); //checkMark NUnit.Framework.Assert.IsFalse(stm.MarkSupported()); stm.Close(); } finally { if (!readCS) { fileSys.SetVerifyChecksum(true); } CleanupFile(fileSys, file); } }
/// <exception cref="System.IO.IOException"/> private long DumpFromOffset(PathData item, long offset) { long fileSize = item.RefreshStatus().GetLen(); if (offset > fileSize) { return(fileSize); } // treat a negative offset as relative to end of the file, floor of 0 if (offset < 0) { offset = Math.Max(fileSize + offset, 0); } FSDataInputStream @in = item.fs.Open(item.path); try { @in.Seek(offset); // use conf so the system configured io block size is used IOUtils.CopyBytes(@in, System.Console.Out, GetConf(), false); offset = @in.GetPos(); } finally { @in.Close(); } return(offset); }
/// <exception cref="System.IO.IOException"/> public virtual void TestVLongRandom() { int count = 1024 * 1024; long[] data = new long[count]; Random rng = new Random(); for (int i = 0; i < data.Length; ++i) { int shift = rng.Next(long.Size) + 1; long mask = (1L << shift) - 1; long a = ((long)rng.Next()) << 32; long b = ((long)rng.Next()) & unchecked ((long)(0xffffffffL)); data[i] = (a + b) & mask; } FSDataOutputStream @out = fs.Create(path); for (int i_1 = 0; i_1 < data.Length; ++i_1) { Utils.WriteVLong(@out, data[i_1]); } @out.Close(); FSDataInputStream @in = fs.Open(path); for (int i_2 = 0; i_2 < data.Length; ++i_2) { Assert.Equal(Utils.ReadVLong(@in), data[i_2]); } @in.Close(); fs.Delete(path, false); }
public virtual void TestShortCircuitTraceHooks() { Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows); conf = new Configuration(); conf.Set(DFSConfigKeys.DfsClientHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix , typeof(TestTracing.SetSpanReceiver).FullName); conf.SetLong("dfs.blocksize", 100 * 1024); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "testShortCircuitTraceHooks._PORT" ); conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C"); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); dfs = cluster.GetFileSystem(); try { DFSTestUtil.CreateFile(dfs, TestPath, TestLength, (short)1, 5678L); TraceScope ts = Trace.StartSpan("testShortCircuitTraceHooks", Sampler.Always); FSDataInputStream stream = dfs.Open(TestPath); byte[] buf = new byte[TestLength]; IOUtils.ReadFully(stream, buf, 0, TestLength); stream.Close(); ts.Close(); string[] expectedSpanNames = new string[] { "OpRequestShortCircuitAccessProto", "ShortCircuitShmRequestProto" }; TestTracing.AssertSpanNamesFound(expectedSpanNames); } finally { dfs.Close(); cluster.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> public SpillRecord(Path indexFileName, JobConf job, Checksum crc, string expectedIndexOwner ) { FileSystem rfs = FileSystem.GetLocal(job).GetRaw(); FSDataInputStream @in = SecureIOUtils.OpenFSDataInputStream(new FilePath(indexFileName .ToUri().GetRawPath()), expectedIndexOwner, null); try { long length = rfs.GetFileStatus(indexFileName).GetLen(); int partitions = (int)length / MapTask.MapOutputIndexRecordLength; int size = partitions * MapTask.MapOutputIndexRecordLength; buf = ByteBuffer.Allocate(size); if (crc != null) { crc.Reset(); CheckedInputStream chk = new CheckedInputStream(@in, crc); IOUtils.ReadFully(chk, ((byte[])buf.Array()), 0, size); if (chk.GetChecksum().GetValue() != @in.ReadLong()) { throw new ChecksumException("Checksum error reading spill index: " + indexFileName , -1); } } else { IOUtils.ReadFully(@in, ((byte[])buf.Array()), 0, size); } entries = buf.AsLongBuffer(); } finally { @in.Close(); } }
public virtual void TestRamDiskShortCircuitRead() { StartUpCluster(ReplFactor, new StorageType[] { StorageType.RamDisk, StorageType.Default }, 2 * BlockSize - 1, true); // 1 replica + delta, SCR read string MethodName = GenericTestUtils.GetMethodName(); int Seed = unchecked ((int)(0xFADED)); Path path = new Path("/" + MethodName + ".dat"); MakeRandomTestFile(path, BlockSize, true, Seed); EnsureFileReplicasOnStorageType(path, StorageType.RamDisk); // Sleep for a short time to allow the lazy writer thread to do its job Sharpen.Thread.Sleep(3 * LazyWriterIntervalSec * 1000); //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true)); FSDataInputStream fis = fs.Open(path); // Verify SCR read counters try { fis = fs.Open(path); byte[] buf = new byte[BufferLength]; fis.Read(0, buf, 0, BufferLength); HdfsDataInputStream dfsis = (HdfsDataInputStream)fis; NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalBytesRead ()); NUnit.Framework.Assert.AreEqual(BufferLength, dfsis.GetReadStatistics().GetTotalShortCircuitBytesRead ()); } finally { fis.Close(); fis = null; } }
/// <exception cref="System.IO.IOException"/> protected internal override void ProcessArguments(List <PathData> items) { base.ProcessArguments(items); if (exitCode != 0) { // check for error collecting paths return; } FSDataOutputStream @out = dst.fs.Create(dst.path); try { foreach (PathData src in srcs) { FSDataInputStream @in = src.fs.Open(src.path); try { IOUtils.CopyBytes(@in, @out, GetConf(), false); if (delimiter != null) { @out.Write(Runtime.GetBytesForString(delimiter, "UTF-8")); } } finally { @in.Close(); } } } finally { @out.Close(); } }
/* * Read some data, skip a few bytes and read more. HADOOP-922. */ /// <exception cref="System.IO.IOException"/> private void SmallReadSeek(FileSystem fileSys, Path name) { if (fileSys is ChecksumFileSystem) { fileSys = ((ChecksumFileSystem)fileSys).GetRawFileSystem(); } // Make the buffer size small to trigger code for HADOOP-922 FSDataInputStream stmRaw = fileSys.Open(name, 1); byte[] expected = new byte[Onemb]; Random rand = new Random(seed); rand.NextBytes(expected); // Issue a simple read first. byte[] actual = new byte[128]; stmRaw.Seek(100000); stmRaw.Read(actual, 0, actual.Length); CheckAndEraseData(actual, 100000, expected, "First Small Read Test"); // now do a small seek of 4 bytes, within the same block. int newpos1 = 100000 + 128 + 4; stmRaw.Seek(newpos1); stmRaw.Read(actual, 0, actual.Length); CheckAndEraseData(actual, newpos1, expected, "Small Seek Bug 1"); // seek another 256 bytes this time int newpos2 = newpos1 + 256; stmRaw.Seek(newpos2); stmRaw.Read(actual, 0, actual.Length); CheckAndEraseData(actual, newpos2, expected, "Small Seek Bug 2"); // all done stmRaw.Close(); }
/// <summary>Read the file and convert to a byte dataset.</summary> /// <remarks> /// Read the file and convert to a byte dataset. /// This implements readfully internally, so that it will read /// in the file without ever having to seek() /// </remarks> /// <param name="fs">filesystem</param> /// <param name="path">path to read from</param> /// <param name="len">length of data to read</param> /// <returns>the bytes</returns> /// <exception cref="System.IO.IOException">IO problems</exception> public static byte[] ReadDataset(FileSystem fs, Path path, int len) { FSDataInputStream @in = fs.Open(path); byte[] dest = new byte[len]; int offset = 0; int nread = 0; try { while (nread < len) { int nbytes = @in.Read(dest, offset + nread, len - nread); if (nbytes < 0) { throw new EOFException("End of file reached before reading fully."); } nread += nbytes; } } finally { @in.Close(); } return(dest); }
/// <exception cref="System.IO.IOException"/> private void CheckFile(FileSystem fileSys, Path name) { BlockLocation[] locations = fileSys.GetFileBlockLocations(fileSys.GetFileStatus(name ), 0, fileSize); NUnit.Framework.Assert.AreEqual("Number of blocks", fileSize, locations.Length); FSDataInputStream stm = fileSys.Open(name); byte[] expected = new byte[fileSize]; if (simulatedStorage) { for (int i = 0; i < expected.Length; ++i) { expected[i] = SimulatedFSDataset.DefaultDatabyte; } } else { Random rand = new Random(seed); rand.NextBytes(expected); } // do a sanity check. Read the file byte[] actual = new byte[fileSize]; stm.ReadFully(0, actual); CheckAndEraseData(actual, 0, expected, "Read Sanity Test"); stm.Close(); }
public virtual void TestAppendLessThanChecksumChunk() { byte[] buf = new byte[1024]; MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).NumDataNodes (1).Build(); cluster.WaitActive(); try { using (DistributedFileSystem fs = cluster.GetFileSystem()) { int len1 = 200; int len2 = 300; Path p = new Path("/foo"); FSDataOutputStream @out = fs.Create(p); @out.Write(buf, 0, len1); @out.Close(); @out = fs.Append(p); @out.Write(buf, 0, len2); // flush but leave open @out.Hflush(); // read data to verify the replica's content and checksum are correct FSDataInputStream @in = fs.Open(p); int length = @in.Read(0, buf, 0, len1 + len2); NUnit.Framework.Assert.IsTrue(length > 0); @in.Close(); @out.Close(); } } finally { cluster.Shutdown(); } }
/// <summary> /// Check file content, reading as user /// <paramref name="readingUser"/> /// /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal static void CheckFileContent(URI uri, Path name, byte[] expected, int readOffset , string readingUser, Configuration conf, bool legacyShortCircuitFails) { // Ensure short circuit is enabled DistributedFileSystem fs = GetFileSystem(readingUser, uri, conf); ClientContext getClientContext = ClientContext.GetFromConf(conf); if (legacyShortCircuitFails) { NUnit.Framework.Assert.IsFalse(getClientContext.GetDisableLegacyBlockReaderLocal( )); } FSDataInputStream stm = fs.Open(name); byte[] actual = new byte[expected.Length - readOffset]; stm.ReadFully(readOffset, actual); CheckData(actual, readOffset, expected, "Read 2"); stm.Close(); // Now read using a different API. actual = new byte[expected.Length - readOffset]; stm = fs.Open(name); IOUtils.SkipFully(stm, readOffset); //Read a small number of bytes first. int nread = stm.Read(actual, 0, 3); nread += stm.Read(actual, nread, 2); //Read across chunk boundary nread += stm.Read(actual, nread, 517); CheckData(actual, readOffset, expected, nread, "A few bytes"); //Now read rest of it while (nread < actual.Length) { int nbytes = stm.Read(actual, nread, actual.Length - nread); if (nbytes < 0) { throw new EOFException("End of file reached before reading fully."); } nread += nbytes; } CheckData(actual, readOffset, expected, "Read 3"); if (legacyShortCircuitFails) { NUnit.Framework.Assert.IsTrue(getClientContext.GetDisableLegacyBlockReaderLocal() ); } stm.Close(); }
public virtual void TestRedirect() { string dir = "/testRedirect/"; string filename = "file"; Path p = new Path(dir, filename); string[] writeStrings = CreateStrings("write to webhdfs ", "write"); string[] appendStrings = CreateStrings("append to webhdfs ", "append"); //test create: create a file for each namenode for (int i = 0; i < webhdfs.Length; i++) { FSDataOutputStream @out = webhdfs[i].Create(p); @out.Write(Sharpen.Runtime.GetBytesForString(writeStrings[i])); @out.Close(); } for (int i_1 = 0; i_1 < webhdfs.Length; i_1++) { //check file length long expected = writeStrings[i_1].Length; NUnit.Framework.Assert.AreEqual(expected, webhdfs[i_1].GetFileStatus(p).GetLen()); } //test read: check file content for each namenode for (int i_2 = 0; i_2 < webhdfs.Length; i_2++) { FSDataInputStream @in = webhdfs[i_2].Open(p); for (int c; (c = @in.Read()) != -1; j++) { NUnit.Framework.Assert.AreEqual(writeStrings[i_2][j], c); } @in.Close(); } //test append: append to the file for each namenode for (int i_3 = 0; i_3 < webhdfs.Length; i_3++) { FSDataOutputStream @out = webhdfs[i_3].Append(p); @out.Write(Sharpen.Runtime.GetBytesForString(appendStrings[i_3])); @out.Close(); } for (int i_4 = 0; i_4 < webhdfs.Length; i_4++) { //check file length long expected = writeStrings[i_4].Length + appendStrings[i_4].Length; NUnit.Framework.Assert.AreEqual(expected, webhdfs[i_4].GetFileStatus(p).GetLen()); } //test read: check file content for each namenode for (int i_5 = 0; i_5 < webhdfs.Length; i_5++) { StringBuilder b = new StringBuilder(); FSDataInputStream @in = webhdfs[i_5].Open(p); for (int c; (c = @in.Read()) != -1;) { b.Append((char)c); } int wlen = writeStrings[i_5].Length; NUnit.Framework.Assert.AreEqual(writeStrings[i_5], b.Substring(0, wlen)); NUnit.Framework.Assert.AreEqual(appendStrings[i_5], b.Substring(wlen)); @in.Close(); } }
/// <exception cref="System.IO.IOException"/> public static JobSplit.TaskSplitMetaInfo[] ReadSplitMetaInfo(JobID jobId, FileSystem fs, Configuration conf, Path jobSubmitDir) { long maxMetaInfoSize = conf.GetLong(MRJobConfig.SplitMetainfoMaxsize, MRJobConfig .DefaultSplitMetainfoMaxsize); Path metaSplitFile = JobSubmissionFiles.GetJobSplitMetaFile(jobSubmitDir); string jobSplitFile = JobSubmissionFiles.GetJobSplitFile(jobSubmitDir).ToString(); FileStatus fStatus = fs.GetFileStatus(metaSplitFile); if (maxMetaInfoSize > 0 && fStatus.GetLen() > maxMetaInfoSize) { throw new IOException("Split metadata size exceeded " + maxMetaInfoSize + ". Aborting job " + jobId); } FSDataInputStream @in = fs.Open(metaSplitFile); byte[] header = new byte[JobSplit.MetaSplitFileHeader.Length]; @in.ReadFully(header); if (!Arrays.Equals(JobSplit.MetaSplitFileHeader, header)) { throw new IOException("Invalid header on split file"); } int vers = WritableUtils.ReadVInt(@in); if (vers != JobSplit.MetaSplitVersion) { @in.Close(); throw new IOException("Unsupported split version " + vers); } int numSplits = WritableUtils.ReadVInt(@in); //TODO: check for insane values JobSplit.TaskSplitMetaInfo[] allSplitMetaInfo = new JobSplit.TaskSplitMetaInfo[numSplits ]; for (int i = 0; i < numSplits; i++) { JobSplit.SplitMetaInfo splitMetaInfo = new JobSplit.SplitMetaInfo(); splitMetaInfo.ReadFields(@in); JobSplit.TaskSplitIndex splitIndex = new JobSplit.TaskSplitIndex(jobSplitFile, splitMetaInfo .GetStartOffset()); allSplitMetaInfo[i] = new JobSplit.TaskSplitMetaInfo(splitIndex, splitMetaInfo.GetLocations (), splitMetaInfo.GetInputDataLength()); } @in.Close(); return(allSplitMetaInfo); }
/// <exception cref="System.IO.IOException"/> public override void Close() { datas.Close(); if (sums != null) { sums.Close(); } Set(fs.verifyChecksum, null, 1, 0); }
/// <exception cref="System.IO.IOException"/> private static string GetJobSummary(FileContext fc, Path path) { Path qPath = fc.MakeQualified(path); FSDataInputStream @in = fc.Open(qPath); string jobSummaryString = @in.ReadUTF(); @in.Close(); return(jobSummaryString); }
/// <exception cref="System.IO.IOException"/> public virtual void TestSeek() { Path dir = new Path("/test/testSeek"); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir)); { //test zero file size Path zero = new Path(dir, "zero"); fs.Create(zero).Close(); int count = 0; FSDataInputStream @in = fs.Open(zero); for (; @in.Read() != -1; count++) { } @in.Close(); NUnit.Framework.Assert.AreEqual(0, count); } byte[] mydata = new byte[1 << 20]; new Random().NextBytes(mydata); Path p = new Path(dir, "file"); FSDataOutputStream @out = fs.Create(p, false, 4096, (short)3, 1L << 17); @out.Write(mydata, 0, mydata.Length); @out.Close(); int one_third = mydata.Length / 3; int two_third = one_third * 2; { //test seek int offset = one_third; int len = mydata.Length - offset; byte[] buf = new byte[len]; FSDataInputStream @in = fs.Open(p); @in.Seek(offset); //read all remaining data @in.ReadFully(buf); @in.Close(); for (int i = 0; i < buf.Length; i++) { NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length=" + len, mydata[i + offset], buf[i]); } } { //test position read (read the data after the two_third location) int offset = two_third; int len = mydata.Length - offset; byte[] buf = new byte[len]; FSDataInputStream @in = fs.Open(p); @in.ReadFully(offset, buf); @in.Close(); for (int i = 0; i < buf.Length; i++) { NUnit.Framework.Assert.AreEqual("Position " + i + ", offset=" + offset + ", length=" + len, mydata[i + offset], buf[i]); } } }
/// <summary> /// Moves files to a bad file directory on the same device, so that their /// storage will not be reused. /// </summary> public override bool ReportChecksumFailure(Path p, FSDataInputStream @in, long inPos , FSDataInputStream sums, long sumsPos) { try { // canonicalize f FilePath f = ((RawLocalFileSystem)fs).PathToFile(p).GetCanonicalFile(); // find highest writable parent dir of f on the same device string device = new DF(f, GetConf()).GetMount(); FilePath parent = f.GetParentFile(); FilePath dir = null; while (parent != null && FileUtil.CanWrite(parent) && parent.ToString().StartsWith (device)) { dir = parent; parent = parent.GetParentFile(); } if (dir == null) { throw new IOException("not able to find the highest writable parent dir"); } // move the file there FilePath badDir = new FilePath(dir, "bad_files"); if (!badDir.Mkdirs()) { if (!badDir.IsDirectory()) { throw new IOException("Mkdirs failed to create " + badDir.ToString()); } } string suffix = "." + rand.Next(); FilePath badFile = new FilePath(badDir, f.GetName() + suffix); Log.Warn("Moving bad file " + f + " to " + badFile); @in.Close(); // close it first bool b = f.RenameTo(badFile); // rename it if (!b) { Log.Warn("Ignoring failure of renameTo"); } // move checksum file too FilePath checkFile = ((RawLocalFileSystem)fs).PathToFile(GetChecksumFile(p)); // close the stream before rename to release the file handle sums.Close(); b = checkFile.RenameTo(new FilePath(badDir, checkFile.GetName() + suffix)); if (!b) { Log.Warn("Ignoring failure of renameTo"); } } catch (IOException e) { Log.Warn("Error moving bad file " + p + ": " + e); } return(false); }
// make sure bytes are available and match expected /// <exception cref="System.IO.IOException"/> private void AssertBytesAvailable(FileSystem fileSystem, Path path, int numBytes) { byte[] buffer = new byte[numBytes]; FSDataInputStream inputStream = fileSystem.Open(path); IOUtils.ReadFully(inputStream, buffer, 0, numBytes); inputStream.Close(); NUnit.Framework.Assert.IsTrue("unable to validate bytes", ValidateSequentialBytes (buffer, 0, numBytes)); }
/// <exception cref="System.Exception"/> private void CheckFile(Path name) { FSDataInputStream stm = fileSys.Open(name); // do a sanity check. Read the file stm.ReadFully(0, actual); CheckAndEraseData(actual, 0, expected, "Read Sanity Test"); stm.Close(); // do a sanity check. Get the file checksum fileSys.GetFileChecksum(name); }
/// <exception cref="System.IO.IOException"/> private void TestSeekAndRead(FileSystem fileSys) { Path file = new Path("try.dat"); WriteFile(fileSys, file); stm = fileSys.Open(file, fileSys.GetConf().GetInt(CommonConfigurationKeys.IoFileBufferSizeKey , 4096)); CheckSeekAndRead(); stm.Close(); CleanupFile(fileSys, file); }