/// <summary>Truncate the given file to the given length</summary> /// <exception cref="System.IO.IOException"/> private void TruncateFile(FilePath logFile, long newLength) { RandomAccessFile raf = new RandomAccessFile(logFile, "rw"); raf.SetLength(newLength); raf.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual void Corrupt(FilePath editFile) { // Corrupt the last edit long fileLen = editFile.Length(); RandomAccessFile rwf = new RandomAccessFile(editFile, "rw"); rwf.SetLength(fileLen - 1); rwf.Close(); }
/// <summary> /// Open a random access file for the given path, more and access. /// </summary> private static RandomAccessFile Open(string path, FileMode mode, FileAccess access) { var file = new Java.Io.File(path); switch (mode) { case FileMode.CreateNew: if (file.Exists()) { throw new IOException("File already exists"); } break; case FileMode.Open: if (!file.Exists()) { throw new FileNotFoundException(path); } break; case FileMode.Append: access = FileAccess.Write; break; } switch (mode) { case FileMode.Create: case FileMode.CreateNew: case FileMode.OpenOrCreate: if (access == FileAccess.Read) { //create empty file, so it can be opened again with read only right, //otherwise an FilNotFoundException is thrown. var additinalAccessFile = new RandomAccessFile(file, "rw"); } break; } var jMode = (access == FileAccess.Read) ? "r" : "rw"; var randomAccessFile = new RandomAccessFile(file, jMode); switch (mode) { case FileMode.Truncate: randomAccessFile.SetLength(0); break; case FileMode.Append: randomAccessFile.Seek(randomAccessFile.Length()); break; } return(randomAccessFile); }
/// <summary>TC7: Corrupted replicas are present.</summary> /// <exception cref="System.IO.IOException">an exception might be thrown</exception> /// <exception cref="System.Exception"/> private void TestTC7(bool appendToNewBlock) { short repl = 2; Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file with replication factor of 2. Write half block of data. Close file. int len1 = (int)(BlockSize / 2); { FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } DFSTestUtil.WaitReplication(fs, p, repl); //b. Log into one datanode that has one replica of this block. // Find the block file on this datanode and truncate it to zero size. LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString() , 0L, len1); NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount()); LocatedBlock lb = locatedblocks.Get(0); ExtendedBlock blk = lb.GetBlock(); NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize()); DatanodeInfo[] datanodeinfos = lb.GetLocations(); NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length); DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort()); FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock ()); RandomAccessFile raf = new RandomAccessFile(f, "rw"); AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes () + ")"); NUnit.Framework.Assert.AreEqual(len1, raf.Length()); raf.SetLength(0); raf.Close(); //c. Open file in "append mode". Append a new block worth of data. Close file. int len2 = (int)BlockSize; { FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append , CreateFlag.NewBlock), 4096, null) : fs.Append(p); AppendTestUtil.Write(@out, len1, len2); @out.Close(); } //d. Reopen file and read two blocks worth of data. AppendTestUtil.Check(fs, p, len1 + len2); }
/// <exception cref="System.IO.IOException"/> private bool Truncate(long newLength, StringBuilder b) { RandomAccessFile raf = new RandomAccessFile(localFile, "rw"); raf.SetLength(newLength); raf.Close(); bool isReady = dfs.Truncate(file, newLength); b.Append(", newLength=").Append(newLength).Append(", isReady=").Append(isReady); if (!isReady) { TestFileTruncate.CheckBlockRecovery(file, dfs, 100, 300L); } return(isReady); }
public virtual void TestBlockRecoveryWithLessMetafile() { Configuration conf = new Configuration(); conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser ().GetShortUserName()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Path file = new Path("/testRecoveryFile"); DistributedFileSystem dfs = cluster.GetFileSystem(); FSDataOutputStream @out = dfs.Create(file); int count = 0; while (count < 2 * 1024 * 1024) { @out.WriteBytes("Data"); count += 4; } @out.Hsync(); // abort the original stream ((DFSOutputStream)@out.GetWrappedStream()).Abort(); LocatedBlocks locations = cluster.GetNameNodeRpc().GetBlockLocations(file.ToString (), 0, count); ExtendedBlock block = locations.Get(0).GetBlock(); DataNode dn = cluster.GetDataNodes()[0]; BlockLocalPathInfo localPathInfo = dn.GetBlockLocalPathInfo(block, null); FilePath metafile = new FilePath(localPathInfo.GetMetaPath()); NUnit.Framework.Assert.IsTrue(metafile.Exists()); // reduce the block meta file size RandomAccessFile raf = new RandomAccessFile(metafile, "rw"); raf.SetLength(metafile.Length() - 20); raf.Close(); // restart DN to make replica to RWR MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0); cluster.RestartDataNode(dnProp, true); // try to recover the lease DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster .GetConfiguration(0)); count = 0; while (++count < 10 && !newdfs.RecoverLease(file)) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsTrue("File should be closed", newdfs.RecoverLease(file)); }
/// <summary> /// Sets the length of the current stream. /// </summary> /// <param name="value">The new length of the stream.</param> public override void SetLength(long value) { file.SetLength(value); }
/// <summary> /// Open a random access file for the given path, more and access. /// </summary> private static RandomAccessFile Open(string path, FileMode mode, FileAccess access) { var file = new Java.Io.File(path); switch (mode) { case FileMode.CreateNew: if (file.Exists()) throw new IOException("File already exists"); break; case FileMode.Open: if (!file.Exists()) throw new FileNotFoundException(path); break; case FileMode.Append: access = FileAccess.Write; break; } switch (mode) { case FileMode.Create: case FileMode.CreateNew: case FileMode.OpenOrCreate: if (access == FileAccess.Read) { //create empty file, so it can be opened again with read only right, //otherwise an FilNotFoundException is thrown. var additinalAccessFile = new RandomAccessFile(file, "rw"); } break; } var jMode = (access == FileAccess.Read) ? "r" : "rw"; var randomAccessFile = new RandomAccessFile(file, jMode); switch (mode) { case FileMode.Truncate: randomAccessFile.SetLength(0); break; case FileMode.Append: randomAccessFile.Seek(randomAccessFile.Length()); break; } return randomAccessFile; }
private void InitializeHeader() { try { lock (_randomAccessLock) { _randomAccessWriter = new RandomAccessFile(_filePath, "rw"); _randomAccessWriter.SetLength(0); // Set file length to 0, to prevent unexpected behavior in case the file already existed _randomAccessWriter.WriteBytes("RIFF"); _randomAccessWriter.WriteInt(0); // Final file size not known yet, write 0 _randomAccessWriter.WriteBytes("WAVE"); _randomAccessWriter.WriteBytes("fmt "); _randomAccessWriter.WriteInt(Integer.ReverseBytes(16)); // Sub-chunk size, 16 for PCM _randomAccessWriter.WriteShort(Short.ReverseBytes(1)); // AudioFormat, 1 for PCM _randomAccessWriter.WriteShort(Short.ReverseBytes(Channels)); // Number of channels, 1 for mono, 2 for stereo _randomAccessWriter.WriteInt(Integer.ReverseBytes(_sampleRate)); // Sample rate _randomAccessWriter.WriteInt(Integer.ReverseBytes(_sampleRate * FmtAvgBps * Channels / 8)); // Byte rate, SampleRate*NumberOfChannels*BitsPerSample/8 _randomAccessWriter.WriteShort(Short.ReverseBytes(Channels * FmtAvgBps / 8)); // Block align, NumberOfChannels*BitsPerSample/8 _randomAccessWriter.WriteShort(Short.ReverseBytes(FmtAvgBps)); // Bits per sample _randomAccessWriter.WriteBytes("data"); _randomAccessWriter.WriteInt(0); // Data chunk size not known yet, write 0 } } catch (NullReferenceException e) { Log.Debug(Tag, e.Message + e.StackTrace); State = WavRecorderState.Error; } }
/// <exception cref="System.IO.IOException"/> public virtual void TestHandleTruncatedBlockFile() { MiniDFSCluster cluster = null; HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/tmp/testHandleTruncatedBlockFile._PORT" ); conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C"); Path TestPath = new Path("/a"); Path TestPath2 = new Path("/b"); long RandomSeed = 4567L; long RandomSeed2 = 4568L; FSDataInputStream fsIn = null; int TestLength = 3456; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestLength, (short)1, RandomSeed); DFSTestUtil.CreateFile(fs, TestPath2, TestLength, (short)1, RandomSeed2); fsIn = cluster.GetFileSystem().Open(TestPath2); byte[] original = new byte[TestLength]; IOUtils.ReadFully(fsIn, original, 0, TestLength); fsIn.Close(); fsIn = null; try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath); FilePath dataFile = cluster.GetBlockFile(0, block); cluster.Shutdown(); cluster = null; RandomAccessFile raf = null; try { raf = new RandomAccessFile(dataFile, "rw"); raf.SetLength(0); } finally { if (raf != null) { raf.Close(); } } cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(false).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); fsIn = fs.Open(TestPath); try { byte[] buf = new byte[100]; fsIn.Seek(2000); fsIn.ReadFully(buf, 0, buf.Length); NUnit.Framework.Assert.Fail("shouldn't be able to read from corrupt 0-length " + "block file."); } catch (IOException e) { DFSClient.Log.Error("caught exception ", e); } fsIn.Close(); fsIn = null; // We should still be able to read the other file. // This is important because it indicates that we detected that the // previous block was corrupt, rather than blaming the problem on // communication. fsIn = fs.Open(TestPath2); byte[] buf_1 = new byte[original.Length]; fsIn.ReadFully(buf_1, 0, buf_1.Length); TestBlockReaderLocal.AssertArrayRegionsEqual(original, 0, buf_1, 0, original.Length ); fsIn.Close(); fsIn = null; } finally { if (fsIn != null) { fsIn.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest test, bool checksum, long readahead) { Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null )); MiniDFSCluster cluster = null; HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum ); conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest .BytesPerChecksum); conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C"); conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead); test.SetConfiguration(conf); FileInputStream dataIn = null; FileInputStream metaIn = null; Path TestPath = new Path("/a"); long RandomSeed = 4567L; BlockReaderLocal blockReaderLocal = null; FSDataInputStream fsIn = null; byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength]; FileSystem fs = null; ShortCircuitShm shm = null; RandomAccessFile raf = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength , (short)1, RandomSeed); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength ); fsIn.Close(); fsIn = null; ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath); FilePath dataFile = cluster.GetBlockFile(0, block); FilePath metaFile = cluster.GetBlockMetadataFile(0, block); ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache (); cluster.Shutdown(); cluster = null; test.Setup(dataFile, checksum); FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile) , new FileInputStream(metaFile) }; dataIn = streams[0]; metaIn = streams[1]; ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId ()); raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID. RandomUUID().ToString()), "rw"); raf.SetLength(8192); FileInputStream shmStream = new FileInputStream(raf.GetFD()); shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream); ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache , Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block)) ); blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename (TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy (new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build(); dataIn = null; metaIn = null; test.DoTest(blockReaderLocal, original); // BlockReaderLocal should not alter the file position. NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position()); NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position()); } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } if (dataIn != null) { dataIn.Close(); } if (metaIn != null) { metaIn.Close(); } if (blockReaderLocal != null) { blockReaderLocal.Close(); } if (shm != null) { shm.Free(); } if (raf != null) { raf.Close(); } } }
public virtual void TestAddHugeFile() { Measure("Commencing test"); FilePath file = new FilePath(db.WorkTree, "a.txt"); RandomAccessFile rf = new RandomAccessFile(file, "rw"); rf.SetLength(4429185024L); rf.Close(); Measure("Created file"); Git git = new Git(db); git.Add().AddFilepattern("a.txt").Call(); Measure("Added file"); NUnit.Framework.Assert.AreEqual("[a.txt, mode:100644, length:134217728, sha1:b8cfba97c2b962a44f080b3ca4e03b3204b6a350]" , IndexState(LENGTH | CONTENT_ID)); Status status = git.Status().Call(); Measure("Status after add"); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded()); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); // Does not change anything, but modified timestamp rf = new RandomAccessFile(file, "rw"); rf.Write(0); rf.Close(); status = git.Status().Call(); Measure("Status after non-modifying update"); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded()); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); // Change something rf = new RandomAccessFile(file, "rw"); rf.Write('a'); rf.Close(); status = git.Status().Call(); Measure("Status after modifying update"); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded()); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified()); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); // Truncate mod 4G and re-establish equality rf = new RandomAccessFile(file, "rw"); rf.SetLength(134217728L); rf.Write(0); rf.Close(); status = git.Status().Call(); Measure("Status after truncating update"); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded()); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified()); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); // Change something rf = new RandomAccessFile(file, "rw"); rf.Write('a'); rf.Close(); status = git.Status().Call(); Measure("Status after modifying and truncating update"); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded()); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified()); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); // Truncate to entry length becomes negative int rf = new RandomAccessFile(file, "rw"); rf.SetLength(3429185024L); rf.Write(0); rf.Close(); git.Add().AddFilepattern("a.txt").Call(); Measure("Added truncated file"); NUnit.Framework.Assert.AreEqual("[a.txt, mode:100644, length:-865782272, sha1:59b3282f8f59f22d953df956ad3511bf2dc660fd]" , IndexState(LENGTH | CONTENT_ID)); status = git.Status().Call(); Measure("Status after status on truncated file"); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded()); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); // Change something rf = new RandomAccessFile(file, "rw"); rf.Write('a'); rf.Close(); status = git.Status().Call(); Measure("Status after modifying and truncating update"); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded()); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified()); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); git.Commit().SetMessage("make a commit").Call(); Measure("After commit"); status = git.Status().Call(); Measure("After status after commit"); NUnit.Framework.Assert.AreEqual(0, status.GetAdded().Count); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified()); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); git.Reset().SetMode(ResetCommand.ResetType.HARD).Call(); Measure("After reset --hard"); NUnit.Framework.Assert.AreEqual("[a.txt, mode:100644, length:-865782272, sha1:59b3282f8f59f22d953df956ad3511bf2dc660fd]" , IndexState(LENGTH | CONTENT_ID)); status = git.Status().Call(); Measure("Status after hard reset"); NUnit.Framework.Assert.AreEqual(0, status.GetAdded().Count); NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count); NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count); NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count); NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count); NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count); NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count); }
/// <summary>Find out the number of bytes in the block that match its crc.</summary> /// <remarks> /// Find out the number of bytes in the block that match its crc. /// This algorithm assumes that data corruption caused by unexpected /// datanode shutdown occurs only in the last crc chunk. So it checks /// only the last chunk. /// </remarks> /// <param name="blockFile">the block file</param> /// <param name="genStamp">generation stamp of the block</param> /// <returns>the number of valid bytes</returns> private long ValidateIntegrityAndSetLength(FilePath blockFile, long genStamp) { DataInputStream checksumIn = null; InputStream blockIn = null; try { FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, genStamp); long blockFileLen = blockFile.Length(); long metaFileLen = metaFile.Length(); int crcHeaderLen = DataChecksum.GetChecksumHeaderSize(); if (!blockFile.Exists() || blockFileLen == 0 || !metaFile.Exists() || metaFileLen < crcHeaderLen) { return(0); } checksumIn = new DataInputStream(new BufferedInputStream(new FileInputStream(metaFile ), HdfsConstants.IoFileBufferSize)); // read and handle the common header here. For now just a version DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(checksumIn, metaFile ); int bytesPerChecksum = checksum.GetBytesPerChecksum(); int checksumSize = checksum.GetChecksumSize(); long numChunks = Math.Min((blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum , (metaFileLen - crcHeaderLen) / checksumSize); if (numChunks == 0) { return(0); } IOUtils.SkipFully(checksumIn, (numChunks - 1) * checksumSize); blockIn = new FileInputStream(blockFile); long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum; IOUtils.SkipFully(blockIn, lastChunkStartPos); int lastChunkSize = (int)Math.Min(bytesPerChecksum, blockFileLen - lastChunkStartPos ); byte[] buf = new byte[lastChunkSize + checksumSize]; checksumIn.ReadFully(buf, lastChunkSize, checksumSize); IOUtils.ReadFully(blockIn, buf, 0, lastChunkSize); checksum.Update(buf, 0, lastChunkSize); long validFileLength; if (checksum.Compare(buf, lastChunkSize)) { // last chunk matches crc validFileLength = lastChunkStartPos + lastChunkSize; } else { // last chunck is corrupt validFileLength = lastChunkStartPos; } // truncate if extra bytes are present without CRC if (blockFile.Length() > validFileLength) { RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw"); try { // truncate blockFile blockRAF.SetLength(validFileLength); } finally { blockRAF.Close(); } } return(validFileLength); } catch (IOException e) { FsDatasetImpl.Log.Warn(e); return(0); } finally { IOUtils.CloseStream(checksumIn); IOUtils.CloseStream(blockIn); } }