/// <summary>Truncate the given file to the given length</summary>
/// <exception cref="System.IO.IOException"/>
private void TruncateFile(FilePath logFile, long newLength)
{
RandomAccessFile raf = new RandomAccessFile(logFile, "rw");
raf.SetLength(newLength);
raf.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void Corrupt(FilePath editFile)
{
// Corrupt the last edit
long fileLen = editFile.Length();
RandomAccessFile rwf = new RandomAccessFile(editFile, "rw");
rwf.SetLength(fileLen - 1);
rwf.Close();
}
/// <summary>
/// Open a random access file for the given path, more and access.
/// </summary>
private static RandomAccessFile Open(string path, FileMode mode, FileAccess access)
{
var file = new Java.Io.File(path);
switch (mode)
{
case FileMode.CreateNew:
if (file.Exists())
{
throw new IOException("File already exists");
}
break;
case FileMode.Open:
if (!file.Exists())
{
throw new FileNotFoundException(path);
}
break;
case FileMode.Append:
access = FileAccess.Write;
break;
}
switch (mode)
{
case FileMode.Create:
case FileMode.CreateNew:
case FileMode.OpenOrCreate:
if (access == FileAccess.Read)
{
//create empty file, so it can be opened again with read only right,
//otherwise an FilNotFoundException is thrown.
var additinalAccessFile = new RandomAccessFile(file, "rw");
}
break;
}
var jMode = (access == FileAccess.Read) ? "r" : "rw";
var randomAccessFile = new RandomAccessFile(file, jMode);
switch (mode)
{
case FileMode.Truncate:
randomAccessFile.SetLength(0);
break;
case FileMode.Append:
randomAccessFile.Seek(randomAccessFile.Length());
break;
}
return(randomAccessFile);
}
/// <summary>TC7: Corrupted replicas are present.</summary>
/// <exception cref="System.IO.IOException">an exception might be thrown</exception>
/// <exception cref="System.Exception"/>
private void TestTC7(bool appendToNewBlock)
{
short repl = 2;
Path p = new Path("/TC7/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file with replication factor of 2. Write half block of data. Close file.
int len1 = (int)(BlockSize / 2);
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, repl, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
DFSTestUtil.WaitReplication(fs, p, repl);
//b. Log into one datanode that has one replica of this block.
// Find the block file on this datanode and truncate it to zero size.
LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(p.ToString()
, 0L, len1);
NUnit.Framework.Assert.AreEqual(1, locatedblocks.LocatedBlockCount());
LocatedBlock lb = locatedblocks.Get(0);
ExtendedBlock blk = lb.GetBlock();
NUnit.Framework.Assert.AreEqual(len1, lb.GetBlockSize());
DatanodeInfo[] datanodeinfos = lb.GetLocations();
NUnit.Framework.Assert.AreEqual(repl, datanodeinfos.Length);
DataNode dn = cluster.GetDataNode(datanodeinfos[0].GetIpcPort());
FilePath f = DataNodeTestUtils.GetBlockFile(dn, blk.GetBlockPoolId(), blk.GetLocalBlock
());
RandomAccessFile raf = new RandomAccessFile(f, "rw");
AppendTestUtil.Log.Info("dn=" + dn + ", blk=" + blk + " (length=" + blk.GetNumBytes
() + ")");
NUnit.Framework.Assert.AreEqual(len1, raf.Length());
raf.SetLength(0);
raf.Close();
//c. Open file in "append mode". Append a new block worth of data. Close file.
int len2 = (int)BlockSize;
{
FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//d. Reopen file and read two blocks worth of data.
AppendTestUtil.Check(fs, p, len1 + len2);
}
/// <exception cref="System.IO.IOException"/>
private bool Truncate(long newLength, StringBuilder b)
{
RandomAccessFile raf = new RandomAccessFile(localFile, "rw");
raf.SetLength(newLength);
raf.Close();
bool isReady = dfs.Truncate(file, newLength);
b.Append(", newLength=").Append(newLength).Append(", isReady=").Append(isReady);
if (!isReady)
{
TestFileTruncate.CheckBlockRecovery(file, dfs, 100, 300L);
}
return(isReady);
}
public virtual void TestBlockRecoveryWithLessMetafile()
{
Configuration conf = new Configuration();
conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser
().GetShortUserName());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
Path file = new Path("/testRecoveryFile");
DistributedFileSystem dfs = cluster.GetFileSystem();
FSDataOutputStream @out = dfs.Create(file);
int count = 0;
while (count < 2 * 1024 * 1024)
{
@out.WriteBytes("Data");
count += 4;
}
@out.Hsync();
// abort the original stream
((DFSOutputStream)@out.GetWrappedStream()).Abort();
LocatedBlocks locations = cluster.GetNameNodeRpc().GetBlockLocations(file.ToString
(), 0, count);
ExtendedBlock block = locations.Get(0).GetBlock();
DataNode dn = cluster.GetDataNodes()[0];
BlockLocalPathInfo localPathInfo = dn.GetBlockLocalPathInfo(block, null);
FilePath metafile = new FilePath(localPathInfo.GetMetaPath());
NUnit.Framework.Assert.IsTrue(metafile.Exists());
// reduce the block meta file size
RandomAccessFile raf = new RandomAccessFile(metafile, "rw");
raf.SetLength(metafile.Length() - 20);
raf.Close();
// restart DN to make replica to RWR
MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0);
cluster.RestartDataNode(dnProp, true);
// try to recover the lease
DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster
.GetConfiguration(0));
count = 0;
while (++count < 10 && !newdfs.RecoverLease(file))
{
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue("File should be closed", newdfs.RecoverLease(file));
}
/// <summary>
/// Sets the length of the current stream.
/// </summary>
/// <param name="value">The new length of the stream.</param>
public override void SetLength(long value)
{
file.SetLength(value);
}
/// <summary>
/// Open a random access file for the given path, more and access.
/// </summary>
private static RandomAccessFile Open(string path, FileMode mode, FileAccess access)
{
var file = new Java.Io.File(path);
switch (mode)
{
case FileMode.CreateNew:
if (file.Exists())
throw new IOException("File already exists");
break;
case FileMode.Open:
if (!file.Exists())
throw new FileNotFoundException(path);
break;
case FileMode.Append:
access = FileAccess.Write;
break;
}
switch (mode)
{
case FileMode.Create:
case FileMode.CreateNew:
case FileMode.OpenOrCreate:
if (access == FileAccess.Read)
{
//create empty file, so it can be opened again with read only right,
//otherwise an FilNotFoundException is thrown.
var additinalAccessFile = new RandomAccessFile(file, "rw");
}
break;
}
var jMode = (access == FileAccess.Read) ? "r" : "rw";
var randomAccessFile = new RandomAccessFile(file, jMode);
switch (mode)
{
case FileMode.Truncate:
randomAccessFile.SetLength(0);
break;
case FileMode.Append:
randomAccessFile.Seek(randomAccessFile.Length());
break;
}
return randomAccessFile;
}
private void InitializeHeader()
{
try
{
lock (_randomAccessLock)
{
_randomAccessWriter = new RandomAccessFile(_filePath, "rw");
_randomAccessWriter.SetLength(0);
// Set file length to 0, to prevent unexpected behavior in case the file already existed
_randomAccessWriter.WriteBytes("RIFF");
_randomAccessWriter.WriteInt(0); // Final file size not known yet, write 0
_randomAccessWriter.WriteBytes("WAVE");
_randomAccessWriter.WriteBytes("fmt ");
_randomAccessWriter.WriteInt(Integer.ReverseBytes(16)); // Sub-chunk size, 16 for PCM
_randomAccessWriter.WriteShort(Short.ReverseBytes(1)); // AudioFormat, 1 for PCM
_randomAccessWriter.WriteShort(Short.ReverseBytes(Channels)); // Number of channels, 1 for mono, 2 for stereo
_randomAccessWriter.WriteInt(Integer.ReverseBytes(_sampleRate)); // Sample rate
_randomAccessWriter.WriteInt(Integer.ReverseBytes(_sampleRate * FmtAvgBps * Channels / 8));
// Byte rate, SampleRate*NumberOfChannels*BitsPerSample/8
_randomAccessWriter.WriteShort(Short.ReverseBytes(Channels * FmtAvgBps / 8));
// Block align, NumberOfChannels*BitsPerSample/8
_randomAccessWriter.WriteShort(Short.ReverseBytes(FmtAvgBps)); // Bits per sample
_randomAccessWriter.WriteBytes("data");
_randomAccessWriter.WriteInt(0); // Data chunk size not known yet, write 0
}
}
catch (NullReferenceException e)
{
Log.Debug(Tag, e.Message + e.StackTrace);
State = WavRecorderState.Error;
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestHandleTruncatedBlockFile()
{
MiniDFSCluster cluster = null;
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, true);
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false);
conf.Set(DFSConfigKeys.DfsDomainSocketPathKey, "/tmp/testHandleTruncatedBlockFile._PORT"
);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
Path TestPath = new Path("/a");
Path TestPath2 = new Path("/b");
long RandomSeed = 4567L;
long RandomSeed2 = 4568L;
FSDataInputStream fsIn = null;
int TestLength = 3456;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestLength, (short)1, RandomSeed);
DFSTestUtil.CreateFile(fs, TestPath2, TestLength, (short)1, RandomSeed2);
fsIn = cluster.GetFileSystem().Open(TestPath2);
byte[] original = new byte[TestLength];
IOUtils.ReadFully(fsIn, original, 0, TestLength);
fsIn.Close();
fsIn = null;
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath);
FilePath dataFile = cluster.GetBlockFile(0, block);
cluster.Shutdown();
cluster = null;
RandomAccessFile raf = null;
try
{
raf = new RandomAccessFile(dataFile, "rw");
raf.SetLength(0);
}
finally
{
if (raf != null)
{
raf.Close();
}
}
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Format(false).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
fsIn = fs.Open(TestPath);
try
{
byte[] buf = new byte[100];
fsIn.Seek(2000);
fsIn.ReadFully(buf, 0, buf.Length);
NUnit.Framework.Assert.Fail("shouldn't be able to read from corrupt 0-length " +
"block file.");
}
catch (IOException e)
{
DFSClient.Log.Error("caught exception ", e);
}
fsIn.Close();
fsIn = null;
// We should still be able to read the other file.
// This is important because it indicates that we detected that the
// previous block was corrupt, rather than blaming the problem on
// communication.
fsIn = fs.Open(TestPath2);
byte[] buf_1 = new byte[original.Length];
fsIn.ReadFully(buf_1, 0, buf_1.Length);
TestBlockReaderLocal.AssertArrayRegionsEqual(original, 0, buf_1, 0, original.Length
);
fsIn.Close();
fsIn = null;
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest
test, bool checksum, long readahead)
{
Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null
));
MiniDFSCluster cluster = null;
HdfsConfiguration conf = new HdfsConfiguration();
conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum
);
conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest
.BytesPerChecksum);
conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C");
conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead);
test.SetConfiguration(conf);
FileInputStream dataIn = null;
FileInputStream metaIn = null;
Path TestPath = new Path("/a");
long RandomSeed = 4567L;
BlockReaderLocal blockReaderLocal = null;
FSDataInputStream fsIn = null;
byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength];
FileSystem fs = null;
ShortCircuitShm shm = null;
RandomAccessFile raf = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
, (short)1, RandomSeed);
try
{
DFSTestUtil.WaitReplication(fs, TestPath, (short)1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: "
+ e);
}
catch (TimeoutException e)
{
NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: "
+ e);
}
fsIn = fs.Open(TestPath);
IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength
);
fsIn.Close();
fsIn = null;
ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath);
FilePath dataFile = cluster.GetBlockFile(0, block);
FilePath metaFile = cluster.GetBlockMetadataFile(0, block);
ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache
();
cluster.Shutdown();
cluster = null;
test.Setup(dataFile, checksum);
FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile)
, new FileInputStream(metaFile) };
dataIn = streams[0];
metaIn = streams[1];
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID.
RandomUUID().ToString()), "rw");
raf.SetLength(8192);
FileInputStream shmStream = new FileInputStream(raf.GetFD());
shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream);
ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache
, Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block))
);
blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename
(TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy
(new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build();
dataIn = null;
metaIn = null;
test.DoTest(blockReaderLocal, original);
// BlockReaderLocal should not alter the file position.
NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position());
NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position());
}
finally
{
if (fsIn != null)
{
fsIn.Close();
}
if (fs != null)
{
fs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
if (dataIn != null)
{
dataIn.Close();
}
if (metaIn != null)
{
metaIn.Close();
}
if (blockReaderLocal != null)
{
blockReaderLocal.Close();
}
if (shm != null)
{
shm.Free();
}
if (raf != null)
{
raf.Close();
}
}
}
public virtual void TestAddHugeFile()
{
Measure("Commencing test");
FilePath file = new FilePath(db.WorkTree, "a.txt");
RandomAccessFile rf = new RandomAccessFile(file, "rw");
rf.SetLength(4429185024L);
rf.Close();
Measure("Created file");
Git git = new Git(db);
git.Add().AddFilepattern("a.txt").Call();
Measure("Added file");
NUnit.Framework.Assert.AreEqual("[a.txt, mode:100644, length:134217728, sha1:b8cfba97c2b962a44f080b3ca4e03b3204b6a350]"
, IndexState(LENGTH | CONTENT_ID));
Status status = git.Status().Call();
Measure("Status after add");
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded());
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
// Does not change anything, but modified timestamp
rf = new RandomAccessFile(file, "rw");
rf.Write(0);
rf.Close();
status = git.Status().Call();
Measure("Status after non-modifying update");
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded());
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
// Change something
rf = new RandomAccessFile(file, "rw");
rf.Write('a');
rf.Close();
status = git.Status().Call();
Measure("Status after modifying update");
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded());
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified());
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
// Truncate mod 4G and re-establish equality
rf = new RandomAccessFile(file, "rw");
rf.SetLength(134217728L);
rf.Write(0);
rf.Close();
status = git.Status().Call();
Measure("Status after truncating update");
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded());
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified());
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
// Change something
rf = new RandomAccessFile(file, "rw");
rf.Write('a');
rf.Close();
status = git.Status().Call();
Measure("Status after modifying and truncating update");
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded());
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified());
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
// Truncate to entry length becomes negative int
rf = new RandomAccessFile(file, "rw");
rf.SetLength(3429185024L);
rf.Write(0);
rf.Close();
git.Add().AddFilepattern("a.txt").Call();
Measure("Added truncated file");
NUnit.Framework.Assert.AreEqual("[a.txt, mode:100644, length:-865782272, sha1:59b3282f8f59f22d953df956ad3511bf2dc660fd]"
, IndexState(LENGTH | CONTENT_ID));
status = git.Status().Call();
Measure("Status after status on truncated file");
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded());
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
// Change something
rf = new RandomAccessFile(file, "rw");
rf.Write('a');
rf.Close();
status = git.Status().Call();
Measure("Status after modifying and truncating update");
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetAdded());
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified());
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
git.Commit().SetMessage("make a commit").Call();
Measure("After commit");
status = git.Status().Call();
Measure("After status after commit");
NUnit.Framework.Assert.AreEqual(0, status.GetAdded().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
AssertCollectionEquals(Arrays.AsList("a.txt"), status.GetModified());
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
git.Reset().SetMode(ResetCommand.ResetType.HARD).Call();
Measure("After reset --hard");
NUnit.Framework.Assert.AreEqual("[a.txt, mode:100644, length:-865782272, sha1:59b3282f8f59f22d953df956ad3511bf2dc660fd]"
, IndexState(LENGTH | CONTENT_ID));
status = git.Status().Call();
Measure("Status after hard reset");
NUnit.Framework.Assert.AreEqual(0, status.GetAdded().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetChanged().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetConflicting().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetMissing().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetModified().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetRemoved().Count);
NUnit.Framework.Assert.AreEqual(0, status.GetUntracked().Count);
}
/// <summary>Find out the number of bytes in the block that match its crc.</summary>
/// <remarks>
/// Find out the number of bytes in the block that match its crc.
/// This algorithm assumes that data corruption caused by unexpected
/// datanode shutdown occurs only in the last crc chunk. So it checks
/// only the last chunk.
/// </remarks>
/// <param name="blockFile">the block file</param>
/// <param name="genStamp">generation stamp of the block</param>
/// <returns>the number of valid bytes</returns>
private long ValidateIntegrityAndSetLength(FilePath blockFile, long genStamp)
{
DataInputStream checksumIn = null;
InputStream blockIn = null;
try
{
FilePath metaFile = FsDatasetUtil.GetMetaFile(blockFile, genStamp);
long blockFileLen = blockFile.Length();
long metaFileLen = metaFile.Length();
int crcHeaderLen = DataChecksum.GetChecksumHeaderSize();
if (!blockFile.Exists() || blockFileLen == 0 || !metaFile.Exists() || metaFileLen
< crcHeaderLen)
{
return(0);
}
checksumIn = new DataInputStream(new BufferedInputStream(new FileInputStream(metaFile
), HdfsConstants.IoFileBufferSize));
// read and handle the common header here. For now just a version
DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(checksumIn, metaFile
);
int bytesPerChecksum = checksum.GetBytesPerChecksum();
int checksumSize = checksum.GetChecksumSize();
long numChunks = Math.Min((blockFileLen + bytesPerChecksum - 1) / bytesPerChecksum
, (metaFileLen - crcHeaderLen) / checksumSize);
if (numChunks == 0)
{
return(0);
}
IOUtils.SkipFully(checksumIn, (numChunks - 1) * checksumSize);
blockIn = new FileInputStream(blockFile);
long lastChunkStartPos = (numChunks - 1) * bytesPerChecksum;
IOUtils.SkipFully(blockIn, lastChunkStartPos);
int lastChunkSize = (int)Math.Min(bytesPerChecksum, blockFileLen - lastChunkStartPos
);
byte[] buf = new byte[lastChunkSize + checksumSize];
checksumIn.ReadFully(buf, lastChunkSize, checksumSize);
IOUtils.ReadFully(blockIn, buf, 0, lastChunkSize);
checksum.Update(buf, 0, lastChunkSize);
long validFileLength;
if (checksum.Compare(buf, lastChunkSize))
{
// last chunk matches crc
validFileLength = lastChunkStartPos + lastChunkSize;
}
else
{
// last chunck is corrupt
validFileLength = lastChunkStartPos;
}
// truncate if extra bytes are present without CRC
if (blockFile.Length() > validFileLength)
{
RandomAccessFile blockRAF = new RandomAccessFile(blockFile, "rw");
try
{
// truncate blockFile
blockRAF.SetLength(validFileLength);
}
finally
{
blockRAF.Close();
}
}
return(validFileLength);
}
catch (IOException e)
{
FsDatasetImpl.Log.Warn(e);
return(0);
}
finally
{
IOUtils.CloseStream(checksumIn);
IOUtils.CloseStream(blockIn);
}
}
