public virtual void TestDeletingBlocks()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).Build
();
try
{
cluster.WaitActive();
DataNode dn = cluster.GetDataNodes()[0];
FsDatasetImpl ds = (FsDatasetImpl)DataNodeTestUtils.GetFSDataset(dn);
FsVolumeImpl vol = ds.GetVolumes()[0];
ExtendedBlock eb;
ReplicaInfo info;
IList <Block> blockList = new AList <Block>();
for (int i = 1; i <= 63; i++)
{
eb = new ExtendedBlock(Blockpool, i, 1, 1000 + i);
info = new FinalizedReplica(eb.GetLocalBlock(), vol, vol.GetCurrentDir().GetParentFile
());
ds.volumeMap.Add(Blockpool, info);
info.GetBlockFile().CreateNewFile();
info.GetMetaFile().CreateNewFile();
blockList.AddItem(info);
}
ds.Invalidate(Blockpool, Sharpen.Collections.ToArray(blockList, new Block[0]));
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
// Nothing to do
NUnit.Framework.Assert.IsTrue(ds.IsDeletingBlock(Blockpool, blockList[0].GetBlockId
()));
blockList.Clear();
eb = new ExtendedBlock(Blockpool, 64, 1, 1064);
info = new FinalizedReplica(eb.GetLocalBlock(), vol, vol.GetCurrentDir().GetParentFile
());
ds.volumeMap.Add(Blockpool, info);
info.GetBlockFile().CreateNewFile();
info.GetMetaFile().CreateNewFile();
blockList.AddItem(info);
ds.Invalidate(Blockpool, Sharpen.Collections.ToArray(blockList, new Block[0]));
try
{
Sharpen.Thread.Sleep(1000);
}
catch (Exception)
{
}
// Nothing to do
NUnit.Framework.Assert.IsFalse(ds.IsDeletingBlock(Blockpool, blockList[0].GetBlockId
()));
}
finally
{
cluster.Shutdown();
}
}
/// <summary>
/// Generate testing environment and return a collection of blocks
/// on which to run the tests.
/// </summary>
/// <param name="bpid">Block pool ID to generate blocks for</param>
/// <param name="dataSet">Namespace in which to insert blocks</param>
/// <returns>Contrived blocks for further testing.</returns>
/// <exception cref="System.IO.IOException"/>
private ExtendedBlock[] Setup(string bpid, FsDatasetImpl dataSet)
{
// setup replicas map
ExtendedBlock[] blocks = new ExtendedBlock[] { new ExtendedBlock(bpid, 1, 1, 2001
), new ExtendedBlock(bpid, 2, 1, 2002), new ExtendedBlock(bpid, 3, 1, 2003), new
ExtendedBlock(bpid, 4, 1, 2004), new ExtendedBlock(bpid, 5, 1, 2005), new ExtendedBlock
(bpid, 6, 1, 2006) };
ReplicaMap replicasMap = dataSet.volumeMap;
FsVolumeImpl vol = (FsVolumeImpl)dataSet.volumes.GetNextVolume(StorageType.Default
, 0).GetVolume();
ReplicaInfo replicaInfo = new FinalizedReplica(blocks[Finalized].GetLocalBlock(),
vol, vol.GetCurrentDir().GetParentFile());
replicasMap.Add(bpid, replicaInfo);
replicaInfo.GetBlockFile().CreateNewFile();
replicaInfo.GetMetaFile().CreateNewFile();
replicasMap.Add(bpid, new ReplicaInPipeline(blocks[Temporary].GetBlockId(), blocks
[Temporary].GetGenerationStamp(), vol, vol.CreateTmpFile(bpid, blocks[Temporary]
.GetLocalBlock()).GetParentFile(), 0));
replicaInfo = new ReplicaBeingWritten(blocks[Rbw].GetLocalBlock(), vol, vol.CreateRbwFile
(bpid, blocks[Rbw].GetLocalBlock()).GetParentFile(), null);
replicasMap.Add(bpid, replicaInfo);
replicaInfo.GetBlockFile().CreateNewFile();
replicaInfo.GetMetaFile().CreateNewFile();
replicasMap.Add(bpid, new ReplicaWaitingToBeRecovered(blocks[Rwr].GetLocalBlock()
, vol, vol.CreateRbwFile(bpid, blocks[Rwr].GetLocalBlock()).GetParentFile()));
replicasMap.Add(bpid, new ReplicaUnderRecovery(new FinalizedReplica(blocks[Rur].GetLocalBlock
(), vol, vol.GetCurrentDir().GetParentFile()), 2007));
return(blocks);
}
/// <exception cref="System.IO.IOException"/>
private void TestRbwReplicas(MiniDFSCluster cluster, bool isCorrupt)
{
FSDataOutputStream @out = null;
FileSystem fs = cluster.GetFileSystem();
Path src = new Path("/test.txt");
try
{
int fileLen = 515;
// create some rbw replicas on disk
byte[] writeBuf = new byte[fileLen];
new Random().NextBytes(writeBuf);
@out = fs.Create(src);
@out.Write(writeBuf);
@out.Hflush();
DataNode dn = cluster.GetDataNodes()[0];
foreach (FsVolumeSpi v in Dataset(dn).GetVolumes())
{
FsVolumeImpl volume = (FsVolumeImpl)v;
FilePath currentDir = volume.GetCurrentDir().GetParentFile().GetParentFile();
FilePath rbwDir = new FilePath(currentDir, "rbw");
foreach (FilePath file in rbwDir.ListFiles())
{
if (isCorrupt && Block.IsBlockFilename(file))
{
new RandomAccessFile(file, "rw").SetLength(fileLen - 1);
}
}
}
// corrupt
cluster.RestartDataNodes();
cluster.WaitActive();
dn = cluster.GetDataNodes()[0];
// check volumeMap: one rwr replica
string bpid = cluster.GetNamesystem().GetBlockPoolId();
ReplicaMap replicas = Dataset(dn).volumeMap;
NUnit.Framework.Assert.AreEqual(1, replicas.Size(bpid));
ReplicaInfo replica = replicas.Replicas(bpid).GetEnumerator().Next();
NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Rwr, replica.GetState
());
if (isCorrupt)
{
NUnit.Framework.Assert.AreEqual((fileLen - 1) / 512 * 512, replica.GetNumBytes());
}
else
{
NUnit.Framework.Assert.AreEqual(fileLen, replica.GetNumBytes());
}
Dataset(dn).Invalidate(bpid, new Block[] { replica });
}
finally
{
IOUtils.CloseStream(@out);
if (fs.Exists(src))
{
fs.Delete(src, false);
}
fs.Close();
}
}
/// <summary>Asynchronously lazy persist the block from the RamDisk to Disk.</summary>
/// <exception cref="System.IO.IOException"/>
internal virtual void SubmitLazyPersistTask(string bpId, long blockId, long genStamp
, long creationTime, FilePath metaFile, FilePath blockFile, FsVolumeReference target
)
{
if (Log.IsDebugEnabled())
{
Log.Debug("LazyWriter schedule async task to persist RamDisk block pool id: " + bpId
+ " block id: " + blockId);
}
FsVolumeImpl volume = (FsVolumeImpl)target.GetVolume();
FilePath lazyPersistDir = volume.GetLazyPersistDir(bpId);
if (!lazyPersistDir.Exists() && !lazyPersistDir.Mkdirs())
{
FsDatasetImpl.Log.Warn("LazyWriter failed to create " + lazyPersistDir);
throw new IOException("LazyWriter fail to find or create lazy persist dir: " + lazyPersistDir
.ToString());
}
RamDiskAsyncLazyPersistService.ReplicaLazyPersistTask lazyPersistTask = new RamDiskAsyncLazyPersistService.ReplicaLazyPersistTask
(this, bpId, blockId, genStamp, creationTime, blockFile, metaFile, target, lazyPersistDir
);
Execute(volume.GetCurrentDir(), lazyPersistTask);
}
public virtual void SubmitSyncFileRangeRequest(FsVolumeImpl volume, FileDescriptor
fd, long offset, long nbytes, int flags)
{
Execute(volume.GetCurrentDir(), new _Runnable_199(fd, offset, nbytes, flags));
}
