public _Supplier_457(DatanodeProtocolClientSideTranslatorPB mockNN, string fakeBlockPoolId
, ArgumentCaptor <StorageReceivedDeletedBlocks[]> captor)
{
this.mockNN = mockNN;
this.fakeBlockPoolId = fakeBlockPoolId;
this.captor = captor;
}
/// <summary>
/// Insert a Mockito spy object between the given DataNode and
/// the given NameNode.
/// </summary>
/// <remarks>
/// Insert a Mockito spy object between the given DataNode and
/// the given NameNode. This can be used to delay or wait for
/// RPC calls on the datanode->NN path.
/// </remarks>
public static DatanodeProtocolClientSideTranslatorPB SpyOnBposToNN(DataNode dn, NameNode
nn)
{
string bpid = nn.GetNamesystem().GetBlockPoolId();
BPOfferService bpos = null;
foreach (BPOfferService thisBpos in dn.GetAllBpOs())
{
if (thisBpos.GetBlockPoolId().Equals(bpid))
{
bpos = thisBpos;
break;
}
}
Preconditions.CheckArgument(bpos != null, "No such bpid: %s", bpid);
BPServiceActor bpsa = null;
foreach (BPServiceActor thisBpsa in bpos.GetBPServiceActors())
{
if (thisBpsa.GetNNSocketAddress().Equals(nn.GetServiceRpcAddress()))
{
bpsa = thisBpsa;
break;
}
}
Preconditions.CheckArgument(bpsa != null, "No service actor to NN at %s", nn.GetServiceRpcAddress
());
DatanodeProtocolClientSideTranslatorPB origNN = bpsa.GetNameNodeProxy();
DatanodeProtocolClientSideTranslatorPB spy = Org.Mockito.Mockito.Spy(origNN);
bpsa.SetNameNode(spy);
return(spy);
}
public virtual void TestStorageReportHasStorageTypeAndState()
{
// Make sure we are not testing with the default type, that would not
// be a very good test.
NUnit.Framework.Assert.AreNotSame(storageType, StorageType.Default);
NameNode nn = cluster.GetNameNode();
DataNode dn = cluster.GetDataNodes()[0];
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy = DataNodeTestUtils.SpyOnBposToNN(dn
, nn);
// Trigger a heartbeat so there is an interaction with the spy
// object.
DataNodeTestUtils.TriggerHeartbeat(dn);
// Verify that the callback passed in the expected parameters.
ArgumentCaptor <StorageReport[]> captor = ArgumentCaptor.ForClass <StorageReport[]>
();
Org.Mockito.Mockito.Verify(nnSpy).SendHeartbeat(Matchers.Any <DatanodeRegistration
>(), captor.Capture(), Matchers.AnyLong(), Matchers.AnyLong(), Matchers.AnyInt()
, Matchers.AnyInt(), Matchers.AnyInt(), Org.Mockito.Mockito.Any <VolumeFailureSummary
>());
StorageReport[] reports = captor.GetValue();
foreach (StorageReport report in reports)
{
Assert.AssertThat(report.GetStorage().GetStorageType(), IS.Is(storageType));
Assert.AssertThat(report.GetStorage().GetState(), IS.Is(DatanodeStorage.State.Normal
));
}
}
public virtual void SetUp()
{
mockDnConf = Org.Mockito.Mockito.Mock <DNConf>();
Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion()).When(mockDnConf).GetMinimumNameNodeVersion
();
DataNode mockDN = Org.Mockito.Mockito.Mock <DataNode>();
Org.Mockito.Mockito.DoReturn(true).When(mockDN).ShouldRun();
Org.Mockito.Mockito.DoReturn(mockDnConf).When(mockDN).GetDnConf();
BPOfferService mockBPOS = Org.Mockito.Mockito.Mock <BPOfferService>();
Org.Mockito.Mockito.DoReturn(mockDN).When(mockBPOS).GetDataNode();
actor = new BPServiceActor(InvalidAddr, mockBPOS);
fakeNsInfo = Org.Mockito.Mockito.Mock <NamespaceInfo>();
// Return a a good software version.
Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion()).When(fakeNsInfo).GetSoftwareVersion
();
// Return a good layout version for now.
Org.Mockito.Mockito.DoReturn(HdfsConstants.NamenodeLayoutVersion).When(fakeNsInfo
).GetLayoutVersion();
DatanodeProtocolClientSideTranslatorPB fakeDnProt = Org.Mockito.Mockito.Mock <DatanodeProtocolClientSideTranslatorPB
>();
Org.Mockito.Mockito.When(fakeDnProt.VersionRequest()).ThenReturn(fakeNsInfo);
actor.SetNameNode(fakeDnProt);
}
/// <summary>Ensure that a delayed IBR is generated for a block deleted on the DN.</summary>
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestReportBlockDeleted()
{
try
{
// Trigger a block report to reset the IBR timer.
DataNodeTestUtils.TriggerBlockReport(singletonDn);
// Spy on calls from the DN to the NN
DatanodeProtocolClientSideTranslatorPB nnSpy = SpyOnDnCallsToNn();
InjectBlockDeleted();
// Sleep for a very short time since IBR is generated
// asynchronously.
Sharpen.Thread.Sleep(2000);
// Ensure that no block report was generated immediately.
// Deleted blocks are reported when the IBR timer elapses.
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.Times(0)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
// Trigger a block report, this also triggers an IBR.
DataNodeTestUtils.TriggerBlockReport(singletonDn);
Sharpen.Thread.Sleep(2000);
// Ensure that the deleted block is reported.
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
finally
{
cluster.Shutdown();
cluster = null;
}
}
public virtual void StartUp()
{
conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, DataDir);
conf.Set(DFSConfigKeys.DfsDatanodeAddressKey, "0.0.0.0:0");
conf.Set(DFSConfigKeys.DfsDatanodeHttpAddressKey, "0.0.0.0:0");
conf.Set(DFSConfigKeys.DfsDatanodeIpcAddressKey, "0.0.0.0:0");
conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 0);
FileSystem.SetDefaultUri(conf, "hdfs://" + NnAddr.GetHostName() + ":" + NnAddr.Port
);
AList <StorageLocation> locations = new AList <StorageLocation>();
FilePath dataDir = new FilePath(DataDir);
FileUtil.FullyDelete(dataDir);
dataDir.Mkdirs();
StorageLocation location = StorageLocation.Parse(dataDir.GetPath());
locations.AddItem(location);
DatanodeProtocolClientSideTranslatorPB namenode = Org.Mockito.Mockito.Mock <DatanodeProtocolClientSideTranslatorPB
>();
Org.Mockito.Mockito.DoAnswer(new _Answer_143()).When(namenode).RegisterDatanode(Org.Mockito.Mockito
.Any <DatanodeRegistration>());
Org.Mockito.Mockito.When(namenode.VersionRequest()).ThenReturn(new NamespaceInfo(
1, ClusterId, PoolId, 1L));
Org.Mockito.Mockito.When(namenode.SendHeartbeat(Org.Mockito.Mockito.Any <DatanodeRegistration
>(), Org.Mockito.Mockito.Any <StorageReport[]>(), Org.Mockito.Mockito.AnyLong(),
Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito
.AnyInt(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.Any <VolumeFailureSummary
>())).ThenReturn(new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat
(HAServiceProtocol.HAServiceState.Active, 1), null));
dn = new _DataNode_169(namenode, conf, locations, null);
// Trigger a heartbeat so that it acknowledges the NN as active.
dn.GetAllBpOs()[0].TriggerHeartbeatForTests();
}
/// <summary>Test that a full block report is sent after hot swapping volumes</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
public virtual void TestFullBlockReportAfterRemovingVolumes()
{
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
// Similar to TestTriggerBlockReport, set a really long value for
// dfs.heartbeat.interval, so that incremental block reports and heartbeats
// won't be sent during this test unless they're triggered
// manually.
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 10800000L);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1080L);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
cluster.WaitActive();
DataNode dn = cluster.GetDataNodes()[0];
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(dn,
cluster.GetNameNode());
// Remove a data dir from datanode
FilePath dataDirToKeep = new FilePath(cluster.GetDataDirectory(), "data1");
dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, dataDirToKeep.ToString
());
// We should get 1 full report
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(1)).BlockReport
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport
[]>(), Matchers.Any <BlockReportContext>());
}
/// <summary>Add a received block entry and then replace it.</summary>
/// <remarks>
/// Add a received block entry and then replace it. Ensure that a single
/// IBR is generated and that pending receive request state is cleared.
/// This test case verifies the failure in HDFS-5922.
/// </remarks>
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestReplaceReceivedBlock()
{
try
{
// Spy on calls from the DN to the NN
DatanodeProtocolClientSideTranslatorPB nnSpy = SpyOnDnCallsToNn();
InjectBlockReceived();
InjectBlockReceived();
// Overwrite the existing entry.
// Sleep for a very short time since IBR is generated
// asynchronously.
Sharpen.Thread.Sleep(2000);
// Ensure that the received block is reported.
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.AtLeastOnce()).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
// Ensure that no more IBRs are pending.
NUnit.Framework.Assert.IsFalse(actor.HasPendingIBR());
}
finally
{
cluster.Shutdown();
cluster = null;
}
}
public virtual void TestDatanodeProtocol()
{
DatanodeProtocolClientSideTranslatorPB translator = new DatanodeProtocolClientSideTranslatorPB
(nnAddress, conf);
NUnit.Framework.Assert.IsTrue(translator.IsMethodSupported("sendHeartbeat"));
}
public _DataNode_169(DatanodeProtocolClientSideTranslatorPB namenode, Configuration
baseArg1, IList <StorageLocation> baseArg2, SecureDataNodeStarter.SecureResources
baseArg3)
: base(baseArg1, baseArg2, baseArg3)
{
this.namenode = namenode;
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Datanode.BPServiceActorActionException
/// "/>
public virtual void ReportTo(DatanodeProtocolClientSideTranslatorPB bpNamenode, DatanodeRegistration
bpRegistration)
{
if (bpRegistration == null)
{
return;
}
DatanodeInfo[] dnArr = new DatanodeInfo[] { new DatanodeInfo(bpRegistration) };
string[] uuids = new string[] { storageUuid };
StorageType[] types = new StorageType[] { storageType };
LocatedBlock[] locatedBlock = new LocatedBlock[] { new LocatedBlock(block, dnArr,
uuids, types) };
try
{
bpNamenode.ReportBadBlocks(locatedBlock);
}
catch (RemoteException re)
{
DataNode.Log.Info("reportBadBlock encountered RemoteException for " + "block: "
+ block, re);
}
catch (IOException)
{
throw new BPServiceActorActionException("Failed to report bad block " + block + " to namenode: "
);
}
}
/// <exception cref="System.Exception"/>
private ReceivedDeletedBlockInfo[] WaitForBlockReceived(ExtendedBlock fakeBlock,
DatanodeProtocolClientSideTranslatorPB mockNN)
{
string fakeBlockPoolId = fakeBlock.GetBlockPoolId();
ArgumentCaptor <StorageReceivedDeletedBlocks[]> captor = ArgumentCaptor.ForClass <StorageReceivedDeletedBlocks
[]>();
GenericTestUtils.WaitFor(new _Supplier_457(mockNN, fakeBlockPoolId, captor), 100,
10000);
return(captor.GetValue()[0].GetBlocks());
}
// return the initial state of the configuration
/// <summary>
/// Test for the case where one of the DNs in the pipeline is in the
/// process of doing a block report exactly when the block is closed.
/// </summary>
/// <remarks>
/// Test for the case where one of the DNs in the pipeline is in the
/// process of doing a block report exactly when the block is closed.
/// In this case, the block report becomes delayed until after the
/// block is marked completed on the NN, and hence it reports an RBW
/// replica for a COMPLETE block. Such a report should not be marked
/// corrupt.
/// This is a regression test for HDFS-2791.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestOneReplicaRbwReportArrivesAfterBlockCompleted()
{
CountDownLatch brFinished = new CountDownLatch(1);
GenericTestUtils.DelayAnswer delayer = new _DelayAnswer_579(brFinished, Log);
// inform the test that our block report went through.
string MethodName = GenericTestUtils.GetMethodName();
Path filePath = new Path("/" + MethodName + ".dat");
// Start a second DN for this test -- we're checking
// what happens when one of the DNs is slowed for some reason.
ReplFactor = 2;
StartDNandWait(null, false);
NameNode nn = cluster.GetNameNode();
FSDataOutputStream @out = fs.Create(filePath, ReplFactor);
try
{
AppendTestUtil.Write(@out, 0, 10);
@out.Hflush();
// Set up a spy so that we can delay the block report coming
// from this node.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(dn,
nn);
Org.Mockito.Mockito.DoAnswer(delayer).When(spy).BlockReport(Org.Mockito.Mockito.AnyObject
<DatanodeRegistration>(), Org.Mockito.Mockito.AnyString(), Org.Mockito.Mockito.AnyObject
<StorageBlockReport[]>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
// Force a block report to be generated. The block report will have
// an RBW replica in it. Wait for the RPC to be sent, but block
// it before it gets to the NN.
dn.ScheduleAllBlockReport(0);
delayer.WaitForCall();
}
finally
{
IOUtils.CloseStream(@out);
}
// Now that the stream is closed, the NN will have the block in COMPLETE
// state.
delayer.Proceed();
brFinished.Await();
// Verify that no replicas are marked corrupt, and that the
// file is still readable.
BlockManagerTestUtil.UpdateState(nn.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, nn.GetNamesystem().GetCorruptReplicaBlocks());
DFSTestUtil.ReadFile(fs, filePath);
// Ensure that the file is readable even from the DN that we futzed with.
cluster.StopDataNode(1);
DFSTestUtil.ReadFile(fs, filePath);
}
private bool Get(DatanodeProtocolClientSideTranslatorPB mockNN)
{
try
{
Org.Mockito.Mockito.Verify(mockNN).BlockReport(Org.Mockito.Mockito.AnyObject <DatanodeRegistration
>(), Org.Mockito.Mockito.Eq(TestBPOfferService.FakeBpid), Org.Mockito.Mockito.AnyObject
<StorageBlockReport[]>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
return(true);
}
catch (Exception t)
{
TestBPOfferService.Log.Info("waiting on block report: " + t.Message);
return(false);
}
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Datanode.BPServiceActorActionException
/// "/>
public virtual void ReportTo(DatanodeProtocolClientSideTranslatorPB bpNamenode, DatanodeRegistration
bpRegistration)
{
try
{
bpNamenode.ErrorReport(bpRegistration, errorCode, errorMessage);
}
catch (RemoteException re)
{
DataNode.Log.Info("trySendErrorReport encountered RemoteException " + "errorMessage: "
+ errorMessage + " errorCode: " + errorCode, re);
}
catch (IOException)
{
throw new BPServiceActorActionException("Error reporting " + "an error to namenode: "
);
}
}
/// <summary>Set up a mock NN with the bare minimum for a DN to register to it.</summary>
/// <exception cref="System.Exception"/>
private DatanodeProtocolClientSideTranslatorPB SetupNNMock(int nnIdx)
{
DatanodeProtocolClientSideTranslatorPB mock = Org.Mockito.Mockito.Mock <DatanodeProtocolClientSideTranslatorPB
>();
Org.Mockito.Mockito.DoReturn(new NamespaceInfo(1, FakeClusterid, FakeBpid, 0)).When
(mock).VersionRequest();
Org.Mockito.Mockito.DoReturn(DFSTestUtil.GetLocalDatanodeRegistration()).When(mock
).RegisterDatanode(Org.Mockito.Mockito.Any <DatanodeRegistration>());
Org.Mockito.Mockito.DoAnswer(new TestBPOfferService.HeartbeatAnswer(this, nnIdx))
.When(mock).SendHeartbeat(Org.Mockito.Mockito.Any <DatanodeRegistration>(), Org.Mockito.Mockito
.Any <StorageReport[]>(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito.AnyLong
(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito
.AnyInt(), Org.Mockito.Mockito.Any <VolumeFailureSummary>());
mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.
Standby, 0);
return(mock);
}
public virtual void TestDatanodeDetect()
{
AtomicReference <DatanodeProtocolProtos.BlockReportRequestProto> request = new AtomicReference
<DatanodeProtocolProtos.BlockReportRequestProto>();
// just capture the outgoing PB
DatanodeProtocolPB mockProxy = Org.Mockito.Mockito.Mock <DatanodeProtocolPB>();
Org.Mockito.Mockito.DoAnswer(new _Answer_205(request)).When(mockProxy).BlockReport
(Matchers.Any <RpcController>(), Matchers.Any <DatanodeProtocolProtos.BlockReportRequestProto
>());
DatanodeProtocolClientSideTranslatorPB nn = new DatanodeProtocolClientSideTranslatorPB
(mockProxy);
DatanodeRegistration reg = DFSTestUtil.GetLocalDatanodeRegistration();
NamespaceInfo nsInfo = new NamespaceInfo(1, "cluster", "bp", 1);
reg.SetNamespaceInfo(nsInfo);
Replica r = new FinalizedReplica(new Block(1, 2, 3), null, null);
BlockListAsLongs bbl = BlockListAsLongs.Encode(Sharpen.Collections.Singleton(r));
DatanodeStorage storage = new DatanodeStorage("s1");
StorageBlockReport[] sbr = new StorageBlockReport[] { new StorageBlockReport(storage
, bbl) };
// check DN sends new-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.StorageBlockReportBuffers.GetMask
());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
DatanodeProtocolProtos.BlockReportRequestProto proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksBuffersList().IsEmpty
());
// back up to prior version and check DN sends old-style BR
request.Set(null);
nsInfo.SetCapabilities(NamespaceInfo.Capability.Unknown.GetMask());
nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime())
);
proto = request.Get();
NUnit.Framework.Assert.IsNotNull(proto);
NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksList().IsEmpty());
NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksBuffersList().IsEmpty(
));
}
/// <summary>Verify the following scenario.</summary>
/// <remarks>
/// Verify the following scenario.
/// 1. The initial DatanodeProtocol.registerDatanode succeeds.
/// 2. DN starts heartbeat process.
/// 3. In the first heartbeat, NN asks DN to reregister.
/// 4. DN calls DatanodeProtocol.registerDatanode.
/// 5. DatanodeProtocol.registerDatanode throws EOFException.
/// 6. DN retries.
/// 7. DatanodeProtocol.registerDatanode succeeds.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDatanodeRegistrationRetry()
{
DatanodeProtocolClientSideTranslatorPB namenode = Org.Mockito.Mockito.Mock <DatanodeProtocolClientSideTranslatorPB
>();
Org.Mockito.Mockito.DoAnswer(new _Answer_166()).When(namenode).RegisterDatanode(Org.Mockito.Mockito
.Any <DatanodeRegistration>());
Org.Mockito.Mockito.When(namenode.VersionRequest()).ThenReturn(new NamespaceInfo(
1, ClusterId, PoolId, 1L));
Org.Mockito.Mockito.DoAnswer(new _Answer_190()).When(namenode).SendHeartbeat(Org.Mockito.Mockito
.Any <DatanodeRegistration>(), Org.Mockito.Mockito.Any <StorageReport[]>(), Org.Mockito.Mockito
.AnyLong(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito
.AnyInt(), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.Any <VolumeFailureSummary
>());
dn = new _DataNode_222(namenode, conf, locations, null);
// Trigger a heartbeat so that it acknowledges the NN as active.
dn.GetAllBpOs()[0].TriggerHeartbeatForTests();
WaitForBlockReport(namenode);
}
public virtual void SetUp()
{
conf = new HdfsConfiguration();
conf.SetLong(DFSConfigKeys.DfsNamenodePathBasedCacheRefreshIntervalMs, 100);
conf.SetLong(DFSConfigKeys.DfsCachereportIntervalMsecKey, 500);
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, CacheCapacity);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1);
prevCacheManipulator = NativeIO.POSIX.GetCacheManipulator();
NativeIO.POSIX.SetCacheManipulator(new NativeIO.POSIX.NoMlockCacheManipulator());
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
fs = cluster.GetFileSystem();
nn = cluster.GetNameNode();
fsImage = nn.GetFSImage();
dn = cluster.GetDataNodes()[0];
fsd = dn.GetFSDataset();
spyNN = DataNodeTestUtils.SpyOnBposToNN(dn, nn);
}
/// <summary>
/// Ensure that an IBR is generated immediately for a block received by
/// the DN.
/// </summary>
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestReportBlockReceived()
{
try
{
DatanodeProtocolClientSideTranslatorPB nnSpy = SpyOnDnCallsToNn();
InjectBlockReceived();
// Sleep for a very short time, this is necessary since the IBR is
// generated asynchronously.
Sharpen.Thread.Sleep(2000);
// Ensure that the received block was reported immediately.
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
finally
{
cluster.Shutdown();
cluster = null;
}
}
public virtual void SetupMocks()
{
mockNN1 = SetupNNMock(0);
mockNN2 = SetupNNMock(1);
// Set up a mock DN with the bare-bones configuration
// objects, etc.
mockDn = Org.Mockito.Mockito.Mock <DataNode>();
Org.Mockito.Mockito.DoReturn(true).When(mockDn).ShouldRun();
Configuration conf = new Configuration();
FilePath dnDataDir = new FilePath(new FilePath(TestBuildData, "dfs"), "data");
conf.Set(DFSConfigKeys.DfsDatanodeDataDirKey, dnDataDir.ToURI().ToString());
Org.Mockito.Mockito.DoReturn(conf).When(mockDn).GetConf();
Org.Mockito.Mockito.DoReturn(new DNConf(conf)).When(mockDn).GetDnConf();
Org.Mockito.Mockito.DoReturn(DataNodeMetrics.Create(conf, "fake dn")).When(mockDn
).GetMetrics();
// Set up a simulated dataset with our fake BP
mockFSDataset = Org.Mockito.Mockito.Spy(new SimulatedFSDataset(null, conf));
mockFSDataset.AddBlockPool(FakeBpid, conf);
// Wire the dataset to the DN.
Org.Mockito.Mockito.DoReturn(mockFSDataset).When(mockDn).GetFSDataset();
}
/// <summary>
/// Test that if splitThreshold is zero, then we always get a separate
/// call per storage.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestAlwaysSplit()
{
StartUpCluster(0);
NameNode nn = cluster.GetNameNode();
DataNode dn = cluster.GetDataNodes()[0];
// Create a file with a few blocks.
CreateFile(GenericTestUtils.GetMethodName(), BlocksInFile);
// Insert a spy object for the NN RPC.
DatanodeProtocolClientSideTranslatorPB nnSpy = DataNodeTestUtils.SpyOnBposToNN(dn
, nn);
// Trigger a block report so there is an interaction with the spy
// object.
DataNodeTestUtils.TriggerBlockReport(dn);
ArgumentCaptor <StorageBlockReport[]> captor = ArgumentCaptor.ForClass <StorageBlockReport
[]>();
Org.Mockito.Mockito.Verify(nnSpy, Org.Mockito.Mockito.Times(cluster.GetStoragesPerDatanode
())).BlockReport(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), captor
.Capture(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
VerifyCapturedArguments(captor, 1, BlocksInFile);
}
public virtual void TestRBWReportArrivesAfterEdits()
{
CountDownLatch brFinished = new CountDownLatch(1);
GenericTestUtils.DelayAnswer delayer = new _DelayAnswer_521(brFinished, Log);
// inform the test that our block report went through.
FSDataOutputStream @out = fs.Create(TestFilePath);
try
{
AppendTestUtil.Write(@out, 0, 10);
@out.Hflush();
DataNode dn = cluster.GetDataNodes()[0];
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(dn,
nn2);
Org.Mockito.Mockito.DoAnswer(delayer).When(spy).BlockReport(Org.Mockito.Mockito.AnyObject
<DatanodeRegistration>(), Org.Mockito.Mockito.AnyString(), Org.Mockito.Mockito.AnyObject
<StorageBlockReport[]>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
dn.ScheduleAllBlockReport(0);
delayer.WaitForCall();
}
finally
{
IOUtils.CloseStream(@out);
}
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
delayer.Proceed();
brFinished.Await();
// Verify that no replicas are marked corrupt, and that the
// file is readable from the failed-over standby.
BlockManagerTestUtil.UpdateState(nn1.GetNamesystem().GetBlockManager());
BlockManagerTestUtil.UpdateState(nn2.GetNamesystem().GetBlockManager());
NUnit.Framework.Assert.AreEqual(0, nn1.GetNamesystem().GetCorruptReplicaBlocks());
NUnit.Framework.Assert.AreEqual(0, nn2.GetNamesystem().GetCorruptReplicaBlocks());
DFSTestUtil.ReadFile(fs, TestFilePath);
}
/// <summary>
/// Test the scenario where the NN fails over after issuing a block
/// synchronization request, but before it is committed.
/// </summary>
/// <remarks>
/// Test the scenario where the NN fails over after issuing a block
/// synchronization request, but before it is committed. The
/// DN running the recovery should then fail to commit the synchronization
/// and a later retry will succeed.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestFailoverRightBeforeCommitSynchronization()
{
Configuration conf = new Configuration();
// Disable permissions so that another user can recover the lease.
conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
FSDataOutputStream stm = null;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(3).Build();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
Sharpen.Thread.Sleep(500);
Log.Info("Starting with NN 0 active");
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
stm = fs.Create(TestPath);
// write a half block
AppendTestUtil.Write(stm, 0, BlockSize / 2);
stm.Hflush();
// Look into the block manager on the active node for the block
// under construction.
NameNode nn0 = cluster.GetNameNode(0);
ExtendedBlock blk = DFSTestUtil.GetFirstBlock(fs, TestPath);
DatanodeDescriptor expectedPrimary = DFSTestUtil.GetExpectedPrimaryNode(nn0, blk);
Log.Info("Expecting block recovery to be triggered on DN " + expectedPrimary);
// Find the corresponding DN daemon, and spy on its connection to the
// active.
DataNode primaryDN = cluster.GetDataNode(expectedPrimary.GetIpcPort());
DatanodeProtocolClientSideTranslatorPB nnSpy = DataNodeTestUtils.SpyOnBposToNN(primaryDN
, nn0);
// Delay the commitBlockSynchronization call
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
Org.Mockito.Mockito.DoAnswer(delayer).When(nnSpy).CommitBlockSynchronization(Org.Mockito.Mockito
.Eq(blk), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.Eq(true), Org.Mockito.Mockito.Eq(false), (DatanodeID[])Org.Mockito.Mockito.AnyObject
(), (string[])Org.Mockito.Mockito.AnyObject());
// new genstamp
// new length
// close file
// delete block
// new targets
// new target storages
DistributedFileSystem fsOtherUser = CreateFsAsOtherUser(cluster, conf);
NUnit.Framework.Assert.IsFalse(fsOtherUser.RecoverLease(TestPath));
Log.Info("Waiting for commitBlockSynchronization call from primary");
delayer.WaitForCall();
Log.Info("Failing over to NN 1");
cluster.TransitionToStandby(0);
cluster.TransitionToActive(1);
// Let the commitBlockSynchronization call go through, and check that
// it failed with the correct exception.
delayer.Proceed();
delayer.WaitForResult();
Exception t = delayer.GetThrown();
if (t == null)
{
NUnit.Framework.Assert.Fail("commitBlockSynchronization call did not fail on standby"
);
}
GenericTestUtils.AssertExceptionContains("Operation category WRITE is not supported"
, t);
// Now, if we try again to recover the block, it should succeed on the new
// active.
LoopRecoverLease(fsOtherUser, TestPath);
AppendTestUtil.Check(fs, TestPath, BlockSize / 2);
}
finally
{
IOUtils.CloseStream(stm);
cluster.Shutdown();
}
}
/// <summary>Test race between delete operation and commitBlockSynchronization method.
/// </summary>
/// <remarks>
/// Test race between delete operation and commitBlockSynchronization method.
/// See HDFS-6825.
/// </remarks>
/// <param name="hasSnapshot"/>
/// <exception cref="System.Exception"/>
private void TestDeleteAndCommitBlockSynchronizationRace(bool hasSnapshot)
{
Log.Info("Start testing, hasSnapshot: " + hasSnapshot);
AList <AbstractMap.SimpleImmutableEntry <string, bool> > testList = new AList <AbstractMap.SimpleImmutableEntry
<string, bool> >();
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/test-file",
false));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/test-file1"
, true));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/testdir/testdir1/test-file"
, false));
testList.AddItem(new AbstractMap.SimpleImmutableEntry <string, bool>("/testdir/testdir1/test-file1"
, true));
Path rootPath = new Path("/");
Configuration conf = new Configuration();
// Disable permissions so that another user can recover the lease.
conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
FSDataOutputStream stm = null;
IDictionary <DataNode, DatanodeProtocolClientSideTranslatorPB> dnMap = new Dictionary
<DataNode, DatanodeProtocolClientSideTranslatorPB>();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
int stId = 0;
foreach (AbstractMap.SimpleImmutableEntry <string, bool> stest in testList)
{
string testPath = stest.Key;
bool mkSameDir = stest.Value;
Log.Info("test on " + testPath + " mkSameDir: " + mkSameDir + " snapshot: " + hasSnapshot
);
Path fPath = new Path(testPath);
//find grandest non-root parent
Path grandestNonRootParent = fPath;
while (!grandestNonRootParent.GetParent().Equals(rootPath))
{
grandestNonRootParent = grandestNonRootParent.GetParent();
}
stm = fs.Create(fPath);
Log.Info("test on " + testPath + " created " + fPath);
// write a half block
AppendTestUtil.Write(stm, 0, BlockSize / 2);
stm.Hflush();
if (hasSnapshot)
{
SnapshotTestHelper.CreateSnapshot(fs, rootPath, "st" + stId.ToString());
++stId;
}
// Look into the block manager on the active node for the block
// under construction.
NameNode nn = cluster.GetNameNode();
ExtendedBlock blk = DFSTestUtil.GetFirstBlock(fs, fPath);
DatanodeDescriptor expectedPrimary = DFSTestUtil.GetExpectedPrimaryNode(nn, blk);
Log.Info("Expecting block recovery to be triggered on DN " + expectedPrimary);
// Find the corresponding DN daemon, and spy on its connection to the
// active.
DataNode primaryDN = cluster.GetDataNode(expectedPrimary.GetIpcPort());
DatanodeProtocolClientSideTranslatorPB nnSpy = dnMap[primaryDN];
if (nnSpy == null)
{
nnSpy = DataNodeTestUtils.SpyOnBposToNN(primaryDN, nn);
dnMap[primaryDN] = nnSpy;
}
// Delay the commitBlockSynchronization call
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
Org.Mockito.Mockito.DoAnswer(delayer).When(nnSpy).CommitBlockSynchronization(Org.Mockito.Mockito
.Eq(blk), Org.Mockito.Mockito.AnyInt(), Org.Mockito.Mockito.AnyLong(), Org.Mockito.Mockito
.Eq(true), Org.Mockito.Mockito.Eq(false), (DatanodeID[])Org.Mockito.Mockito.AnyObject
(), (string[])Org.Mockito.Mockito.AnyObject());
// new genstamp
// new length
// close file
// delete block
// new targets
// new target storages
fs.RecoverLease(fPath);
Log.Info("Waiting for commitBlockSynchronization call from primary");
delayer.WaitForCall();
Log.Info("Deleting recursively " + grandestNonRootParent);
fs.Delete(grandestNonRootParent, true);
if (mkSameDir && !grandestNonRootParent.ToString().Equals(testPath))
{
Log.Info("Recreate dir " + grandestNonRootParent + " testpath: " + testPath);
fs.Mkdirs(grandestNonRootParent);
}
delayer.Proceed();
Log.Info("Now wait for result");
delayer.WaitForResult();
Exception t = delayer.GetThrown();
if (t != null)
{
Log.Info("Result exception (snapshot: " + hasSnapshot + "): " + t);
}
}
// end of loop each fPath
Log.Info("Now check we can restart");
cluster.RestartNameNodes();
Log.Info("Restart finished");
}
finally
{
if (stm != null)
{
IOUtils.CloseStream(stm);
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
private void TestTriggerBlockReport(bool incremental)
{
Configuration conf = new HdfsConfiguration();
// Set a really long value for dfs.blockreport.intervalMsec and
// dfs.heartbeat.interval, so that incremental block reports and heartbeats
// won't be sent during this test unless they're triggered
// manually.
conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 10800000L);
conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1080L);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
FileSystem fs = cluster.GetFileSystem();
DatanodeProtocolClientSideTranslatorPB spy = DataNodeTestUtils.SpyOnBposToNN(cluster
.GetDataNodes()[0], cluster.GetNameNode());
DFSTestUtil.CreateFile(fs, new Path("/abc"), 16, (short)1, 1L);
// We should get 1 incremental block report.
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
// We should not receive any more incremental or incremental block reports,
// since the interval we configured is so long.
for (int i = 0; i < 3; i++)
{
Sharpen.Thread.Sleep(10);
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Times(0)).BlockReport(Matchers.Any
<DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport[]
>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Times(1)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
// Create a fake block deletion notification on the DataNode.
// This will be sent with the next incremental block report.
ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(new Block(5678, 512,
1000), ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null);
DataNode datanode = cluster.GetDataNodes()[0];
BPServiceActor actor = datanode.GetAllBpOs()[0].GetBPServiceActors()[0];
string storageUuid = datanode.GetFSDataset().GetVolumes()[0].GetStorageID();
actor.NotifyNamenodeDeletedBlock(rdbi, storageUuid);
// Manually trigger a block report.
datanode.TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental(incremental
).Build());
// triggerBlockReport returns before the block report is
// actually sent. Wait for it to be sent here.
if (incremental)
{
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000).Times(2)).BlockReceivedAndDeleted
(Matchers.Any <DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageReceivedDeletedBlocks
[]>());
}
else
{
Org.Mockito.Mockito.Verify(spy, Org.Mockito.Mockito.Timeout(60000)).BlockReport(Matchers.Any
<DatanodeRegistration>(), Matchers.AnyString(), Matchers.Any <StorageBlockReport[]
>(), Org.Mockito.Mockito.AnyObject <BlockReportContext>());
}
cluster.Shutdown();
}
public _Supplier_407(DatanodeProtocolClientSideTranslatorPB mockNN)
{
this.mockNN = mockNN;
}
/// <exception cref="System.Exception"/>
private void WaitForBlockReport(DatanodeProtocolClientSideTranslatorPB mockNN1, DatanodeProtocolClientSideTranslatorPB
mockNN2)
{
GenericTestUtils.WaitFor(new _Supplier_429(mockNN1, mockNN2), 500, 10000);
}
public _Supplier_429(DatanodeProtocolClientSideTranslatorPB mockNN1, DatanodeProtocolClientSideTranslatorPB
mockNN2)
{
this.mockNN1 = mockNN1;
this.mockNN2 = mockNN2;
}
