public virtual void TestScheduleSameBlock()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string file = "/testScheduleSameBlock/file";
{
FSDataOutputStream @out = dfs.Create(new Path(file));
@out.WriteChars("testScheduleSameBlock");
@out.Close();
}
Org.Apache.Hadoop.Hdfs.Server.Mover.Mover mover = NewMover(conf);
mover.Init();
Mover.Processor processor = new Mover.Processor(this);
LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb);
Mover.MLocation ml = locations[0];
Dispatcher.DBlock db = mover.NewDBlock(lb.GetBlock().GetLocalBlock(), locations);
IList <StorageType> storageTypes = new AList <StorageType>(Arrays.AsList(StorageType
.Default, StorageType.Default));
NUnit.Framework.Assert.IsTrue(processor.ScheduleMoveReplica(db, ml, storageTypes)
);
NUnit.Framework.Assert.IsFalse(processor.ScheduleMoveReplica(db, ml, storageTypes
));
}
finally
{
cluster.Shutdown();
}
}
public LocatedBlock AddBlock(string srcFile)
{
var randomDataNodeID = _dataNodeRepository.GetRandomDataNodeId();
var dataNodeDescriptor = _dataNodeRepository.GetDataNodeDescriptorById(randomDataNodeID);
var dataNodeID = new DataNodeId {
HostName = dataNodeDescriptor.HostName, IPAddress = dataNodeDescriptor.IPAddress
};
var dataNodeIds = new List <DataNodeId> {
dataNodeID
};
var blockId = Guid.NewGuid();
var block = new Block(blockId, 0, DateTime.Now);
var blockInfo = new BlockInfo(block, dataNodeIds);
var node = _fileSystem.GetFile(srcFile);
node.AddBlock(blockInfo);
var locatedBlock = new LocatedBlock
{
Block = block,
Locations = dataNodeIds
};
return(locatedBlock);
}
/// <summary>Create a file with one block and corrupt some/all of the block replicas.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.TimeoutException"/>
private void CreateAFileWithCorruptedBlockReplicas(Path filePath, short repl, int
corruptBlockCount)
{
DFSTestUtil.CreateFile(dfs, filePath, BlockSize, repl, 0);
DFSTestUtil.WaitReplication(dfs, filePath, repl);
// Locate the file blocks by asking name node
LocatedBlocks locatedblocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToString
(), 0L, BlockSize);
NUnit.Framework.Assert.AreEqual(repl, locatedblocks.Get(0).GetLocations().Length);
// The file only has one block
LocatedBlock lblock = locatedblocks.Get(0);
DatanodeInfo[] datanodeinfos = lblock.GetLocations();
ExtendedBlock block = lblock.GetBlock();
// corrupt some /all of the block replicas
for (int i = 0; i < corruptBlockCount; i++)
{
DatanodeInfo dninfo = datanodeinfos[i];
DataNode dn = cluster.GetDataNode(dninfo.GetIpcPort());
CorruptBlock(block, dn);
Log.Debug("Corrupted block " + block.GetBlockName() + " on data node " + dninfo);
}
}
/// <exception cref="System.IO.IOException"/>
internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID
datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname,
LocatedBlock locatedBlock)
{
string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname);
IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr);
if (Log.IsDebugEnabled())
{
Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey
, 0);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock.
GetBlock().GetLocalBlock().ToString());
ticket.AddToken(locatedBlock.GetBlockToken());
return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils
.GetDefaultSocketFactory(conf), socketTimeout));
}
public virtual void TestMissingBlock()
{
// Create a file with single block with two replicas
Path file = GetTestPath("testMissingBlocks");
CreateFile(file, 100, (short)1);
// Corrupt the only replica of the block to result in a missing block
LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), file
.ToString(), 0, 1).Get(0);
cluster.GetNamesystem().WriteLock();
try
{
bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID"
, "TEST");
}
finally
{
cluster.GetNamesystem().WriteUnlock();
}
UpdateMetrics();
MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(NsMetrics);
MetricsAsserts.AssertGauge("UnderReplicatedBlocks", 1L, rb);
MetricsAsserts.AssertGauge("MissingBlocks", 1L, rb);
MetricsAsserts.AssertGauge("MissingReplOneBlocks", 1L, rb);
fs.Delete(file, true);
WaitForDnMetricValue(NsMetrics, "UnderReplicatedBlocks", 0L);
}
internal Pipeline(LocatedBlock lb)
{
foreach (DatanodeInfo d in lb.GetLocations())
{
datanodes.AddItem(d.GetName());
}
}
/// <summary>Get a DataNode that serves our testBlock.</summary>
public virtual DataNode GetDataNode(LocatedBlock testBlock)
{
DatanodeInfo[] nodes = testBlock.GetLocations();
int ipcport = nodes[0].GetIpcPort();
return(cluster.GetDataNode(ipcport));
}
public virtual void TestLocatedBlocks2Locations()
{
DatanodeInfo d = DFSTestUtil.GetLocalDatanodeInfo();
DatanodeInfo[] ds = new DatanodeInfo[1];
ds[0] = d;
// ok
ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1);
LocatedBlock l1 = new LocatedBlock(b1, ds, 0, false);
// corrupt
ExtendedBlock b2 = new ExtendedBlock("bpid", 2, 1, 1);
LocatedBlock l2 = new LocatedBlock(b2, ds, 0, true);
IList <LocatedBlock> ls = Arrays.AsList(l1, l2);
LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null);
BlockLocation[] bs = DFSUtil.LocatedBlocks2Locations(lbs);
NUnit.Framework.Assert.IsTrue("expected 2 blocks but got " + bs.Length, bs.Length
== 2);
int corruptCount = 0;
foreach (BlockLocation b in bs)
{
if (b.IsCorrupt())
{
corruptCount++;
}
}
NUnit.Framework.Assert.IsTrue("expected 1 corrupt files but got " + corruptCount,
corruptCount == 1);
// test an empty location
bs = DFSUtil.LocatedBlocks2Locations(new LocatedBlocks());
NUnit.Framework.Assert.AreEqual(0, bs.Length);
}
internal virtual bool ScheduleMoves4Block(Mover.StorageTypeDiff diff, LocatedBlock
lb)
{
IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb);
Sharpen.Collections.Shuffle(locations);
Dispatcher.DBlock db = this._enclosing.NewDBlock(lb.GetBlock().GetLocalBlock(), locations
);
foreach (StorageType t in diff.existing)
{
foreach (Mover.MLocation ml in locations)
{
Dispatcher.Source source = this._enclosing.storages.GetSource(ml);
if (ml.storageType == t && source != null)
{
// try to schedule one replica move.
if (this.ScheduleMoveReplica(db, source, diff.expected))
{
return(true);
}
}
}
}
return(false);
}
public virtual void TestAddBlockRetryShouldReturnBlockWithLocations()
{
string src = "/testAddBlockRetryShouldReturnBlockWithLocations";
NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc();
// create file
nameNodeRpc.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable
<CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null);
// start first addBlock()
Log.Info("Starting first addBlock for " + src);
LocatedBlock lb1 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId
, null);
NUnit.Framework.Assert.IsTrue("Block locations should be present", lb1.GetLocations
().Length > 0);
cluster.RestartNameNode();
nameNodeRpc = cluster.GetNameNodeRpc();
LocatedBlock lb2 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId
, null);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock
());
NUnit.Framework.Assert.IsTrue("Wrong locations with retry", lb2.GetLocations().Length
> 0);
}
public _Thread_612(LocatedBlock block, DataNode dataNode, AtomicBoolean recoveryInitResult
)
{
this.block = block;
this.dataNode = dataNode;
this.recoveryInitResult = recoveryInitResult;
}
public void Handle_WithPutCommand_CallsPutAPI()
{
const string testFilePath = "TestFilePath";
const string testSrcFile = "TestSrcFile";
// Arrange
var mockClientProtocol = new Mock <IRestClientProtocol>();
var locatedBlock = new LocatedBlock()
{
Locations = new DataNodeId[] { new DataNodeId {
IPAddress = "http://localhost"
} }
};
mockClientProtocol.Setup(x => x.AddBlock(It.IsAny <string>())).Returns(locatedBlock);
var stubDataTransferProtocol = new Mock <IRestDataTransferProtocol>();
var sut = new PutCommandHandler(mockClientProtocol.Object, stubDataTransferProtocol.Object);
var stubPutCommand = new PutCommand()
{
FilePath = testFilePath, SrcFile = testSrcFile
};
// Act
sut.Handle(stubPutCommand);
// Assert
mockClientProtocol.Verify(x => x.Create(testSrcFile, testFilePath));
}
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Datanode.BPServiceActorActionException
/// "/>
public virtual void ReportTo(DatanodeProtocolClientSideTranslatorPB bpNamenode, DatanodeRegistration
bpRegistration)
{
if (bpRegistration == null)
{
return;
}
DatanodeInfo[] dnArr = new DatanodeInfo[] { new DatanodeInfo(bpRegistration) };
string[] uuids = new string[] { storageUuid };
StorageType[] types = new StorageType[] { storageType };
LocatedBlock[] locatedBlock = new LocatedBlock[] { new LocatedBlock(block, dnArr,
uuids, types) };
try
{
bpNamenode.ReportBadBlocks(locatedBlock);
}
catch (RemoteException re)
{
DataNode.Log.Info("reportBadBlock encountered RemoteException for " + "block: "
+ block, re);
}
catch (IOException)
{
throw new BPServiceActorActionException("Failed to report bad block " + block + " to namenode: "
);
}
}
public virtual void TestMoverFailedRetry()
{
// HDFS-8147
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsMoverRetryMaxAttemptsKey, "2");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StorageTypes
(new StorageType[][] { new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType
[] { StorageType.Disk, StorageType.Archive } }).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
string file = "/testMoverFailedRetry";
// write to DISK
FSDataOutputStream @out = dfs.Create(new Path(file), (short)2);
@out.WriteChars("testMoverFailedRetry");
@out.Close();
// Delete block file so, block move will fail with FileNotFoundException
LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0);
cluster.CorruptBlockOnDataNodesByDeletingBlockFile(lb.GetBlock());
// move to ARCHIVE
dfs.SetStoragePolicy(new Path(file), "COLD");
int rc = ToolRunner.Run(conf, new Mover.Cli(), new string[] { "-p", file.ToString
() });
NUnit.Framework.Assert.AreEqual("Movement should fail after some retry", ExitStatus
.IoException.GetExitCode(), rc);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestAbandonBlock()
{
string src = FileNamePrefix + "foo";
// Start writing a file but do not close it
FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L);
for (int i = 0; i < 1024; i++)
{
fout.Write(123);
}
fout.Hflush();
long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId();
// Now abandon the last block
DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs);
LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue
);
int orginalNumBlocks = blocks.LocatedBlockCount();
LocatedBlock b = blocks.GetLastLocatedBlock();
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// call abandonBlock again to make sure the operation is idempotent
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
);
// And close the file
fout.Close();
// Close cluster and check the block has been abandoned after restart
cluster.RestartNameNode();
blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue);
NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks
, blocks.LocatedBlockCount() + 1);
}
public virtual void TestBlockTokenRpcLeak()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
UserGroupInformation.SetConfiguration(conf);
Assume.AssumeTrue(FdDir.Exists());
BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval,
blockTokenLifetime, 0, "fake-pool", null);
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken
(block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>());
Server server = CreateMockDatanode(sm, token, conf);
server.Start();
IPEndPoint addr = NetUtils.GetConnectAddress(server);
DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port);
ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block
(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
fakeBlock.SetBlockToken(token);
// Create another RPC proxy with the same configuration - this will never
// attempt to connect anywhere -- but it causes the refcount on the
// RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
// actually close the TCP connections to the real target DN.
ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol
.versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser(
"junk"), conf, NetUtils.GetDefaultSocketFactory(conf));
ClientDatanodeProtocol proxy = null;
int fdsAtStart = CountOpenFileDescriptors();
try
{
long endTime = Time.Now() + 3000;
while (Time.Now() < endTime)
{
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock
);
NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength
(block3));
if (proxy != null)
{
RPC.StopProxy(proxy);
}
Log.Info("Num open fds:" + CountOpenFileDescriptors());
}
int fdsAtEnd = CountOpenFileDescriptors();
if (fdsAtEnd - fdsAtStart > 50)
{
NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
}
}
finally
{
server.Stop();
}
RPC.StopProxy(proxyToNoWhere);
}
/// <summary>Verify the first block of the file is corrupted (for all its replica).</summary>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="System.IO.IOException"/>
private void VerifyFirstBlockCorrupted(Path filePath, bool isCorrupted)
{
LocatedBlocks locatedBlocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToUri
().GetPath(), 0, long.MaxValue);
LocatedBlock firstLocatedBlock = locatedBlocks.Get(0);
NUnit.Framework.Assert.AreEqual(isCorrupted, firstLocatedBlock.IsCorrupt());
}
public virtual void TestConvertLocatedBlockNoStorageMedia()
{
LocatedBlock lb = CreateLocatedBlockNoStorageMedia();
HdfsProtos.LocatedBlockProto lbProto = PBHelper.Convert(lb);
LocatedBlock lb2 = PBHelper.Convert(lbProto);
Compare(lb, lb2);
}
public _Runnable_471(MiniDFSCluster cluster, LocatedBlock lblock, int TestFileLen
, Semaphore sem, byte[] buf, AtomicBoolean testFailed)
{
this.cluster = cluster;
this.lblock = lblock;
this.TestFileLen = TestFileLen;
this.sem = sem;
this.buf = buf;
this.testFailed = testFailed;
}
public static void SetupCluster()
{
int ReplicationFactor = 1;
util = new BlockReaderTestUtil(ReplicationFactor);
util.WriteFile(TestFile, FileSizeK);
IList <LocatedBlock> blkList = util.GetFileBlocks(TestFile, FileSizeK);
testBlock = blkList[0];
}
/// <summary>
/// Verify the number of corrupted block replicas by fetching the block
/// location from name node.
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
/// <exception cref="System.IO.IOException"/>
private void VerifyCorruptedBlockCount(Path filePath, int expectedReplicas)
{
LocatedBlocks lBlocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToUri().
GetPath(), 0, long.MaxValue);
// we expect only the first block of the file is used for this test
LocatedBlock firstLocatedBlock = lBlocks.Get(0);
NUnit.Framework.Assert.AreEqual(expectedReplicas, firstLocatedBlock.GetLocations(
).Length);
}
public virtual void TestReplicationError()
{
// create a file of replication factor of 1
Path fileName = new Path("/test.txt");
int fileLen = 1;
DFSTestUtil.CreateFile(fs, fileName, 1, (short)1, 1L);
DFSTestUtil.WaitReplication(fs, fileName, (short)1);
// get the block belonged to the created file
LocatedBlocks blocks = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), fileName
.ToString(), 0, (long)fileLen);
NUnit.Framework.Assert.AreEqual("Should only find 1 block", blocks.LocatedBlockCount
(), 1);
LocatedBlock block = blocks.Get(0);
// bring up a second datanode
cluster.StartDataNodes(conf, 1, true, null, null);
cluster.WaitActive();
int sndNode = 1;
DataNode datanode = cluster.GetDataNodes()[sndNode];
// replicate the block to the second datanode
IPEndPoint target = datanode.GetXferAddress();
Socket s = Sharpen.Extensions.CreateSocket(target.Address, target.Port);
// write the header.
DataOutputStream @out = new DataOutputStream(s.GetOutputStream());
DataChecksum checksum = DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, 512
);
new Sender(@out).WriteBlock(block.GetBlock(), StorageType.Default, BlockTokenSecretManager
.DummyToken, string.Empty, new DatanodeInfo[0], new StorageType[0], null, BlockConstructionStage
.PipelineSetupCreate, 1, 0L, 0L, 0L, checksum, CachingStrategy.NewDefaultStrategy
(), false, false, null);
@out.Flush();
// close the connection before sending the content of the block
@out.Close();
// the temporary block & meta files should be deleted
string bpid = cluster.GetNamesystem().GetBlockPoolId();
FilePath storageDir = cluster.GetInstanceStorageDir(sndNode, 0);
FilePath dir1 = MiniDFSCluster.GetRbwDir(storageDir, bpid);
storageDir = cluster.GetInstanceStorageDir(sndNode, 1);
FilePath dir2 = MiniDFSCluster.GetRbwDir(storageDir, bpid);
while (dir1.ListFiles().Length != 0 || dir2.ListFiles().Length != 0)
{
Sharpen.Thread.Sleep(100);
}
// then increase the file's replication factor
fs.SetReplication(fileName, (short)2);
// replication should succeed
DFSTestUtil.WaitReplication(fs, fileName, (short)1);
// clean up the file
fs.Delete(fileName, false);
}
/// <exception cref="System.Exception"/>
public virtual void TestBlockReaderLocalLegacyWithAppend()
{
short ReplFactor = 1;
HdfsConfiguration conf = GetConfiguration(null);
conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
Path path = new Path("/testBlockReaderLocalLegacy");
DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0);
DFSTestUtil.WaitReplication(dfs, path, ReplFactor);
ClientDatanodeProtocol proxy;
Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token;
ExtendedBlock originalBlock;
long originalGS;
{
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000
, false);
token = lb.GetBlockToken();
// get block and generation stamp
ExtendedBlock blk = new ExtendedBlock(lb.GetBlock());
originalBlock = new ExtendedBlock(blk);
originalGS = originalBlock.GetGenerationStamp();
// test getBlockLocalPathInfo
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token);
NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp());
}
{
// append one byte
FSDataOutputStream @out = dfs.Append(path);
@out.Write(1);
@out.Close();
}
{
// get new generation stamp
LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString
(), 0, 1).Get(0);
long newGS = lb.GetBlock().GetGenerationStamp();
NUnit.Framework.Assert.IsTrue(newGS > originalGS);
// getBlockLocalPathInfo using the original block.
NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp());
BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token);
NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp());
}
cluster.Shutdown();
}
public virtual void TestLocatedBlockConstructorWithNullCachedLocs()
{
DatanodeInfo d = DFSTestUtil.GetLocalDatanodeInfo();
DatanodeInfo[] ds = new DatanodeInfo[1];
ds[0] = d;
ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1);
LocatedBlock l1 = new LocatedBlock(b1, ds, null, null, 0, false, null);
DatanodeInfo[] cachedLocs = l1.GetCachedLocations();
NUnit.Framework.Assert.IsTrue(cachedLocs.Length == 0);
}
/// <exception cref="System.IO.IOException"/>
private Block FindBlock(Path path, long size)
{
Block ret;
IList <LocatedBlock> lbs = cluster.GetNameNodeRpc().GetBlockLocations(path.ToString
(), FileStart, size).GetLocatedBlocks();
LocatedBlock lb = lbs[lbs.Count - 1];
// Get block from the first DN
ret = cluster.GetDataNodes()[DnN0].data.GetStoredBlock(lb.GetBlock().GetBlockPoolId
(), lb.GetBlock().GetBlockId());
return(ret);
}
internal static IList <Mover.MLocation> ToLocations(LocatedBlock lb)
{
DatanodeInfo[] datanodeInfos = lb.GetLocations();
StorageType[] storageTypes = lb.GetStorageTypes();
long size = lb.GetBlockSize();
IList <Mover.MLocation> locations = new List <Mover.MLocation>();
for (int i = 0; i < datanodeInfos.Length; i++)
{
locations.AddItem(new Mover.MLocation(datanodeInfos[i], storageTypes[i], size));
}
return(locations);
}
/// <summary>Initialize the pipeline.</summary>
public virtual Pipeline InitPipeline(LocatedBlock lb)
{
lock (this)
{
Pipeline pl = new Pipeline(lb);
if (pipelines.Contains(pl))
{
throw new InvalidOperationException("thepipeline != null");
}
pipelines.AddItem(pl);
return(pl);
}
}
public void Run()
{
try
{
IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations
(TestFile, 0, TestFileLen).GetLocatedBlocks();
LocatedBlock lblock = locatedBlocks[0];
BlockReader blockReader = null;
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
NUnit.Framework.Assert.Fail("expected getBlockReader to fail the first time.");
}
catch (Exception t)
{
NUnit.Framework.Assert.IsTrue("expected to see 'TCP reads were disabled " + "for testing' in exception "
+ t, t.Message.Contains("TCP reads were disabled for testing"));
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
gotFailureLatch.CountDown();
shouldRetryLatch.Await();
try
{
blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
}
catch (Exception t)
{
TestBlockReaderFactory.Log.Error("error trying to retrieve a block reader " + "the second time."
, t);
throw;
}
finally
{
if (blockReader != null)
{
blockReader.Close();
}
}
}
catch (Exception t)
{
TestBlockReaderFactory.Log.Error("getBlockReader failure", t);
testFailed.Set(true);
}
}
/// <summary>Test to verify the race between finalizeBlock and Lease recovery</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRaceBetweenReplicaRecoveryAndFinalizeBlock()
{
TearDown();
// Stop the Mocked DN started in startup()
Configuration conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisKey, "1000");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
try
{
cluster.WaitClusterUp();
DistributedFileSystem fs = cluster.GetFileSystem();
Path path = new Path("/test");
FSDataOutputStream @out = fs.Create(path);
@out.WriteBytes("data");
@out.Hsync();
IList <LocatedBlock> blocks = DFSTestUtil.GetAllBlocks(fs.Open(path));
LocatedBlock block = blocks[0];
DataNode dataNode = cluster.GetDataNodes()[0];
AtomicBoolean recoveryInitResult = new AtomicBoolean(true);
Sharpen.Thread recoveryThread = new _Thread_612(block, dataNode, recoveryInitResult
);
recoveryThread.Start();
try
{
@out.Close();
}
catch (IOException e)
{
NUnit.Framework.Assert.IsTrue("Writing should fail", e.Message.Contains("are bad. Aborting..."
));
}
finally
{
recoveryThread.Join();
}
NUnit.Framework.Assert.IsTrue("Recovery should be initiated successfully", recoveryInitResult
.Get());
dataNode.UpdateReplicaUnderRecovery(block.GetBlock(), block.GetBlock().GetGenerationStamp
() + 1, block.GetBlock().GetBlockId(), block.GetBlockSize());
}
finally
{
if (null != cluster)
{
cluster.Shutdown();
cluster = null;
}
}
}
// try reading a block using a BlockReader directly
private static void TryRead(Configuration conf, LocatedBlock lblock, bool shouldSucceed
)
{
IPEndPoint targetAddr = null;
IOException ioe = null;
BlockReader blockReader = null;
ExtendedBlock block = lblock.GetBlock();
try
{
DatanodeInfo[] nodes = lblock.GetLocations();
targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr());
blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetFileName(BlockReaderFactory
.GetFileName(targetAddr, "test-blockpoolid", block.GetBlockId())).SetBlock(block
).SetBlockToken(lblock.GetBlockToken()).SetInetSocketAddress(targetAddr).SetStartOffset
(0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestBlockTokenWithDFS")
.SetDatanodeInfo(nodes[0]).SetCachingStrategy(CachingStrategy.NewDefaultStrategy
()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf
).SetRemotePeerFactory(new _RemotePeerFactory_162(conf)).Build();
}
catch (IOException ex)
{
ioe = ex;
}
finally
{
if (blockReader != null)
{
try
{
blockReader.Close();
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}
if (shouldSucceed)
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid"
, blockReader);
}
else
{
NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid"
, ioe);
NUnit.Framework.Assert.IsTrue("OP_READ_BLOCK failed due to reasons other than access token: "
, ioe is InvalidBlockTokenException);
}
}