/// <summary>TC12: Append to partial CRC chunk</summary>
/// <exception cref="System.Exception"/>
private void TestTC12(bool appendToNewBlock)
{
Path p = new Path("/TC12/foo" + (appendToNewBlock ? "0" : "1"));
System.Console.Out.WriteLine("p=" + p);
//a. Create file with a block size of 64KB
// and a default io.bytes.per.checksum of 512 bytes.
// Write 25687 bytes of data. Close file.
int len1 = 25687;
{
FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize);
AppendTestUtil.Write(@out, 0, len1);
@out.Close();
}
//b. Reopen file in "append" mode. Append another 5877 bytes of data. Close file.
int len2 = 5877;
{
FSDataOutputStream @out = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null) : fs.Append(p);
AppendTestUtil.Write(@out, len1, len2);
@out.Close();
}
//c. Reopen file and read 25687+5877 bytes of data from file. Close file.
AppendTestUtil.Check(fs, p, len1 + len2);
if (appendToNewBlock)
{
LocatedBlocks blks = fs.dfs.GetLocatedBlocks(p.ToString(), 0);
NUnit.Framework.Assert.AreEqual(2, blks.GetLocatedBlocks().Count);
NUnit.Framework.Assert.AreEqual(len1, blks.GetLocatedBlocks()[0].GetBlockSize());
NUnit.Framework.Assert.AreEqual(len2, blks.GetLocatedBlocks()[1].GetBlockSize());
AppendTestUtil.Check(fs, p, 0, len1);
AppendTestUtil.Check(fs, p, len1, len2);
}
}
/// <exception cref="System.IO.IOException"/>
private LocatedBlock GetLocatedBlock()
{
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(Path.ToString(), 0, BlockSize
);
Assert.AssertThat(locatedBlocks.GetLocatedBlocks().Count, CoreMatchers.Is(1));
return(Iterables.GetOnlyElement(locatedBlocks.GetLocatedBlocks()));
}
public virtual void HSyncEndBlock_00()
{
int preferredBlockSize = 1024;
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsBlockSizeKey, preferredBlockSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
DistributedFileSystem fileSystem = cluster.GetFileSystem();
FSDataOutputStream stm = null;
try
{
Path path = new Path("/" + fName);
stm = fileSystem.Create(path, true, 4096, (short)2, AppendTestUtil.BlockSize);
System.Console.Out.WriteLine("Created file " + path.ToString());
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
long currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(0L, currentFileLength);
LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(0, blocks.GetLocatedBlocks().Count);
// write a block and call hsync(end_block) at the block boundary
stm.Write(new byte[preferredBlockSize]);
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize, currentFileLength);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(1, blocks.GetLocatedBlocks().Count);
// call hsync then call hsync(end_block) immediately
stm.Write(new byte[preferredBlockSize / 2]);
stm.Hsync();
((DFSOutputStream)stm.GetWrappedStream()).Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag
.EndBlock));
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2, currentFileLength
);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(2, blocks.GetLocatedBlocks().Count);
stm.Write(new byte[preferredBlockSize / 4]);
stm.Hsync();
currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual(preferredBlockSize + preferredBlockSize / 2 + preferredBlockSize
/ 4, currentFileLength);
blocks = fileSystem.dfs.GetLocatedBlocks(path.ToString(), 0);
NUnit.Framework.Assert.AreEqual(3, blocks.GetLocatedBlocks().Count);
}
finally
{
IOUtils.Cleanup(null, stm, fileSystem);
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Test assumes that the file has a single block</summary>
/// <exception cref="System.IO.IOException"/>
private FilePath GetBlockForFile(Path path, bool exists)
{
LocatedBlocks blocks = nn.GetRpcServer().GetBlockLocations(path.ToString(), 0, long.MaxValue
);
NUnit.Framework.Assert.AreEqual("The test helper functions assume that each file has a single block"
, 1, blocks.GetLocatedBlocks().Count);
ExtendedBlock block = blocks.GetLocatedBlocks()[0].GetBlock();
BlockLocalPathInfo bInfo = dn0.GetFSDataset().GetBlockLocalPathInfo(block);
FilePath blockFile = new FilePath(bInfo.GetBlockPath());
NUnit.Framework.Assert.AreEqual(exists, blockFile.Exists());
return(blockFile);
}
public virtual void TestAppend2AfterSoftLimit()
{
Configuration conf = new HdfsConfiguration();
conf.SetInt(DFSConfigKeys.DfsReplicationKey, 1);
//Set small soft-limit for lease
long softLimit = 1L;
long hardLimit = 9999999L;
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.SetLeasePeriod(softLimit, hardLimit);
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
DistributedFileSystem fs2 = new DistributedFileSystem();
fs2.Initialize(fs.GetUri(), conf);
Path testPath = new Path("/testAppendAfterSoftLimit");
byte[] fileContents = AppendTestUtil.InitBuffer(32);
// create a new file without closing
FSDataOutputStream @out = fs.Create(testPath);
@out.Write(fileContents);
//Wait for > soft-limit
Sharpen.Thread.Sleep(250);
try
{
FSDataOutputStream appendStream2 = fs2.Append(testPath, EnumSet.Of(CreateFlag.Append
, CreateFlag.NewBlock), 4096, null);
appendStream2.Write(fileContents);
appendStream2.Close();
NUnit.Framework.Assert.AreEqual(fileContents.Length, fs.GetFileStatus(testPath).GetLen
());
// make sure we now have 1 block since the first writer was revoked
LocatedBlocks blks = fs.GetClient().GetLocatedBlocks(testPath.ToString(), 0L);
NUnit.Framework.Assert.AreEqual(1, blks.GetLocatedBlocks().Count);
foreach (LocatedBlock blk in blks.GetLocatedBlocks())
{
NUnit.Framework.Assert.AreEqual(fileContents.Length, blk.GetBlockSize());
}
}
finally
{
fs.Close();
fs2.Close();
cluster.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
public static DatanodeInfo BestNode(LocatedBlocks blks, Configuration conf)
{
Dictionary <DatanodeInfo, JspHelper.NodeRecord> map = new Dictionary <DatanodeInfo,
JspHelper.NodeRecord>();
foreach (LocatedBlock block in blks.GetLocatedBlocks())
{
DatanodeInfo[] nodes = block.GetLocations();
foreach (DatanodeInfo node in nodes)
{
JspHelper.NodeRecord record = map[node];
if (record == null)
{
map[node] = new JspHelper.NodeRecord(node, 1);
}
else
{
record.frequency++;
}
}
}
JspHelper.NodeRecord[] nodes_1 = Sharpen.Collections.ToArray(map.Values, new JspHelper.NodeRecord
[map.Count]);
Arrays.Sort(nodes_1, new JspHelper.NodeRecordComparator());
return(BestNode(nodes_1, false));
}
public virtual void TestFallbackToDiskPartial()
{
StartUpCluster(true, 2);
string MethodName = GenericTestUtils.GetMethodName();
Path path = new Path("/" + MethodName + ".dat");
MakeTestFile(path, BlockSize * 5, true);
// Sleep for a short time to allow the lazy writer thread to do its job
Sharpen.Thread.Sleep(6 * LazyWriterIntervalSec * 1000);
TriggerBlockReport();
int numBlocksOnRamDisk = 0;
int numBlocksOnDisk = 0;
long fileLength = client.GetFileInfo(path.ToString()).GetLen();
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(path.ToString(), 0, fileLength
);
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
if (locatedBlock.GetStorageTypes()[0] == StorageType.RamDisk)
{
numBlocksOnRamDisk++;
}
else
{
if (locatedBlock.GetStorageTypes()[0] == StorageType.Default)
{
numBlocksOnDisk++;
}
}
}
// Since eviction is asynchronous, depending on the timing of eviction
// wrt writes, we may get 2 or less blocks on RAM disk.
System.Diagnostics.Debug.Assert((numBlocksOnRamDisk <= 2));
System.Diagnostics.Debug.Assert((numBlocksOnDisk >= 3));
}
public virtual void TestGetBlockLocations()
{
NamenodeProtocols namenode = cluster.GetNameNodeRpc();
Path p = new Path(BaseDir, "file2.dat");
string src = p.ToString();
FSDataOutputStream @out = TestFileCreation.CreateFile(hdfs, p, 3);
// write a half block
int len = (int)(((uint)BlockSize) >> 1);
WriteFile(p, @out, len);
for (int i = 1; i < NumBlocks;)
{
// verify consistency
LocatedBlocks lb = namenode.GetBlockLocations(src, 0, len);
IList <LocatedBlock> blocks = lb.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(i, blocks.Count);
Block b = blocks[blocks.Count - 1].GetBlock().GetLocalBlock();
NUnit.Framework.Assert.IsTrue(b is BlockInfoContiguousUnderConstruction);
if (++i < NumBlocks)
{
// write one more block
WriteFile(p, @out, BlockSize);
len += BlockSize;
}
}
// close file
@out.Close();
}
public virtual void TestBlockHasMultipleReplicasOnSameDN()
{
string filename = MakeFileName(GenericTestUtils.GetMethodName());
Path filePath = new Path(filename);
// Write out a file with a few blocks.
DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize,
NumDatanodes, seed);
// Get the block list for the file with the block locations.
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize
* NumBlocks);
// Generate a fake block report from one of the DataNodes, such
// that it reports one copy of each block on either storage.
DataNode dn = cluster.GetDataNodes()[0];
DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid);
StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode
()];
AList <Replica> blocks = new AList <Replica>();
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
Block localBlock = locatedBlock.GetBlock().GetLocalBlock();
blocks.AddItem(new FinalizedReplica(localBlock, null, null));
}
BlockListAsLongs bll = BlockListAsLongs.Encode(blocks);
for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i)
{
FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i];
DatanodeStorage dns = new DatanodeStorage(v.GetStorageID());
reports[i] = new StorageBlockReport(dns, bll);
}
// Should not assert!
cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext
(1, 0, Runtime.NanoTime()));
// Get the block locations once again.
locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks);
// Make sure that each block has two replicas, one on each DataNode.
foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks())
{
DatanodeInfo[] locations = locatedBlock_1.GetLocations();
Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes));
Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid
()));
}
}
/// <summary>
/// Create a cluster with even distribution, and a new empty node is added to
/// the cluster, then test rack locality for balancer policy.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestBalancerWithRackLocality()
{
Configuration conf = CreateConf();
long[] capacities = new long[] { Capacity, Capacity };
string[] racks = new string[] { Rack0, Rack1 };
string[] nodeGroups = new string[] { Nodegroup0, Nodegroup1 };
int numOfDatanodes = capacities.Length;
NUnit.Framework.Assert.AreEqual(numOfDatanodes, racks.Length);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf).NumDataNodes(capacities
.Length).Racks(racks).SimulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.SetNodeGroups(nodeGroups);
cluster = new MiniDFSClusterWithNodeGroup(builder);
try
{
cluster.WaitActive();
client = NameNodeProxies.CreateProxy <ClientProtocol>(conf, cluster.GetFileSystem(
0).GetUri()).GetProxy();
long totalCapacity = TestBalancer.Sum(capacities);
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity * 3 / 10;
long length = totalUsedSpace / numOfDatanodes;
TestBalancer.CreateFile(cluster, filePath, length, (short)numOfDatanodes, 0);
LocatedBlocks lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length
);
ICollection <ExtendedBlock> before = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0
);
long newCapacity = Capacity;
string newRack = Rack1;
string newNodeGroup = Nodegroup2;
// start up an empty node with the same capacity and on the same rack
cluster.StartDataNodes(conf, 1, true, null, new string[] { newRack }, new long[]
{ newCapacity }, new string[] { newNodeGroup });
totalCapacity += newCapacity;
// run balancer and validate results
RunBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
lbs = client.GetBlockLocations(filePath.ToUri().GetPath(), 0, length);
ICollection <ExtendedBlock> after = GetBlocksOnRack(lbs.GetLocatedBlocks(), Rack0);
NUnit.Framework.Assert.AreEqual(before, after);
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestRetryAddBlockWhileInChooseTarget()
{
string src = "/testRetryAddBlockWhileInChooseTarget";
FSNamesystem ns = cluster.GetNamesystem();
NamenodeProtocols nn = cluster.GetNameNodeRpc();
// create file
nn.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable <CreateFlag
>(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null);
// start first addBlock()
Log.Info("Starting first addBlock for " + src);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
DatanodeStorageInfo[] targets = ns.GetNewBlockTargets(src, INodeId.GrandfatherInodeId
, "clientName", null, null, null, onRetryBlock);
NUnit.Framework.Assert.IsNotNull("Targets must be generated", targets);
// run second addBlock()
Log.Info("Starting second addBlock for " + src);
nn.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId, null);
NUnit.Framework.Assert.IsTrue("Penultimate block must be complete", CheckFileProgress
(src, false));
LocatedBlocks lbs = nn.GetBlockLocations(src, 0, long.MaxValue);
NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count
);
LocatedBlock lb2 = lbs.Get(0);
NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb2.GetLocations
().Length);
// continue first addBlock()
LocatedBlock newBlock = ns.StoreAllocatedBlock(src, INodeId.GrandfatherInodeId, "clientName"
, null, targets);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb2.GetBlock(), newBlock.
GetBlock());
// check locations
lbs = nn.GetBlockLocations(src, 0, long.MaxValue);
NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count
);
LocatedBlock lb1 = lbs.Get(0);
NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb1.GetLocations
().Length);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock
());
}
/// <summary>
/// Check the blocks of dst file are cleaned after rename with overwrite
/// Restart NN to check the rename successfully
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestRenameWithOverwrite()
{
short replFactor = 2;
long blockSize = 512;
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(replFactor
).Build();
DistributedFileSystem dfs = cluster.GetFileSystem();
try
{
long fileLen = blockSize * 3;
string src = "/foo/src";
string dst = "/foo/dst";
Path srcPath = new Path(src);
Path dstPath = new Path(dst);
DFSTestUtil.CreateFile(dfs, srcPath, fileLen, replFactor, 1);
DFSTestUtil.CreateFile(dfs, dstPath, fileLen, replFactor, 1);
LocatedBlocks lbs = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), dst,
0, fileLen);
BlockManager bm = NameNodeAdapter.GetNamesystem(cluster.GetNameNode()).GetBlockManager
();
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock
().GetLocalBlock()) != null);
dfs.Rename(srcPath, dstPath, Options.Rename.Overwrite);
NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(lbs.GetLocatedBlocks()[0].GetBlock
().GetLocalBlock()) == null);
// Restart NN and check the rename successfully
cluster.RestartNameNodes();
NUnit.Framework.Assert.IsFalse(dfs.Exists(srcPath));
NUnit.Framework.Assert.IsTrue(dfs.Exists(dstPath));
}
finally
{
if (dfs != null)
{
dfs.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Select a datanode to service this request.</summary>
/// <remarks>
/// Select a datanode to service this request.
/// Currently, this looks at no more than the first five blocks of a file,
/// selecting a datanode randomly from the most represented.
/// </remarks>
/// <param name="conf"></param>
/// <exception cref="System.IO.IOException"/>
private DatanodeID PickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration
conf)
{
if (i.GetLen() == 0 || blks.GetLocatedBlocks().Count <= 0)
{
// pick a random datanode
NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext());
return(NamenodeJspHelper.GetRandomDatanode(nn));
}
return(JspHelper.BestNode(blks, conf));
}
/// <exception cref="System.IO.IOException"/>
public static LocatedBlock GetLastLocatedBlock(ClientProtocol namenode, string src
)
{
//get block info for the last block
LocatedBlocks locations = namenode.GetBlockLocations(src, 0, long.MaxValue);
IList <LocatedBlock> blocks = locations.GetLocatedBlocks();
DataNode.Log.Info("blocks.size()=" + blocks.Count);
NUnit.Framework.Assert.IsTrue(blocks.Count > 0);
return(blocks[blocks.Count - 1]);
}
/// <summary>Make sure at least one non-transient volume has a saved copy of the replica.
/// </summary>
/// <remarks>
/// Make sure at least one non-transient volume has a saved copy of the replica.
/// An infinite loop is used to ensure the async lazy persist tasks are completely
/// done before verification. Caller of ensureLazyPersistBlocksAreSaved expects
/// either a successful pass or timeout failure.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal void EnsureLazyPersistBlocksAreSaved(LocatedBlocks locatedBlocks
)
{
string bpid = cluster.GetNamesystem().GetBlockPoolId();
IList <FsVolumeSpi> volumes = cluster.GetDataNodes()[0].GetFSDataset().GetVolumes(
);
ICollection <long> persistedBlockIds = new HashSet <long>();
while (persistedBlockIds.Count < locatedBlocks.GetLocatedBlocks().Count)
{
// Take 1 second sleep before each verification iteration
Sharpen.Thread.Sleep(1000);
foreach (LocatedBlock lb in locatedBlocks.GetLocatedBlocks())
{
foreach (FsVolumeSpi v in volumes)
{
if (v.IsTransientStorage())
{
continue;
}
FsVolumeImpl volume = (FsVolumeImpl)v;
FilePath lazyPersistDir = volume.GetBlockPoolSlice(bpid).GetLazypersistDir();
long blockId = lb.GetBlock().GetBlockId();
FilePath targetDir = DatanodeUtil.IdToBlockDir(lazyPersistDir, blockId);
FilePath blockFile = new FilePath(targetDir, lb.GetBlock().GetBlockName());
if (blockFile.Exists())
{
// Found a persisted copy for this block and added to the Set
persistedBlockIds.AddItem(blockId);
}
}
}
}
// We should have found a persisted copy for each located block.
Assert.AssertThat(persistedBlockIds.Count, IS.Is(locatedBlocks.GetLocatedBlocks()
.Count));
}
/// <summary>Convert LocatedBlocks to a Json string.</summary>
/// <exception cref="System.IO.IOException"/>
public static string ToJsonString(LocatedBlocks locatedblocks)
{
if (locatedblocks == null)
{
return(null);
}
IDictionary <string, object> m = new SortedDictionary <string, object>();
m["fileLength"] = locatedblocks.GetFileLength();
m["isUnderConstruction"] = locatedblocks.IsUnderConstruction();
m["locatedBlocks"] = ToJsonArray(locatedblocks.GetLocatedBlocks());
m["lastLocatedBlock"] = ToJsonMap(locatedblocks.GetLastLocatedBlock());
m["isLastBlockComplete"] = locatedblocks.IsLastBlockComplete();
return(ToJsonString(typeof(LocatedBlocks), m));
}
/// <exception cref="System.IO.IOException"/>
protected internal LocatedBlocks EnsureFileReplicasOnStorageType(Path path, StorageType
storageType)
{
// Ensure that returned block locations returned are correct!
Log.Info("Ensure path: " + path + " is on StorageType: " + storageType);
Assert.AssertThat(fs.Exists(path), IS.Is(true));
long fileLength = client.GetFileInfo(path.ToString()).GetLen();
LocatedBlocks locatedBlocks = client.GetLocatedBlocks(path.ToString(), 0, fileLength
);
foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks())
{
Assert.AssertThat(locatedBlock.GetStorageTypes()[0], IS.Is(storageType));
}
return(locatedBlocks);
}
/// <exception cref="System.IO.IOException"/>
public virtual void VerifyIncrementalBlockReports(bool splitReports)
{
// Get the block list for the file with the block locations.
LocatedBlocks blocks = CreateFileGetBlocks(GenericTestUtils.GetMethodName());
// We will send 'fake' incremental block reports to the NN that look
// like they originated from DN 0.
StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[dn0.GetFSDataset
().GetVolumes().Count];
// Lie to the NN that one block on each storage has been deleted.
for (int i = 0; i < reports.Length; ++i)
{
FsVolumeSpi volume = dn0.GetFSDataset().GetVolumes()[i];
bool foundBlockOnStorage = false;
ReceivedDeletedBlockInfo[] rdbi = new ReceivedDeletedBlockInfo[1];
// Find the first block on this storage and mark it as deleted for the
// report.
foreach (LocatedBlock block in blocks.GetLocatedBlocks())
{
if (block.GetStorageIDs()[0].Equals(volume.GetStorageID()))
{
rdbi[0] = new ReceivedDeletedBlockInfo(block.GetBlock().GetLocalBlock(), ReceivedDeletedBlockInfo.BlockStatus
.DeletedBlock, null);
foundBlockOnStorage = true;
break;
}
}
NUnit.Framework.Assert.IsTrue(foundBlockOnStorage);
reports[i] = new StorageReceivedDeletedBlocks(volume.GetStorageID(), rdbi);
if (splitReports)
{
// If we are splitting reports then send the report for this storage now.
StorageReceivedDeletedBlocks[] singletonReport = new StorageReceivedDeletedBlocks
[] { reports[i] };
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, singletonReport);
}
}
if (!splitReports)
{
// Send a combined report.
cluster.GetNameNodeRpc().BlockReceivedAndDeleted(dn0Reg, poolId, reports);
}
// Make sure that the deleted block from each storage was picked up
// by the NameNode.
Assert.AssertThat(cluster.GetNamesystem().GetMissingBlocksCount(), IS.Is((long)reports
.Length));
}
/// <returns>true if it is necessary to run another round of migration</returns>
private bool ProcessFile(string fullPath, HdfsLocatedFileStatus status)
{
byte policyId = status.GetStoragePolicy();
// currently we ignore files with unspecified storage policy
if (policyId == BlockStoragePolicySuite.IdUnspecified)
{
return(false);
}
BlockStoragePolicy policy = this._enclosing.blockStoragePolicies[policyId];
if (policy == null)
{
Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to get the storage policy of file "
+ fullPath);
return(false);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
LocatedBlocks locatedBlocks = status.GetBlockLocations();
bool hasRemaining = false;
bool lastBlkComplete = locatedBlocks.IsLastBlockComplete();
IList <LocatedBlock> lbs = locatedBlocks.GetLocatedBlocks();
for (int i = 0; i < lbs.Count; i++)
{
if (i == lbs.Count - 1 && !lastBlkComplete)
{
// last block is incomplete, skip it
continue;
}
LocatedBlock lb = lbs[i];
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
if (!diff.RemoveOverlap(true))
{
if (this.ScheduleMoves4Block(diff, lb))
{
hasRemaining |= (diff.existing.Count > 1 && diff.expected.Count > 1);
}
}
}
return(hasRemaining);
}
// Waits for all of the blocks to have expected replication
/// <exception cref="System.IO.IOException"/>
private void WaitForBlockReplication(string filename, ClientProtocol namenode, int
expected, long maxWaitSec)
{
long start = Time.MonotonicNow();
//wait for all the blocks to be replicated;
Log.Info("Checking for block replication for " + filename);
while (true)
{
bool replOk = true;
LocatedBlocks blocks = namenode.GetBlockLocations(filename, 0, long.MaxValue);
for (IEnumerator <LocatedBlock> iter = blocks.GetLocatedBlocks().GetEnumerator();
iter.HasNext();)
{
LocatedBlock block = iter.Next();
int actual = block.GetLocations().Length;
if (actual < expected)
{
Log.Info("Not enough replicas for " + block.GetBlock() + " yet. Expecting " + expected
+ ", got " + actual + ".");
replOk = false;
break;
}
}
if (replOk)
{
return;
}
if (maxWaitSec > 0 && (Time.MonotonicNow() - start) > (maxWaitSec * 1000))
{
throw new IOException("Timedout while waiting for all blocks to " + " be replicated for "
+ filename);
}
try
{
Sharpen.Thread.Sleep(500);
}
catch (Exception)
{
}
}
}
protected internal bool VerifyBlockDeletedFromDir(FilePath dir, LocatedBlocks locatedBlocks
)
{
foreach (LocatedBlock lb in locatedBlocks.GetLocatedBlocks())
{
FilePath targetDir = DatanodeUtil.IdToBlockDir(dir, lb.GetBlock().GetBlockId());
FilePath blockFile = new FilePath(targetDir, lb.GetBlock().GetBlockName());
if (blockFile.Exists())
{
Log.Warn("blockFile: " + blockFile.GetAbsolutePath() + " exists after deletion.");
return(false);
}
FilePath metaFile = new FilePath(targetDir, DatanodeUtil.GetMetaName(lb.GetBlock(
).GetBlockName(), lb.GetBlock().GetGenerationStamp()));
if (metaFile.Exists())
{
Log.Warn("metaFile: " + metaFile.GetAbsolutePath() + " exists after deletion.");
return(false);
}
}
return(true);
}
/// <summary>
/// Verify that the DataNode sends a single incremental block report for all
/// storages.
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDataNodeDoesNotSplitReports()
{
LocatedBlocks blocks = CreateFileGetBlocks(GenericTestUtils.GetMethodName());
Assert.AssertThat(cluster.GetDataNodes().Count, IS.Is(1));
// Remove all blocks from the DataNode.
foreach (LocatedBlock block in blocks.GetLocatedBlocks())
{
dn0.NotifyNamenodeDeletedBlock(block.GetBlock(), block.GetStorageIDs()[0]);
}
Log.Info("Triggering report after deleting blocks");
long ops = MetricsAsserts.GetLongCounter("BlockReceivedAndDeletedOps", MetricsAsserts.GetMetrics
(NnMetrics));
// Trigger a report to the NameNode and give it a few seconds.
DataNodeTestUtils.TriggerBlockReport(dn0);
Sharpen.Thread.Sleep(5000);
// Ensure that NameNodeRpcServer.blockReceivedAndDeletes is invoked
// exactly once after we triggered the report.
MetricsAsserts.AssertCounter("BlockReceivedAndDeletedOps", ops + 1, MetricsAsserts.GetMetrics
(NnMetrics));
}
public virtual void TestGetBlockLocations()
{
Path root = new Path("/");
Path file = new Path("/file");
DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed);
// take a snapshot on root
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1");
Path fileInSnapshot = SnapshotTestHelper.GetSnapshotPath(root, "s1", file.GetName
());
FileStatus status = hdfs.GetFileStatus(fileInSnapshot);
// make sure we record the size for the file
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// append data to file
DFSTestUtil.AppendFile(hdfs, file, Blocksize - 1);
status = hdfs.GetFileStatus(fileInSnapshot);
// the size of snapshot file should still be BLOCKSIZE
NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen());
// the size of the file should be (2 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// call DFSClient#callGetBlockLocations for the file in snapshot
LocatedBlocks blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc
(), fileInSnapshot.ToString(), 0, long.MaxValue);
IList <LocatedBlock> blockList = blocks.GetLocatedBlocks();
// should be only one block
NUnit.Framework.Assert.AreEqual(Blocksize, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check the last block
LocatedBlock lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(0, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
// take another snapshot
SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2");
Path fileInSnapshot2 = SnapshotTestHelper.GetSnapshotPath(root, "s2", file.GetName
());
// append data to file without closing
HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize);
@out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength));
status = hdfs.GetFileStatus(fileInSnapshot2);
// the size of snapshot file should be BLOCKSIZE*2-1
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen());
// the size of the file should be (3 * BLOCKSIZE - 1)
status = hdfs.GetFileStatus(file);
NUnit.Framework.Assert.AreEqual(Blocksize * 3 - 1, status.GetLen());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), 0, long.MaxValue);
NUnit.Framework.Assert.IsFalse(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsTrue(blocks.IsLastBlockComplete());
blockList = blocks.GetLocatedBlocks();
// should be 2 blocks
NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, blocks.GetFileLength());
NUnit.Framework.Assert.AreEqual(2, blockList.Count);
// check the last block
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize());
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2
.ToString(), Blocksize, 0);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, blockList.Count);
// check blocks for file being written
blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), file.ToString
(), 0, long.MaxValue);
blockList = blocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(3, blockList.Count);
NUnit.Framework.Assert.IsTrue(blocks.IsUnderConstruction());
NUnit.Framework.Assert.IsFalse(blocks.IsLastBlockComplete());
lastBlock = blocks.GetLastLocatedBlock();
NUnit.Framework.Assert.AreEqual(Blocksize * 2, lastBlock.GetStartOffset());
NUnit.Framework.Assert.AreEqual(Blocksize - 1, lastBlock.GetBlockSize());
@out.Close();
}
/// <summary>
/// The method starts new cluster with defined Configuration; creates a file
/// with specified block_size and writes 10 equal sections in it; it also calls
/// hflush/hsync after each write and throws an IOException in case of an error.
/// </summary>
/// <param name="conf">cluster configuration</param>
/// <param name="fileName">of the file to be created and processed as required</param>
/// <param name="block_size">value to be used for the file's creation</param>
/// <param name="replicas">is the number of replicas</param>
/// <param name="isSync">hsync or hflush</param>
/// <param name="syncFlags">specify the semantic of the sync/flush</param>
/// <exception cref="System.IO.IOException">in case of any errors</exception>
public static void DoTheJob(Configuration conf, string fileName, long block_size,
short replicas, bool isSync, EnumSet <HdfsDataOutputStream.SyncFlag> syncFlags)
{
byte[] fileContent;
int Sections = 10;
fileContent = AppendTestUtil.InitBuffer(AppendTestUtil.FileSize);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(replicas).
Build();
// Make sure we work with DFS in order to utilize all its functionality
DistributedFileSystem fileSystem = cluster.GetFileSystem();
FSDataInputStream @is;
try
{
Path path = new Path(fileName);
string pathName = new Path(fileSystem.GetWorkingDirectory(), path).ToUri().GetPath
();
FSDataOutputStream stm = fileSystem.Create(path, false, 4096, replicas, block_size
);
System.Console.Out.WriteLine("Created file " + fileName);
int tenth = AppendTestUtil.FileSize / Sections;
int rounding = AppendTestUtil.FileSize - tenth * Sections;
for (int i = 0; i < Sections; i++)
{
System.Console.Out.WriteLine("Writing " + (tenth * i) + " to " + (tenth * (i + 1)
) + " section to file " + fileName);
// write to the file
stm.Write(fileContent, tenth * i, tenth);
// Wait while hflush/hsync pushes all packets through built pipeline
if (isSync)
{
((DFSOutputStream)stm.GetWrappedStream()).Hsync(syncFlags);
}
else
{
((DFSOutputStream)stm.GetWrappedStream()).Hflush();
}
// Check file length if updatelength is required
if (isSync && syncFlags.Contains(HdfsDataOutputStream.SyncFlag.UpdateLength))
{
long currentFileLength = fileSystem.GetFileStatus(path).GetLen();
NUnit.Framework.Assert.AreEqual("File size doesn't match for hsync/hflush with updating the length"
, tenth * (i + 1), currentFileLength);
}
else
{
if (isSync && syncFlags.Contains(HdfsDataOutputStream.SyncFlag.EndBlock))
{
LocatedBlocks blocks = fileSystem.dfs.GetLocatedBlocks(pathName, 0);
NUnit.Framework.Assert.AreEqual(i + 1, blocks.GetLocatedBlocks().Count);
}
}
byte[] toRead = new byte[tenth];
byte[] expected = new byte[tenth];
System.Array.Copy(fileContent, tenth * i, expected, 0, tenth);
// Open the same file for read. Need to create new reader after every write operation(!)
@is = fileSystem.Open(path);
@is.Seek(tenth * i);
int readBytes = @is.Read(toRead, 0, tenth);
System.Console.Out.WriteLine("Has read " + readBytes);
NUnit.Framework.Assert.IsTrue("Should've get more bytes", (readBytes > 0) && (readBytes
<= tenth));
@is.Close();
CheckData(toRead, 0, readBytes, expected, "Partial verification");
}
System.Console.Out.WriteLine("Writing " + (tenth * Sections) + " to " + (tenth *
Sections + rounding) + " section to file " + fileName);
stm.Write(fileContent, tenth * Sections, rounding);
stm.Close();
NUnit.Framework.Assert.AreEqual("File size doesn't match ", AppendTestUtil.FileSize
, fileSystem.GetFileStatus(path).GetLen());
AppendTestUtil.CheckFullFile(fileSystem, path, fileContent.Length, fileContent, "hflush()"
);
}
finally
{
fileSystem.Close();
cluster.Shutdown();
}
}
public virtual void TestCopyOnWrite()
{
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort());
DFSClient client = new DFSClient(addr, conf);
try
{
// create a new file, write to it and close it.
//
Path file1 = new Path("/filestatus.dat");
FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
WriteFile(stm);
stm.Close();
// Get a handle to the datanode
DataNode[] dn = cluster.ListDataNodes();
NUnit.Framework.Assert.IsTrue("There should be only one datanode but found " + dn
.Length, dn.Length == 1);
LocatedBlocks locations = client.GetNamenode().GetBlockLocations(file1.ToString()
, 0, long.MaxValue);
IList <LocatedBlock> blocks = locations.GetLocatedBlocks();
//
// Create hard links for a few of the blocks
//
for (int i = 0; i < blocks.Count; i = i + 2)
{
ExtendedBlock b = blocks[i].GetBlock();
FilePath f = DataNodeTestUtils.GetFile(dn[0], b.GetBlockPoolId(), b.GetLocalBlock
().GetBlockId());
FilePath link = new FilePath(f.ToString() + ".link");
System.Console.Out.WriteLine("Creating hardlink for File " + f + " to " + link);
HardLink.CreateHardLink(f, link);
}
//
// Detach all blocks. This should remove hardlinks (if any)
//
for (int i_1 = 0; i_1 < blocks.Count; i_1++)
{
ExtendedBlock b = blocks[i_1].GetBlock();
System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned true"
, DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
}
// Since the blocks were already detached earlier, these calls should
// return false
//
for (int i_2 = 0; i_2 < blocks.Count; i_2++)
{
ExtendedBlock b = blocks[i_2].GetBlock();
System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned false"
, !DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
}
}
finally
{
client.Close();
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestMigrateOpenFileToArchival()
{
Log.Info("testMigrateOpenFileToArchival");
Path fooDir = new Path("/foo");
IDictionary <Path, BlockStoragePolicy> policyMap = Maps.NewHashMap();
policyMap[fooDir] = Cold;
TestStorageMover.NamespaceScheme nsScheme = new TestStorageMover.NamespaceScheme(
Arrays.AsList(fooDir), null, BlockSize, null, policyMap);
TestStorageMover.ClusterScheme clusterScheme = new TestStorageMover.ClusterScheme
(DefaultConf, NumDatanodes, Repl, GenStorageTypes(NumDatanodes), null);
TestStorageMover.MigrationTest test = new TestStorageMover.MigrationTest(this, clusterScheme
, nsScheme);
test.SetupCluster();
// create an open file
Banner("writing to file /foo/bar");
Path barFile = new Path(fooDir, "bar");
DFSTestUtil.CreateFile(test.dfs, barFile, BlockSize, (short)1, 0L);
FSDataOutputStream @out = test.dfs.Append(barFile);
@out.WriteBytes("hello, ");
((DFSOutputStream)@out.GetWrappedStream()).Hsync();
try
{
Banner("start data migration");
test.SetStoragePolicy();
// set /foo to COLD
test.Migrate();
// make sure the under construction block has not been migrated
LocatedBlocks lbs = test.dfs.GetClient().GetLocatedBlocks(barFile.ToString(), BlockSize
);
Log.Info("Locations: " + lbs);
IList <LocatedBlock> blks = lbs.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Count);
NUnit.Framework.Assert.AreEqual(1, blks[0].GetLocations().Length);
Banner("finish the migration, continue writing");
// make sure the writing can continue
@out.WriteBytes("world!");
((DFSOutputStream)@out.GetWrappedStream()).Hsync();
IOUtils.Cleanup(Log, @out);
lbs = test.dfs.GetClient().GetLocatedBlocks(barFile.ToString(), BlockSize);
Log.Info("Locations: " + lbs);
blks = lbs.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, blks.Count);
NUnit.Framework.Assert.AreEqual(1, blks[0].GetLocations().Length);
Banner("finish writing, starting reading");
// check the content of /foo/bar
FSDataInputStream @in = test.dfs.Open(barFile);
byte[] buf = new byte[13];
// read from offset 1024
@in.ReadFully(BlockSize, buf, 0, buf.Length);
IOUtils.Cleanup(Log, @in);
NUnit.Framework.Assert.AreEqual("hello, world!", Sharpen.Runtime.GetStringForBytes
(buf));
}
finally
{
test.ShutdownCluster();
}
}
public virtual void TestRestartDfsWithAbandonedBlock()
{
Configuration conf = new HdfsConfiguration();
// Turn off persistent IPC, so that the DFSClient can survive NN restart
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
MiniDFSCluster cluster = null;
long len = 0;
FSDataOutputStream stream;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
FileSystem fs = cluster.GetFileSystem();
// Creating a file with 4096 blockSize to write multiple blocks
stream = fs.Create(FilePath, true, BlockSize, (short)1, BlockSize);
stream.Write(DataBeforeRestart);
stream.Hflush();
// Wait for all of the blocks to get through
while (len < BlockSize * (NumBlocks - 1))
{
FileStatus status = fs.GetFileStatus(FilePath);
len = status.GetLen();
Sharpen.Thread.Sleep(100);
}
// Abandon the last block
DFSClient dfsclient = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
HdfsFileStatus fileStatus = dfsclient.GetNamenode().GetFileInfo(FileName);
LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(FileName, 0, BlockSize
* NumBlocks);
NUnit.Framework.Assert.AreEqual(NumBlocks, blocks.GetLocatedBlocks().Count);
LocatedBlock b = blocks.GetLastLocatedBlock();
dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileStatus.GetFileId(), FileName
, dfsclient.clientName);
// explicitly do NOT close the file.
cluster.RestartNameNode();
// Check that the file has no less bytes than before the restart
// This would mean that blocks were successfully persisted to the log
FileStatus status_1 = fs.GetFileStatus(FilePath);
NUnit.Framework.Assert.IsTrue("Length incorrect: " + status_1.GetLen(), status_1.
GetLen() == len - BlockSize);
// Verify the data showed up from before restart, sans abandoned block.
FSDataInputStream readStream = fs.Open(FilePath);
try
{
byte[] verifyBuf = new byte[DataBeforeRestart.Length - BlockSize];
IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length);
byte[] expectedBuf = new byte[DataBeforeRestart.Length - BlockSize];
System.Array.Copy(DataBeforeRestart, 0, expectedBuf, 0, expectedBuf.Length);
Assert.AssertArrayEquals(expectedBuf, verifyBuf);
}
finally
{
IOUtils.CloseStream(readStream);
}
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/* check if there are at least two nodes are on the same rack */
/// <exception cref="System.IO.IOException"/>
private void CheckFile(FileSystem fileSys, Path name, int repl)
{
Configuration conf = fileSys.GetConf();
ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys
.GetUri()).GetProxy();
WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl),
-1);
LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue
);
FileStatus stat = fileSys.GetFileStatus(name);
BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue
);
// verify that rack locations match
NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount
());
for (int i = 0; i < blockLocations.Length; i++)
{
LocatedBlock blk = locations.Get(i);
DatanodeInfo[] datanodes = blk.GetLocations();
string[] topologyPaths = blockLocations[i].GetTopologyPaths();
NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length);
for (int j = 0; j < topologyPaths.Length; j++)
{
bool found = false;
for (int k = 0; k < racks.Length; k++)
{
if (topologyPaths[j].StartsWith(racks[k]))
{
found = true;
break;
}
}
NUnit.Framework.Assert.IsTrue(found);
}
}
bool isOnSameRack = true;
bool isNotOnSameRack = true;
foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks())
{
DatanodeInfo[] datanodes = blk_1.GetLocations();
if (datanodes.Length <= 1)
{
break;
}
if (datanodes.Length == 2)
{
isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation
()));
break;
}
isOnSameRack = false;
isNotOnSameRack = false;
for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++)
{
Log.Info("datanode " + i_1 + ": " + datanodes[i_1]);
bool onRack = false;
for (int j = i_1 + 1; j < datanodes.Length; j++)
{
if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation()))
{
onRack = true;
}
}
if (onRack)
{
isOnSameRack = true;
}
if (!onRack)
{
isNotOnSameRack = true;
}
if (isOnSameRack && isNotOnSameRack)
{
break;
}
}
if (!isOnSameRack || !isNotOnSameRack)
{
break;
}
}
NUnit.Framework.Assert.IsTrue(isOnSameRack);
NUnit.Framework.Assert.IsTrue(isNotOnSameRack);
}
public virtual void TestExcludeDataNodes()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 };
string[] hosts = new string[] { "DataNode1", "DataNode2", "DataNode3", "DataNode4"
, "DataNode5", "DataNode6" };
int nDataNodes = hosts.Length;
Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks) + ", hosts="
+ Arrays.AsList(hosts));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Hosts(hosts).NumDataNodes
(nDataNodes).Racks(racks).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
NameNode namenode = cluster.GetNameNode();
DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager
();
Log.Info("dm=" + dm);
long blocksize = DFSConfigKeys.DfsBlockSizeDefault;
string f = "/foo";
//create a file with three replica.
Path p = new Path(f);
FSDataOutputStream @out = dfs.Create(p, (short)3);
@out.Write(1);
@out.Close();
//get replica location.
LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1
);
IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, lb.Count);
DatanodeInfo[] locations = lb[0].GetLocations();
NUnit.Framework.Assert.AreEqual(3, locations.Length);
//For GETFILECHECKSUM, OPEN and APPEND,
//the chosen datanode must be different with exclude nodes.
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 2; i++)
{
sb.Append(locations[i].GetXferAddr());
{
// test GETFILECHECKSUM
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Getfilechecksum, -1L, blocksize, sb.ToString());
for (int j = 0; j <= i; j++)
{
Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName());
}
}
{
// test OPEN
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Open, 0, blocksize, sb.ToString());
for (int j = 0; j <= i; j++)
{
Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName());
}
}
{
// test APPEND
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP
.Append, -1L, blocksize, sb.ToString());
for (int j = 0; j <= i; j++)
{
Assert.AssertNotEquals(locations[j].GetHostName(), chosen.GetHostName());
}
}
sb.Append(",");
}
}
finally
{
cluster.Shutdown();
}
}
public virtual void TestDataLocality()
{
Configuration conf = WebHdfsTestUtil.CreateConf();
string[] racks = new string[] { Rack0, Rack0, Rack1, Rack1, Rack2, Rack2 };
int nDataNodes = racks.Length;
Log.Info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.AsList(racks));
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(nDataNodes
).Racks(racks).Build();
try
{
cluster.WaitActive();
DistributedFileSystem dfs = cluster.GetFileSystem();
NameNode namenode = cluster.GetNameNode();
DatanodeManager dm = namenode.GetNamesystem().GetBlockManager().GetDatanodeManager
();
Log.Info("dm=" + dm);
long blocksize = DFSConfigKeys.DfsBlockSizeDefault;
string f = "/foo";
{
//test CREATE
for (int i = 0; i < nDataNodes; i++)
{
//set client address to a particular datanode
DataNode dn = cluster.GetDataNodes()[i];
string ipAddr = dm.GetDatanode(dn.GetDatanodeId()).GetIpAddr();
//The chosen datanode must be the same as the client address
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PutOpParam.OP
.Create, -1L, blocksize, null);
NUnit.Framework.Assert.AreEqual(ipAddr, chosen.GetIpAddr());
}
}
//create a file with one replica.
Path p = new Path(f);
FSDataOutputStream @out = dfs.Create(p, (short)1);
@out.Write(1);
@out.Close();
//get replica location.
LocatedBlocks locatedblocks = NameNodeAdapter.GetBlockLocations(namenode, f, 0, 1
);
IList <LocatedBlock> lb = locatedblocks.GetLocatedBlocks();
NUnit.Framework.Assert.AreEqual(1, lb.Count);
DatanodeInfo[] locations = lb[0].GetLocations();
NUnit.Framework.Assert.AreEqual(1, locations.Length);
DatanodeInfo expected = locations[0];
{
//For GETFILECHECKSUM, OPEN and APPEND,
//the chosen datanode must be the same as the replica location.
//test GETFILECHECKSUM
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Getfilechecksum, -1L, blocksize, null);
NUnit.Framework.Assert.AreEqual(expected, chosen);
}
{
//test OPEN
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, GetOpParam.OP
.Open, 0, blocksize, null);
NUnit.Framework.Assert.AreEqual(expected, chosen);
}
{
//test APPEND
DatanodeInfo chosen = NamenodeWebHdfsMethods.ChooseDatanode(namenode, f, PostOpParam.OP
.Append, -1L, blocksize, null);
NUnit.Framework.Assert.AreEqual(expected, chosen);
}
}
finally
{
cluster.Shutdown();
}
}
