public virtual void TestSetAndGetStoragePolicy()
{
Path foo = new Path("/foo");
Path bar = new Path(foo, "bar");
DFSTestUtil.CreateFile(fs, bar, Size, Repl, 0);
StoragePolicyAdmin admin = new StoragePolicyAdmin(conf);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo", 0, "The storage policy of "
+ foo.ToString() + " is unspecified");
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo/bar", 0, "The storage policy of "
+ bar.ToString() + " is unspecified");
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /foo -policy WARM", 0, "Set storage policy WARM on "
+ foo.ToString());
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /foo/bar -policy COLD", 0, "Set storage policy COLD on "
+ bar.ToString());
DFSTestUtil.ToolRun(admin, "-setStoragePolicy -path /fooz -policy WARM", 2, "File/Directory does not exist: /fooz"
);
BlockStoragePolicySuite suite = BlockStoragePolicySuite.CreateDefaultSuite();
BlockStoragePolicy warm = suite.GetPolicy("WARM");
BlockStoragePolicy cold = suite.GetPolicy("COLD");
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo", 0, "The storage policy of "
+ foo.ToString() + ":\n" + warm);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /foo/bar", 0, "The storage policy of "
+ bar.ToString() + ":\n" + cold);
DFSTestUtil.ToolRun(admin, "-getStoragePolicy -path /fooz", 2, "File/Directory does not exist: /fooz"
);
}
public virtual void TestMultipleWarmsInDifferentOrder()
{
BlockStoragePolicySuite bsps = BlockStoragePolicySuite.CreateDefaultSuite();
StoragePolicySummary sts = new StoragePolicySummary(bsps.GetAllPolicies());
BlockStoragePolicy warm = bsps.GetPolicy("WARM");
//DISK:1,ARCHIVE:1
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Disk }, warm);
//DISK:2,ARCHIVE:1
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Disk, StorageType.Disk }, warm);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Archive, StorageType.Disk }, warm);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Archive }, warm);
//DISK:1,ARCHIVE:2
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Archive, StorageType.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Disk, StorageType.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Disk }, warm);
//DISK:2,ARCHIVE:2
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Disk, StorageType.Disk }, warm);
IDictionary <string, long> actualOutput = ConvertToStringMap(sts);
NUnit.Framework.Assert.AreEqual(4, actualOutput.Count);
IDictionary <string, long> expectedOutput = new Dictionary <string, long>();
expectedOutput["WARM|DISK:1,ARCHIVE:1(WARM)"] = 2l;
expectedOutput["WARM|DISK:2,ARCHIVE:1"] = 3l;
expectedOutput["WARM|DISK:1,ARCHIVE:2(WARM)"] = 3l;
expectedOutput["WARM|DISK:2,ARCHIVE:2"] = 1l;
NUnit.Framework.Assert.AreEqual(expectedOutput, actualOutput);
}
public virtual void TestMultipleHotsWithDifferentCounts()
{
BlockStoragePolicySuite bsps = BlockStoragePolicySuite.CreateDefaultSuite();
StoragePolicySummary sts = new StoragePolicySummary(bsps.GetAllPolicies());
BlockStoragePolicy hot = bsps.GetPolicy("HOT");
sts.Add(new StorageType[] { StorageType.Disk }, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk }, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk }, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk,
StorageType.Disk }, hot);
IDictionary <string, long> actualOutput = ConvertToStringMap(sts);
NUnit.Framework.Assert.AreEqual(4, actualOutput.Count);
IDictionary <string, long> expectedOutput = new Dictionary <string, long>();
expectedOutput["HOT|DISK:1(HOT)"] = 1l;
expectedOutput["HOT|DISK:2(HOT)"] = 2l;
expectedOutput["HOT|DISK:3(HOT)"] = 2l;
expectedOutput["HOT|DISK:4(HOT)"] = 1l;
NUnit.Framework.Assert.AreEqual(expectedOutput, actualOutput);
}
internal StorageTypeAllocation(StorageType[] storageTypes, BlockStoragePolicy specifiedStoragePolicy
)
{
Arrays.Sort(storageTypes);
this.storageTypes = storageTypes;
this.specifiedStoragePolicy = specifiedStoragePolicy;
}
// This is the only place that needs to use the BlockStoragePolicySuite to
// derive the intended storage type usage for quota by storage type
public sealed override QuotaCounts ComputeQuotaUsage(BlockStoragePolicySuite bsps
, byte blockStoragePolicyId, QuotaCounts counts, bool useCache, int lastSnapshotId
)
{
long nsDelta = 1;
long ssDeltaNoReplication;
short replication;
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
if (sf != null)
{
FileDiffList fileDiffList = sf.GetDiffs();
int last = fileDiffList.GetLastSnapshotId();
if (lastSnapshotId == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId ||
last == Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId)
{
ssDeltaNoReplication = StoragespaceConsumedNoReplication();
replication = GetBlockReplication();
}
else
{
if (last < lastSnapshotId)
{
ssDeltaNoReplication = ComputeFileSize(true, false);
replication = GetFileReplication();
}
else
{
int sid = fileDiffList.GetSnapshotById(lastSnapshotId);
ssDeltaNoReplication = StoragespaceConsumedNoReplication(sid);
replication = GetReplication(sid);
}
}
}
else
{
ssDeltaNoReplication = StoragespaceConsumedNoReplication();
replication = GetBlockReplication();
}
counts.AddNameSpace(nsDelta);
counts.AddStorageSpace(ssDeltaNoReplication * replication);
if (blockStoragePolicyId != BlockStoragePolicySuite.IdUnspecified)
{
BlockStoragePolicy bsp = bsps.GetPolicy(blockStoragePolicyId);
IList <StorageType> storageTypes = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in storageTypes)
{
if (!t.SupportTypeQuota())
{
continue;
}
counts.AddTypeSpace(t, ssDeltaNoReplication);
}
}
return(counts);
}
static TestStorageMover()
{
DefaultConf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize);
DefaultConf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L);
DefaultConf.SetLong(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 2L);
DefaultConf.SetLong(DFSConfigKeys.DfsMoverMovedwinwidthKey, 2000L);
DefaultPolicies = BlockStoragePolicySuite.CreateDefaultSuite();
Hot = DefaultPolicies.GetPolicy(HdfsConstants.HotStoragePolicyName);
Warm = DefaultPolicies.GetPolicy(HdfsConstants.WarmStoragePolicyName);
Cold = DefaultPolicies.GetPolicy(HdfsConstants.ColdStoragePolicyName);
TestBalancer.InitTestSetup();
Dispatcher.SetDelayAfterErrors(1000L);
}
public virtual void TestDifferentSpecifiedPolicies()
{
BlockStoragePolicySuite bsps = BlockStoragePolicySuite.CreateDefaultSuite();
StoragePolicySummary sts = new StoragePolicySummary(bsps.GetAllPolicies());
BlockStoragePolicy hot = bsps.GetPolicy("HOT");
BlockStoragePolicy warm = bsps.GetPolicy("WARM");
BlockStoragePolicy cold = bsps.GetPolicy("COLD");
//DISK:3
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, hot);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, warm);
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Disk, StorageType.Disk }
, cold);
//DISK:1,ARCHIVE:2
sts.Add(new StorageType[] { StorageType.Disk, StorageType.Archive, StorageType.Archive }, hot);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Disk, StorageType.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Disk }, cold);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Disk }, cold);
//ARCHIVE:3
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Archive }, hot);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Archive }, hot);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Archive }, warm);
sts.Add(new StorageType[] { StorageType.Archive, StorageType.Archive, StorageType
.Archive }, cold);
IDictionary <string, long> actualOutput = ConvertToStringMap(sts);
NUnit.Framework.Assert.AreEqual(9, actualOutput.Count);
IDictionary <string, long> expectedOutput = new Dictionary <string, long>();
expectedOutput["HOT|DISK:3(HOT)"] = 2l;
expectedOutput["COLD|DISK:1,ARCHIVE:2(WARM)"] = 2l;
expectedOutput["HOT|ARCHIVE:3(COLD)"] = 2l;
expectedOutput["WARM|DISK:3(HOT)"] = 1l;
expectedOutput["COLD|DISK:3(HOT)"] = 1l;
expectedOutput["WARM|ARCHIVE:3(COLD)"] = 1l;
expectedOutput["WARM|DISK:1,ARCHIVE:2(WARM)"] = 1l;
expectedOutput["COLD|ARCHIVE:3(COLD)"] = 1l;
expectedOutput["HOT|DISK:1,ARCHIVE:2(WARM)"] = 1l;
NUnit.Framework.Assert.AreEqual(expectedOutput, actualOutput);
}
// Add a storage type combination
internal virtual void Add(StorageType[] storageTypes, BlockStoragePolicy policy)
{
StoragePolicySummary.StorageTypeAllocation storageCombo = new StoragePolicySummary.StorageTypeAllocation
(storageTypes, policy);
long count = storageComboCounts[storageCombo];
if (count == null)
{
storageComboCounts[storageCombo] = 1l;
storageCombo.SetActualStoragePolicy(GetStoragePolicy(storageCombo.GetStorageTypes
()));
}
else
{
storageComboCounts[storageCombo] = count + 1;
}
totalBlocks++;
}
public sealed override ContentSummaryComputationContext ComputeContentSummary(ContentSummaryComputationContext
summary)
{
ContentCounts counts = summary.GetCounts();
FileWithSnapshotFeature sf = GetFileWithSnapshotFeature();
long fileLen = 0;
if (sf == null)
{
fileLen = ComputeFileSize();
counts.AddContent(Content.File, 1);
}
else
{
FileDiffList diffs = sf.GetDiffs();
int n = diffs.AsList().Count;
counts.AddContent(Content.File, n);
if (n > 0 && sf.IsCurrentFileDeleted())
{
fileLen = diffs.GetLast().GetFileSize();
}
else
{
fileLen = ComputeFileSize();
}
}
counts.AddContent(Content.Length, fileLen);
counts.AddContent(Content.Diskspace, StoragespaceConsumed());
if (GetStoragePolicyID() != BlockStoragePolicySuite.IdUnspecified)
{
BlockStoragePolicy bsp = summary.GetBlockStoragePolicySuite().GetPolicy(GetStoragePolicyID
());
IList <StorageType> storageTypes = bsp.ChooseStorageTypes(GetFileReplication());
foreach (StorageType t in storageTypes)
{
if (!t.SupportTypeQuota())
{
continue;
}
counts.AddTypeSpace(t, fileLen);
}
}
return(summary);
}
/// <returns>true if it is necessary to run another round of migration</returns>
private bool ProcessFile(string fullPath, HdfsLocatedFileStatus status)
{
byte policyId = status.GetStoragePolicy();
// currently we ignore files with unspecified storage policy
if (policyId == BlockStoragePolicySuite.IdUnspecified)
{
return(false);
}
BlockStoragePolicy policy = this._enclosing.blockStoragePolicies[policyId];
if (policy == null)
{
Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to get the storage policy of file "
+ fullPath);
return(false);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
LocatedBlocks locatedBlocks = status.GetBlockLocations();
bool hasRemaining = false;
bool lastBlkComplete = locatedBlocks.IsLastBlockComplete();
IList <LocatedBlock> lbs = locatedBlocks.GetLocatedBlocks();
for (int i = 0; i < lbs.Count; i++)
{
if (i == lbs.Count - 1 && !lastBlkComplete)
{
// last block is incomplete, skip it
continue;
}
LocatedBlock lb = lbs[i];
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
if (!diff.RemoveOverlap(true))
{
if (this.ScheduleMoves4Block(diff, lb))
{
hasRemaining |= (diff.existing.Count > 1 && diff.expected.Count > 1);
}
}
}
return(hasRemaining);
}
/// <exception cref="System.IO.IOException"/>
internal static void UnprotectedSetStoragePolicy(FSDirectory fsd, BlockManager bm
, INodesInPath iip, byte policyId)
{
System.Diagnostics.Debug.Assert(fsd.HasWriteLock());
INode inode = iip.GetLastINode();
if (inode == null)
{
throw new FileNotFoundException("File/Directory does not exist: " + iip.GetPath()
);
}
int snapshotId = iip.GetLatestSnapshotId();
if (inode.IsFile())
{
BlockStoragePolicy newPolicy = bm.GetStoragePolicy(policyId);
if (newPolicy.IsCopyOnCreateFile())
{
throw new HadoopIllegalArgumentException("Policy " + newPolicy + " cannot be set after file creation."
);
}
BlockStoragePolicy currentPolicy = bm.GetStoragePolicy(inode.GetLocalStoragePolicyID
());
if (currentPolicy != null && currentPolicy.IsCopyOnCreateFile())
{
throw new HadoopIllegalArgumentException("Existing policy " + currentPolicy.GetName
() + " cannot be changed after file creation.");
}
inode.AsFile().SetStoragePolicyID(policyId, snapshotId);
}
else
{
if (inode.IsDirectory())
{
SetDirStoragePolicy(fsd, inode.AsDirectory(), policyId, snapshotId);
}
else
{
throw new FileNotFoundException(iip.GetPath() + " is not a file or directory");
}
}
}
CreateDefaultSuite()
{
BlockStoragePolicy[] policies = new BlockStoragePolicy[1 << IdBitLength];
byte lazyPersistId = HdfsConstants.MemoryStoragePolicyId;
policies[lazyPersistId] = new BlockStoragePolicy(lazyPersistId, HdfsConstants.MemoryStoragePolicyName
, new StorageType[] { StorageType.RamDisk, StorageType.Disk }, new StorageType[]
{ StorageType.Disk }, new StorageType[] { StorageType.Disk }, true);
// Cannot be changed on regular files, but inherited.
byte allssdId = HdfsConstants.AllssdStoragePolicyId;
policies[allssdId] = new BlockStoragePolicy(allssdId, HdfsConstants.AllssdStoragePolicyName
, new StorageType[] { StorageType.Ssd }, new StorageType[] { StorageType.Disk },
new StorageType[] { StorageType.Disk });
byte onessdId = HdfsConstants.OnessdStoragePolicyId;
policies[onessdId] = new BlockStoragePolicy(onessdId, HdfsConstants.OnessdStoragePolicyName
, new StorageType[] { StorageType.Ssd, StorageType.Disk }, new StorageType[] { StorageType
.Ssd, StorageType.Disk }, new StorageType[] { StorageType.Ssd, StorageType.Disk }
);
byte hotId = HdfsConstants.HotStoragePolicyId;
policies[hotId] = new BlockStoragePolicy(hotId, HdfsConstants.HotStoragePolicyName
, new StorageType[] { StorageType.Disk }, StorageType.EmptyArray, new StorageType
[] { StorageType.Archive });
byte warmId = HdfsConstants.WarmStoragePolicyId;
policies[warmId] = new BlockStoragePolicy(warmId, HdfsConstants.WarmStoragePolicyName
, new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType[]
{ StorageType.Disk, StorageType.Archive }, new StorageType[] { StorageType.Disk
, StorageType.Archive });
byte coldId = HdfsConstants.ColdStoragePolicyId;
policies[coldId] = new BlockStoragePolicy(coldId, HdfsConstants.ColdStoragePolicyName
, new StorageType[] { StorageType.Archive }, StorageType.EmptyArray, StorageType
.EmptyArray);
return(new Org.Apache.Hadoop.Hdfs.Server.Blockmanagement.BlockStoragePolicySuite(
hotId, policies));
}
/// <exception cref="System.IO.IOException"/>
internal static HdfsFileStatus SetStoragePolicy(FSDirectory fsd, BlockManager bm,
string src, string policyName)
{
if (!fsd.IsStoragePolicyEnabled())
{
throw new IOException("Failed to set storage policy since " + DFSConfigKeys.DfsStoragePolicyEnabledKey
+ " is set to false.");
}
FSPermissionChecker pc = fsd.GetPermissionChecker();
byte[][] pathComponents = FSDirectory.GetPathComponentsForReservedPath(src);
INodesInPath iip;
fsd.WriteLock();
try
{
src = FSDirectory.ResolvePath(src, pathComponents, fsd);
iip = fsd.GetINodesInPath4Write(src);
if (fsd.IsPermissionEnabled())
{
fsd.CheckPathAccess(pc, iip, FsAction.Write);
}
// get the corresponding policy and make sure the policy name is valid
BlockStoragePolicy policy = bm.GetStoragePolicy(policyName);
if (policy == null)
{
throw new HadoopIllegalArgumentException("Cannot find a block policy with the name "
+ policyName);
}
UnprotectedSetStoragePolicy(fsd, bm, iip, policy.GetId());
fsd.GetEditLog().LogSetStoragePolicy(src, policy.GetId());
}
finally
{
fsd.WriteUnlock();
}
return(fsd.GetAuditFileInfo(iip));
}
private static QuotaCounts ComputeQuotaDeltas(FSDirectory fsd, INodeFile target,
INodeFile[] srcList)
{
QuotaCounts deltas = new QuotaCounts.Builder().Build();
short targetRepl = target.GetBlockReplication();
foreach (INodeFile src in srcList)
{
short srcRepl = src.GetBlockReplication();
long fileSize = src.ComputeFileSize();
if (targetRepl != srcRepl)
{
deltas.AddStorageSpace(fileSize * (targetRepl - srcRepl));
BlockStoragePolicy bsp = fsd.GetBlockStoragePolicySuite().GetPolicy(src.GetStoragePolicyID
());
if (bsp != null)
{
IList <StorageType> srcTypeChosen = bsp.ChooseStorageTypes(srcRepl);
foreach (StorageType t in srcTypeChosen)
{
if (t.SupportTypeQuota())
{
deltas.AddTypeSpace(t, -fileSize);
}
}
IList <StorageType> targetTypeChosen = bsp.ChooseStorageTypes(targetRepl);
foreach (StorageType t_1 in targetTypeChosen)
{
if (t_1.SupportTypeQuota())
{
deltas.AddTypeSpace(t_1, fileSize);
}
}
}
}
}
return(deltas);
}
/// <exception cref="System.Exception"/>
private void VerifyFile(Path parent, HdfsFileStatus status, byte expectedPolicyId
)
{
HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus)status;
byte policyId = fileStatus.GetStoragePolicy();
BlockStoragePolicy policy = this.policies.GetPolicy(policyId);
if (expectedPolicyId != null)
{
NUnit.Framework.Assert.AreEqual(unchecked ((byte)expectedPolicyId), policy.GetId()
);
}
IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());
foreach (LocatedBlock lb in fileStatus.GetBlockLocations().GetLocatedBlocks())
{
Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
));
NUnit.Framework.Assert.IsTrue(fileStatus.GetFullName(parent.ToString()) + " with policy "
+ policy + " has non-empty overlap: " + diff + ", the corresponding block is "
+ lb.GetBlock().GetLocalBlock(), diff.RemoveOverlap(true));
}
}
public virtual QuotaCounts UpdateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps
, INodeFile file, FileDiff removed, INode.BlocksMapUpdateInfo collectedBlocks, IList
<INode> removedINodes)
{
long oldStoragespace = file.StoragespaceConsumed();
byte storagePolicyID = file.GetStoragePolicyID();
BlockStoragePolicy bsp = null;
EnumCounters <StorageType> typeSpaces = new EnumCounters <StorageType>(typeof(StorageType
));
if (storagePolicyID != BlockStoragePolicySuite.IdUnspecified)
{
bsp = bsps.GetPolicy(file.GetStoragePolicyID());
}
if (removed.snapshotINode != null)
{
short replication = removed.snapshotINode.GetFileReplication();
short currentRepl = file.GetBlockReplication();
if (currentRepl == 0)
{
long oldFileSizeNoRep = file.ComputeFileSize(true, true);
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
}
}
else
{
if (replication > currentRepl)
{
long oldFileSizeNoRep = file.StoragespaceConsumedNoReplication();
oldStoragespace = oldFileSizeNoRep * replication;
if (bsp != null)
{
IList <StorageType> oldTypeChosen = bsp.ChooseStorageTypes(replication);
foreach (StorageType t in oldTypeChosen)
{
if (t.SupportTypeQuota())
{
typeSpaces.Add(t, -oldFileSizeNoRep);
}
}
IList <StorageType> newTypeChosen = bsp.ChooseStorageTypes(currentRepl);
foreach (StorageType t_1 in newTypeChosen)
{
if (t_1.SupportTypeQuota())
{
typeSpaces.Add(t_1, oldFileSizeNoRep);
}
}
}
}
}
AclFeature aclFeature = removed.GetSnapshotINode().GetAclFeature();
if (aclFeature != null)
{
AclStorage.RemoveAclFeature(aclFeature);
}
}
GetDiffs().CombineAndCollectSnapshotBlocks(bsps, file, removed, collectedBlocks,
removedINodes);
long ssDelta = oldStoragespace - file.StoragespaceConsumed();
return(new QuotaCounts.Builder().StorageSpace(ssDelta).TypeSpaces(typeSpaces).Build
());
}
public override DatanodeStorageInfo[] ChooseTarget(string srcPath, int numOfReplicas
, Node writer, IList <DatanodeStorageInfo> chosenNodes, bool returnChosenNodes, ICollection
<Node> excludedNodes, long blocksize, BlockStoragePolicy storagePolicy)
{
DatanodeStorageInfo[] results = base.ChooseTarget(srcPath, numOfReplicas, writer,
chosenNodes, returnChosenNodes, excludedNodes, blocksize, storagePolicy);
try
{
Sharpen.Thread.Sleep(3000);
}
catch (Exception)
{
}
return(results);
}
/// <summary>
/// choose <i>numOfReplicas</i> data nodes for <i>writer</i>
/// to re-replicate a block with size <i>blocksize</i>
/// If not, return as many as we can.
/// </summary>
/// <param name="srcPath">the file to which this chooseTargets is being invoked.</param>
/// <param name="numOfReplicas">additional number of replicas wanted.</param>
/// <param name="writer">the writer's machine, null if not in the cluster.</param>
/// <param name="chosen">datanodes that have been chosen as targets.</param>
/// <param name="returnChosenNodes">decide if the chosenNodes are returned.</param>
/// <param name="excludedNodes">datanodes that should not be considered as targets.</param>
/// <param name="blocksize">size of the data to be written.</param>
/// <returns>
/// array of DatanodeDescriptor instances chosen as target
/// and sorted as a pipeline.
/// </returns>
public abstract DatanodeStorageInfo[] ChooseTarget(string srcPath, int numOfReplicas
, Node writer, IList <DatanodeStorageInfo> chosen, bool returnChosenNodes, ICollection
<Node> excludedNodes, long blocksize, BlockStoragePolicy storagePolicy);
/// <summary>
/// Same as
/// <see cref="ChooseTarget(string, int, Org.Apache.Hadoop.Net.Node, System.Collections.Generic.ICollection{E}, long, System.Collections.Generic.IList{E}, Org.Apache.Hadoop.Hdfs.Protocol.BlockStoragePolicy)
/// "/>
/// with added parameter
/// <c>favoredDatanodes</c>
/// </summary>
/// <param name="favoredNodes">
/// datanodes that should be favored as targets. This
/// is only a hint and due to cluster state, namenode may not be
/// able to place the blocks on these datanodes.
/// </param>
internal virtual DatanodeStorageInfo[] ChooseTarget(string src, int numOfReplicas
, Node writer, ICollection <Node> excludedNodes, long blocksize, IList <DatanodeDescriptor
> favoredNodes, BlockStoragePolicy storagePolicy)
{
// This class does not provide the functionality of placing
// a block in favored datanodes. The implementations of this class
// are expected to provide this functionality
return(ChooseTarget(src, numOfReplicas, writer, new AList <DatanodeStorageInfo>(numOfReplicas
), false, excludedNodes, blocksize, storagePolicy));
}
internal virtual void SetActualStoragePolicy(BlockStoragePolicy actualStoragePolicy
)
{
this.actualStoragePolicy = actualStoragePolicy;
}
