コード例 #1
0
ファイル: Mover.cs プロジェクト: orf53975/hadoop.net
            /// <returns>true if it is necessary to run another round of migration</returns>
            private bool ProcessFile(string fullPath, HdfsLocatedFileStatus status)
            {
                byte policyId = status.GetStoragePolicy();

                // currently we ignore files with unspecified storage policy
                if (policyId == BlockStoragePolicySuite.IdUnspecified)
                {
                    return(false);
                }
                BlockStoragePolicy policy = this._enclosing.blockStoragePolicies[policyId];

                if (policy == null)
                {
                    Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to get the storage policy of file "
                                                                       + fullPath);
                    return(false);
                }
                IList <StorageType> types         = policy.ChooseStorageTypes(status.GetReplication());
                LocatedBlocks       locatedBlocks = status.GetBlockLocations();
                bool hasRemaining        = false;
                bool lastBlkComplete     = locatedBlocks.IsLastBlockComplete();
                IList <LocatedBlock> lbs = locatedBlocks.GetLocatedBlocks();

                for (int i = 0; i < lbs.Count; i++)
                {
                    if (i == lbs.Count - 1 && !lastBlkComplete)
                    {
                        // last block is incomplete, skip it
                        continue;
                    }
                    LocatedBlock          lb   = lbs[i];
                    Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
                                                                               ));
                    if (!diff.RemoveOverlap(true))
                    {
                        if (this.ScheduleMoves4Block(diff, lb))
                        {
                            hasRemaining |= (diff.existing.Count > 1 && diff.expected.Count > 1);
                        }
                    }
                }
                return(hasRemaining);
            }
コード例 #2
0
            /// <exception cref="System.Exception"/>
            private void VerifyFile(Path parent, HdfsFileStatus status, byte expectedPolicyId
                                    )
            {
                HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus)status;
                byte policyId             = fileStatus.GetStoragePolicy();
                BlockStoragePolicy policy = this.policies.GetPolicy(policyId);

                if (expectedPolicyId != null)
                {
                    NUnit.Framework.Assert.AreEqual(unchecked ((byte)expectedPolicyId), policy.GetId()
                                                    );
                }
                IList <StorageType> types = policy.ChooseStorageTypes(status.GetReplication());

                foreach (LocatedBlock lb in fileStatus.GetBlockLocations().GetLocatedBlocks())
                {
                    Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types, lb.GetStorageTypes(
                                                                               ));
                    NUnit.Framework.Assert.IsTrue(fileStatus.GetFullName(parent.ToString()) + " with policy "
                                                  + policy + " has non-empty overlap: " + diff + ", the corresponding block is "
                                                  + lb.GetBlock().GetLocalBlock(), diff.RemoveOverlap(true));
                }
            }