/// <exception cref="System.IO.IOException"/> internal virtual void GetAllVolumesMap(string bpid, ReplicaMap volumeMap, RamDiskReplicaTracker ramDiskReplicaMap) { long totalStartTime = Time.MonotonicNow(); IList <IOException> exceptions = Sharpen.Collections.SynchronizedList(new AList <IOException >()); IList <Sharpen.Thread> replicaAddingThreads = new AList <Sharpen.Thread>(); foreach (FsVolumeImpl v in volumes.Get()) { Sharpen.Thread t = new _Thread_186(v, bpid, volumeMap, ramDiskReplicaMap, exceptions ); replicaAddingThreads.AddItem(t); t.Start(); } foreach (Sharpen.Thread t_1 in replicaAddingThreads) { try { t_1.Join(); } catch (Exception ie) { throw new IOException(ie); } } if (!exceptions.IsEmpty()) { throw exceptions[0]; } long totalTimeTaken = Time.MonotonicNow() - totalStartTime; FsDatasetImpl.Log.Info("Total time to add all replicas to map: " + totalTimeTaken + "ms"); }
/// <exception cref="System.IO.IOException"/> internal virtual void GetVolumeMap(ReplicaMap volumeMap, RamDiskReplicaTracker ramDiskReplicaMap ) { foreach (BlockPoolSlice s in bpSlices.Values) { s.GetVolumeMap(volumeMap, ramDiskReplicaMap); } }
public _Thread_186(FsVolumeImpl v, string bpid, ReplicaMap volumeMap, RamDiskReplicaTracker ramDiskReplicaMap, IList <IOException> exceptions) { this.v = v; this.bpid = bpid; this.volumeMap = volumeMap; this.ramDiskReplicaMap = ramDiskReplicaMap; this.exceptions = exceptions; }
/// <summary> /// Get an instance of the configured RamDiskReplicaTracker based on the /// the configuration property /// <see cref="Org.Apache.Hadoop.Hdfs.DFSConfigKeys.DfsDatanodeRamDiskReplicaTrackerKey /// "/> /// . /// </summary> /// <param name="conf">the configuration to be used</param> /// <param name="dataset">the FsDataset object.</param> /// <returns>an instance of RamDiskReplicaTracker</returns> internal static RamDiskReplicaTracker GetInstance(Configuration conf, FsDatasetImpl fsDataset) { Type trackerClass = conf.GetClass <RamDiskReplicaTracker>(DFSConfigKeys.DfsDatanodeRamDiskReplicaTrackerKey , DFSConfigKeys.DfsDatanodeRamDiskReplicaTrackerDefault); RamDiskReplicaTracker tracker = ReflectionUtils.NewInstance(trackerClass, conf); tracker.Initialize(fsDataset); return(tracker); }
/// <exception cref="System.IO.IOException"/> internal virtual void GetVolumeMap(ReplicaMap volumeMap, RamDiskReplicaTracker lazyWriteReplicaMap ) { // Recover lazy persist replicas, they will be added to the volumeMap // when we scan the finalized directory. if (lazypersistDir.Exists()) { int numRecovered = MoveLazyPersistReplicasToFinalized(lazypersistDir); FsDatasetImpl.Log.Info("Recovered " + numRecovered + " replicas from " + lazypersistDir ); } // add finalized replicas AddToReplicasMap(volumeMap, finalizedDir, lazyWriteReplicaMap, true); // add rbw replicas AddToReplicasMap(volumeMap, rbwDir, lazyWriteReplicaMap, false); }
/// <exception cref="System.IO.IOException"/> internal virtual void GetVolumeMap(string bpid, ReplicaMap volumeMap, RamDiskReplicaTracker ramDiskReplicaMap) { GetBlockPoolSlice(bpid).GetVolumeMap(volumeMap, ramDiskReplicaMap); }
/// <summary>Add replicas under the given directory to the volume map</summary> /// <param name="volumeMap">the replicas map</param> /// <param name="dir">an input directory</param> /// <param name="lazyWriteReplicaMap"> /// Map of replicas on transient /// storage. /// </param> /// <param name="isFinalized"> /// true if the directory has finalized replicas; /// false if the directory has rbw replicas /// </param> /// <exception cref="System.IO.IOException"/> internal virtual void AddToReplicasMap(ReplicaMap volumeMap, FilePath dir, RamDiskReplicaTracker lazyWriteReplicaMap, bool isFinalized) { FilePath[] files = FileUtil.ListFiles(dir); foreach (FilePath file in files) { if (file.IsDirectory()) { AddToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized); } if (isFinalized && FsDatasetUtil.IsUnlinkTmpFile(file)) { file = RecoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it // in another iteration and can continue here continue; } } if (!Block.IsBlockFilename(file)) { continue; } long genStamp = FsDatasetUtil.GetGenerationStampFromFile(files, file); long blockId = Block.Filename2id(file.GetName()); ReplicaInfo newReplica = null; if (isFinalized) { newReplica = new FinalizedReplica(blockId, file.Length(), genStamp, volume, file. GetParentFile()); } else { bool loadRwr = true; FilePath restartMeta = new FilePath(file.GetParent() + FilePath.pathSeparator + "." + file.GetName() + ".restart"); Scanner sc = null; try { sc = new Scanner(restartMeta, "UTF-8"); // The restart meta file exists if (sc.HasNextLong() && (sc.NextLong() > Time.Now())) { // It didn't expire. Load the replica as a RBW. // We don't know the expected block length, so just use 0 // and don't reserve any more space for writes. newReplica = new ReplicaBeingWritten(blockId, ValidateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.GetParentFile(), null, 0); loadRwr = false; } sc.Close(); if (!restartMeta.Delete()) { FsDatasetImpl.Log.Warn("Failed to delete restart meta file: " + restartMeta.GetPath ()); } } catch (FileNotFoundException) { } finally { // nothing to do hereFile dir = if (sc != null) { sc.Close(); } } // Restart meta doesn't exist or expired. if (loadRwr) { newReplica = new ReplicaWaitingToBeRecovered(blockId, ValidateIntegrityAndSetLength (file, genStamp), genStamp, volume, file.GetParentFile()); } } ReplicaInfo oldReplica = volumeMap.Get(bpid, newReplica.GetBlockId()); if (oldReplica == null) { volumeMap.Add(bpid, newReplica); } else { // We have multiple replicas of the same block so decide which one // to keep. newReplica = ResolveDuplicateReplicas(newReplica, oldReplica, volumeMap); } // If we are retaining a replica on transient storage make sure // it is in the lazyWriteReplicaMap so it can be persisted // eventually. if (newReplica.GetVolume().IsTransientStorage()) { lazyWriteReplicaMap.AddReplica(bpid, blockId, (FsVolumeImpl)newReplica.GetVolume( )); } else { lazyWriteReplicaMap.DiscardReplica(bpid, blockId, false); } } }