/// <exception cref="System.IO.IOException"/> private void AddToLocalResources(FileSystem fs, string fileSrcPath, string fileDstPath , string appId, IDictionary <string, LocalResource> localResources, string resources ) { string suffix = appName + "/" + appId + "/" + fileDstPath; Path dst = new Path(fs.GetHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem.Create(fs, dst, new FsPermission((short)0x1c8)); ostream.WriteUTF(resources); } finally { IOUtils.CloseQuietly(ostream); } } else { fs.CopyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.GetFileStatus(dst); LocalResource scRsrc = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI (dst.ToUri()), LocalResourceType.File, LocalResourceVisibility.Application, scFileStatus .GetLen(), scFileStatus.GetModificationTime()); localResources[fileDstPath] = scRsrc; }
/// <returns> /// /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// absolute path for localization which includes local /// directory path and the relative hierarchical path (if use local /// cache directory manager is enabled) /// </returns> /// <?/> /// <?/> /// <?/> public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath , DeletionService delService) { Path rPath = localDirPath; if (useLocalCacheDirectoryManager && localDirPath != null) { if (!directoryManagers.Contains(localDirPath)) { directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf)); } LocalCacheDirectoryManager dir = directoryManagers[localDirPath]; rPath = localDirPath; string hierarchicalPath = dir.GetRelativePathForLocalization(); // For most of the scenarios we will get root path only which // is an empty string if (!hierarchicalPath.IsEmpty()) { rPath = new Path(localDirPath, hierarchicalPath); } inProgressLocalResourcesMap[req] = rPath; } while (true) { Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FilePath file = new FilePath(uniquePath.ToUri().GetRawPath()); if (!file.Exists()) { rPath = uniquePath; break; } // If the directory already exists, delete it and move to next one. Log.Warn("Directory " + uniquePath + " already exists, " + "try next one."); if (delService != null) { delService.Delete(GetUser(), uniquePath); } } Path localPath = new Path(rPath, req.GetPath().GetName()); LocalizedResource rsrc = localrsrc[req]; rsrc.SetLocalPath(localPath); LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req .GetVisibility(), req.GetSize(), req.GetTimestamp()); try { stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto (), localPath); } catch (IOException e) { Log.Error("Unable to record localization start for " + rsrc, e); } return(rPath); }
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr is LocalResourcePBImpl)) { lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility() , lr.GetSize(), lr.GetTimestamp(), lr.GetPattern()); } lrpb = (LocalResourcePBImpl)lr; return(lrpb.GetProto()); }
// TODO - Move this to MR! // Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[], // long[], boolean[], Path[], FileType) /// <exception cref="System.IO.IOException"/> private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary <string, LocalResource> localResources, LocalResourceType type, URI[] uris, long [] timestamps, long[] sizes, bool[] visibilities) { if (uris != null) { // Sanity check if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris. Length != visibilities.Length)) { throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type " + type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length + " #visibilities=" + visibilities.Length); } for (int i = 0; i < uris.Length; ++i) { URI u = uris[i]; Path p = new Path(u); FileSystem remoteFS = p.GetFileSystem(conf); p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory ())); // Add URI fragment or just the filename Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment()); if (name.IsAbsolute()) { throw new ArgumentException("Resource name must be relative"); } string linkName = name.ToUri().GetPath(); LocalResource orig = localResources[linkName]; URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri()); if (orig != null && !orig.GetResource().Equals(url)) { Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) + " conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0" ); continue; } localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI (p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility .Private, sizes[i], timestamps[i]); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.Exception"/> public virtual void TestClearLocalDirWhenNodeReboot() { nm = new TestNodeManagerReboot.MyNodeManager(this); nm.Start(); ContainerManagementProtocol containerManager = nm.GetContainerManager(); // create files under fileCache CreateFiles(nmLocalDir.GetAbsolutePath(), ContainerLocalizer.Filecache, 100); localResourceDir.Mkdirs(); ContainerLaunchContext containerLaunchContext = Records.NewRecord <ContainerLaunchContext >(); // Construct the Container-id ContainerId cId = CreateContainerId(); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(localResourceDir.GetAbsolutePath()))); LocalResource localResource = LocalResource.NewInstance(localResourceUri, LocalResourceType .File, LocalResourceVisibility.Application, -1, localResourceDir.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = new AList <string>(); containerLaunchContext.SetCommands(commands); NodeId nodeId = nm.GetNMContext().GetNodeId(); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, destinationFile, nm. GetNMContext().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.GetApplicationAttemptId ().ToString()); NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.GetApplicationAttemptId (), nodeId, user, 123); currentUser.AddTokenIdentifier(nmIdentifier); currentUser.DoAs(new _PrivilegedExceptionAction_152(this, allRequests)); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = nm.GetNMContext().GetContainers()[request.GetContainerIds()[0]]; int MaxTries = 20; int numTries = 0; while (!container.GetContainerState().Equals(ContainerState.Done) && numTries <= MaxTries) { try { Sharpen.Thread.Sleep(500); } catch (Exception) { } // Do nothing numTries++; } NUnit.Framework.Assert.AreEqual(ContainerState.Done, container.GetContainerState( )); NUnit.Framework.Assert.IsTrue("The container should create a subDir named currentUser: "******"under localDir/usercache", NumOfLocalDirs(nmLocalDir.GetAbsolutePath( ), ContainerLocalizer.Usercache) > 0); NUnit.Framework.Assert.IsTrue("There should be files or Dirs under nm_private when " + "container is launched", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(), ResourceLocalizationService .NmPrivateDir) > 0); // restart the NodeManager RestartNM(MaxTries); CheckNumOfLocalDirs(); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string )Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this, ResourceLocalizationService.NmPrivateDir + "_DEL_"))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string )Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this, ContainerLocalizer.Filecache + "_DEL_"))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask (Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, user, null , new string[] { destinationFile }))); Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask (Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, null, ContainerLocalizer .Usercache + "_DEL_", new string[] { }))); // restart the NodeManager again // this time usercache directory should be empty RestartNM(MaxTries); CheckNumOfLocalDirs(); }