/// <exception cref="System.IO.IOException"/>
private void AddToLocalResources(FileSystem fs, string fileSrcPath, string fileDstPath
, string appId, IDictionary <string, LocalResource> localResources, string resources
)
{
string suffix = appName + "/" + appId + "/" + fileDstPath;
Path dst = new Path(fs.GetHomeDirectory(), suffix);
if (fileSrcPath == null)
{
FSDataOutputStream ostream = null;
try
{
ostream = FileSystem.Create(fs, dst, new FsPermission((short)0x1c8));
ostream.WriteUTF(resources);
}
finally
{
IOUtils.CloseQuietly(ostream);
}
}
else
{
fs.CopyFromLocalFile(new Path(fileSrcPath), dst);
}
FileStatus scFileStatus = fs.GetFileStatus(dst);
LocalResource scRsrc = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI
(dst.ToUri()), LocalResourceType.File, LocalResourceVisibility.Application, scFileStatus
.GetLen(), scFileStatus.GetModificationTime());
localResources[fileDstPath] = scRsrc;
}
/// <returns>
///
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// absolute path for localization which includes local
/// directory path and the relative hierarchical path (if use local
/// cache directory manager is enabled)
/// </returns>
/// <?/>
/// <?/>
/// <?/>
public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath
, DeletionService delService)
{
Path rPath = localDirPath;
if (useLocalCacheDirectoryManager && localDirPath != null)
{
if (!directoryManagers.Contains(localDirPath))
{
directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf));
}
LocalCacheDirectoryManager dir = directoryManagers[localDirPath];
rPath = localDirPath;
string hierarchicalPath = dir.GetRelativePathForLocalization();
// For most of the scenarios we will get root path only which
// is an empty string
if (!hierarchicalPath.IsEmpty())
{
rPath = new Path(localDirPath, hierarchicalPath);
}
inProgressLocalResourcesMap[req] = rPath;
}
while (true)
{
Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FilePath file = new FilePath(uniquePath.ToUri().GetRawPath());
if (!file.Exists())
{
rPath = uniquePath;
break;
}
// If the directory already exists, delete it and move to next one.
Log.Warn("Directory " + uniquePath + " already exists, " + "try next one.");
if (delService != null)
{
delService.Delete(GetUser(), uniquePath);
}
}
Path localPath = new Path(rPath, req.GetPath().GetName());
LocalizedResource rsrc = localrsrc[req];
rsrc.SetLocalPath(localPath);
LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req
.GetVisibility(), req.GetSize(), req.GetTimestamp());
try
{
stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto
(), localPath);
}
catch (IOException e)
{
Log.Error("Unable to record localization start for " + rsrc, e);
}
return(rPath);
}
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr)
{
LocalResourcePBImpl lrpb;
if (!(lr is LocalResourcePBImpl))
{
lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility()
, lr.GetSize(), lr.GetTimestamp(), lr.GetPattern());
}
lrpb = (LocalResourcePBImpl)lr;
return(lrpb.GetProto());
}
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
/// <exception cref="System.IO.IOException"/>
private static void ParseDistributedCacheArtifacts(Configuration conf, IDictionary
<string, LocalResource> localResources, LocalResourceType type, URI[] uris, long
[] timestamps, long[] sizes, bool[] visibilities)
{
if (uris != null)
{
// Sanity check
if ((uris.Length != timestamps.Length) || (uris.Length != sizes.Length) || (uris.
Length != visibilities.Length))
{
throw new ArgumentException("Invalid specification for " + "distributed-cache artifacts of type "
+ type + " :" + " #uris=" + uris.Length + " #timestamps=" + timestamps.Length +
" #visibilities=" + visibilities.Length);
}
for (int i = 0; i < uris.Length; ++i)
{
URI u = uris[i];
Path p = new Path(u);
FileSystem remoteFS = p.GetFileSystem(conf);
p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
// Add URI fragment or just the filename
Path name = new Path((null == u.GetFragment()) ? p.GetName() : u.GetFragment());
if (name.IsAbsolute())
{
throw new ArgumentException("Resource name must be relative");
}
string linkName = name.ToUri().GetPath();
LocalResource orig = localResources[linkName];
URL url = ConverterUtils.GetYarnUrlFromURI(p.ToUri());
if (orig != null && !orig.GetResource().Equals(url))
{
Log.Warn(GetResourceDescription(orig.GetType()) + ToString(orig.GetResource()) +
" conflicts with " + GetResourceDescription(type) + ToString(url) + " This will be an error in Hadoop 2.0"
);
continue;
}
localResources[linkName] = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI
(p.ToUri()), type, visibilities[i] ? LocalResourceVisibility.Public : LocalResourceVisibility
.Private, sizes[i], timestamps[i]);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.Exception"/>
public virtual void TestClearLocalDirWhenNodeReboot()
{
nm = new TestNodeManagerReboot.MyNodeManager(this);
nm.Start();
ContainerManagementProtocol containerManager = nm.GetContainerManager();
// create files under fileCache
CreateFiles(nmLocalDir.GetAbsolutePath(), ContainerLocalizer.Filecache, 100);
localResourceDir.Mkdirs();
ContainerLaunchContext containerLaunchContext = Records.NewRecord <ContainerLaunchContext
>();
// Construct the Container-id
ContainerId cId = CreateContainerId();
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(localResourceDir.GetAbsolutePath())));
LocalResource localResource = LocalResource.NewInstance(localResourceUri, LocalResourceType
.File, LocalResourceVisibility.Application, -1, localResourceDir.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = new AList <string>();
containerLaunchContext.SetCommands(commands);
NodeId nodeId = nm.GetNMContext().GetNodeId();
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, destinationFile, nm.
GetNMContext().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.GetApplicationAttemptId
().ToString());
NMTokenIdentifier nmIdentifier = new NMTokenIdentifier(cId.GetApplicationAttemptId
(), nodeId, user, 123);
currentUser.AddTokenIdentifier(nmIdentifier);
currentUser.DoAs(new _PrivilegedExceptionAction_152(this, allRequests));
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
= nm.GetNMContext().GetContainers()[request.GetContainerIds()[0]];
int MaxTries = 20;
int numTries = 0;
while (!container.GetContainerState().Equals(ContainerState.Done) && numTries <=
MaxTries)
{
try
{
Sharpen.Thread.Sleep(500);
}
catch (Exception)
{
}
// Do nothing
numTries++;
}
NUnit.Framework.Assert.AreEqual(ContainerState.Done, container.GetContainerState(
));
NUnit.Framework.Assert.IsTrue("The container should create a subDir named currentUser: "******"under localDir/usercache", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(
), ContainerLocalizer.Usercache) > 0);
NUnit.Framework.Assert.IsTrue("There should be files or Dirs under nm_private when "
+ "container is launched", NumOfLocalDirs(nmLocalDir.GetAbsolutePath(), ResourceLocalizationService
.NmPrivateDir) > 0);
// restart the NodeManager
RestartNM(MaxTries);
CheckNumOfLocalDirs();
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ResourceLocalizationService.NmPrivateDir + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).Delete((string
)Matchers.IsNull(), Matchers.ArgThat(new TestNodeManagerReboot.PathInclude(this,
ContainerLocalizer.Filecache + "_DEL_")));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, user, null
, new string[] { destinationFile })));
Org.Mockito.Mockito.Verify(delService, Org.Mockito.Mockito.Times(1)).ScheduleFileDeletionTask
(Matchers.ArgThat(new TestNodeManagerReboot.FileDeletionInclude(this, null, ContainerLocalizer
.Usercache + "_DEL_", new string[] { })));
// restart the NodeManager again
// this time usercache directory should be empty
RestartNM(MaxTries);
CheckNumOfLocalDirs();
}
