/// <exception cref="System.Exception"/>
public virtual void TestSetupDistributedCacheConflictsFiles()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem
));
URI mockUri = URI.Create("mockfs://mock/");
FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem
();
URI file = new URI("mockfs://mock/tmp/something.zip#something");
Path filePath = new Path(file);
URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
Path file2Path = new Path(file2);
Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath);
Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path);
DistributedCache.AddCacheFile(file, conf);
DistributedCache.AddCacheFile(file2, conf);
conf.Set(MRJobConfig.CacheFileTimestamps, "10,11");
conf.Set(MRJobConfig.CacheFilesSizes, "10,11");
conf.Set(MRJobConfig.CacheFileVisibilities, "true,true");
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
NUnit.Framework.Assert.AreEqual(1, localResources.Count);
LocalResource lr = localResources["something"];
//First one wins
NUnit.Framework.Assert.IsNotNull(lr);
NUnit.Framework.Assert.AreEqual(10l, lr.GetSize());
NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp());
NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType());
}
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr)
{
LocalResourcePBImpl lrpb;
if (!(lr is LocalResourcePBImpl))
{
lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility()
, lr.GetSize(), lr.GetTimestamp(), lr.GetPattern());
}
lrpb = (LocalResourcePBImpl)lr;
return(lrpb.GetProto());
}
internal static LocalResource CreateMockResource()
{
// mock rsrc location
URL uriA = Org.Mockito.Mockito.Mock <URL>();
Org.Mockito.Mockito.When(uriA.GetScheme()).ThenReturn("file");
Org.Mockito.Mockito.When(uriA.GetHost()).ThenReturn(null);
Org.Mockito.Mockito.When(uriA.GetFile()).ThenReturn("/localA/rsrc");
LocalResource apiRsrc = Org.Mockito.Mockito.Mock <LocalResource>();
Org.Mockito.Mockito.When(apiRsrc.GetResource()).ThenReturn(uriA);
Org.Mockito.Mockito.When(apiRsrc.GetTimestamp()).ThenReturn(4344L);
Org.Mockito.Mockito.When(apiRsrc.GetType()).ThenReturn(LocalResourceType.File);
return(apiRsrc);
}
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility
vis, Path p)
{
ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec
>();
LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>();
string name = long.ToHexString(r.NextLong());
URL uri = Org.Mockito.Mockito.Mock <URL>();
Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file");
Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null);
Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name);
Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri);
Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L);
Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L);
Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File);
Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn
(ConverterUtils.GetYarnUrlFromPath(p));
return(resourceLocalizationSpec);
}
internal virtual bool VerifyAccess()
{
// if it is in the public cache, it's trivially OK
if (resource.GetVisibility() == LocalResourceVisibility.Public)
{
return(true);
}
Path remotePath;
try
{
remotePath = ConverterUtils.GetPathFromYarnURL(resource.GetResource());
}
catch (URISyntaxException e)
{
throw new IOException("Invalid resource", e);
}
// get the file status of the HDFS file
FileSystem remoteFs = remotePath.GetFileSystem(conf);
FileStatus status = remoteFs.GetFileStatus(remotePath);
// check to see if the file has been modified in any way
if (status.GetModificationTime() != resource.GetTimestamp())
{
Log.Warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload"
);
return(false);
}
// check for the user ownership
if (status.GetOwner().Equals(user))
{
return(true);
}
// the user owns the file
// check if the file is publicly readable otherwise
return(FileIsPublic(remotePath, remoteFs, status));
}
/// <summary>Wrap API resource to match against cache of localized resources.</summary>
/// <param name="resource">Resource requested by container</param>
/// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception>
public LocalResourceRequest(LocalResource resource)
: this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp
(), resource.GetType(), resource.GetVisibility(), resource.GetPattern())
{
}
