/// <exception cref="System.Exception"/> public virtual void TestSetupDistributedCacheConflictsFiles() { Configuration conf = new Configuration(); conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem )); URI mockUri = URI.Create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem (); URI file = new URI("mockfs://mock/tmp/something.zip#something"); Path filePath = new Path(file); URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); Path file2Path = new Path(file2); Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath); Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path); DistributedCache.AddCacheFile(file, conf); DistributedCache.AddCacheFile(file2, conf); conf.Set(MRJobConfig.CacheFileTimestamps, "10,11"); conf.Set(MRJobConfig.CacheFilesSizes, "10,11"); conf.Set(MRJobConfig.CacheFileVisibilities, "true,true"); IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); NUnit.Framework.Assert.AreEqual(1, localResources.Count); LocalResource lr = localResources["something"]; //First one wins NUnit.Framework.Assert.IsNotNull(lr); NUnit.Framework.Assert.AreEqual(10l, lr.GetSize()); NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp()); NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType()); }
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr is LocalResourcePBImpl)) { lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility() , lr.GetSize(), lr.GetTimestamp(), lr.GetPattern()); } lrpb = (LocalResourcePBImpl)lr; return(lrpb.GetProto()); }
internal static LocalResource CreateMockResource() { // mock rsrc location URL uriA = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uriA.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uriA.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uriA.GetFile()).ThenReturn("/localA/rsrc"); LocalResource apiRsrc = Org.Mockito.Mockito.Mock <LocalResource>(); Org.Mockito.Mockito.When(apiRsrc.GetResource()).ThenReturn(uriA); Org.Mockito.Mockito.When(apiRsrc.GetTimestamp()).ThenReturn(4344L); Org.Mockito.Mockito.When(apiRsrc.GetType()).ThenReturn(LocalResourceType.File); return(apiRsrc); }
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec >(); LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>(); string name = long.ToHexString(r.NextLong()); URL uri = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name); Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri); Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L); Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L); Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File); Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn (ConverterUtils.GetYarnUrlFromPath(p)); return(resourceLocalizationSpec); }
internal virtual bool VerifyAccess() { // if it is in the public cache, it's trivially OK if (resource.GetVisibility() == LocalResourceVisibility.Public) { return(true); } Path remotePath; try { remotePath = ConverterUtils.GetPathFromYarnURL(resource.GetResource()); } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); } // get the file status of the HDFS file FileSystem remoteFs = remotePath.GetFileSystem(conf); FileStatus status = remoteFs.GetFileStatus(remotePath); // check to see if the file has been modified in any way if (status.GetModificationTime() != resource.GetTimestamp()) { Log.Warn("The remote file " + remotePath + " has changed since it's localized; will not consider it for upload" ); return(false); } // check for the user ownership if (status.GetOwner().Equals(user)) { return(true); } // the user owns the file // check if the file is publicly readable otherwise return(FileIsPublic(remotePath, remoteFs, status)); }
/// <summary>Wrap API resource to match against cache of localized resources.</summary> /// <param name="resource">Resource requested by container</param> /// <exception cref="Sharpen.URISyntaxException">If the path is malformed</exception> public LocalResourceRequest(LocalResource resource) : this(ConverterUtils.GetPathFromYarnURL(resource.GetResource()), resource.GetTimestamp (), resource.GetType(), resource.GetVisibility(), resource.GetPattern()) { }