/// <exception cref="System.Exception"/> public virtual void TestSetupDistributedCacheConflictsFiles() { Configuration conf = new Configuration(); conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem )); URI mockUri = URI.Create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem (); URI file = new URI("mockfs://mock/tmp/something.zip#something"); Path filePath = new Path(file); URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); Path file2Path = new Path(file2); Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath); Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path); DistributedCache.AddCacheFile(file, conf); DistributedCache.AddCacheFile(file2, conf); conf.Set(MRJobConfig.CacheFileTimestamps, "10,11"); conf.Set(MRJobConfig.CacheFilesSizes, "10,11"); conf.Set(MRJobConfig.CacheFileVisibilities, "true,true"); IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); NUnit.Framework.Assert.AreEqual(1, localResources.Count); LocalResource lr = localResources["something"]; //First one wins NUnit.Framework.Assert.IsNotNull(lr); NUnit.Framework.Assert.AreEqual(10l, lr.GetSize()); NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp()); NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType()); }
internal static long GetEstimatedSize(LocalResource rsrc) { if (rsrc.GetSize() < 0) { return(-1); } switch (rsrc.GetType()) { case LocalResourceType.Archive: case LocalResourceType.Pattern: { return(5 * rsrc.GetSize()); } case LocalResourceType.File: default: { return(rsrc.GetSize()); } } }
private YarnProtos.LocalResourceProto BuildLocalResourceProto(LocalResource lr) { LocalResourcePBImpl lrpb; if (!(lr is LocalResourcePBImpl)) { lr = LocalResource.NewInstance(lr.GetResource(), lr.GetType(), lr.GetVisibility() , lr.GetSize(), lr.GetTimestamp(), lr.GetPattern()); } lrpb = (LocalResourcePBImpl)lr; return(lrpb.GetProto()); }
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility vis, Path p) { ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec >(); LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>(); string name = long.ToHexString(r.NextLong()); URL uri = Org.Mockito.Mockito.Mock <URL>(); Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file"); Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null); Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name); Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri); Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L); Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L); Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File); Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc); Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn (ConverterUtils.GetYarnUrlFromPath(p)); return(resourceLocalizationSpec); }