/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
internal static LocalResource CreateZipFile(FileContext files, Path p, int len, Random
r, LocalResourceVisibility vis)
{
byte[] bytes = new byte[len];
r.NextBytes(bytes);
FilePath archiveFile = new FilePath(p.ToUri().GetPath() + ".ZIP");
archiveFile.CreateNewFile();
ZipOutputStream @out = new ZipOutputStream(new FileOutputStream(archiveFile));
@out.PutNextEntry(new ZipEntry(p.GetName()));
@out.Write(bytes);
@out.CloseEntry();
@out.Close();
LocalResource ret = recordFactory.NewRecordInstance <LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromPath(new Path(p.ToString() + ".ZIP")
));
ret.SetSize(len);
ret.SetType(LocalResourceType.Archive);
ret.SetVisibility(vis);
ret.SetTimestamp(files.GetFileStatus(new Path(p.ToString() + ".ZIP")).GetModificationTime
());
return(ret);
}
public ResourceRequestEvent(LocalResourceRequest resource, LocalResourceVisibility
vis, LocalizerContext context)
: base(resource, ResourceEventType.Request)
{
this.vis = vis;
this.context = context;
}
/// <exception cref="System.Exception"/>
public virtual void TestUniqueDestinationPath()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
Path p = new Path(basedir, "dir" + 0 + ".jar");
LocalResourceVisibility vis = LocalResourceVisibility.Private;
LocalResource rsrc = CreateJar(files, p, vis);
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
Future <Path> rPath = singleThreadedExec.Submit(fsd);
singleThreadedExec.Shutdown();
while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(rPath.IsDone());
// Now FSDownload will not create a random directory to localize the
// resource. Therefore the final localizedPath for the resource should be
// destination directory (passed as an argument) + file name.
NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent());
}
public LocalizerEventMatcher(ContainerId idRef, Credentials creds, LocalResourceVisibility
vis, LocalizerEventType type)
{
this.vis = vis;
this.type = type;
this.creds = creds;
this.idRef = idRef;
}
internal LocalResourceRequest(Path loc, long timestamp, LocalResourceType type, LocalResourceVisibility
visibility, string pattern)
{
this.loc = loc;
this.timestamp = timestamp;
this.type = type;
this.visibility = visibility;
this.pattern = pattern;
}
public LocalizerResourceRequestEvent(LocalizedResource resource, LocalResourceVisibility
vis, LocalizerContext context, string pattern)
: base(LocalizerEventType.RequestResourceLocalization, ConverterUtils.ToString(context
.GetContainerId()))
{
this.vis = vis;
this.context = context;
this.resource = resource;
this.pattern = pattern;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDownloadBadPublic()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource
, LocalResourceVisibility>();
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
int size = 512;
LocalResourceVisibility vis = LocalResourceVisibility.Public;
Path path = new Path(basedir, "test-file");
LocalResource rsrc = CreateFile(files, path, size, rand, vis);
rsrcVis[rsrc] = vis;
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(pending[rsrc].IsDone());
try
{
foreach (KeyValuePair <LocalResource, Future <Path> > p in pending)
{
p.Value.Get();
NUnit.Framework.Assert.Fail("We localized a file that is not public.");
}
}
catch (ExecutionException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is IOException);
}
}
public override void SetVisibility(LocalResourceVisibility visibility)
{
lock (this)
{
MaybeInitBuilder();
if (visibility == null)
{
builder.ClearVisibility();
return;
}
builder.SetVisibility(ConvertToProtoFormat(visibility));
}
}
/// <exception cref="System.IO.IOException"/>
internal static LocalResource CreateFile(FileContext files, Path p, int len, Random
r, LocalResourceVisibility vis)
{
CreateFile(files, p, len, r);
LocalResource ret = recordFactory.NewRecordInstance <LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromPath(p));
ret.SetSize(len);
ret.SetType(LocalResourceType.File);
ret.SetVisibility(vis);
ret.SetTimestamp(files.GetFileStatus(p).GetModificationTime());
return(ret);
}
/// <exception cref="Sharpen.URISyntaxException"/>
internal static LocalResource GetYarnResource(Path p, long size, long timestamp,
LocalResourceType type, LocalResourceVisibility state, string pattern)
{
LocalResource ret = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromURI(p.ToUri()));
ret.SetSize(size);
ret.SetTimestamp(timestamp);
ret.SetType(type);
ret.SetVisibility(state);
ret.SetPattern(pattern);
return(ret);
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, string pattern, bool shouldBeUploadedToSharedCache
)
{
LocalResource resource = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <LocalResource
>();
resource.SetResource(url);
resource.SetType(type);
resource.SetVisibility(visibility);
resource.SetSize(size);
resource.SetTimestamp(timestamp);
resource.SetPattern(pattern);
resource.SetShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
return(resource);
}
internal static ResourceLocalizationSpec GetMockRsrc(Random r, LocalResourceVisibility
vis, Path p)
{
ResourceLocalizationSpec resourceLocalizationSpec = Org.Mockito.Mockito.Mock <ResourceLocalizationSpec
>();
LocalResource rsrc = Org.Mockito.Mockito.Mock <LocalResource>();
string name = long.ToHexString(r.NextLong());
URL uri = Org.Mockito.Mockito.Mock <URL>();
Org.Mockito.Mockito.When(uri.GetScheme()).ThenReturn("file");
Org.Mockito.Mockito.When(uri.GetHost()).ThenReturn(null);
Org.Mockito.Mockito.When(uri.GetFile()).ThenReturn("/local/" + vis + "/" + name);
Org.Mockito.Mockito.When(rsrc.GetResource()).ThenReturn(uri);
Org.Mockito.Mockito.When(rsrc.GetSize()).ThenReturn(r.Next(1024) + 1024L);
Org.Mockito.Mockito.When(rsrc.GetTimestamp()).ThenReturn(r.Next(1024) + 2048L);
Org.Mockito.Mockito.When(rsrc.GetType()).ThenReturn(LocalResourceType.File);
Org.Mockito.Mockito.When(rsrc.GetVisibility()).ThenReturn(vis);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetResource()).ThenReturn(rsrc);
Org.Mockito.Mockito.When(resourceLocalizationSpec.GetDestinationDirectory()).ThenReturn
(ConverterUtils.GetYarnUrlFromPath(p));
return(resourceLocalizationSpec);
}
/// <exception cref="System.IO.IOException"/>
internal static LocalResource CreateJar(FileContext files, Path p, LocalResourceVisibility
vis)
{
Log.Info("Create jar file " + p);
FilePath jarFile = new FilePath((files.MakeQualified(p)).ToUri());
FileOutputStream stream = new FileOutputStream(jarFile);
Log.Info("Create jar out stream ");
JarOutputStream @out = new JarOutputStream(stream, new Manifest());
Log.Info("Done writing jar stream ");
@out.Close();
LocalResource ret = recordFactory.NewRecordInstance <LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromPath(p));
FileStatus status = files.GetFileStatus(p);
ret.SetSize(status.GetLen());
ret.SetTimestamp(status.GetModificationTime());
ret.SetType(LocalResourceType.Pattern);
ret.SetVisibility(vis);
ret.SetPattern("classes/.*");
return(ret);
}
public abstract void SetVisibility(LocalResourceVisibility visibility);
public static LocalResource NewLocalResource(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache)
{
LocalResource resource = recordFactory.NewRecordInstance <LocalResource>();
resource.SetResource(url);
resource.SetType(type);
resource.SetVisibility(visibility);
resource.SetSize(size);
resource.SetTimestamp(timestamp);
resource.SetShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
return(resource);
}
/*
* LocalResourceVisibility
*/
public static YarnProtos.LocalResourceVisibilityProto ConvertToProtoFormat(LocalResourceVisibility
e)
{
return(YarnProtos.LocalResourceVisibilityProto.ValueOf(e.ToString()));
}
public override void SetVisibility(LocalResourceVisibility visibility)
{
throw new NotSupportedException();
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp)
{
return(NewInstance(url, type, visibility, size, timestamp, null));
}
private YarnProtos.LocalResourceVisibilityProto ConvertToProtoFormat(LocalResourceVisibility
e)
{
return(ProtoUtils.ConvertToProtoFormat(e));
}
public static LocalResource NewLocalResource(URI uri, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache)
{
return(NewLocalResource(ConverterUtils.GetYarnUrlFromURI(uri), type, visibility,
size, timestamp, shouldBeUploadedToSharedCache));
}
public virtual void TestNotification()
{
// mocked generic
DrainDispatcher dispatcher = new DrainDispatcher();
dispatcher.Init(new Configuration());
try
{
dispatcher.Start();
EventHandler <ContainerEvent> containerBus = Org.Mockito.Mockito.Mock <EventHandler
>();
EventHandler <LocalizerEvent> localizerBus = Org.Mockito.Mockito.Mock <EventHandler
>();
dispatcher.Register(typeof(ContainerEventType), containerBus);
dispatcher.Register(typeof(LocalizerEventType), localizerBus);
// mock resource
LocalResource apiRsrc = CreateMockResource();
ContainerId container0 = GetMockContainer(0L);
Credentials creds0 = new Credentials();
LocalResourceVisibility vis0 = LocalResourceVisibility.Private;
LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0);
LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
local.Handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
dispatcher.Await();
// Register C0, verify request event
TestLocalizedResource.LocalizerEventMatcher matchesL0Req = new TestLocalizedResource.LocalizerEventMatcher
(container0, creds0, vis0, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL0Req));
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Register C1, verify request event
Credentials creds1 = new Credentials();
ContainerId container1 = GetMockContainer(1L);
LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1);
LocalResourceVisibility vis1 = LocalResourceVisibility.Public;
local.Handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
dispatcher.Await();
TestLocalizedResource.LocalizerEventMatcher matchesL1Req = new TestLocalizedResource.LocalizerEventMatcher
(container1, creds1, vis1, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL1Req));
// Release C0 container localization, verify no notification
local.Handle(new ResourceReleaseEvent(rsrcA, container0));
dispatcher.Await();
Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA
<ContainerEvent>());
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Release C1 container localization, verify no notification
local.Handle(new ResourceReleaseEvent(rsrcA, container1));
dispatcher.Await();
Org.Mockito.Mockito.Verify(containerBus, Org.Mockito.Mockito.Never()).Handle(Matchers.IsA
<ContainerEvent>());
NUnit.Framework.Assert.AreEqual(ResourceState.Downloading, local.GetState());
// Register C2, C3
ContainerId container2 = GetMockContainer(2L);
LocalResourceVisibility vis2 = LocalResourceVisibility.Private;
Credentials creds2 = new Credentials();
LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2);
ContainerId container3 = GetMockContainer(3L);
LocalResourceVisibility vis3 = LocalResourceVisibility.Private;
Credentials creds3 = new Credentials();
LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3);
local.Handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
local.Handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
dispatcher.Await();
TestLocalizedResource.LocalizerEventMatcher matchesL2Req = new TestLocalizedResource.LocalizerEventMatcher
(container2, creds2, vis2, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL2Req));
TestLocalizedResource.LocalizerEventMatcher matchesL3Req = new TestLocalizedResource.LocalizerEventMatcher
(container3, creds3, vis3, LocalizerEventType.RequestResourceLocalization);
Org.Mockito.Mockito.Verify(localizerBus).Handle(Matchers.ArgThat(matchesL3Req));
// Successful localization. verify notification C2, C3
Path locA = new Path("file:///cache/rsrcA");
local.Handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
dispatcher.Await();
TestLocalizedResource.ContainerEventMatcher matchesC2Localized = new TestLocalizedResource.ContainerEventMatcher
(container2, ContainerEventType.ResourceLocalized);
TestLocalizedResource.ContainerEventMatcher matchesC3Localized = new TestLocalizedResource.ContainerEventMatcher
(container3, ContainerEventType.ResourceLocalized);
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC2Localized
));
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC3Localized
));
NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState());
// Register C4, verify notification
ContainerId container4 = GetMockContainer(4L);
Credentials creds4 = new Credentials();
LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4);
LocalResourceVisibility vis4 = LocalResourceVisibility.Private;
local.Handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
dispatcher.Await();
TestLocalizedResource.ContainerEventMatcher matchesC4Localized = new TestLocalizedResource.ContainerEventMatcher
(container4, ContainerEventType.ResourceLocalized);
Org.Mockito.Mockito.Verify(containerBus).Handle(Matchers.ArgThat(matchesC4Localized
));
NUnit.Framework.Assert.AreEqual(ResourceState.Localized, local.GetState());
}
finally
{
dispatcher.Stop();
}
}
public virtual void SetVisibility(LocalResourceVisibility visibility)
{
this.visibility = visibility;
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, string pattern)
{
return(NewInstance(url, type, visibility, size, timestamp, pattern, false));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
private void DownloadWithFileType(TestFSDownload.TEST_FILE_TYPE fileType)
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
int size = rand.Next(512) + 512;
LocalResourceVisibility vis = LocalResourceVisibility.Private;
Path p = new Path(basedir, string.Empty + 1);
string strFileName = string.Empty;
LocalResource rsrc = null;
switch (fileType)
{
case TestFSDownload.TEST_FILE_TYPE.Tar:
{
rsrc = CreateTarFile(files, p, size, rand, vis);
break;
}
case TestFSDownload.TEST_FILE_TYPE.Jar:
{
rsrc = CreateJarFile(files, p, size, rand, vis);
rsrc.SetType(LocalResourceType.Pattern);
break;
}
case TestFSDownload.TEST_FILE_TYPE.Zip:
{
rsrc = CreateZipFile(files, p, size, rand, vis);
strFileName = p.GetName() + ".ZIP";
break;
}
case TestFSDownload.TEST_FILE_TYPE.Tgz:
{
rsrc = CreateTgzFile(files, p, size, rand, vis);
break;
}
}
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
try
{
pending[rsrc].Get();
// see if there was an Exception during download
FileStatus[] filesstatus = files.GetDefaultFileSystem().ListStatus(basedir);
foreach (FileStatus filestatus in filesstatus)
{
if (filestatus.IsDirectory())
{
FileStatus[] childFiles = files.GetDefaultFileSystem().ListStatus(filestatus.GetPath
());
foreach (FileStatus childfile in childFiles)
{
if (strFileName.EndsWith(".ZIP") && childfile.GetPath().GetName().Equals(strFileName
) && !childfile.IsDirectory())
{
NUnit.Framework.Assert.Fail("Failure...After unzip, there should have been a" + " directory formed with zip file name but found a file. "
+ childfile.GetPath());
}
if (childfile.GetPath().GetName().StartsWith("tmp"))
{
NUnit.Framework.Assert.Fail("Tmp File should not have been there " + childfile.GetPath
());
}
}
}
}
}
catch (Exception e)
{
throw new IOException("Failed exec", e);
}
}
/// <exception cref="System.IO.IOException"/>
private void VerifyPermsRecursively(FileSystem fs, FileContext files, Path p, LocalResourceVisibility
vis)
{
FileStatus status = files.GetFileStatus(p);
if (status.IsDirectory())
{
if (vis == LocalResourceVisibility.Public)
{
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PublicDirPerms
.ToShort());
}
else
{
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PrivateDirPerms
.ToShort());
}
if (!status.IsSymlink())
{
FileStatus[] statuses = fs.ListStatus(p);
foreach (FileStatus stat in statuses)
{
VerifyPermsRecursively(fs, files, stat.GetPath(), vis);
}
}
}
else
{
if (vis == LocalResourceVisibility.Public)
{
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PublicFilePerms
.ToShort());
}
else
{
NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PrivateFilePerms
.ToShort());
}
}
}
public static LocalResource NewInstance(URL url, LocalResourceType type, LocalResourceVisibility
visibility, long size, long timestamp, bool shouldBeUploadedToSharedCache)
{
return(NewInstance(url, type, visibility, size, timestamp, null, shouldBeUploadedToSharedCache
));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDirDownload()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource
, LocalResourceVisibility>();
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
for (int i = 0; i < 5; ++i)
{
LocalResourceVisibility vis = LocalResourceVisibility.Private;
if (i % 2 == 1)
{
vis = LocalResourceVisibility.Application;
}
Path p = new Path(basedir, "dir" + i + ".jar");
LocalResource rsrc = CreateJar(files, p, vis);
rsrcVis[rsrc] = vis;
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
}
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
foreach (Future <Path> path in pending.Values)
{
NUnit.Framework.Assert.IsTrue(path.IsDone());
}
try
{
foreach (KeyValuePair <LocalResource, Future <Path> > p in pending)
{
Path localized = p.Value.Get();
FileStatus status = files.GetFileStatus(localized);
System.Console.Out.WriteLine("Testing path " + localized);
System.Diagnostics.Debug.Assert((status.IsDirectory()));
System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key)));
VerifyPermsRecursively(localized.GetFileSystem(conf), files, localized, rsrcVis[p
.Key]);
}
}
catch (ExecutionException e)
{
throw new IOException("Failed exec", e);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDownload()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource
, LocalResourceVisibility>();
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
int[] sizes = new int[10];
for (int i = 0; i < 10; ++i)
{
sizes[i] = rand.Next(512) + 512;
LocalResourceVisibility vis = LocalResourceVisibility.Private;
if (i % 2 == 1)
{
vis = LocalResourceVisibility.Application;
}
Path p = new Path(basedir, string.Empty + i);
LocalResource rsrc = CreateFile(files, p, sizes[i], rand, vis);
rsrcVis[rsrc] = vis;
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), sizes[i], conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
}
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
foreach (Future <Path> path in pending.Values)
{
NUnit.Framework.Assert.IsTrue(path.IsDone());
}
try
{
foreach (KeyValuePair <LocalResource, Future <Path> > p in pending)
{
Path localized = p.Value.Get();
NUnit.Framework.Assert.AreEqual(sizes[Sharpen.Extensions.ValueOf(localized.GetName
())], p.Key.GetSize());
FileStatus status = files.GetFileStatus(localized.GetParent());
FsPermission perm = status.GetPermission();
NUnit.Framework.Assert.AreEqual("Cache directory permissions are incorrect", new
FsPermission((short)0x1ed), perm);
status = files.GetFileStatus(localized);
perm = status.GetPermission();
System.Console.Out.WriteLine("File permission " + perm + " for rsrc vis " + p.Key
.GetVisibility().ToString());
System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key)));
NUnit.Framework.Assert.IsTrue("Private file should be 500", perm.ToShort() == FSDownload
.PrivateFilePerms.ToShort());
}
}
catch (ExecutionException e)
{
throw new IOException("Failed exec", e);
}
}
public static LocalResourceVisibility ConvertFromProtoFormat(YarnProtos.LocalResourceVisibilityProto
e)
{
return(LocalResourceVisibility.ValueOf(e.ToString()));
}
