public _PrivilegedExceptionAction_359(FSDownload _enclosing, Path sCopy, Path dst_work ) { this._enclosing = _enclosing; this.sCopy = sCopy; this.dst_work = dst_work; }
public _PrivilegedExceptionAction_419(FSDownload _enclosing, Path path, FsPermission fPerm) { this._enclosing = _enclosing; this.path = path; this.fPerm = fPerm; }
/// <exception cref="System.Exception"/> public virtual void TestUniqueDestinationPath() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); Path p = new Path(basedir, "dir" + 0 + ".jar"); LocalResourceVisibility vis = LocalResourceVisibility.Private; LocalResource rsrc = CreateJar(files, p, vis); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); Future <Path> rPath = singleThreadedExec.Submit(fsd); singleThreadedExec.Shutdown(); while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } NUnit.Framework.Assert.IsTrue(rPath.IsDone()); // Now FSDownload will not create a random directory to localize the // resource. Therefore the final localizedPath for the resource should be // destination directory (passed as an argument) + file name. NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.ExecutionException"/> public virtual void TestDownloadPublicWithStatCache() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); // if test directory doesn't have ancestor permission, skip this test FileSystem f = basedir.GetFileSystem(conf); Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null)); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); int size = 512; ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger >(); CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader (conf); LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build (new _CacheLoader_328(counts, loader)); // increment the count // use the default loader // test FSDownload.isPublic() concurrently int fileCount = 3; IList <Callable <bool> > tasks = new AList <Callable <bool> >(); for (int i = 0; i < fileCount; i++) { Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); Path path = new Path(basedir, "test-file-" + i); CreateFile(files, path, size, rand); FileSystem fs = path.GetFileSystem(conf); FileStatus sStat = fs.GetFileStatus(path); tasks.AddItem(new _Callable_358(fs, path, sStat, statCache)); } ExecutorService exec = Executors.NewFixedThreadPool(fileCount); try { IList <Future <bool> > futures = exec.InvokeAll(tasks); // files should be public foreach (Future <bool> future in futures) { NUnit.Framework.Assert.IsTrue(future.Get()); } // for each path exactly one file status call should be made foreach (AtomicInteger count in counts.Values) { NUnit.Framework.Assert.AreSame(count.Get(), 1); } } finally { exec.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> public virtual void TestDownloadBadPublic() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); int size = 512; LocalResourceVisibility vis = LocalResourceVisibility.Public; Path path = new Path(basedir, "test-file"); LocalResource rsrc = CreateFile(files, path, size, rand, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } NUnit.Framework.Assert.IsTrue(pending[rsrc].IsDone()); try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { p.Value.Get(); NUnit.Framework.Assert.Fail("We localized a file that is not public."); } } catch (ExecutionException e) { NUnit.Framework.Assert.IsTrue(e.InnerException is IOException); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestDirDownload() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); for (int i = 0; i < 5; ++i) { LocalResourceVisibility vis = LocalResourceVisibility.Private; if (i % 2 == 1) { vis = LocalResourceVisibility.Application; } Path p = new Path(basedir, "dir" + i + ".jar"); LocalResource rsrc = CreateJar(files, p, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); } exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } foreach (Future <Path> path in pending.Values) { NUnit.Framework.Assert.IsTrue(path.IsDone()); } try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { Path localized = p.Value.Get(); FileStatus status = files.GetFileStatus(localized); System.Console.Out.WriteLine("Testing path " + localized); System.Diagnostics.Debug.Assert((status.IsDirectory())); System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key))); VerifyPermsRecursively(localized.GetFileSystem(conf), files, localized, rsrcVis[p .Key]); } } catch (ExecutionException e) { throw new IOException("Failed exec", e); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> private void DownloadWithFileType(TestFSDownload.TEST_FILE_TYPE fileType) { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); int size = rand.Next(512) + 512; LocalResourceVisibility vis = LocalResourceVisibility.Private; Path p = new Path(basedir, string.Empty + 1); string strFileName = string.Empty; LocalResource rsrc = null; switch (fileType) { case TestFSDownload.TEST_FILE_TYPE.Tar: { rsrc = CreateTarFile(files, p, size, rand, vis); break; } case TestFSDownload.TEST_FILE_TYPE.Jar: { rsrc = CreateJarFile(files, p, size, rand, vis); rsrc.SetType(LocalResourceType.Pattern); break; } case TestFSDownload.TEST_FILE_TYPE.Zip: { rsrc = CreateZipFile(files, p, size, rand, vis); strFileName = p.GetName() + ".ZIP"; break; } case TestFSDownload.TEST_FILE_TYPE.Tgz: { rsrc = CreateTgzFile(files, p, size, rand, vis); break; } } Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } try { pending[rsrc].Get(); // see if there was an Exception during download FileStatus[] filesstatus = files.GetDefaultFileSystem().ListStatus(basedir); foreach (FileStatus filestatus in filesstatus) { if (filestatus.IsDirectory()) { FileStatus[] childFiles = files.GetDefaultFileSystem().ListStatus(filestatus.GetPath ()); foreach (FileStatus childfile in childFiles) { if (strFileName.EndsWith(".ZIP") && childfile.GetPath().GetName().Equals(strFileName ) && !childfile.IsDirectory()) { NUnit.Framework.Assert.Fail("Failure...After unzip, there should have been a" + " directory formed with zip file name but found a file. " + childfile.GetPath()); } if (childfile.GetPath().GetName().StartsWith("tmp")) { NUnit.Framework.Assert.Fail("Tmp File should not have been there " + childfile.GetPath ()); } } } } } catch (Exception e) { throw new IOException("Failed exec", e); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> public virtual void TestDownload() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); int[] sizes = new int[10]; for (int i = 0; i < 10; ++i) { sizes[i] = rand.Next(512) + 512; LocalResourceVisibility vis = LocalResourceVisibility.Private; if (i % 2 == 1) { vis = LocalResourceVisibility.Application; } Path p = new Path(basedir, string.Empty + i); LocalResource rsrc = CreateFile(files, p, sizes[i], rand, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), sizes[i], conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); } exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } foreach (Future <Path> path in pending.Values) { NUnit.Framework.Assert.IsTrue(path.IsDone()); } try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { Path localized = p.Value.Get(); NUnit.Framework.Assert.AreEqual(sizes[Sharpen.Extensions.ValueOf(localized.GetName ())], p.Key.GetSize()); FileStatus status = files.GetFileStatus(localized.GetParent()); FsPermission perm = status.GetPermission(); NUnit.Framework.Assert.AreEqual("Cache directory permissions are incorrect", new FsPermission((short)0x1ed), perm); status = files.GetFileStatus(localized); perm = status.GetPermission(); System.Console.Out.WriteLine("File permission " + perm + " for rsrc vis " + p.Key .GetVisibility().ToString()); System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key))); NUnit.Framework.Assert.IsTrue("Private file should be 500", perm.ToShort() == FSDownload .PrivateFilePerms.ToShort()); } } catch (ExecutionException e) { throw new IOException("Failed exec", e); } }
/// <exception cref="System.IO.IOException"/> public bool Call() { return(FSDownload.IsPublic(fs, path, sStat, statCache)); }