public virtual void TestAddResourceConcurrency() { StartEmptyStore(); string key = "key1"; int count = 5; ExecutorService exec = Executors.NewFixedThreadPool(count); IList <Future <string> > futures = new AList <Future <string> >(count); CountDownLatch start = new CountDownLatch(1); for (int i = 0; i < count; i++) { string fileName = "foo-" + i + ".jar"; Callable <string> task = new _Callable_129(this, start, key, fileName); futures.AddItem(exec.Submit(task)); } // start them all at the same time start.CountDown(); // check the result; they should all agree with the value ICollection <string> results = new HashSet <string>(); foreach (Future <string> future in futures) { results.AddItem(future.Get()); } NUnit.Framework.Assert.AreSame(1, results.Count); exec.Shutdown(); }
/// <exception cref="System.Exception"/> private void SingleRun() { FsUrlStreamHandlerFactory factory = new FsUrlStreamHandlerFactory(); Random random = new Random(); ExecutorService executor = Executors.NewFixedThreadPool(Threads); AList <Future <object> > futures = new AList <Future <object> >(Tasks); for (int i = 0; i < Tasks; i++) { int aux = i; futures.AddItem(executor.Submit(new _Runnable_55(aux, random, factory))); } executor.Shutdown(); try { executor.AwaitTermination(Timeout, TimeUnit.Seconds); executor.ShutdownNow(); } catch (Exception) { } // pass // check for exceptions foreach (Future future in futures) { if (!future.IsDone()) { break; } // timed out future.Get(); } }
public virtual void TestCounter() { long countResetTimePeriodMs = 200L; ByteArrayManager.Counter c = new ByteArrayManager.Counter(countResetTimePeriodMs); int n = DFSUtil.GetRandom().Next(512) + 512; IList <Future <int> > futures = new AList <Future <int> >(n); ExecutorService pool = Executors.NewFixedThreadPool(32); try { // increment for (int i = 0; i < n; i++) { futures.AddItem(pool.Submit(new _Callable_82(c))); } // sort and wait for the futures futures.Sort(Cmp); } finally { pool.Shutdown(); } // check futures NUnit.Framework.Assert.AreEqual(n, futures.Count); for (int i_1 = 0; i_1 < n; i_1++) { NUnit.Framework.Assert.AreEqual(i_1 + 1, futures[i_1].Get()); } NUnit.Framework.Assert.AreEqual(n, c.GetCount()); // test auto-reset Sharpen.Thread.Sleep(countResetTimePeriodMs + 100); NUnit.Framework.Assert.AreEqual(1, c.Increment()); }
public virtual void TestAddResourceRefAddResourceConcurrency() { StartEmptyStore(); string key = "key1"; string fileName = "foo.jar"; string user = "******"; ApplicationId id = CreateAppId(1, 1L); // add the resource and add the resource ref at the same time ExecutorService exec = Executors.NewFixedThreadPool(2); CountDownLatch start = new CountDownLatch(1); Callable <string> addKeyTask = new _Callable_240(this, start, key, fileName); Callable <string> addAppIdTask = new _Callable_246(this, start, key, id, user); Future <string> addAppIdFuture = exec.Submit(addAppIdTask); Future <string> addKeyFuture = exec.Submit(addKeyTask); // start them at the same time start.CountDown(); // get the results string addKeyResult = addKeyFuture.Get(); string addAppIdResult = addAppIdFuture.Get(); NUnit.Framework.Assert.AreEqual(fileName, addKeyResult); System.Console.Out.WriteLine("addAppId() result: " + addAppIdResult); // it may be null or the fileName depending on the timing NUnit.Framework.Assert.IsTrue(addAppIdResult == null || addAppIdResult.Equals(fileName )); exec.Shutdown(); }
/// <summary>Run a set of tasks and waits for them to complete.</summary> /// <exception cref="System.Exception"/> private void RunTasks(IList <LocalJobRunner.Job.RunnableWithThrowable> runnables, ExecutorService service, string taskType) { // Start populating the executor with work units. // They may begin running immediately (in other threads). foreach (Runnable r in runnables) { service.Submit(r); } try { service.Shutdown(); // Instructs queue to drain. // Wait for tasks to finish; do not use a time-based timeout. // (See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6179024) LocalJobRunner.Log.Info("Waiting for " + taskType + " tasks"); service.AwaitTermination(long.MaxValue, TimeUnit.Nanoseconds); } catch (Exception ie) { // Cancel all threads. service.ShutdownNow(); throw; } LocalJobRunner.Log.Info(taskType + " task executor complete."); // After waiting for the tasks to complete, if any of these // have thrown an exception, rethrow it now in the main thread context. foreach (LocalJobRunner.Job.RunnableWithThrowable r_1 in runnables) { if (r_1.storedException != null) { throw new Exception(r_1.storedException); } } }
/// <exception cref="System.Exception"/> public virtual void TestUniqueDestinationPath() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); Path p = new Path(basedir, "dir" + 0 + ".jar"); LocalResourceVisibility vis = LocalResourceVisibility.Private; LocalResource rsrc = CreateJar(files, p, vis); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); Future <Path> rPath = singleThreadedExec.Submit(fsd); singleThreadedExec.Shutdown(); while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } NUnit.Framework.Assert.IsTrue(rPath.IsDone()); // Now FSDownload will not create a random directory to localize the // resource. Therefore the final localizedPath for the resource should be // destination directory (passed as an argument) + file name. NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent()); }
public virtual void ConcurrentRepack() { // // leave the syncPoint in broken state so any awaiting // threads and any threads that call await in the future get // the BrokenBarrierException // CyclicBarrier syncPoint = new CyclicBarrier(1); RevBlob a = tr.Blob("a"); tr.LightweightTag("t", a); ExecutorService pool = Executors.NewFixedThreadPool(2); try { _T187790690 repack1 = new _T187790690(this) { syncPoint = syncPoint }; //_T187790690 repack2 = new _T187790690(this) { syncPoint = syncPoint }; Future <int> result1 = pool.Submit(repack1); //Future<int> result2 = pool.Submit(repack2); NUnit.Framework.Assert.AreEqual(0, result1.Get()); // + result2.Get()); } finally { pool.Shutdown(); pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS); } }
/// <exception cref="System.Exception"/> public virtual void TestThreadSafety() { // Test for thread safety by starting multiple threads that mutate the same // StartupProgress instance in various ways. We expect no internal // corruption of data structures and no lost updates on counter increments. int numThreads = 100; // Data tables used by each thread to determine values to pass to APIs. Phase[] phases = new Phase[] { Phase.LoadingFsimage, Phase.LoadingFsimage, Phase. LoadingEdits, Phase.LoadingEdits }; Step[] steps = new Step[] { new Step(StepType.Inodes), new Step(StepType.DelegationKeys ), new Step(StepType.Inodes), new Step(StepType.DelegationKeys) }; string[] files = new string[] { "file1", "file1", "file2", "file2" }; long[] sizes = new long[] { 1000L, 1000L, 2000L, 2000L }; long[] totals = new long[] { 10000L, 20000L, 30000L, 40000L }; ExecutorService exec = Executors.NewFixedThreadPool(numThreads); try { for (int i = 0; i < numThreads; ++i) { Phase phase = phases[i % phases.Length]; Step step = steps[i % steps.Length]; string file = files[i % files.Length]; long size = sizes[i % sizes.Length]; long total = totals[i % totals.Length]; exec.Submit(new _Callable_369(this, phase, file, size, step, total)); } } finally { exec.Shutdown(); NUnit.Framework.Assert.IsTrue(exec.AwaitTermination(10000L, TimeUnit.Milliseconds )); } StartupProgressView view = startupProgress.CreateView(); NUnit.Framework.Assert.IsNotNull(view); NUnit.Framework.Assert.AreEqual("file1", view.GetFile(Phase.LoadingFsimage)); NUnit.Framework.Assert.AreEqual(1000L, view.GetSize(Phase.LoadingFsimage)); NUnit.Framework.Assert.AreEqual(10000L, view.GetTotal(Phase.LoadingFsimage, new Step (StepType.Inodes))); NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingFsimage, new Step (StepType.Inodes))); NUnit.Framework.Assert.AreEqual(20000L, view.GetTotal(Phase.LoadingFsimage, new Step (StepType.DelegationKeys))); NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingFsimage, new Step (StepType.DelegationKeys))); NUnit.Framework.Assert.AreEqual("file2", view.GetFile(Phase.LoadingEdits)); NUnit.Framework.Assert.AreEqual(2000L, view.GetSize(Phase.LoadingEdits)); NUnit.Framework.Assert.AreEqual(30000L, view.GetTotal(Phase.LoadingEdits, new Step (StepType.Inodes))); NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingEdits, new Step (StepType.Inodes))); NUnit.Framework.Assert.AreEqual(40000L, view.GetTotal(Phase.LoadingEdits, new Step (StepType.DelegationKeys))); NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingEdits, new Step (StepType.DelegationKeys))); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.ExecutionException"/> public virtual void TestDownloadPublicWithStatCache() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); // if test directory doesn't have ancestor permission, skip this test FileSystem f = basedir.GetFileSystem(conf); Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null)); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); int size = 512; ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger >(); CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader (conf); LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build (new _CacheLoader_328(counts, loader)); // increment the count // use the default loader // test FSDownload.isPublic() concurrently int fileCount = 3; IList <Callable <bool> > tasks = new AList <Callable <bool> >(); for (int i = 0; i < fileCount; i++) { Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); Path path = new Path(basedir, "test-file-" + i); CreateFile(files, path, size, rand); FileSystem fs = path.GetFileSystem(conf); FileStatus sStat = fs.GetFileStatus(path); tasks.AddItem(new _Callable_358(fs, path, sStat, statCache)); } ExecutorService exec = Executors.NewFixedThreadPool(fileCount); try { IList <Future <bool> > futures = exec.InvokeAll(tasks); // files should be public foreach (Future <bool> future in futures) { NUnit.Framework.Assert.IsTrue(future.Get()); } // for each path exactly one file status call should be made foreach (AtomicInteger count in counts.Values) { NUnit.Framework.Assert.AreSame(count.Get(), 1); } } finally { exec.Shutdown(); } }
/// <summary>Verify the RPC server can shutdown properly when callQueue is full.</summary> /// <exception cref="System.Exception"/> public virtual void TestRPCServerShutdown() { int numClients = 3; IList <Future <Void> > res = new AList <Future <Void> >(); ExecutorService executorService = Executors.NewFixedThreadPool(numClients); Configuration conf = new Configuration(); conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 0); Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance (new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetQueueSizePerHandler (1).SetNumHandlers(1).SetVerbose(true).Build(); server.Start(); TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol .versionID, NetUtils.GetConnectAddress(server), conf); try { // start a sleep RPC call to consume the only handler thread. // Start another sleep RPC call to make callQueue full. // Start another sleep RPC call to make reader thread block on CallQueue. for (int i = 0; i < numClients; i++) { res.AddItem(executorService.Submit(new _Callable_1046(proxy))); } while (server.GetCallQueueLen() != 1 && CountThreads(typeof(CallQueueManager).FullName ) != 1 && CountThreads(typeof(TestRPC.TestProtocol).FullName) != 1) { Thread.Sleep(100); } } finally { try { server.Stop(); Assert.Equal("Not enough clients", numClients, res.Count); foreach (Future <Void> f in res) { try { f.Get(); NUnit.Framework.Assert.Fail("Future get should not return"); } catch (ExecutionException e) { Assert.True("Unexpected exception: " + e, e.InnerException is IOException ); Log.Info("Expected exception", e.InnerException); } } } finally { RPC.StopProxy(proxy); executorService.Shutdown(); } } }
/// <exception cref="System.Exception"/> protected override void ServiceStop() { if (enabled) { uploaderPool.Shutdown(); RPC.StopProxy(scmClient); } base.ServiceStop(); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> public virtual void TestDownloadBadPublic() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); int size = 512; LocalResourceVisibility vis = LocalResourceVisibility.Public; Path path = new Path(basedir, "test-file"); LocalResource rsrc = CreateFile(files, path, size, rand, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } NUnit.Framework.Assert.IsTrue(pending[rsrc].IsDone()); try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { p.Value.Get(); NUnit.Framework.Assert.Fail("We localized a file that is not public."); } } catch (ExecutionException e) { NUnit.Framework.Assert.IsTrue(e.InnerException is IOException); } }
public void IsShutdownReturnsFalseBeforeShutdownTrueAfter() { //ExecutorService = NewThreadPoolExecutor(1, 1); Assert.IsFalse(ExecutorService.IsShutdown); ExecutorService.Execute(_mediumInterruptableAction); ExecutorService.Shutdown(); Assert.IsTrue(ExecutorService.IsShutdown); InterruptAndJoinPool(ExecutorService); }
public void ExecuteRejectsTaskOnShutdown() { var runnable = MockRepository.GenerateStub <IRunnable>(); ExecutorService.Shutdown(); Assert.Throws <RejectedExecutionException>(() => ExecutorService.Execute(runnable)); JoinPool(ExecutorService); runnable.AssertWasNotCalled(r => r.Run()); }
/// <exception cref="System.Exception"/> private void TestConcurrentCommitTaskWithSubDir(int version) { Job job = Job.GetInstance(); FileOutputFormat.SetOutputPath(job, outDir); Configuration conf = job.GetConfiguration(); conf.Set(MRJobConfig.TaskAttemptId, attempt); conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version); conf.SetClass("fs.file.impl", typeof(TestFileOutputCommitter.RLFS), typeof(FileSystem )); FileSystem.CloseAll(); JobContext jContext = new JobContextImpl(conf, taskID.GetJobID()); FileOutputCommitter amCommitter = new FileOutputCommitter(outDir, jContext); amCommitter.SetupJob(jContext); TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2]; taCtx[0] = new TaskAttemptContextImpl(conf, taskID); taCtx[1] = new TaskAttemptContextImpl(conf, taskID1); TextOutputFormat[] tof = new TextOutputFormat[2]; for (int i = 0; i < tof.Length; i++) { tof[i] = new _TextOutputFormat_508(this); } ExecutorService executor = Executors.NewFixedThreadPool(2); try { for (int i_1 = 0; i_1 < taCtx.Length; i_1++) { int taskIdx = i_1; executor.Submit(new _Callable_524(this, tof, taskIdx, taCtx)); } } finally { executor.Shutdown(); while (!executor.AwaitTermination(1, TimeUnit.Seconds)) { Log.Info("Awaiting thread termination!"); } } amCommitter.CommitJob(jContext); RawLocalFileSystem lfs = new RawLocalFileSystem(); lfs.SetConf(conf); NUnit.Framework.Assert.IsFalse("Must not end up with sub_dir/sub_dir", lfs.Exists (new Path(OutSubDir, SubDir))); // validate output ValidateContent(OutSubDir); FileUtil.FullyDelete(new FilePath(outDir.ToString())); }
private void Destroy() { _state = 2; // .NET-specific: if (_cts != null) { _cts.Cancel(); _cts = null; } if (_executor != null) { _executor.Shutdown(); } }
private void Destroy() { _state = 2; // .NET-specific: if (_timer != null) { ExecutorService.Cancel(_timer); _timer = null; } if (_executor != null) { _executor.Shutdown(); } }
private void StopAggregators() { threadPool.Shutdown(); // if recovery on restart is supported then leave outstanding aggregations // to the next restart bool shouldAbort = context.GetNMStateStore().CanRecover() && !context.GetDecommissioned (); // politely ask to finish foreach (AppLogAggregator aggregator in appLogAggregators.Values) { if (shouldAbort) { aggregator.AbortLogAggregation(); } else { aggregator.FinishLogAggregation(); } } while (!threadPool.IsTerminated()) { // wait for all threads to finish foreach (ApplicationId appId in appLogAggregators.Keys) { Log.Info("Waiting for aggregation to complete for " + appId); } try { if (!threadPool.AwaitTermination(30, TimeUnit.Seconds)) { threadPool.ShutdownNow(); } } catch (Exception) { // send interrupt to hurry them along Log.Warn("Aggregation stop interrupted!"); break; } } foreach (ApplicationId appId_1 in appLogAggregators.Keys) { Log.Warn("Some logs may not have been aggregated for " + appId_1); } }
/// <param name="service"> /// /// <see cref="ExecutorService">to be shutdown</see> /// </param> /// <param name="timeoutInMs"> /// time to wait for /// <see cref="ExecutorService.AwaitTermination(long, TimeUnit)"/> /// calls in milli seconds. /// </param> /// <returns> /// <tt>true</tt> if the service is terminated, /// <tt>false</tt> otherwise /// </returns> /// <exception cref="System.Exception"/> public static bool ShutdownExecutorService(ExecutorService service, long timeoutInMs ) { if (service == null) { return(true); } service.Shutdown(); if (!service.AwaitTermination(timeoutInMs, TimeUnit.Milliseconds)) { service.ShutdownNow(); return(service.AwaitTermination(timeoutInMs, TimeUnit.Milliseconds)); } else { return(true); } }
/// <exception cref="System.Exception"/> public virtual void TestMultiThreadedCompressorPool() { int iterations = 4; ExecutorService threadpool = Executors.NewFixedThreadPool(3); LinkedBlockingDeque <Compressor> queue = new LinkedBlockingDeque <Compressor>(2 * iterations ); Callable <bool> consumer = new _Callable_110(queue); Callable <bool> producer = new _Callable_119(this, queue); for (int i = 0; i < iterations; i++) { threadpool.Submit(consumer); threadpool.Submit(producer); } // wait for completion threadpool.Shutdown(); threadpool.AwaitTermination(1000, TimeUnit.Seconds); Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount (codec)); }
/// <exception cref="System.Exception"/> internal static long PerformanceTest(int arrayLength, int maxArrays, int nThreads , int[] sleepTimeMSs, ByteArrayManager impl) { ExecutorService pool = Executors.NewFixedThreadPool(nThreads); IList <Future <Void> > futures = new AList <Future <Void> >(sleepTimeMSs.Length); long startTime = Time.MonotonicNow(); for (int i = 0; i < sleepTimeMSs.Length; i++) { long sleepTime = sleepTimeMSs[i]; futures.AddItem(pool.Submit(new _Callable_628(impl, arrayLength, sleepTime))); } foreach (Future <Void> f in futures) { f.Get(); } long endTime = Time.MonotonicNow(); pool.Shutdown(); return(endTime - startTime); }
public virtual void ConcurrentPackRefs_onlyOneWritesPackedRefs() { RevBlob a = tr.Blob("a"); tr.LightweightTag("t", a); CyclicBarrier syncPoint = new CyclicBarrier(2); Callable <int> packRefs = new _Callable_131(this, syncPoint); ExecutorService pool = Executors.NewFixedThreadPool(2); try { Future <int> p1 = pool.Submit(packRefs); Future <int> p2 = pool.Submit(packRefs); NUnit.Framework.Assert.AreEqual(1, p1.Get() + p2.Get()); } finally { pool.Shutdown(); pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS); } }
public virtual void TestAddResourceRefConcurrency() { StartEmptyStore(); string key = "key1"; string user = "******"; string fileName = "foo.jar"; // first add the resource store.AddResource(key, fileName); // make concurrent addResourceRef calls (clients) int count = 5; ExecutorService exec = Executors.NewFixedThreadPool(count); IList <Future <string> > futures = new AList <Future <string> >(count); CountDownLatch start = new CountDownLatch(1); for (int i = 0; i < count; i++) { ApplicationId id = CreateAppId(i, i); Callable <string> task = new _Callable_205(this, start, key, id, user); futures.AddItem(exec.Submit(task)); } // start them all at the same time start.CountDown(); // check the result ICollection <string> results = new HashSet <string>(); foreach (Future <string> future in futures) { results.AddItem(future.Get()); } // they should all have the same file name NUnit.Framework.Assert.AreSame(1, results.Count); NUnit.Framework.Assert.AreEqual(Sharpen.Collections.Singleton(fileName), results); // there should be 5 refs as a result ICollection <SharedCacheResourceReference> refs = store.GetResourceReferences(key); NUnit.Framework.Assert.AreSame(count, refs.Count); exec.Shutdown(); }
public virtual void TestParallelThreadsWithDifferentLocales() { CyclicBarrier barrier = new CyclicBarrier(2); // wait for the other thread to set its locale ExecutorService pool = Executors.NewFixedThreadPool(2); try { Future <TranslationBundle> root = pool.Submit(new _T879158014(this, NLS.ROOT_LOCALE, barrier)); Future <TranslationBundle> german = pool.Submit(new _T879158014(this, Sharpen.Extensions.GetGermanCulture(), barrier)); NUnit.Framework.Assert.AreEqual(NLS.ROOT_LOCALE, root.Get().EffectiveLocale()); NUnit.Framework.Assert.AreEqual(Sharpen.Extensions.GetGermanCulture(), german.Get ().EffectiveLocale()); } finally { pool.Shutdown(); pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestDirDownload() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); for (int i = 0; i < 5; ++i) { LocalResourceVisibility vis = LocalResourceVisibility.Private; if (i % 2 == 1) { vis = LocalResourceVisibility.Application; } Path p = new Path(basedir, "dir" + i + ".jar"); LocalResource rsrc = CreateJar(files, p, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); } exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } foreach (Future <Path> path in pending.Values) { NUnit.Framework.Assert.IsTrue(path.IsDone()); } try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { Path localized = p.Value.Get(); FileStatus status = files.GetFileStatus(localized); System.Console.Out.WriteLine("Testing path " + localized); System.Diagnostics.Debug.Assert((status.IsDirectory())); System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key))); VerifyPermsRecursively(localized.GetFileSystem(conf), files, localized, rsrcVis[p .Key]); } } catch (ExecutionException e) { throw new IOException("Failed exec", e); } }
/// <summary> /// Set up the distributed cache by localizing the resources, and updating /// the configuration with references to the localized resources. /// </summary> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> public virtual void Setup(JobConf conf) { FilePath workDir = new FilePath(Runtime.GetProperty("user.dir")); // Generate YARN local resources objects corresponding to the distributed // cache configuration IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); // Generating unique numbers for FSDownload. AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis()); // Find which resources are to be put on the local classpath IDictionary <string, Path> classpaths = new Dictionary <string, Path>(); Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf); if (archiveClassPaths != null) { foreach (Path p in archiveClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf); if (fileClassPaths != null) { foreach (Path p in fileClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } // Localize the resources LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir); FileContext localFSFileContext = FileContext.GetLocalFSFileContext(); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ExecutorService exec = null; try { ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d" ).Build(); exec = Executors.NewCachedThreadPool(tf); Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf); IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap(); foreach (LocalResource resource in localResources.Values) { Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path( destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource ); Future <Path> future = exec.Submit(download); resourcesToPaths[resource] = future; } foreach (KeyValuePair <string, LocalResource> entry in localResources) { LocalResource resource_1 = entry.Value; Path path; try { path = resourcesToPaths[resource_1].Get(); } catch (Exception e) { throw new IOException(e); } catch (ExecutionException e) { throw new IOException(e); } string pathString = path.ToUri().ToString(); string link = entry.Key; string target = new FilePath(path.ToUri()).GetPath(); Symlink(workDir, target, link); if (resource_1.GetType() == LocalResourceType.Archive) { localArchives.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.File) { localFiles.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.Pattern) { //PATTERN is not currently used in local mode throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. " + resource_1.GetResource()); } } } Path resourcePath; try { resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource()); } catch (URISyntaxException e) { throw new IOException(e); } Log.Info(string.Format("Localized %s as %s", resourcePath, path)); string cp = resourcePath.ToUri().GetPath(); if (classpaths.Keys.Contains(cp)) { localClasspaths.AddItem(path.ToUri().GetPath().ToString()); } } } finally { if (exec != null) { exec.Shutdown(); } } // Update the configuration object with localized data. if (!localArchives.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localArchives, new string[localArchives.Count]))); } if (!localFiles.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localFiles, new string[localArchives.Count]))); } setupCalled = true; }
/// <summary> /// Handles the DomainUnload event of the CurrentDomain control. /// </summary> /// <param name="sender">The source of the event.</param> /// <param name="e">The <see cref="System.EventArgs"/> instance containing the event data.</param> static void CurrentDomain_DomainUnload(object sender, EventArgs e) { ExecutorService.Shutdown(); }
public override void Shutdown() { e.Shutdown(); }
/// <exception cref="System.Exception"/> /// <exception cref="System.IO.IOException"/> private void DoCheckpoint() { System.Diagnostics.Debug.Assert(canceler != null); long txid; NNStorage.NameNodeFile imageType; // Acquire cpLock to make sure no one is modifying the name system. // It does not need the full namesystem write lock, since the only thing // that modifies namesystem on standby node is edit log replaying. namesystem.CpLockInterruptibly(); try { System.Diagnostics.Debug.Assert(namesystem.GetEditLog().IsOpenForRead(), "Standby Checkpointer should only attempt a checkpoint when " + "NN is in standby mode, but the edit logs are in an unexpected state"); FSImage img = namesystem.GetFSImage(); long prevCheckpointTxId = img.GetStorage().GetMostRecentCheckpointTxId(); long thisCheckpointTxId = img.GetLastAppliedOrWrittenTxId(); System.Diagnostics.Debug.Assert(thisCheckpointTxId >= prevCheckpointTxId); if (thisCheckpointTxId == prevCheckpointTxId) { Log.Info("A checkpoint was triggered but the Standby Node has not " + "received any transactions since the last checkpoint at txid " + thisCheckpointTxId + ". Skipping..."); return; } if (namesystem.IsRollingUpgrade() && !namesystem.GetFSImage().HasRollbackFSImage( )) { // if we will do rolling upgrade but have not created the rollback image // yet, name this checkpoint as fsimage_rollback imageType = NNStorage.NameNodeFile.ImageRollback; } else { imageType = NNStorage.NameNodeFile.Image; } img.SaveNamespace(namesystem, imageType, canceler); txid = img.GetStorage().GetMostRecentCheckpointTxId(); System.Diagnostics.Debug.Assert(txid == thisCheckpointTxId, "expected to save checkpoint at txid=" + thisCheckpointTxId + " but instead saved at txid=" + txid); // Save the legacy OIV image, if the output dir is defined. string outputDir = checkpointConf.GetLegacyOivImageDir(); if (outputDir != null && !outputDir.IsEmpty()) { img.SaveLegacyOIVImage(namesystem, outputDir, canceler); } } finally { namesystem.CpUnlock(); } // Upload the saved checkpoint back to the active // Do this in a separate thread to avoid blocking transition to active // See HDFS-4816 ExecutorService executor = Executors.NewSingleThreadExecutor(uploadThreadFactory); Future <Void> upload = executor.Submit(new _Callable_204(this, imageType, txid)); executor.Shutdown(); try { upload.Get(); } catch (Exception e) { // The background thread may be blocked waiting in the throttler, so // interrupt it. upload.Cancel(true); throw; } catch (ExecutionException e) { throw new IOException("Exception during image upload: " + e.Message, e.InnerException ); } }
public virtual void TestMaxOutHedgedReadPool() { isHedgedRead = true; Configuration conf = new Configuration(); int numHedgedReadPoolThreads = 5; int initialHedgedReadTimeoutMillis = 50000; int fixedSleepIntervalMillis = 50; conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads ); conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, initialHedgedReadTimeoutMillis ); // Set up the InjectionHandler DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector >(); DFSClientFaultInjector injector = DFSClientFaultInjector.instance; // make preads sleep for 50ms Org.Mockito.Mockito.DoAnswer(new _Answer_372(fixedSleepIntervalMillis)).When(injector ).StartFetchFromDatanode(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Format( true).Build(); DistributedFileSystem fileSys = cluster.GetFileSystem(); DFSClient dfsClient = fileSys.GetClient(); DFSHedgedReadMetrics metrics = dfsClient.GetHedgedReadMetrics(); // Metrics instance is static, so we need to reset counts from prior tests. metrics.hedgedReadOps.Set(0); metrics.hedgedReadOpsWin.Set(0); metrics.hedgedReadOpsInCurThread.Set(0); try { Path file1 = new Path("hedgedReadMaxOut.dat"); WriteFile(fileSys, file1); // Basic test. Reads complete within timeout. Assert that there were no // hedged reads. PReadFile(fileSys, file1); // assert that there were no hedged reads. 50ms + delta < 500ms NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOps() == 0); NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOpsInCurThread() == 0); /* * Reads take longer than timeout. But, only one thread reading. Assert * that there were hedged reads. But, none of the reads had to run in the * current thread. */ dfsClient.SetHedgedReadTimeout(50); // 50ms PReadFile(fileSys, file1); // assert that there were hedged reads NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOps() > 0); NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOpsInCurThread() == 0); /* * Multiple threads reading. Reads take longer than timeout. Assert that * there were hedged reads. And that reads had to run in the current * thread. */ int factor = 10; int numHedgedReads = numHedgedReadPoolThreads * factor; long initialReadOpsValue = metrics.GetHedgedReadOps(); ExecutorService executor = Executors.NewFixedThreadPool(numHedgedReads); AList <Future <Void> > futures = new AList <Future <Void> >(); for (int i = 0; i < numHedgedReads; i++) { futures.AddItem(executor.Submit(GetPReadFileCallable(fileSys, file1))); } for (int i_1 = 0; i_1 < numHedgedReads; i_1++) { futures[i_1].Get(); } NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOps() > initialReadOpsValue); NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOpsInCurThread() > 0); CleanupFile(fileSys, file1); executor.Shutdown(); } finally { fileSys.Close(); cluster.Shutdown(); Org.Mockito.Mockito.Reset(injector); } }