public virtual void TestAddResourceConcurrency()
{
StartEmptyStore();
string key = "key1";
int count = 5;
ExecutorService exec = Executors.NewFixedThreadPool(count);
IList <Future <string> > futures = new AList <Future <string> >(count);
CountDownLatch start = new CountDownLatch(1);
for (int i = 0; i < count; i++)
{
string fileName = "foo-" + i + ".jar";
Callable <string> task = new _Callable_129(this, start, key, fileName);
futures.AddItem(exec.Submit(task));
}
// start them all at the same time
start.CountDown();
// check the result; they should all agree with the value
ICollection <string> results = new HashSet <string>();
foreach (Future <string> future in futures)
{
results.AddItem(future.Get());
}
NUnit.Framework.Assert.AreSame(1, results.Count);
exec.Shutdown();
}
/// <exception cref="System.Exception"/>
private void SingleRun()
{
FsUrlStreamHandlerFactory factory = new FsUrlStreamHandlerFactory();
Random random = new Random();
ExecutorService executor = Executors.NewFixedThreadPool(Threads);
AList <Future <object> > futures = new AList <Future <object> >(Tasks);
for (int i = 0; i < Tasks; i++)
{
int aux = i;
futures.AddItem(executor.Submit(new _Runnable_55(aux, random, factory)));
}
executor.Shutdown();
try
{
executor.AwaitTermination(Timeout, TimeUnit.Seconds);
executor.ShutdownNow();
}
catch (Exception)
{
}
// pass
// check for exceptions
foreach (Future future in futures)
{
if (!future.IsDone())
{
break;
}
// timed out
future.Get();
}
}
public virtual void TestCounter()
{
long countResetTimePeriodMs = 200L;
ByteArrayManager.Counter c = new ByteArrayManager.Counter(countResetTimePeriodMs);
int n = DFSUtil.GetRandom().Next(512) + 512;
IList <Future <int> > futures = new AList <Future <int> >(n);
ExecutorService pool = Executors.NewFixedThreadPool(32);
try
{
// increment
for (int i = 0; i < n; i++)
{
futures.AddItem(pool.Submit(new _Callable_82(c)));
}
// sort and wait for the futures
futures.Sort(Cmp);
}
finally
{
pool.Shutdown();
}
// check futures
NUnit.Framework.Assert.AreEqual(n, futures.Count);
for (int i_1 = 0; i_1 < n; i_1++)
{
NUnit.Framework.Assert.AreEqual(i_1 + 1, futures[i_1].Get());
}
NUnit.Framework.Assert.AreEqual(n, c.GetCount());
// test auto-reset
Sharpen.Thread.Sleep(countResetTimePeriodMs + 100);
NUnit.Framework.Assert.AreEqual(1, c.Increment());
}
public virtual void TestAddResourceRefAddResourceConcurrency()
{
StartEmptyStore();
string key = "key1";
string fileName = "foo.jar";
string user = "******";
ApplicationId id = CreateAppId(1, 1L);
// add the resource and add the resource ref at the same time
ExecutorService exec = Executors.NewFixedThreadPool(2);
CountDownLatch start = new CountDownLatch(1);
Callable <string> addKeyTask = new _Callable_240(this, start, key, fileName);
Callable <string> addAppIdTask = new _Callable_246(this, start, key, id, user);
Future <string> addAppIdFuture = exec.Submit(addAppIdTask);
Future <string> addKeyFuture = exec.Submit(addKeyTask);
// start them at the same time
start.CountDown();
// get the results
string addKeyResult = addKeyFuture.Get();
string addAppIdResult = addAppIdFuture.Get();
NUnit.Framework.Assert.AreEqual(fileName, addKeyResult);
System.Console.Out.WriteLine("addAppId() result: " + addAppIdResult);
// it may be null or the fileName depending on the timing
NUnit.Framework.Assert.IsTrue(addAppIdResult == null || addAppIdResult.Equals(fileName
));
exec.Shutdown();
}
/// <summary>Run a set of tasks and waits for them to complete.</summary>
/// <exception cref="System.Exception"/>
private void RunTasks(IList <LocalJobRunner.Job.RunnableWithThrowable> runnables,
ExecutorService service, string taskType)
{
// Start populating the executor with work units.
// They may begin running immediately (in other threads).
foreach (Runnable r in runnables)
{
service.Submit(r);
}
try
{
service.Shutdown();
// Instructs queue to drain.
// Wait for tasks to finish; do not use a time-based timeout.
// (See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6179024)
LocalJobRunner.Log.Info("Waiting for " + taskType + " tasks");
service.AwaitTermination(long.MaxValue, TimeUnit.Nanoseconds);
}
catch (Exception ie)
{
// Cancel all threads.
service.ShutdownNow();
throw;
}
LocalJobRunner.Log.Info(taskType + " task executor complete.");
// After waiting for the tasks to complete, if any of these
// have thrown an exception, rethrow it now in the main thread context.
foreach (LocalJobRunner.Job.RunnableWithThrowable r_1 in runnables)
{
if (r_1.storedException != null)
{
throw new Exception(r_1.storedException);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestUniqueDestinationPath()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
Path p = new Path(basedir, "dir" + 0 + ".jar");
LocalResourceVisibility vis = LocalResourceVisibility.Private;
LocalResource rsrc = CreateJar(files, p, vis);
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
Future <Path> rPath = singleThreadedExec.Submit(fsd);
singleThreadedExec.Shutdown();
while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(rPath.IsDone());
// Now FSDownload will not create a random directory to localize the
// resource. Therefore the final localizedPath for the resource should be
// destination directory (passed as an argument) + file name.
NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent());
}
public virtual void ConcurrentRepack()
{
//
// leave the syncPoint in broken state so any awaiting
// threads and any threads that call await in the future get
// the BrokenBarrierException
//
CyclicBarrier syncPoint = new CyclicBarrier(1);
RevBlob a = tr.Blob("a");
tr.LightweightTag("t", a);
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
_T187790690 repack1 = new _T187790690(this)
{
syncPoint = syncPoint
};
//_T187790690 repack2 = new _T187790690(this) { syncPoint = syncPoint };
Future <int> result1 = pool.Submit(repack1);
//Future<int> result2 = pool.Submit(repack2);
NUnit.Framework.Assert.AreEqual(0, result1.Get()); // + result2.Get());
}
finally
{
pool.Shutdown();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestThreadSafety()
{
// Test for thread safety by starting multiple threads that mutate the same
// StartupProgress instance in various ways. We expect no internal
// corruption of data structures and no lost updates on counter increments.
int numThreads = 100;
// Data tables used by each thread to determine values to pass to APIs.
Phase[] phases = new Phase[] { Phase.LoadingFsimage, Phase.LoadingFsimage, Phase.
LoadingEdits, Phase.LoadingEdits };
Step[] steps = new Step[] { new Step(StepType.Inodes), new Step(StepType.DelegationKeys
), new Step(StepType.Inodes), new Step(StepType.DelegationKeys) };
string[] files = new string[] { "file1", "file1", "file2", "file2" };
long[] sizes = new long[] { 1000L, 1000L, 2000L, 2000L };
long[] totals = new long[] { 10000L, 20000L, 30000L, 40000L };
ExecutorService exec = Executors.NewFixedThreadPool(numThreads);
try
{
for (int i = 0; i < numThreads; ++i)
{
Phase phase = phases[i % phases.Length];
Step step = steps[i % steps.Length];
string file = files[i % files.Length];
long size = sizes[i % sizes.Length];
long total = totals[i % totals.Length];
exec.Submit(new _Callable_369(this, phase, file, size, step, total));
}
}
finally
{
exec.Shutdown();
NUnit.Framework.Assert.IsTrue(exec.AwaitTermination(10000L, TimeUnit.Milliseconds
));
}
StartupProgressView view = startupProgress.CreateView();
NUnit.Framework.Assert.IsNotNull(view);
NUnit.Framework.Assert.AreEqual("file1", view.GetFile(Phase.LoadingFsimage));
NUnit.Framework.Assert.AreEqual(1000L, view.GetSize(Phase.LoadingFsimage));
NUnit.Framework.Assert.AreEqual(10000L, view.GetTotal(Phase.LoadingFsimage, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingFsimage, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(20000L, view.GetTotal(Phase.LoadingFsimage, new Step
(StepType.DelegationKeys)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingFsimage, new Step
(StepType.DelegationKeys)));
NUnit.Framework.Assert.AreEqual("file2", view.GetFile(Phase.LoadingEdits));
NUnit.Framework.Assert.AreEqual(2000L, view.GetSize(Phase.LoadingEdits));
NUnit.Framework.Assert.AreEqual(30000L, view.GetTotal(Phase.LoadingEdits, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingEdits, new Step
(StepType.Inodes)));
NUnit.Framework.Assert.AreEqual(40000L, view.GetTotal(Phase.LoadingEdits, new Step
(StepType.DelegationKeys)));
NUnit.Framework.Assert.AreEqual(2500L, view.GetCount(Phase.LoadingEdits, new Step
(StepType.DelegationKeys)));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.ExecutionException"/>
public virtual void TestDownloadPublicWithStatCache()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
// if test directory doesn't have ancestor permission, skip this test
FileSystem f = basedir.GetFileSystem(conf);
Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
int size = 512;
ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger
>();
CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader
(conf);
LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build
(new _CacheLoader_328(counts, loader));
// increment the count
// use the default loader
// test FSDownload.isPublic() concurrently
int fileCount = 3;
IList <Callable <bool> > tasks = new AList <Callable <bool> >();
for (int i = 0; i < fileCount; i++)
{
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
Path path = new Path(basedir, "test-file-" + i);
CreateFile(files, path, size, rand);
FileSystem fs = path.GetFileSystem(conf);
FileStatus sStat = fs.GetFileStatus(path);
tasks.AddItem(new _Callable_358(fs, path, sStat, statCache));
}
ExecutorService exec = Executors.NewFixedThreadPool(fileCount);
try
{
IList <Future <bool> > futures = exec.InvokeAll(tasks);
// files should be public
foreach (Future <bool> future in futures)
{
NUnit.Framework.Assert.IsTrue(future.Get());
}
// for each path exactly one file status call should be made
foreach (AtomicInteger count in counts.Values)
{
NUnit.Framework.Assert.AreSame(count.Get(), 1);
}
}
finally
{
exec.Shutdown();
}
}
/// <summary>Verify the RPC server can shutdown properly when callQueue is full.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestRPCServerShutdown()
{
int numClients = 3;
IList <Future <Void> > res = new AList <Future <Void> >();
ExecutorService executorService = Executors.NewFixedThreadPool(numClients);
Configuration conf = new Configuration();
conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 0);
Server server = new RPC.Builder(conf).SetProtocol(typeof(TestRPC.TestProtocol)).SetInstance
(new TestRPC.TestImpl()).SetBindAddress(Address).SetPort(0).SetQueueSizePerHandler
(1).SetNumHandlers(1).SetVerbose(true).Build();
server.Start();
TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol
.versionID, NetUtils.GetConnectAddress(server), conf);
try
{
// start a sleep RPC call to consume the only handler thread.
// Start another sleep RPC call to make callQueue full.
// Start another sleep RPC call to make reader thread block on CallQueue.
for (int i = 0; i < numClients; i++)
{
res.AddItem(executorService.Submit(new _Callable_1046(proxy)));
}
while (server.GetCallQueueLen() != 1 && CountThreads(typeof(CallQueueManager).FullName
) != 1 && CountThreads(typeof(TestRPC.TestProtocol).FullName) != 1)
{
Thread.Sleep(100);
}
}
finally
{
try
{
server.Stop();
Assert.Equal("Not enough clients", numClients, res.Count);
foreach (Future <Void> f in res)
{
try
{
f.Get();
NUnit.Framework.Assert.Fail("Future get should not return");
}
catch (ExecutionException e)
{
Assert.True("Unexpected exception: " + e, e.InnerException is IOException
);
Log.Info("Expected exception", e.InnerException);
}
}
}
finally
{
RPC.StopProxy(proxy);
executorService.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
protected override void ServiceStop()
{
if (enabled)
{
uploaderPool.Shutdown();
RPC.StopProxy(scmClient);
}
base.ServiceStop();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDownloadBadPublic()
{
Configuration conf = new Configuration();
conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource
, LocalResourceVisibility>();
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
int size = 512;
LocalResourceVisibility vis = LocalResourceVisibility.Public;
Path path = new Path(basedir, "test-file");
LocalResource rsrc = CreateFile(files, path, size, rand, vis);
rsrcVis[rsrc] = vis;
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(pending[rsrc].IsDone());
try
{
foreach (KeyValuePair <LocalResource, Future <Path> > p in pending)
{
p.Value.Get();
NUnit.Framework.Assert.Fail("We localized a file that is not public.");
}
}
catch (ExecutionException e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is IOException);
}
}
public void IsShutdownReturnsFalseBeforeShutdownTrueAfter()
{
//ExecutorService = NewThreadPoolExecutor(1, 1);
Assert.IsFalse(ExecutorService.IsShutdown);
ExecutorService.Execute(_mediumInterruptableAction);
ExecutorService.Shutdown();
Assert.IsTrue(ExecutorService.IsShutdown);
InterruptAndJoinPool(ExecutorService);
}
public void ExecuteRejectsTaskOnShutdown()
{
var runnable = MockRepository.GenerateStub <IRunnable>();
ExecutorService.Shutdown();
Assert.Throws <RejectedExecutionException>(() => ExecutorService.Execute(runnable));
JoinPool(ExecutorService);
runnable.AssertWasNotCalled(r => r.Run());
}
/// <exception cref="System.Exception"/>
private void TestConcurrentCommitTaskWithSubDir(int version)
{
Job job = Job.GetInstance();
FileOutputFormat.SetOutputPath(job, outDir);
Configuration conf = job.GetConfiguration();
conf.Set(MRJobConfig.TaskAttemptId, attempt);
conf.SetInt(FileOutputCommitter.FileoutputcommitterAlgorithmVersion, version);
conf.SetClass("fs.file.impl", typeof(TestFileOutputCommitter.RLFS), typeof(FileSystem
));
FileSystem.CloseAll();
JobContext jContext = new JobContextImpl(conf, taskID.GetJobID());
FileOutputCommitter amCommitter = new FileOutputCommitter(outDir, jContext);
amCommitter.SetupJob(jContext);
TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
taCtx[0] = new TaskAttemptContextImpl(conf, taskID);
taCtx[1] = new TaskAttemptContextImpl(conf, taskID1);
TextOutputFormat[] tof = new TextOutputFormat[2];
for (int i = 0; i < tof.Length; i++)
{
tof[i] = new _TextOutputFormat_508(this);
}
ExecutorService executor = Executors.NewFixedThreadPool(2);
try
{
for (int i_1 = 0; i_1 < taCtx.Length; i_1++)
{
int taskIdx = i_1;
executor.Submit(new _Callable_524(this, tof, taskIdx, taCtx));
}
}
finally
{
executor.Shutdown();
while (!executor.AwaitTermination(1, TimeUnit.Seconds))
{
Log.Info("Awaiting thread termination!");
}
}
amCommitter.CommitJob(jContext);
RawLocalFileSystem lfs = new RawLocalFileSystem();
lfs.SetConf(conf);
NUnit.Framework.Assert.IsFalse("Must not end up with sub_dir/sub_dir", lfs.Exists
(new Path(OutSubDir, SubDir)));
// validate output
ValidateContent(OutSubDir);
FileUtil.FullyDelete(new FilePath(outDir.ToString()));
}
private void Destroy()
{
_state = 2;
// .NET-specific:
if (_cts != null)
{
_cts.Cancel();
_cts = null;
}
if (_executor != null)
{
_executor.Shutdown();
}
}
private void Destroy()
{
_state = 2;
// .NET-specific:
if (_timer != null)
{
ExecutorService.Cancel(_timer);
_timer = null;
}
if (_executor != null)
{
_executor.Shutdown();
}
}
private void StopAggregators()
{
threadPool.Shutdown();
// if recovery on restart is supported then leave outstanding aggregations
// to the next restart
bool shouldAbort = context.GetNMStateStore().CanRecover() && !context.GetDecommissioned
();
// politely ask to finish
foreach (AppLogAggregator aggregator in appLogAggregators.Values)
{
if (shouldAbort)
{
aggregator.AbortLogAggregation();
}
else
{
aggregator.FinishLogAggregation();
}
}
while (!threadPool.IsTerminated())
{
// wait for all threads to finish
foreach (ApplicationId appId in appLogAggregators.Keys)
{
Log.Info("Waiting for aggregation to complete for " + appId);
}
try
{
if (!threadPool.AwaitTermination(30, TimeUnit.Seconds))
{
threadPool.ShutdownNow();
}
}
catch (Exception)
{
// send interrupt to hurry them along
Log.Warn("Aggregation stop interrupted!");
break;
}
}
foreach (ApplicationId appId_1 in appLogAggregators.Keys)
{
Log.Warn("Some logs may not have been aggregated for " + appId_1);
}
}
/// <param name="service">
///
/// <see cref="ExecutorService">to be shutdown</see>
/// </param>
/// <param name="timeoutInMs">
/// time to wait for
/// <see cref="ExecutorService.AwaitTermination(long, TimeUnit)"/>
/// calls in milli seconds.
/// </param>
/// <returns>
/// <tt>true</tt> if the service is terminated,
/// <tt>false</tt> otherwise
/// </returns>
/// <exception cref="System.Exception"/>
public static bool ShutdownExecutorService(ExecutorService service, long timeoutInMs
)
{
if (service == null)
{
return(true);
}
service.Shutdown();
if (!service.AwaitTermination(timeoutInMs, TimeUnit.Milliseconds))
{
service.ShutdownNow();
return(service.AwaitTermination(timeoutInMs, TimeUnit.Milliseconds));
}
else
{
return(true);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMultiThreadedCompressorPool()
{
int iterations = 4;
ExecutorService threadpool = Executors.NewFixedThreadPool(3);
LinkedBlockingDeque <Compressor> queue = new LinkedBlockingDeque <Compressor>(2 * iterations
);
Callable <bool> consumer = new _Callable_110(queue);
Callable <bool> producer = new _Callable_119(this, queue);
for (int i = 0; i < iterations; i++)
{
threadpool.Submit(consumer);
threadpool.Submit(producer);
}
// wait for completion
threadpool.Shutdown();
threadpool.AwaitTermination(1000, TimeUnit.Seconds);
Assert.Equal(LeaseCountErr, 0, CodecPool.GetLeasedCompressorsCount
(codec));
}
/// <exception cref="System.Exception"/>
internal static long PerformanceTest(int arrayLength, int maxArrays, int nThreads
, int[] sleepTimeMSs, ByteArrayManager impl)
{
ExecutorService pool = Executors.NewFixedThreadPool(nThreads);
IList <Future <Void> > futures = new AList <Future <Void> >(sleepTimeMSs.Length);
long startTime = Time.MonotonicNow();
for (int i = 0; i < sleepTimeMSs.Length; i++)
{
long sleepTime = sleepTimeMSs[i];
futures.AddItem(pool.Submit(new _Callable_628(impl, arrayLength, sleepTime)));
}
foreach (Future <Void> f in futures)
{
f.Get();
}
long endTime = Time.MonotonicNow();
pool.Shutdown();
return(endTime - startTime);
}
public virtual void ConcurrentPackRefs_onlyOneWritesPackedRefs()
{
RevBlob a = tr.Blob("a");
tr.LightweightTag("t", a);
CyclicBarrier syncPoint = new CyclicBarrier(2);
Callable <int> packRefs = new _Callable_131(this, syncPoint);
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
Future <int> p1 = pool.Submit(packRefs);
Future <int> p2 = pool.Submit(packRefs);
NUnit.Framework.Assert.AreEqual(1, p1.Get() + p2.Get());
}
finally
{
pool.Shutdown();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
}
public virtual void TestAddResourceRefConcurrency()
{
StartEmptyStore();
string key = "key1";
string user = "******";
string fileName = "foo.jar";
// first add the resource
store.AddResource(key, fileName);
// make concurrent addResourceRef calls (clients)
int count = 5;
ExecutorService exec = Executors.NewFixedThreadPool(count);
IList <Future <string> > futures = new AList <Future <string> >(count);
CountDownLatch start = new CountDownLatch(1);
for (int i = 0; i < count; i++)
{
ApplicationId id = CreateAppId(i, i);
Callable <string> task = new _Callable_205(this, start, key, id, user);
futures.AddItem(exec.Submit(task));
}
// start them all at the same time
start.CountDown();
// check the result
ICollection <string> results = new HashSet <string>();
foreach (Future <string> future in futures)
{
results.AddItem(future.Get());
}
// they should all have the same file name
NUnit.Framework.Assert.AreSame(1, results.Count);
NUnit.Framework.Assert.AreEqual(Sharpen.Collections.Singleton(fileName), results);
// there should be 5 refs as a result
ICollection <SharedCacheResourceReference> refs = store.GetResourceReferences(key);
NUnit.Framework.Assert.AreSame(count, refs.Count);
exec.Shutdown();
}
public virtual void TestParallelThreadsWithDifferentLocales()
{
CyclicBarrier barrier = new CyclicBarrier(2);
// wait for the other thread to set its locale
ExecutorService pool = Executors.NewFixedThreadPool(2);
try
{
Future <TranslationBundle> root = pool.Submit(new _T879158014(this, NLS.ROOT_LOCALE,
barrier));
Future <TranslationBundle> german = pool.Submit(new _T879158014(this, Sharpen.Extensions.GetGermanCulture(),
barrier));
NUnit.Framework.Assert.AreEqual(NLS.ROOT_LOCALE, root.Get().EffectiveLocale());
NUnit.Framework.Assert.AreEqual(Sharpen.Extensions.GetGermanCulture(), german.Get
().EffectiveLocale());
}
finally
{
pool.Shutdown();
pool.AwaitTermination(long.MaxValue, TimeUnit.SECONDS);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestDirDownload()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource
, LocalResourceVisibility>();
Random rand = new Random();
long sharedSeed = rand.NextLong();
rand.SetSeed(sharedSeed);
System.Console.Out.WriteLine("SEED: " + sharedSeed);
IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource,
Future <Path> >();
ExecutorService exec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
for (int i = 0; i < 5; ++i)
{
LocalResourceVisibility vis = LocalResourceVisibility.Private;
if (i % 2 == 1)
{
vis = LocalResourceVisibility.Application;
}
Path p = new Path(basedir, "dir" + i + ".jar");
LocalResource rsrc = CreateJar(files, p, vis);
rsrcVis[rsrc] = vis;
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
pending[rsrc] = exec.Submit(fsd);
}
exec.Shutdown();
while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
foreach (Future <Path> path in pending.Values)
{
NUnit.Framework.Assert.IsTrue(path.IsDone());
}
try
{
foreach (KeyValuePair <LocalResource, Future <Path> > p in pending)
{
Path localized = p.Value.Get();
FileStatus status = files.GetFileStatus(localized);
System.Console.Out.WriteLine("Testing path " + localized);
System.Diagnostics.Debug.Assert((status.IsDirectory()));
System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key)));
VerifyPermsRecursively(localized.GetFileSystem(conf), files, localized, rsrcVis[p
.Key]);
}
}
catch (ExecutionException e)
{
throw new IOException("Failed exec", e);
}
}
/// <summary>
/// Set up the distributed cache by localizing the resources, and updating
/// the configuration with references to the localized resources.
/// </summary>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
public virtual void Setup(JobConf conf)
{
FilePath workDir = new FilePath(Runtime.GetProperty("user.dir"));
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
// Generating unique numbers for FSDownload.
AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis());
// Find which resources are to be put on the local classpath
IDictionary <string, Path> classpaths = new Dictionary <string, Path>();
Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf);
if (archiveClassPaths != null)
{
foreach (Path p in archiveClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf);
if (fileClassPaths != null)
{
foreach (Path p in fileClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
// Localize the resources
LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir);
FileContext localFSFileContext = FileContext.GetLocalFSFileContext();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ExecutorService exec = null;
try
{
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d"
).Build();
exec = Executors.NewCachedThreadPool(tf);
Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf);
IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap();
foreach (LocalResource resource in localResources.Values)
{
Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path(
destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource
);
Future <Path> future = exec.Submit(download);
resourcesToPaths[resource] = future;
}
foreach (KeyValuePair <string, LocalResource> entry in localResources)
{
LocalResource resource_1 = entry.Value;
Path path;
try
{
path = resourcesToPaths[resource_1].Get();
}
catch (Exception e)
{
throw new IOException(e);
}
catch (ExecutionException e)
{
throw new IOException(e);
}
string pathString = path.ToUri().ToString();
string link = entry.Key;
string target = new FilePath(path.ToUri()).GetPath();
Symlink(workDir, target, link);
if (resource_1.GetType() == LocalResourceType.Archive)
{
localArchives.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.File)
{
localFiles.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.Pattern)
{
//PATTERN is not currently used in local mode
throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. "
+ resource_1.GetResource());
}
}
}
Path resourcePath;
try
{
resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource());
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
Log.Info(string.Format("Localized %s as %s", resourcePath, path));
string cp = resourcePath.ToUri().GetPath();
if (classpaths.Keys.Contains(cp))
{
localClasspaths.AddItem(path.ToUri().GetPath().ToString());
}
}
}
finally
{
if (exec != null)
{
exec.Shutdown();
}
}
// Update the configuration object with localized data.
if (!localArchives.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localArchives, new string[localArchives.Count])));
}
if (!localFiles.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localFiles, new string[localArchives.Count])));
}
setupCalled = true;
}
/// <summary>
/// Handles the DomainUnload event of the CurrentDomain control.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="System.EventArgs"/> instance containing the event data.</param>
static void CurrentDomain_DomainUnload(object sender, EventArgs e)
{
ExecutorService.Shutdown();
}
public override void Shutdown()
{
e.Shutdown();
}
/// <exception cref="System.Exception"/>
/// <exception cref="System.IO.IOException"/>
private void DoCheckpoint()
{
System.Diagnostics.Debug.Assert(canceler != null);
long txid;
NNStorage.NameNodeFile imageType;
// Acquire cpLock to make sure no one is modifying the name system.
// It does not need the full namesystem write lock, since the only thing
// that modifies namesystem on standby node is edit log replaying.
namesystem.CpLockInterruptibly();
try
{
System.Diagnostics.Debug.Assert(namesystem.GetEditLog().IsOpenForRead(), "Standby Checkpointer should only attempt a checkpoint when "
+ "NN is in standby mode, but the edit logs are in an unexpected state");
FSImage img = namesystem.GetFSImage();
long prevCheckpointTxId = img.GetStorage().GetMostRecentCheckpointTxId();
long thisCheckpointTxId = img.GetLastAppliedOrWrittenTxId();
System.Diagnostics.Debug.Assert(thisCheckpointTxId >= prevCheckpointTxId);
if (thisCheckpointTxId == prevCheckpointTxId)
{
Log.Info("A checkpoint was triggered but the Standby Node has not " + "received any transactions since the last checkpoint at txid "
+ thisCheckpointTxId + ". Skipping...");
return;
}
if (namesystem.IsRollingUpgrade() && !namesystem.GetFSImage().HasRollbackFSImage(
))
{
// if we will do rolling upgrade but have not created the rollback image
// yet, name this checkpoint as fsimage_rollback
imageType = NNStorage.NameNodeFile.ImageRollback;
}
else
{
imageType = NNStorage.NameNodeFile.Image;
}
img.SaveNamespace(namesystem, imageType, canceler);
txid = img.GetStorage().GetMostRecentCheckpointTxId();
System.Diagnostics.Debug.Assert(txid == thisCheckpointTxId, "expected to save checkpoint at txid="
+ thisCheckpointTxId + " but instead saved at txid=" + txid);
// Save the legacy OIV image, if the output dir is defined.
string outputDir = checkpointConf.GetLegacyOivImageDir();
if (outputDir != null && !outputDir.IsEmpty())
{
img.SaveLegacyOIVImage(namesystem, outputDir, canceler);
}
}
finally
{
namesystem.CpUnlock();
}
// Upload the saved checkpoint back to the active
// Do this in a separate thread to avoid blocking transition to active
// See HDFS-4816
ExecutorService executor = Executors.NewSingleThreadExecutor(uploadThreadFactory);
Future <Void> upload = executor.Submit(new _Callable_204(this, imageType, txid));
executor.Shutdown();
try
{
upload.Get();
}
catch (Exception e)
{
// The background thread may be blocked waiting in the throttler, so
// interrupt it.
upload.Cancel(true);
throw;
}
catch (ExecutionException e)
{
throw new IOException("Exception during image upload: " + e.Message, e.InnerException
);
}
}
public virtual void TestMaxOutHedgedReadPool()
{
isHedgedRead = true;
Configuration conf = new Configuration();
int numHedgedReadPoolThreads = 5;
int initialHedgedReadTimeoutMillis = 50000;
int fixedSleepIntervalMillis = 50;
conf.SetInt(DFSConfigKeys.DfsDfsclientHedgedReadThreadpoolSize, numHedgedReadPoolThreads
);
conf.SetLong(DFSConfigKeys.DfsDfsclientHedgedReadThresholdMillis, initialHedgedReadTimeoutMillis
);
// Set up the InjectionHandler
DFSClientFaultInjector.instance = Org.Mockito.Mockito.Mock <DFSClientFaultInjector
>();
DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
// make preads sleep for 50ms
Org.Mockito.Mockito.DoAnswer(new _Answer_372(fixedSleepIntervalMillis)).When(injector
).StartFetchFromDatanode();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Format(
true).Build();
DistributedFileSystem fileSys = cluster.GetFileSystem();
DFSClient dfsClient = fileSys.GetClient();
DFSHedgedReadMetrics metrics = dfsClient.GetHedgedReadMetrics();
// Metrics instance is static, so we need to reset counts from prior tests.
metrics.hedgedReadOps.Set(0);
metrics.hedgedReadOpsWin.Set(0);
metrics.hedgedReadOpsInCurThread.Set(0);
try
{
Path file1 = new Path("hedgedReadMaxOut.dat");
WriteFile(fileSys, file1);
// Basic test. Reads complete within timeout. Assert that there were no
// hedged reads.
PReadFile(fileSys, file1);
// assert that there were no hedged reads. 50ms + delta < 500ms
NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOps() == 0);
NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOpsInCurThread() == 0);
/*
* Reads take longer than timeout. But, only one thread reading. Assert
* that there were hedged reads. But, none of the reads had to run in the
* current thread.
*/
dfsClient.SetHedgedReadTimeout(50);
// 50ms
PReadFile(fileSys, file1);
// assert that there were hedged reads
NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOps() > 0);
NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOpsInCurThread() == 0);
/*
* Multiple threads reading. Reads take longer than timeout. Assert that
* there were hedged reads. And that reads had to run in the current
* thread.
*/
int factor = 10;
int numHedgedReads = numHedgedReadPoolThreads * factor;
long initialReadOpsValue = metrics.GetHedgedReadOps();
ExecutorService executor = Executors.NewFixedThreadPool(numHedgedReads);
AList <Future <Void> > futures = new AList <Future <Void> >();
for (int i = 0; i < numHedgedReads; i++)
{
futures.AddItem(executor.Submit(GetPReadFileCallable(fileSys, file1)));
}
for (int i_1 = 0; i_1 < numHedgedReads; i_1++)
{
futures[i_1].Get();
}
NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOps() > initialReadOpsValue);
NUnit.Framework.Assert.IsTrue(metrics.GetHedgedReadOpsInCurThread() > 0);
CleanupFile(fileSys, file1);
executor.Shutdown();
}
finally
{
fileSys.Close();
cluster.Shutdown();
Org.Mockito.Mockito.Reset(injector);
}
}