public void Run()
{
try
{
barrier.Await();
long i = iters;
int y = 0;
int succ = 0;
while (i > 0)
{
for (int k = 0; k < CASLoops.innerPerOuter; ++k)
{
int x = obj.get_Renamed();
int z = y + LoopHelpers.compute6(x);
if (obj.compareAndSet(x, z))
{
++succ;
}
y = LoopHelpers.compute7(z);
}
i -= CASLoops.innerPerOuter;
}
CASLoops.sum.AddDeltaAndReturnPreviousValue(obj.get_Renamed());
CASLoops.successes.AddDeltaAndReturnPreviousValue(succ);
barrier.Await();
}
catch (System.Exception ie)
{
return;
}
}
internal static void runCalibration(int n, long nms)
{
long now = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
long endTime = now + nms;
CyclicBarrier b = new CyclicBarrier(n + 1);
totalIters.Value = (0);
NonAtomicInteger a = new NonAtomicInteger();
for (int j = 0; j < n; ++j)
{
new SupportClass.ThreadClass(new System.Threading.ThreadStart(new NACalibrationLoop(endTime, a, b).Run)).Start();
}
b.Await();
b.Await();
long ipt = totalIters.Value / n;
if (ipt > loopIters[n])
{
loopIters[n] = ipt;
}
if (sum.Value == 0)
{
System.Console.Out.Write(" ");
}
}
public override void BeginTask(string title, int totalWork)
{
if (title.Equals(JGitText.Get().writingObjects))
{
try
{
syncPoint.Await();
}
catch (Exception)
{
Sharpen.Thread.CurrentThread().Interrupt();
}
}
}
internal static long runLocked(int n, long iters)
{
LoopHelpers.BarrierTimer timer = new LoopHelpers.BarrierTimer();
CyclicBarrier b = new CyclicBarrier(n + 1, timer);
LockedInteger a = new LockedInteger();
for (int j = 0; j < n; ++j)
{
new SupportClass.ThreadClass(new System.Threading.ThreadStart(new LockedLoop(iters, a, b).Run)).Start();
}
b.Await();
b.Await();
if (sum.Value == 0)
{
System.Console.Out.Write(" ");
}
return(timer.Time);
}
/// <summary>
/// Test the case that remove a data volume on a particular DataNode when the
/// volume is actively being written.
/// </summary>
/// <param name="dataNodeIdx">the index of the DataNode to remove a volume.</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Conf.ReconfigurationException"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="Sharpen.BrokenBarrierException"/>
private void TestRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx)
{
// Starts DFS cluster with 3 DataNodes to form a pipeline.
StartDFSCluster(1, 3);
short Replication = 3;
DataNode dn = cluster.GetDataNodes()[dataNodeIdx];
FileSystem fs = cluster.GetFileSystem();
Path testFile = new Path("/test");
long lastTimeDiskErrorCheck = dn.GetLastDiskErrorCheck();
FSDataOutputStream @out = fs.Create(testFile, Replication);
Random rb = new Random(0);
byte[] writeBuf = new byte[BlockSize / 2];
// half of the block.
rb.NextBytes(writeBuf);
@out.Write(writeBuf);
@out.Hflush();
// Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
// BlockReceiver releases volume reference before finalizeBlock(), the blocks
// on the volume will be removed, and finalizeBlock() throws IOE.
FsDatasetSpi <FsVolumeSpi> data = dn.data;
dn.data = Org.Mockito.Mockito.Spy(data);
Org.Mockito.Mockito.DoAnswer(new _Answer_599(data)).When(dn.data).FinalizeBlock(Matchers.Any
<ExtendedBlock>());
// Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
// the block is not removed, since the volume reference should not
// be released at this point.
CyclicBarrier barrier = new CyclicBarrier(2);
IList <string> oldDirs = GetDataDirs(dn);
string newDirs = oldDirs[1];
// Remove the first volume.
IList <Exception> exceptions = new AList <Exception>();
Sharpen.Thread reconfigThread = new _Thread_616(barrier, dn, newDirs, exceptions);
reconfigThread.Start();
barrier.Await();
rb.NextBytes(writeBuf);
@out.Write(writeBuf);
@out.Hflush();
@out.Close();
reconfigThread.Join();
// Verify the file has sufficient replications.
DFSTestUtil.WaitReplication(fs, testFile, Replication);
// Read the content back
byte[] content = DFSTestUtil.ReadFileBuffer(fs, testFile);
NUnit.Framework.Assert.AreEqual(BlockSize, content.Length);
// If an IOException thrown from BlockReceiver#run, it triggers
// DataNode#checkDiskError(). So we can test whether checkDiskError() is called,
// to see whether there is IOException in BlockReceiver#run().
NUnit.Framework.Assert.AreEqual(lastTimeDiskErrorCheck, dn.GetLastDiskErrorCheck(
));
if (!exceptions.IsEmpty())
{
throw new IOException(exceptions[0].InnerException);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.Exception"/>
protected internal virtual void TestContainerPreservationOnResyncImpl(TestNodeManagerResync.TestNodeManager1
nm, bool isWorkPreservingRestartEnabled)
{
YarnConfiguration conf = CreateNMConfig();
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, isWorkPreservingRestartEnabled
);
try
{
nm.Init(conf);
nm.Start();
ContainerId cId = TestNodeManagerShutdown.CreateContainerId();
TestNodeManagerShutdown.StartContainer(nm, cId, localFS, tmpDir, processStartFile
);
nm.SetExistingContainerId(cId);
NUnit.Framework.Assert.AreEqual(1, ((TestNodeManagerResync.TestNodeManager1)nm).GetNMRegistrationCount
());
nm.GetNMDispatcher().GetEventHandler().Handle(resyncEvent);
try
{
syncBarrier.Await();
}
catch (BrokenBarrierException)
{
}
NUnit.Framework.Assert.AreEqual(2, ((TestNodeManagerResync.TestNodeManager1)nm).GetNMRegistrationCount
());
// Only containers should be killed on resync, apps should lie around.
// That way local resources for apps can be used beyond resync without
// relocalization
NUnit.Framework.Assert.IsTrue(nm.GetNMContext().GetApplications().Contains(cId.GetApplicationAttemptId
().GetApplicationId()));
NUnit.Framework.Assert.IsFalse(assertionFailedInThread.Get());
}
finally
{
nm.Stop();
}
}
public void Run()
{
try
{
barrier.Await();
long iters = 0;
int y = 0;
int succ = 0;
do
{
int i = CASLoops.loopsPerTimeCheck;
while (i > 0)
{
for (int k = 0; k < CASLoops.innerPerOuter; ++k)
{
int x = obj.get_Renamed();
int z = y + LoopHelpers.compute6(x);
if (obj.compareAndSet(x, z))
{
++succ;
}
y = LoopHelpers.compute7(z);
}
i -= CASLoops.innerPerOuter;
}
iters += CASLoops.loopsPerTimeCheck;
}while ((System.DateTime.Now.Ticks - 621355968000000000) / 10000 < endTime);
CASLoops.totalIters.AddDeltaAndReturnPreviousValue(iters);
CASLoops.sum.AddDeltaAndReturnPreviousValue(obj.get_Renamed());
CASLoops.successes.AddDeltaAndReturnPreviousValue(succ);
barrier.Await();
}
catch (System.Exception ie)
{
return;
}
}
/// <exception cref="System.IO.IOException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest request
)
{
try
{
startLaunchBarrier.Await();
completeLaunchBarrier.Await();
//To ensure the kill is started before the launch
Sharpen.Thread.Sleep(100);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
catch (BrokenBarrierException e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
throw new IOException(new TestContainerLauncherImpl.ContainerException("Force fail CM"
));
}
/// <exception cref="System.IO.IOException"/>
public override void Merge(IList <InMemoryMapOutput <Text, Text> > inputs)
{
lock (this)
{
numMerges.IncrementAndGet();
foreach (InMemoryMapOutput <Text, Text> input in inputs)
{
manager.Unreserve(input.GetSize());
}
}
try
{
mergeStart.Await();
mergeComplete.Await();
}
catch (Exception)
{
}
catch (BrokenBarrierException)
{
}
}
private bool SafeAwait(int mySource, CyclicBarrier barrier)
{
try
{
barrier.Await(2, TimeUnit.Seconds);
}
catch (Exception)
{
results[mySource] = "Interrupted";
return(false);
}
catch (BrokenBarrierException)
{
results[mySource] = "Broken Barrier";
return(false);
}
catch (TimeoutException)
{
results[mySource] = "Timed out on barrier";
return(false);
}
return(true);
}
/// <exception cref="System.Exception"/>
public virtual void TestOutOfOrder()
{
CyclicBarrier barrierA = new CyclicBarrier(2);
CyclicBarrier barrierB = new CyclicBarrier(2);
CyclicBarrier barrierC = new CyclicBarrier(2);
asyncClient = new TestNMClientAsync.MockNMClientAsync2(this, barrierA, barrierB,
barrierC);
asyncClient.Init(new Configuration());
asyncClient.Start();
Container container = MockContainer(1);
ContainerLaunchContext clc = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
// start container from another thread
Sharpen.Thread t = new _Thread_434(this, container, clc);
t.Start();
barrierA.Await();
asyncClient.StopContainerAsync(container.GetId(), container.GetNodeId());
barrierC.Await();
NUnit.Framework.Assert.IsFalse("Starting and stopping should be out of order", ((
TestNMClientAsync.TestCallbackHandler2)asyncClient.GetCallbackHandler()).exceptionOccurred
.Get());
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerCleaned()
{
Log.Info("STARTING testContainerCleaned");
CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);
AppContext mockContext = Org.Mockito.Mockito.Mock <AppContext>();
EventHandler mockEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
Org.Mockito.Mockito.When(mockContext.GetEventHandler()).ThenReturn(mockEventHandler
);
TestContainerLauncherImpl.ContainerManagementProtocolClient mockCM = new TestContainerLauncherImpl.ContainerManagerForTest
(startLaunchBarrier, completeLaunchBarrier);
TestContainerLauncherImpl.ContainerLauncherImplUnderTest ut = new TestContainerLauncherImpl.ContainerLauncherImplUnderTest
(mockContext, mockCM);
Configuration conf = new Configuration();
ut.Init(conf);
ut.Start();
try
{
ContainerId contId = MakeContainerId(0l, 0, 0, 1);
TaskAttemptId taskAttemptId = MakeTaskAttemptId(0l, 0, 0, TaskType.Map, 0);
string cmAddress = "127.0.0.1:8000";
StartContainersResponse startResp = recordFactory.NewRecordInstance <StartContainersResponse
>();
startResp.SetAllServicesMetaData(serviceResponse);
Log.Info("inserting launch event");
ContainerRemoteLaunchEvent mockLaunchEvent = Org.Mockito.Mockito.Mock <ContainerRemoteLaunchEvent
>();
Org.Mockito.Mockito.When(mockLaunchEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteLaunch);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockLaunchEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
Org.Mockito.Mockito.When(mockLaunchEvent.GetContainerToken()).ThenReturn(CreateNewContainerToken
(contId, cmAddress));
ut.Handle(mockLaunchEvent);
startLaunchBarrier.Await();
Log.Info("inserting cleanup event");
ContainerLauncherEvent mockCleanupEvent = Org.Mockito.Mockito.Mock <ContainerLauncherEvent
>();
Org.Mockito.Mockito.When(mockCleanupEvent.GetType()).ThenReturn(ContainerLauncher.EventType
.ContainerRemoteCleanup);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerID()).ThenReturn(contId);
Org.Mockito.Mockito.When(mockCleanupEvent.GetTaskAttemptID()).ThenReturn(taskAttemptId
);
Org.Mockito.Mockito.When(mockCleanupEvent.GetContainerMgrAddress()).ThenReturn(cmAddress
);
ut.Handle(mockCleanupEvent);
completeLaunchBarrier.Await();
ut.WaitForPoolToIdle();
ArgumentCaptor <Org.Apache.Hadoop.Yarn.Event.Event> arg = ArgumentCaptor.ForClass <
Org.Apache.Hadoop.Yarn.Event.Event>();
Org.Mockito.Mockito.Verify(mockEventHandler, Org.Mockito.Mockito.AtLeast(2)).Handle
(arg.Capture());
bool containerCleaned = false;
for (int i = 0; i < arg.GetAllValues().Count; i++)
{
Log.Info(arg.GetAllValues()[i].ToString());
Org.Apache.Hadoop.Yarn.Event.Event currentEvent = arg.GetAllValues()[i];
if (currentEvent.GetType() == TaskAttemptEventType.TaContainerCleaned)
{
containerCleaned = true;
}
}
System.Diagnostics.Debug.Assert((containerCleaned));
}
finally
{
ut.Stop();
}
}
/// <exception cref="System.Exception"></exception>
public virtual TranslationBundle Call()
{
NLS.SetLocale(this.locale);
barrier.Await();
return(GermanTranslatedBundle.Get());
}
/// <exception cref="System.Exception"/>
public virtual void TestMemoryMerge()
{
int TotalMemBytes = 10000;
int OutputSize = 7950;
JobConf conf = new JobConf();
conf.SetFloat(MRJobConfig.ShuffleInputBufferPercent, 1.0f);
conf.SetLong(MRJobConfig.ReduceMemoryTotalBytes, TotalMemBytes);
conf.SetFloat(MRJobConfig.ShuffleMemoryLimitPercent, 0.8f);
conf.SetFloat(MRJobConfig.ShuffleMergePercent, 0.9f);
TestMergeManager.TestExceptionReporter reporter = new TestMergeManager.TestExceptionReporter
();
CyclicBarrier mergeStart = new CyclicBarrier(2);
CyclicBarrier mergeComplete = new CyclicBarrier(2);
TestMergeManager.StubbedMergeManager mgr = new TestMergeManager.StubbedMergeManager
(conf, reporter, mergeStart, mergeComplete);
// reserve enough map output to cause a merge when it is committed
MapOutput <Text, Text> out1 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out1 is InMemoryMapOutput
));
InMemoryMapOutput <Text, Text> mout1 = (InMemoryMapOutput <Text, Text>)out1;
FillOutput(mout1);
MapOutput <Text, Text> out2 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out2 is InMemoryMapOutput
));
InMemoryMapOutput <Text, Text> mout2 = (InMemoryMapOutput <Text, Text>)out2;
FillOutput(mout2);
// next reservation should be a WAIT
MapOutput <Text, Text> out3 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.AreEqual("Should be told to wait", null, out3);
// trigger the first merge and wait for merge thread to start merging
// and free enough output to reserve more
mout1.Commit();
mout2.Commit();
mergeStart.Await();
NUnit.Framework.Assert.AreEqual(1, mgr.GetNumMerges());
// reserve enough map output to cause another merge when committed
out1 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out1 is InMemoryMapOutput
));
mout1 = (InMemoryMapOutput <Text, Text>)out1;
FillOutput(mout1);
out2 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.IsTrue("Should be a memory merge", (out2 is InMemoryMapOutput
));
mout2 = (InMemoryMapOutput <Text, Text>)out2;
FillOutput(mout2);
// next reservation should be null
out3 = mgr.Reserve(null, OutputSize, 0);
NUnit.Framework.Assert.AreEqual("Should be told to wait", null, out3);
// commit output *before* merge thread completes
mout1.Commit();
mout2.Commit();
// allow the first merge to complete
mergeComplete.Await();
// start the second merge and verify
mergeStart.Await();
NUnit.Framework.Assert.AreEqual(2, mgr.GetNumMerges());
// trigger the end of the second merge
mergeComplete.Await();
NUnit.Framework.Assert.AreEqual(2, mgr.GetNumMerges());
NUnit.Framework.Assert.AreEqual("exception reporter invoked", 0, reporter.GetNumExceptions
());
}