public void Resume(string name)
{
if (_suspendedDeliveryOverrides.Get() !.Pop(name))
{
_dispatcher.Execute(this);
}
}
public virtual void TestCloseTwice()
{
DistributedFileSystem fs = cluster.GetFileSystem();
FSDataOutputStream os = fs.Create(new Path("/test"));
DFSOutputStream dos = (DFSOutputStream)Whitebox.GetInternalState(os, "wrappedStream"
);
AtomicReference <IOException> ex = (AtomicReference <IOException>)Whitebox.GetInternalState
(dos, "lastException");
NUnit.Framework.Assert.AreEqual(null, ex.Get());
dos.Close();
IOException dummy = new IOException("dummy");
ex.Set(dummy);
try
{
dos.Close();
}
catch (IOException e)
{
NUnit.Framework.Assert.AreEqual(e, dummy);
}
NUnit.Framework.Assert.AreEqual(null, ex.Get());
dos.Close();
}
// use a small block size and a large write so that DN is busy creating
// new blocks. This makes it almost 100% sure we can reproduce
// case of client getting a DN that hasn't yet created the blocks
/// <exception cref="System.IO.IOException"/>
public virtual void TestImmediateReadOfNewFile()
{
int blockSize = 64 * 1024;
int writeSize = 10 * blockSize;
Configuration conf = new Configuration();
conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, blockSize);
Init(conf);
int requiredSuccessfulOpens = 100;
Path file = new Path("/file1");
AtomicBoolean openerDone = new AtomicBoolean(false);
AtomicReference <string> errorMessage = new AtomicReference <string>();
FSDataOutputStream @out = fileSystem.Create(file);
Sharpen.Thread writer = new Sharpen.Thread(new _Runnable_219(openerDone, @out, writeSize
));
Sharpen.Thread opener = new Sharpen.Thread(new _Runnable_239(this, requiredSuccessfulOpens
, file, openerDone, errorMessage, writer));
writer.Start();
opener.Start();
try
{
writer.Join();
opener.Join();
}
catch (Exception)
{
Sharpen.Thread.CurrentThread().Interrupt();
}
NUnit.Framework.Assert.IsNull(errorMessage.Get(), errorMessage.Get());
}
/// <summary>
/// Test for races in fstat usage
/// NOTE: this test is likely to fail on RHEL 6.0 which has a non-threadsafe
/// implementation of getpwuid_r.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestMultiThreadedFstat()
{
if (Path.Windows)
{
return;
}
FileOutputStream fos = new FileOutputStream(new FilePath(TestDir, "testfstat"));
AtomicReference <Exception> thrown = new AtomicReference <Exception>();
IList <Thread> statters = new AList <Thread>();
for (int i = 0; i < 10; i++)
{
Thread statter = new _Thread_120(fos, thrown);
statters.AddItem(statter);
statter.Start();
}
foreach (Thread t in statters)
{
t.Join();
}
fos.Close();
if (thrown.Get() != null)
{
throw new RuntimeException(thrown.Get());
}
}
/**
* Pass any caught exceptions here
*
* @param exception the exception
* @throws Exception if not retry-able or the retry policy returned negative
*/
public void takeException(Exception exception)
{
bool rethrow = true;
if (isRetryException(exception))
{
log.Debug(exception, "Retry-able exception received");
long elapsedTimeMs = (DateTime.Now.Ticks - startTimeMs) / 1000;
if (retryPolicy.allowRetry(retryCount++, elapsedTimeMs, sleeper))
{
tracer.Get().addCount("retries-allowed", 1);
log.Debug("Retrying operation");
rethrow = false;
}
else
{
tracer.Get().addCount("retries-disallowed", 1);
log.Debug("Retry policy not allowing retry");
}
}
if (rethrow)
{
throw exception;
}
}
public override object deref()
{
registerDep(this);
if (!dirty.get())
{
return(state.Get());
}
try
{
Var.pushThreadBindings(RT.map(REGISTER_DEP, registerDepInst));
for (;;)
{
dirty.set(false);
var v = state.Get();
var newv = compute();
Validate(newv);
if (state.CompareAndSet(v, newv))
{
NotifyWatches(v, newv);
return(newv);
}
}
}
finally
{
Var.popThreadBindings();
}
}
internal void HandlePartitionsViewEvent(Connection connection, ICollection <KeyValuePair <Guid, IList <int> > > partitions,
int partitionStateVersion)
{
if (Logger.IsFinestEnabled)
{
Logger.Finest($"Handling new partition table with partitionStateVersion: {partitionStateVersion}");
}
while (true)
{
var current = _partitionTable.Get();
if (!ShouldBeApplied(connection, partitions, partitionStateVersion, current))
{
return;
}
var newPartitions = ConvertToMap(partitions);
var newMetaData = new PartitionTable
{
Connection = connection, PartitionSateVersion = partitionStateVersion, Partitions = newPartitions
};
if (_partitionTable.CompareAndSet(current, newMetaData))
{
if (Logger.IsFinestEnabled)
{
Logger.Finest("Applied partition table with partitionStateVersion : " + partitionStateVersion);
}
return;
}
}
}
private void IncorporateReading(TaskAttemptId attemptID, float newProgress, long
newTime)
{
//TODO: Refactor this method, it seems more complicated than necessary.
AtomicReference <ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector> vectorRef
= estimates[attemptID];
if (vectorRef == null)
{
estimates.PutIfAbsent(attemptID, new AtomicReference <ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector
>(null));
IncorporateReading(attemptID, newProgress, newTime);
return;
}
ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector oldVector = vectorRef.Get
();
if (oldVector == null)
{
if (vectorRef.CompareAndSet(null, new ExponentiallySmoothedTaskRuntimeEstimator.EstimateVector
(this, -1.0, 0.0F, long.MinValue)))
{
return;
}
IncorporateReading(attemptID, newProgress, newTime);
return;
}
while (!vectorRef.CompareAndSet(oldVector, oldVector.Incorporate(newProgress, newTime
)))
{
oldVector = vectorRef.Get();
}
}
public virtual void TestSaveImageWhileSyncInProgress()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);
try
{
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = fsimage.GetEditLog();
JournalSet.JournalAndStream jas = editLog.GetJournals()[0];
EditLogFileOutputStream spyElos = Org.Mockito.Mockito.Spy((EditLogFileOutputStream
)jas.GetCurrentStream());
jas.SetCurrentStreamForTests(spyElos);
AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
CountDownLatch waitToEnterFlush = new CountDownLatch(1);
Sharpen.Thread doAnEditThread = new _Thread_371(namesystem, deferredException, waitToEnterFlush
);
Answer <Void> blockingFlush = new _Answer_388(doAnEditThread, waitToEnterFlush);
// Signal to main thread that the edit thread is in the racy section
Org.Mockito.Mockito.DoAnswer(blockingFlush).When(spyElos).Flush();
doAnEditThread.Start();
// Wait for the edit thread to get to the logsync unsynchronized section
Log.Info("Main thread: waiting to enter flush...");
waitToEnterFlush.Await();
NUnit.Framework.Assert.IsNull(deferredException.Get());
Log.Info("Main thread: detected that logSync is in unsynchronized section.");
Log.Info("Trying to enter safe mode.");
Log.Info("This should block for " + BlockTime + "sec, since flush will sleep that long"
);
long st = Time.Now();
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
long et = Time.Now();
Log.Info("Entered safe mode");
// Make sure we really waited for the flush to complete!
NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.SaveNamespace();
Log.Info("Joining on edit thread...");
doAnEditThread.Join();
NUnit.Framework.Assert.IsNull(deferredException.Get());
// We did 3 edits: begin, txn, and end
NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetFinalizedEditsFileName(1, 3), 1));
// after the save, just the one "begin"
NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetInProgressEditsFileName(4), 4));
}
finally
{
Log.Info("Closing nn");
if (namesystem != null)
{
namesystem.Close();
}
}
}
public void Resume(string name)
{
if (suspendedOverrides.Get().Count > 0)
{
suspendedOverrides.Get().Pop();
}
ResumeAll();
}
public virtual void TestEditLogRolling()
{
// start a cluster
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
FileSystem fileSys = null;
AtomicReference <Exception> caughtErr = new AtomicReference <Exception>();
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Build();
cluster.WaitActive();
fileSys = cluster.GetFileSystem();
NamenodeProtocols nn = cluster.GetNameNode().GetRpcServer();
FSImage fsimage = cluster.GetNamesystem().GetFSImage();
Storage.StorageDirectory sd = fsimage.GetStorage().GetStorageDir(0);
StartTransactionWorkers(nn, caughtErr);
long previousLogTxId = 1;
for (int i = 0; i < NumRolls && caughtErr.Get() == null; i++)
{
try
{
Sharpen.Thread.Sleep(20);
}
catch (Exception)
{
}
Log.Info("Starting roll " + i + ".");
CheckpointSignature sig = nn.RollEditLog();
long nextLog = sig.curSegmentTxId;
string logFileName = NNStorage.GetFinalizedEditsFileName(previousLogTxId, nextLog
- 1);
previousLogTxId += VerifyEditLogs(cluster.GetNamesystem(), fsimage, logFileName,
previousLogTxId);
NUnit.Framework.Assert.AreEqual(previousLogTxId, nextLog);
FilePath expectedLog = NNStorage.GetInProgressEditsFile(sd, previousLogTxId);
NUnit.Framework.Assert.IsTrue("Expect " + expectedLog + " to exist", expectedLog.
Exists());
}
}
finally
{
StopTransactionWorkers();
if (caughtErr.Get() != null)
{
throw new RuntimeException(caughtErr.Get());
}
if (fileSys != null)
{
fileSys.Close();
}
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Retrieve an E from the backing queue or block until we can.</summary>
/// <remarks>
/// Retrieve an E from the backing queue or block until we can.
/// Guaranteed to return an element from the current queue.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual E Take()
{
E e = null;
while (e == null)
{
e = takeRef.Get().Poll(1000L, TimeUnit.Milliseconds);
}
return(e);
}
public IMember GetMember(Address address)
{
var members = _membersRef.Get();
if (members == null)
{
return(null);
}
return(members.ContainsKey(address) ? members[address] : null);
}
internal virtual void Start()
{
Preconditions.CheckState(state.CompareAndSet(TestAppendSnapshotTruncate.Worker.State
.Idle, TestAppendSnapshotTruncate.Worker.State.Running));
if (thread.Get() == null)
{
Sharpen.Thread t = new Sharpen.Thread(null, new _Runnable_446(this), name);
Preconditions.CheckState(thread.CompareAndSet(null, t));
t.Start();
}
}
public virtual void TestSaveRightBeforeSync()
{
Configuration conf = GetConf();
NameNode.InitMetrics(conf, HdfsServerConstants.NamenodeRole.Namenode);
DFSTestUtil.FormatNameNode(conf);
FSNamesystem namesystem = FSNamesystem.LoadFromDisk(conf);
try
{
FSImage fsimage = namesystem.GetFSImage();
FSEditLog editLog = Org.Mockito.Mockito.Spy(fsimage.GetEditLog());
DFSTestUtil.SetEditLogForTesting(namesystem, editLog);
AtomicReference <Exception> deferredException = new AtomicReference <Exception>();
CountDownLatch waitToEnterSync = new CountDownLatch(1);
Sharpen.Thread doAnEditThread = new _Thread_467(namesystem, deferredException, waitToEnterSync
);
Answer <Void> blockingSync = new _Answer_484(doAnEditThread, waitToEnterSync);
Org.Mockito.Mockito.DoAnswer(blockingSync).When(editLog).LogSync();
doAnEditThread.Start();
Log.Info("Main thread: waiting to just before logSync...");
waitToEnterSync.Await();
NUnit.Framework.Assert.IsNull(deferredException.Get());
Log.Info("Main thread: detected that logSync about to be called.");
Log.Info("Trying to enter safe mode.");
Log.Info("This should block for " + BlockTime + "sec, since we have pending edits"
);
long st = Time.Now();
namesystem.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
long et = Time.Now();
Log.Info("Entered safe mode");
// Make sure we really waited for the flush to complete!
NUnit.Framework.Assert.IsTrue(et - st > (BlockTime - 1) * 1000);
// Once we're in safe mode, save namespace.
namesystem.SaveNamespace();
Log.Info("Joining on edit thread...");
doAnEditThread.Join();
NUnit.Framework.Assert.IsNull(deferredException.Get());
// We did 3 edits: begin, txn, and end
NUnit.Framework.Assert.AreEqual(3, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetFinalizedEditsFileName(1, 3), 1));
// after the save, just the one "begin"
NUnit.Framework.Assert.AreEqual(1, VerifyEditLogs(namesystem, fsimage, NNStorage.
GetInProgressEditsFileName(4), 4));
}
finally
{
Log.Info("Closing nn");
if (namesystem != null)
{
namesystem.Close();
}
}
}
public virtual E Get(int i)
{
object[] arr = index2Item.Get();
if (i < indexSize)
{
// arr.length guaranteed to be == to size() given the
// implementation of indexOf below.
return((E)arr[i]);
}
throw new IndexOutOfRangeException(string.Format("Out of bounds: %d >= %d", i, indexSize));
}
protected void onAcknowledge(Acknowledgement acknowledgement)
{
try
{
waitForAckLatch.Get().CountDown();
waitForSeqAckLatch.Get().CountDown();
CongestionControl cc = session.getCongestionControl();
long rtt = acknowledgement.getRoundTripTime();
if (rtt > 0)
{
long rttVar = acknowledgement.getRoundTripTimeVar();
cc.setRTT(rtt, rttVar);
statistics.setRTT(rtt, rttVar);
}
long rate = acknowledgement.getPacketReceiveRate();
if (rate > 0)
{
long linkCapacity = acknowledgement.getEstimatedLinkCapacity();
cc.updatePacketArrivalRate(rate, linkCapacity);
statistics.setPacketArrivalRate(cc.getPacketArrivalRate(), cc.getEstimatedLinkCapacity());
}
long ackNumber = acknowledgement.getAckNumber();
cc.onACK(ackNumber);
statistics.setCongestionWindowSize((long)cc.getCongestionWindowSize());
//need to remove all sequence numbers up the ack number from the sendBuffer
bool removed = false;
for (long s = lastAckSequenceNumber; s < ackNumber; s++)
{
lock (sendLock)
{
removed = sendBuffer.Remove(s);
}
if (removed)
{
unacknowledged.DecrementAndGet();
}
}
lastAckSequenceNumber = (int)Math.Max(lastAckSequenceNumber, ackNumber);
//send ACK2 packet to the receiver
sendAck2(ackNumber);
statistics.incNumberOfACKReceived();
if (storeStatistics)
{
statistics.storeParameters();
}
}
catch (Exception ex)
{
Log.Write(this.ToString(), ex);
}
}
public IUnlaunchDataStore GetDataStore()
{
if (_refreshableUnlaunchFetcherRef.Get() != null)
{
return(_refreshableUnlaunchFetcherRef.Get());
}
var dataStore = new UnlaunchHttpDataStore(_restWrapper, _s3BucketClient, _initialDownloadDoneEvent, _downloadSuccessful, _dataStoreRefreshDelay);
_refreshableUnlaunchFetcherRef.Set(dataStore);
return(dataStore);
}
/// <exception cref="CertificateException"/>
public void CheckClientTrusted(X509Certificate[] chain, string authType)
{
X509TrustManager tm = trustManagerRef.Get();
if (tm != null)
{
tm.CheckClientTrusted(chain, authType);
}
else
{
throw new CertificateException("Unknown client chain certificate: " + chain[0].ToString
());
}
}
/// <summary>Get the start state for a specific precedence value.</summary>
/// <param name="precedence">The current precedence.</param>
/// <returns>
/// The start state corresponding to the specified precedence, or
/// <see langword="null"/>
/// if no start state exists for the specified precedence.
/// </returns>
/// <exception cref="System.InvalidOperationException">if this is not a precedence DFA.</exception>
/// <seealso cref="IsPrecedenceDfa()"/>
public DFAState GetPrecedenceStartState(int precedence, bool fullContext)
{
if (!IsPrecedenceDfa)
{
throw new InvalidOperationException("Only precedence DFAs may contain a precedence start state.");
}
if (fullContext)
{
return(s0full.Get().GetTarget(precedence));
}
else
{
return(s0.Get().GetTarget(precedence));
}
}
public AccessSafely AfterCompleting(int times)
{
Access = AccessSafely.AfterCompleting(times);
Access
.WritingWith <int>("confirmDispatched", i => _confirmDispatchedResultedIn.IncrementAndGet())
.WritingWith <int>("readTextResultedIn", i => _readTextResultedIn.IncrementAndGet())
.WritingWith <int>("writeTextResultedIn", i => _writeTextResultedIn.IncrementAndGet())
.WritingWith <Result>("textReadResult", i => _textReadResult.Set(i))
.WritingWith <Result>("textWriteResult", i => _textWriteResult.Set(i))
.WritingWith <object>("stateHolder", i => _stateHolder.Set(i))
.WritingWith <Result>("textWriteAccumulatedResults", i => _textWriteAccumulatedResults.Enqueue(i))
.WritingWith <Metadata>("metadataHolder", i => _metadataHolder.Set(i))
.WritingWith <StorageException>("errorCauses", i => _errorCauses.Enqueue(i))
.ReadingWith("confirmDispatched", () => _confirmDispatchedResultedIn.Get())
.ReadingWith("readTextResultedIn", () => _readTextResultedIn.IncrementAndGet())
.ReadingWith("writeTextResultedIn", () => _writeTextResultedIn.IncrementAndGet())
.ReadingWith("textReadResult", () => _textReadResult.Get())
.ReadingWith("textWriteResult", () => _textWriteResult.Get())
.ReadingWith("stateHolder", () => _stateHolder.Get())
.ReadingWith("textWriteAccumulatedResults", () =>
{
_textWriteAccumulatedResults.TryDequeue(out var writeResult);
return(writeResult);
})
.ReadingWith("metadataHolder", () => _metadataHolder.Get())
.ReadingWith("errorCauses", () =>
{
_errorCauses.TryDequeue(out var exception);
return(exception);
});
return(Access);
}
public AccessSafely AfterCompleting <TState, TSource>(int times)
{
_access = AccessSafely.AfterCompleting(times);
_access
.WritingWith <int>("confirmDispatchedResultedIn", increment => _confirmDispatchedResultedIn.AddAndGet(increment))
.ReadingWith("confirmDispatchedResultedIn", () => _confirmDispatchedResultedIn.Get())
.WritingWith <StoreData <TSource> >("writeStoreData", data =>
{
_writeObjectResultedIn.AddAndGet(data.ResultedIn);
_objectWriteResult.Set(data.Result);
_objectWriteAccumulatedResults.Enqueue(data.Result);
_objectState.Set(data.State);
data.Sources.ForEach(source => _sources.Enqueue(source));
_metadataHolder.Set(data.Metadata);
if (data.ErrorCauses != null)
{
_errorCauses.Enqueue(data.ErrorCauses);
}
})
.WritingWith <StoreData <TSource> >("readStoreData", data =>
{
_readObjectResultedIn.AddAndGet(data.ResultedIn);
_objectReadResult.Set(data.Result);
_objectWriteAccumulatedResults.Enqueue(data.Result);
_objectState.Set(data.State);
data.Sources.ForEach(source => _sources.Enqueue(source));
_metadataHolder.Set(data.Metadata);
if (data.ErrorCauses != null)
{
_errorCauses.Enqueue(data.ErrorCauses);
}
})
.ReadingWith("readObjectResultedIn", () => _readObjectResultedIn.Get())
.ReadingWith("objectReadResult", () => _objectReadResult.Get())
.ReadingWith("objectWriteResult", () => _objectWriteResult.Get())
.ReadingWith("objectWriteAccumulatedResults", () =>
{
_objectWriteAccumulatedResults.TryDequeue(out var result);
return(result);
})
.ReadingWith("objectWriteAccumulatedResultsCount", () => _objectWriteAccumulatedResults.Count)
.ReadingWith("metadataHolder", () => _metadataHolder.Get())
.ReadingWith("objectState", () => (TState)_objectState.Get())
.ReadingWith("sources", () =>
{
_sources.TryDequeue(out var result);
return(result);
})
.ReadingWith("errorCauses", () =>
{
_errorCauses.TryDequeue(out var result);
return(result);
})
.ReadingWith("errorCausesCount", () => _errorCauses.Count)
.ReadingWith("writeObjectResultedIn", () => _writeObjectResultedIn.Get());
return(_access);
}
/// <exception cref="System.IO.IOException"></exception>
internal override ICollection <CachedPack> GetCachedPacks()
{
ObjectDirectory.CachedPackList list = cachedPacks.Get();
if (list == null || list.snapshot.IsModified(cachedPacksFile))
{
list = ScanCachedPacks(list);
}
ICollection <CachedPack> result = list.GetCachedPacks();
bool resultIsCopy = false;
foreach (FileObjectDatabase.AlternateHandle h in MyAlternates())
{
ICollection <CachedPack> altPacks = h.GetCachedPacks();
if (altPacks.IsEmpty())
{
continue;
}
if (result.IsEmpty())
{
result = altPacks;
continue;
}
if (!resultIsCopy)
{
result = new AList <CachedPack>(result);
resultIsCopy = true;
}
Sharpen.Collections.AddAll(result, altPacks);
}
return(result);
}
//====================================
// ChannelReaderConsumer
//====================================
public void Consume(RawMessage message)
{
var incoming = message.AsTextMessage();
Consumed.Get()?.Add(incoming);
var registerService = RegisterService.From(incoming);
if (registerService.IsValid)
{
var attributeSetName = _serviceNamePrefix + registerService.Name.Value;
foreach (var address in registerService.Addresses)
{
var fullAddress = address.Full;
_attributesClient?.Add(attributeSetName, fullAddress, fullAddress);
}
}
else
{
var unregisterService = UnregisterService.From(incoming);
if (unregisterService.IsValid)
{
var attributeSetName = _serviceNamePrefix + unregisterService.Name.Value;
_attributesClient?.RemoveAll(attributeSetName);
_attributesClient?.Add(_unregisteredServiceNamePrefix + unregisterService.Name.Value, _unregisteredCount, _unpublishedNotifications);
}
else
{
Logger.Warn($"DIRECTORY: RECEIVED UNKNOWN: {incoming}");
}
}
}
/// <summary>
/// Answer my reference values as a <typeparamref name="T"/>.
/// </summary>
/// <typeparam name="T">The value type.</typeparam>
/// <returns></returns>
public virtual T ReferenceValue <T>()
{
var result = reference.Get();
if (result == null)
{
return(default !);
private void UpdateVariance(long value)
{
while (true)
{
double[] oldValues = variance.Get();
var newValues = new double[2];
if (oldValues[0] == -1)
{
newValues[0] = value;
newValues[1] = 0;
}
else
{
var oldM = oldValues[0];
var oldS = oldValues[1];
var newM = oldM + (value - oldM) / Count;
var newS = oldS + (value - oldM) * (value - newM);
newValues[0] = newM;
newValues[1] = newS;
}
if (variance.CompareAndSet(oldValues, newValues))
{
return;
}
}
}
public void ThrowIfException()
{
var t = _exception.Get();
if (t != null)
{
throw new InvalidOperationException(string.Concat("Append failed because: ", t.Message), t);
}
}
/// <summary>
/// Test case that stops a writer after finalizing a block but
/// before calling completeFile, recovers a file from another writer,
/// starts writing from that writer, and then has the old lease holder
/// call completeFile
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestCompleteOtherLeaseHoldersFile()
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Build();
try
{
cluster.WaitActive();
NamenodeProtocols preSpyNN = cluster.GetNameNodeRpc();
NamenodeProtocols spyNN = Org.Mockito.Mockito.Spy(preSpyNN);
// Delay completeFile
GenericTestUtils.DelayAnswer delayer = new GenericTestUtils.DelayAnswer(Log);
Org.Mockito.Mockito.DoAnswer(delayer).When(spyNN).Complete(Matchers.AnyString(),
Matchers.AnyString(), (ExtendedBlock)Matchers.AnyObject(), Matchers.AnyLong());
DFSClient client = new DFSClient(null, spyNN, conf, null);
file1 = new Path("/testCompleteOtherLease");
OutputStream stm = client.Create("/testCompleteOtherLease", true);
// write 1/2 block
AppendTestUtil.Write(stm, 0, 4096);
AtomicReference <Exception> err = new AtomicReference <Exception>();
Sharpen.Thread t = new _Thread_242(stm, err);
t.Start();
Log.Info("Waiting for close to get to latch...");
delayer.WaitForCall();
// At this point, the block is finalized on the DNs, but the file
// has not been completed in the NN.
// Lose the leases
Log.Info("Killing lease checker");
client.GetLeaseRenewer().InterruptAndJoin();
FileSystem fs1 = cluster.GetFileSystem();
FileSystem fs2 = AppendTestUtil.CreateHdfsWithDifferentUsername(fs1.GetConf());
Log.Info("Recovering file");
RecoverFile(fs2);
Log.Info("Opening file for append from new fs");
FSDataOutputStream appenderStream = fs2.Append(file1);
Log.Info("Writing some data from new appender");
AppendTestUtil.Write(appenderStream, 0, 4096);
Log.Info("Telling old close to proceed.");
delayer.Proceed();
Log.Info("Waiting for close to finish.");
t.Join();
Log.Info("Close finished.");
// We expect that close will get a "Lease mismatch"
// error.
Exception thrownByClose = err.Get();
NUnit.Framework.Assert.IsNotNull(thrownByClose);
NUnit.Framework.Assert.IsTrue(thrownByClose is IOException);
if (!thrownByClose.Message.Contains("Lease mismatch"))
{
throw thrownByClose;
}
// The appender should be able to close properly
appenderStream.Close();
}
finally
{
cluster.Shutdown();
}
}
internal virtual TestAppendSnapshotTruncate.Worker.State CheckErrorState()
{
TestAppendSnapshotTruncate.Worker.State s = state.Get();
if (s == TestAppendSnapshotTruncate.Worker.State.Error)
{
throw new InvalidOperationException(name + " has " + s, thrown);
}
return(s);
}
public virtual void TestHflushWhileClosing()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fs = cluster.GetFileSystem();
Path p = new Path("/hflush-and-close.dat");
FSDataOutputStream stm = CreateFile(fs, p, 1);
AList <Sharpen.Thread> flushers = new AList <Sharpen.Thread>();
AtomicReference <Exception> thrown = new AtomicReference <Exception>();
try
{
for (int i = 0; i < 10; i++)
{
Sharpen.Thread flusher = new _Thread_165(stm, thrown);
// Expected exception caught. Ignoring.
flusher.Start();
flushers.AddItem(flusher);
}
// Write some data
for (int i_1 = 0; i_1 < 10000; i_1++)
{
stm.Write(1);
}
// Close it while the flushing threads are still flushing
stm.Close();
// Wait for the flushers to all die.
foreach (Sharpen.Thread t in flushers)
{
t.Join();
}
// They should have all gotten the expected exception, not anything
// else.
if (thrown.Get() != null)
{
throw thrown.Get();
}
}
finally
{
fs.Close();
cluster.Shutdown();
}
}
public virtual void TestRefsChangedStackOverflow()
{
FileRepository newRepo = CreateBareRepository();
RefDatabase refDb = newRepo.RefDatabase;
FilePath packedRefs = new FilePath(newRepo.Directory, "packed-refs");
NUnit.Framework.Assert.IsTrue(packedRefs.CreateNewFile());
AtomicReference<StackOverflowError> error = new AtomicReference<StackOverflowError
>();
AtomicReference<IOException> exception = new AtomicReference<IOException>();
AtomicInteger changeCount = new AtomicInteger();
newRepo.Listeners.AddRefsChangedListener(new _RefsChangedListener_1156(refDb, changeCount
, error, exception));
refDb.GetRefs("ref");
refDb.GetRefs("ref");
NUnit.Framework.Assert.IsNull(error.Get());
NUnit.Framework.Assert.IsNull(exception.Get());
NUnit.Framework.Assert.AreEqual(1, changeCount.Get());
}
