public async Task Run(CancellationToken token)
{
var prevStats = new Dictionary <string, ClusterNodeStatusReport>();
while (token.IsCancellationRequested == false)
{
var delay = TimeoutManager.WaitFor(SupervisorSamplePeriod, token);
try
{
if (Suspended == false)
{
_iteration++;
var newStats = _maintenance.GetStats();
await AnalyzeLatestStats(newStats, prevStats);
prevStats = newStats;
}
await delay;
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"An error occurred while analyzing maintenance stats on node {_nodeTag}.", e);
}
}
finally
{
await delay;
}
}
}
public void RunComponentsDefault()
{
TimeoutManager = new TimeoutManager();
CallbackBufferBlock = new ConsumerQueue <IActionItem>();
ActionCaseConsumer = new ConsumerQueue <IActionCase>();
RunComponents();
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
isDisposing = true;
// TODO: dispose managed state (managed objects).
_actionStartSignal.Set();
_actionStartSignal.Close();
CallbackBufferBlock.Stop();
TimeoutManager.Stop();
ActionCaseConsumer.Stop();
RunningActions.ForEach(action =>
{
action.Token.SetCancelled();
action.Token.SetCompletedSignal();
return(false);
});
}
// TODO: free unmanaged resources (unmanaged objects) and override a finalizer below.
// TODO: set large fields to null.
disposedValue = true;
}
}
public void Write(T value, int timeoutMillis)
{
using (TimeoutManager timeout = TimeoutManager.Start(timeoutMillis))
{
try
{
if (!Monitor.TryEnter(_subscribers, timeout))
{
throw new TimeoutException("Timeout locking queue.");
}
if (_closed)
{
throw new ChannelClosedException("Write end closed. Impossible to write.");
}
foreach (IChannelWriter <T> item in _subscribers.WriteBarrierWith(timeout))
{
if (!item.Closed)
{
item.Write(value, timeout);
}
}
}
finally
{
Monitor.Exit(_subscribers);
}
}
}
protected async Task WaitOrThrowOperationCanceled(TimeSpan time)
{
try
{
if (time < TimeSpan.Zero)
{
ThrowOperationCanceledExceptionIfNeeded();
return;
}
// if cancellation requested then it will throw TaskCancelledException and we stop the work
await TimeoutManager.WaitFor(time, CancellationToken).ConfigureAwait(false);
ThrowOperationCanceledExceptionIfNeeded();
}
catch (Exception e) when(e is OperationCanceledException == false)
{
// can happen if there is an invalid timespan
if (Logger.IsOperationsEnabled && e is ObjectDisposedException == false)
{
Logger.Operations($"Error in the background worker when {nameof(WaitOrThrowOperationCanceled)} was called", e);
}
throw new OperationCanceledException(); // throw OperationCanceled so we stop the work
}
void ThrowOperationCanceledExceptionIfNeeded()
{
if (CancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException(); //If we are disposed we need to throw OCE because this is the expected behavior
}
}
}
public async Task TestConnection()
{
var url = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
DynamicJsonValue result;
try
{
var timeout = TimeoutManager.WaitFor(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
var connectionInfo = ReplicationUtils.GetTcpInfoAsync(url, null, "Test-Connection", Server.ClusterCertificateHolder.Certificate);
if (await Task.WhenAny(timeout, connectionInfo) == timeout)
{
throw new TimeoutException($"Waited for {ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan} to receive tcp info from {url} and got no response");
}
result = await ConnectToClientNodeAsync(connectionInfo.Result, ServerStore.Engine.TcpConnectionTimeout, LoggingSource.Instance.GetLogger("testing-connection", "testing-connection"));
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.Error)] = $"An exception was thrown while trying to connect to {url} : {e}"
};
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, result);
}
}
private async Task <(Task <SubscriptionConnectionClientMessage> ReplyFromClientTask, string SubscriptionChangeVectorBeforeCurrentBatch)> WaitForClientAck(
Task <SubscriptionConnectionClientMessage> replyFromClientTask,
string subscriptionChangeVectorBeforeCurrentBatch)
{
SubscriptionConnectionClientMessage clientReply;
while (true)
{
var result = await Task.WhenAny(replyFromClientTask,
TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(5000), CancellationTokenSource.Token)).ConfigureAwait(false);
CancellationTokenSource.Token.ThrowIfCancellationRequested();
if (result == replyFromClientTask)
{
clientReply = await replyFromClientTask;
if (clientReply.Type == SubscriptionConnectionClientMessage.MessageType.DisposedNotification)
{
CancellationTokenSource.Cancel();
break;
}
replyFromClientTask = GetReplyFromClientAsync();
break;
}
await SendHeartBeat();
await SendNoopAck();
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
switch (clientReply.Type)
{
case SubscriptionConnectionClientMessage.MessageType.Acknowledge:
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(
SubscriptionId,
Options.SubscriptionName,
_lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = _lastChangeVector;
Stats.LastAckReceivedAt = DateTime.UtcNow;
Stats.AckRate.Mark();
await WriteJsonAsync(new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.Confirm)
});
break;
//precaution, should not reach this case...
case SubscriptionConnectionClientMessage.MessageType.DisposedNotification:
CancellationTokenSource.Cancel();
break;
default:
throw new ArgumentException("Unknown message type from client " +
clientReply.Type);
}
return(replyFromClientTask, subscriptionChangeVectorBeforeCurrentBatch);
}
public void Run()
{
if (TimeoutManager == null)
{
return;
}
TimeoutManager.SagaTimedOut +=
(o, e) =>
{
using (var scope = new TransactionScope(TransactionScopeOption.Required))
{
MessageSender.Send(MapToTransportMessage(e), e.Destination);
Persister.Remove(e.SagaId);
scope.Complete();
}
};
Persister.GetAll().ToList().ForEach(td =>
TimeoutManager.PushTimeout(td));
thread = new Thread(Poll);
thread.Start();
}
public void Write(T value, int timeoutMillis)
{
using (TimeoutManager timeout = TimeoutManager.Start(timeoutMillis))
{
try
{
if (!Monitor.TryEnter(_channel, timeout))
{
throw new TimeoutException("Timeout locking queue.");
}
if (Closed)
{
throw new ChannelClosedException("Impossible to send on a colsed channel.");
}
IBasicProperties prop = _channel.CreateBasicProperties();
prop.ContentEncoding = "utf-8";
prop.ContentType = "application/json";
prop.Persistent = true;
prop.Headers = new Dictionary <string, object>();
prop.Headers.Add(ChannelsRabbitMQManager.Ask().HeadersNameWriterClosed, false);
prop.Headers.Add(ChannelsRabbitMQManager.Ask().HeadersNameWriterMessageType, typeof(T).AssemblyQualifiedName);
byte[] data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(value));
_channel.BasicPublish(ChannelsRabbitMQManager.Ask().SubscribableChannelCollectorExchange, $"{Name}.{SHA1HashStringForUTF8String(typeof(T).FullName)}", prop, data);
}
finally
{
Monitor.Exit(_channel);
}
}
}
void Poll()
{
while (!stopRequested)
{
TimeoutManager.PopTimeout();
}
}
public async Task <ResponseTimeInformation> ExecuteAllPendingLazyOperationsAsync(CancellationToken token = default(CancellationToken))
{
using (AsyncTaskHolder())
{
var requests = new List <GetRequest>();
for (int i = 0; i < PendingLazyOperations.Count; i++)
{
var req = PendingLazyOperations[i].CreateRequest(Context);
if (req == null)
{
PendingLazyOperations.RemoveAt(i);
i--; // so we'll recheck this index
continue;
}
requests.Add(req);
}
if (requests.Count == 0)
{
return(new ResponseTimeInformation());
}
try
{
var sw = Stopwatch.StartNew();
IncrementRequestCount();
var responseTimeDuration = new ResponseTimeInformation();
while (await ExecuteLazyOperationsSingleStep(responseTimeDuration, requests, sw, token).ConfigureAwait(false))
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(100), token).ConfigureAwait(false);
}
responseTimeDuration.ComputeServerTotal();
foreach (var pendingLazyOperation in PendingLazyOperations)
{
Action <object> value;
if (OnEvaluateLazy.TryGetValue(pendingLazyOperation, out value))
{
value(pendingLazyOperation.Result);
}
}
sw.Stop();
responseTimeDuration.TotalClientDuration = sw.Elapsed;
return(responseTimeDuration);
}
finally
{
PendingLazyOperations.Clear();
}
}
}
protected async Task WaitForExecutionOnRelevantNodes(JsonOperationContext context, string database, ClusterTopology clusterTopology, List <string> members, long index)
{
if (members.Count == 0)
{
throw new InvalidOperationException("Cannot wait for execution when there are no nodes to execute ON.");
}
var executors = new List <ClusterRequestExecutor>();
var timeoutTask = TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(10000));
var waitingTasks = new List <Task>
{
timeoutTask
};
var cts = CancellationTokenSource.CreateLinkedTokenSource(ServerStore.ServerShutdown);
try
{
foreach (var member in members)
{
var url = member == ServerStore.NodeTag ?
Server.ServerStore.GetNodeHttpServerUrl() :
clusterTopology.GetUrlFromTag(member);
var requester = ClusterRequestExecutor.CreateForSingleNode(url, ServerStore.Server.Certificate.Certificate);
executors.Add(requester);
waitingTasks.Add(requester.ExecuteAsync(new WaitForRaftIndexCommand(index), context, token: cts.Token));
}
while (true)
{
var task = await Task.WhenAny(waitingTasks);
if (task == timeoutTask)
{
throw new TimeoutException($"Waited too long for the raft command (number {index}) to be executed on any of the relevant nodes to this command.");
}
if (task.IsCompletedSuccessfully)
{
break;
}
waitingTasks.Remove(task);
if (waitingTasks.Count == 1) // only the timeout task is left
{
throw new InvalidDataException($"The database '{database}' was created but is not accessible, because all of the nodes on which this database was supposed to reside on, threw an exception.", task.Exception);
}
}
}
finally
{
cts.Cancel();
foreach (var clusterRequestExecutor in executors)
{
clusterRequestExecutor.Dispose();
}
cts.Dispose();
}
}
private async Task StartCollectingStats()
{
_database.ReplicationLoader.IncomingReplicationAdded += IncomingHandlerAdded;
_database.ReplicationLoader.IncomingReplicationRemoved += IncomingHandlerRemoved;
_database.ReplicationLoader.OutgoingReplicationAdded += OutgoingHandlerAdded;
_database.ReplicationLoader.OutgoingReplicationRemoved += OutgoingHandlerRemoved;
foreach (var handler in _database.ReplicationLoader.IncomingHandlers)
{
IncomingHandlerAdded(handler);
}
foreach (var handler in _database.ReplicationLoader.OutgoingHandlers)
{
OutgoingHandlerAdded(handler);
}
var token = _cts.Token;
try
{
while (token.IsCancellationRequested == false)
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(3000), token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
break;
}
var performanceStats = PreparePerformanceStats().ToList();
if (performanceStats.Count > 0)
{
Stats.Enqueue(performanceStats);
}
}
}
finally
{
_database.ReplicationLoader.OutgoingReplicationRemoved -= OutgoingHandlerRemoved;
_database.ReplicationLoader.OutgoingReplicationAdded -= OutgoingHandlerAdded;
_database.ReplicationLoader.IncomingReplicationRemoved -= IncomingHandlerRemoved;
_database.ReplicationLoader.IncomingReplicationAdded -= IncomingHandlerAdded;
foreach (var kvp in _incoming)
{
IncomingHandlerRemoved(kvp.Value.Handler);
}
foreach (var kvp in _outgoing)
{
OutgoingHandlerRemoved(kvp.Key);
}
}
}
void HandleInternal(TransportMessage message)
{
var sagaId = Guid.Empty;
string sagaIdString;
if (message.Headers.TryGetValue(Headers.SagaId, out sagaIdString))
{
sagaId = Guid.Parse(sagaIdString);
}
if (message.Headers.ContainsKey(TimeoutManagerHeaders.ClearTimeouts))
{
if (sagaId == Guid.Empty)
{
throw new InvalidOperationException("Invalid saga id specified, clear timeouts is only supported for saga instances");
}
TimeoutManager.RemoveTimeoutBy(sagaId);
}
else
{
string expire;
if (!message.Headers.TryGetValue(TimeoutManagerHeaders.Expire, out expire))
{
throw new InvalidOperationException("Non timeout message arrived at the timeout manager, id:" + message.Id);
}
var destination = message.ReplyToAddress;
string routeExpiredTimeoutTo;
if (message.Headers.TryGetValue(TimeoutManagerHeaders.RouteExpiredTimeoutTo, out routeExpiredTimeoutTo))
{
destination = Address.Parse(routeExpiredTimeoutTo);
}
var data = new TimeoutData
{
Destination = destination,
SagaId = sagaId,
State = message.Body,
Time = DateTimeExtensions.ToUtcDateTime(expire),
CorrelationId = GetCorrelationIdToStore(message),
Headers = message.Headers,
OwningTimeoutManager = Configure.EndpointName
};
//add a temp header so that we can make sure to restore the ReplyToAddress
if (message.ReplyToAddress != null)
{
data.Headers[TimeoutData.OriginalReplyToAddress] = message.ReplyToAddress.ToString();
}
TimeoutManager.PushTimeout(data);
}
}
public static async Task <bool> WaitWithTimeout(this Task task, TimeSpan?timeout)
{
if (timeout == null)
{
await task.ConfigureAwait(false);
return(true);
}
return(task == await Task.WhenAny(task, TimeoutManager.WaitFor(timeout.Value)).ConfigureAwait(false));
}
public void ClickAndWaitForNavigation(Func <bool> navigationHasOccured, TimeSpan timeout = default(TimeSpan))
{
if (timeout == default(TimeSpan))
{
timeout = GlobalConfiguration.Configuration.WaitTimeout;
}
Click();
TimeoutManager.Execute(timeout, navigationHasOccured, new[] { typeof(StaleElementReferenceException) });
}
public async Task ReplaceClusterCert()
{
var replaceImmediately = GetBoolValueQueryString("replaceImmediately", required: false) ?? false;
ServerStore.EnsureNotPassive();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "replace-cluster-cert"))
{
try
{
var certificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
if (string.IsNullOrWhiteSpace(certificate.Name))
{
certificate.Name = "Cluster-Wide Certificate";
}
// This restriction should be removed when updating to .net core 2.1 when export of collection is fixed in Linux.
// With export, we'll be able to load the certificate and export it without a password, and propogate it through the cluster.
if (string.IsNullOrWhiteSpace(certificate.Password) == false)
{
throw new NotSupportedException("Replacing the cluster certificate with a password protected certificates is currently not supported.");
}
if (string.IsNullOrWhiteSpace(certificate.Certificate))
{
throw new ArgumentException($"{nameof(certificate.Certificate)} is a required field in the certificate definition.");
}
if (IsClusterAdmin() == false)
{
throw new InvalidOperationException("Cannot replace the server certificate. Only a ClusterAdmin can do this.");
}
var timeoutTask = TimeoutManager.WaitFor(TimeSpan.FromSeconds(60), ServerStore.ServerShutdown);
var replicationTask = Server.StartCertificateReplicationAsync(certificate.Certificate, certificate.Name, replaceImmediately);
await Task.WhenAny(replicationTask, timeoutTask);
if (replicationTask.IsCompleted == false)
{
throw new TimeoutException("Timeout when trying to replace the server certificate.");
}
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to replace the server certificate.", e);
}
}
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
internal static async Task WaitAndThrowOnTimeout(this Task task, TimeSpan timeout)
{
var result = await Task.WhenAny(task, TimeoutManager.WaitFor(timeout)).ConfigureAwait(false);
if (result != task)
{
throw new TimeoutException($"Task wasn't completed within {timeout}.");
}
await result.ConfigureAwait(false);
}
public IAbortableOperation <T> Consume(int timeoutMillis)
{
try
{
using (TimeoutManager timeout = TimeoutManager.Start(timeoutMillis))
{
if (!Monitor.TryEnter(_buffer, timeout))
{
throw new TimeoutException("Timeout locking queue.");
}
while (_buffer.Count == 0)
{
if (Drained)
{
throw new ChannelDrainedException("Impossible to read other data, channel drained.");
}
_waitReadable.Reset();
if (!Monitor.Wait(_buffer, timeout))
{
throw new TimeoutException("Timeout waiting readable state.");
}
}
return(new AbortableOperationImpl <T>(_buffer.Peek(),
() =>
{
_buffer.Dequeue();
Monitor.PulseAll(_buffer);
_waitWriteable.Set();
if (_buffer.Count == 0)
{
_waitReadable.Reset();
}
Monitor.Exit(_buffer);
},
() =>
{
Monitor.Exit(_buffer);
}));
}
}
catch (Exception e)
{
if (Monitor.IsEntered(_buffer))
{
Monitor.Exit(_buffer);
}
throw e;
}
}
public void Timeout_60000()
{
// Arrange
var timeoutManager = new TimeoutManager(60000);
// Act
var result = timeoutManager.CalculateAdaptiveDelay();
// Assert
MbUnit.Framework.Assert.AreEqual(1000, result);
Assert.Equal(1000, result);
}
public void Timeout_100()
{
// Arrange
var timeoutManager = new TimeoutManager(100);
// Act
var result = timeoutManager.CalculateAdaptiveDelay();
// Assert
MbUnit.Framework.Assert.AreEqual(Preferences.OnSleepDelay, result);
Assert.Equal(Preferences.OnSleepDelay, result);
}
private async Task SendHeartBeat()
{
// Todo: this is temporary, we should try using TcpConnection's receive and send timeout properties
var writeAsync = TcpConnection.Stream.WriteAsync(Heartbeat, 0, Heartbeat.Length);
if (writeAsync != await Task.WhenAny(writeAsync, TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(3000))).ConfigureAwait(false))
{
throw new SubscriptionClosedException($"Cannot contact client anymore, closing subscription ({Options?.SubscriptionName})");
}
TcpConnection.RegisterBytesSent(Heartbeat.Length);
}
public void RunComponents()
{
TimeoutManager.Start(this);
CallbackBufferBlock.Start("CallbackBufferBlock", action =>
{
if (action.CompletedCallback != null)
{
action.CompletedCallback(action);
}
});
ActionCaseConsumer.Start("ActionCaseConsumer", HandleActionCaseInner);
}
public IAbortableOperation <T> Consume(int timeoutMillis)
{
try
{
using (TimeoutManager timeout = TimeoutManager.Start(timeoutMillis))
{
if (!Monitor.TryEnter(_channel, timeout))
{
throw new TimeoutException("Timeout locking queue.");
}
while (_queue == null)
{
if (Drained)
{
throw new ChannelDrainedException("Impossible to read other data, channel drained.");
}
_waitReadable.Reset();
if (!Monitor.Wait(_channel, timeout))
{
throw new TimeoutException("Timeout waiting readable state.");
}
}
T temp = JsonConvert.DeserializeObject <T>(Encoding.UTF8.GetString(_queue.Body));
return(new AbortableOperationImpl <T>(temp,
() =>
{
_channel.BasicAck(_queue.DeliveryTag, false);
_queue = null;
_waitReadable.Reset();
Monitor.Exit(_channel);
},
() =>
{
_channel.BasicNack(_queue.DeliveryTag, false, true);
_queue = null;
_waitReadable.Reset();
Monitor.Exit(_channel);
}));
}
}
catch (Exception e)
{
if (Monitor.IsEntered(_channel))
{
Monitor.Exit(_channel);
}
throw e;
}
}
public IAbortableOperation <T> Consume(int timeoutMillis)
{
using (TimeoutManager timeout = TimeoutManager.Start(timeoutMillis))
{
IAbortableOperation <T> rd = _source.Consume(timeout);
_destiantion.Write(Pipe(rd.Value), timeout);
IAbortableOperation <T> wr = _destiantion.Consume(timeout);
return(new AbortableOperationImpl <T>(wr.Value, () => { rd.Commit(); wr.Commit(); }, () => { wr.Abort(); rd.Abort(); }));
}
}
public void Consume(Action <T> action, int timeoutMillis)
{
using (TimeoutManager timeout = TimeoutManager.Start(timeoutMillis))
{
_source.Consume(t =>
{
T temp = Pipe(t);
_destiantion.Write(temp, timeout);
}, timeout);
_destiantion.Consume(action, timeout);
}
}
public async Task <bool> WaitAsync(TimeSpan timeout)
{
var waitAsync = _tcs.Task;
_parent._token.ThrowIfCancellationRequested();
var result = await Task.WhenAny(waitAsync, TimeoutManager.WaitFor(timeout, _parent._token)).ConfigureAwait(false);
if (_parent._token != CancellationToken.None)
{
return(result == waitAsync && !_parent._token.IsCancellationRequested);
}
return(result == waitAsync);
}
public T Read(int timeoutMillis)
{
using (TimeoutManager timeout = TimeoutManager.Start(timeoutMillis))
{
T temp;
_source.Consume(t =>
{
temp = Pipe(t);
_destiantion.Write(temp, timeout);
}, timeout);
return(_destiantion.Read(timeout));
}
}
protected override async Task Process()
{
_work = true;
while (_work)
{
await FetchOperationStatus().ConfigureAwait(false);
if (_work == false)
{
break;
}
await TimeoutManager.WaitFor(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
void Poll()
{
while (!stopRequested)
{
try
{
TimeoutManager.PopTimeout();
}
catch (Exception ex)
{
//intentionally swallow here to avoid this bringing the entire endpoint down.
//remove this when our sattelite support is introduced
Logger.ErrorFormat("Failed to pop timeouts - " + ex);
}
}
}
