private async Task <(Task <SubscriptionConnectionClientMessage> ReplyFromClientTask, string SubscriptionChangeVectorBeforeCurrentBatch)> WaitForClientAck(
Task <SubscriptionConnectionClientMessage> replyFromClientTask,
string subscriptionChangeVectorBeforeCurrentBatch)
{
SubscriptionConnectionClientMessage clientReply;
while (true)
{
var result = await Task.WhenAny(replyFromClientTask,
TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(5000), CancellationTokenSource.Token)).ConfigureAwait(false);
CancellationTokenSource.Token.ThrowIfCancellationRequested();
if (result == replyFromClientTask)
{
clientReply = await replyFromClientTask;
if (clientReply.Type == SubscriptionConnectionClientMessage.MessageType.DisposedNotification)
{
CancellationTokenSource.Cancel();
break;
}
replyFromClientTask = GetReplyFromClientAsync();
break;
}
await SendHeartBeat();
await SendNoopAck();
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
switch (clientReply.Type)
{
case SubscriptionConnectionClientMessage.MessageType.Acknowledge:
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(
SubscriptionId,
Options.SubscriptionName,
_lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = _lastChangeVector;
Stats.LastAckReceivedAt = DateTime.UtcNow;
Stats.AckRate.Mark();
await WriteJsonAsync(new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.Confirm)
});
break;
//precaution, should not reach this case...
case SubscriptionConnectionClientMessage.MessageType.DisposedNotification:
CancellationTokenSource.Cancel();
break;
default:
throw new ArgumentException("Unknown message type from client " +
clientReply.Type);
}
return(replyFromClientTask, subscriptionChangeVectorBeforeCurrentBatch);
}
public async Task TestConnection()
{
var url = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
DynamicJsonValue result;
try
{
var timeout = TimeoutManager.WaitFor(ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
var connectionInfo = ReplicationUtils.GetTcpInfoAsync(url, null, "Test-Connection", Server.ClusterCertificateHolder.Certificate);
if (await Task.WhenAny(timeout, connectionInfo) == timeout)
{
throw new TimeoutException($"Waited for {ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan} to receive tcp info from {url} and got no response");
}
result = await ConnectToClientNodeAsync(connectionInfo.Result, ServerStore.Engine.TcpConnectionTimeout, LoggingSource.Instance.GetLogger("testing-connection", "testing-connection"));
}
catch (Exception e)
{
result = new DynamicJsonValue
{
[nameof(NodeConnectionTestResult.Success)] = false,
[nameof(NodeConnectionTestResult.Error)] = $"An exception was thrown while trying to connect to {url} : {e}"
};
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, result);
}
}
protected async Task WaitOrThrowOperationCanceled(TimeSpan time)
{
try
{
if (time < TimeSpan.Zero)
{
ThrowOperationCanceledExceptionIfNeeded();
return;
}
// if cancellation requested then it will throw TaskCancelledException and we stop the work
await TimeoutManager.WaitFor(time, CancellationToken).ConfigureAwait(false);
ThrowOperationCanceledExceptionIfNeeded();
}
catch (Exception e) when(e is OperationCanceledException == false)
{
// can happen if there is an invalid timespan
if (Logger.IsOperationsEnabled && e is ObjectDisposedException == false)
{
Logger.Operations($"Error in the background worker when {nameof(WaitOrThrowOperationCanceled)} was called", e);
}
throw new OperationCanceledException(); // throw OperationCanceled so we stop the work
}
void ThrowOperationCanceledExceptionIfNeeded()
{
if (CancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException(); //If we are disposed we need to throw OCE because this is the expected behavior
}
}
}
public async Task Run(CancellationToken token)
{
var prevStats = new Dictionary <string, ClusterNodeStatusReport>();
while (token.IsCancellationRequested == false)
{
var delay = TimeoutManager.WaitFor(SupervisorSamplePeriod, token);
try
{
if (Suspended == false)
{
_iteration++;
var newStats = _maintenance.GetStats();
await AnalyzeLatestStats(newStats, prevStats);
prevStats = newStats;
}
await delay;
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"An error occurred while analyzing maintenance stats on node {_nodeTag}.", e);
}
}
finally
{
await delay;
}
}
}
public async Task <ResponseTimeInformation> ExecuteAllPendingLazyOperationsAsync(CancellationToken token = default(CancellationToken))
{
using (AsyncTaskHolder())
{
var requests = new List <GetRequest>();
for (int i = 0; i < PendingLazyOperations.Count; i++)
{
var req = PendingLazyOperations[i].CreateRequest(Context);
if (req == null)
{
PendingLazyOperations.RemoveAt(i);
i--; // so we'll recheck this index
continue;
}
requests.Add(req);
}
if (requests.Count == 0)
{
return(new ResponseTimeInformation());
}
try
{
var sw = Stopwatch.StartNew();
IncrementRequestCount();
var responseTimeDuration = new ResponseTimeInformation();
while (await ExecuteLazyOperationsSingleStep(responseTimeDuration, requests, sw, token).ConfigureAwait(false))
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(100), token).ConfigureAwait(false);
}
responseTimeDuration.ComputeServerTotal();
foreach (var pendingLazyOperation in PendingLazyOperations)
{
Action <object> value;
if (OnEvaluateLazy.TryGetValue(pendingLazyOperation, out value))
{
value(pendingLazyOperation.Result);
}
}
sw.Stop();
responseTimeDuration.TotalClientDuration = sw.Elapsed;
return(responseTimeDuration);
}
finally
{
PendingLazyOperations.Clear();
}
}
}
private async Task StartCollectingStats()
{
_database.ReplicationLoader.IncomingReplicationAdded += IncomingHandlerAdded;
_database.ReplicationLoader.IncomingReplicationRemoved += IncomingHandlerRemoved;
_database.ReplicationLoader.OutgoingReplicationAdded += OutgoingHandlerAdded;
_database.ReplicationLoader.OutgoingReplicationRemoved += OutgoingHandlerRemoved;
foreach (var handler in _database.ReplicationLoader.IncomingHandlers)
{
IncomingHandlerAdded(handler);
}
foreach (var handler in _database.ReplicationLoader.OutgoingHandlers)
{
OutgoingHandlerAdded(handler);
}
var token = _cts.Token;
try
{
while (token.IsCancellationRequested == false)
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(3000), token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
break;
}
var performanceStats = PreparePerformanceStats().ToList();
if (performanceStats.Count > 0)
{
Stats.Enqueue(performanceStats);
}
}
}
finally
{
_database.ReplicationLoader.OutgoingReplicationRemoved -= OutgoingHandlerRemoved;
_database.ReplicationLoader.OutgoingReplicationAdded -= OutgoingHandlerAdded;
_database.ReplicationLoader.IncomingReplicationRemoved -= IncomingHandlerRemoved;
_database.ReplicationLoader.IncomingReplicationAdded -= IncomingHandlerAdded;
foreach (var kvp in _incoming)
{
IncomingHandlerRemoved(kvp.Value.Handler);
}
foreach (var kvp in _outgoing)
{
OutgoingHandlerRemoved(kvp.Key);
}
}
}
protected async Task WaitForExecutionOnRelevantNodes(JsonOperationContext context, string database, ClusterTopology clusterTopology, List <string> members, long index)
{
if (members.Count == 0)
{
throw new InvalidOperationException("Cannot wait for execution when there are no nodes to execute ON.");
}
var executors = new List <ClusterRequestExecutor>();
var timeoutTask = TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(10000));
var waitingTasks = new List <Task>
{
timeoutTask
};
var cts = CancellationTokenSource.CreateLinkedTokenSource(ServerStore.ServerShutdown);
try
{
foreach (var member in members)
{
var url = member == ServerStore.NodeTag ?
Server.ServerStore.GetNodeHttpServerUrl() :
clusterTopology.GetUrlFromTag(member);
var requester = ClusterRequestExecutor.CreateForSingleNode(url, ServerStore.Server.Certificate.Certificate);
executors.Add(requester);
waitingTasks.Add(requester.ExecuteAsync(new WaitForRaftIndexCommand(index), context, token: cts.Token));
}
while (true)
{
var task = await Task.WhenAny(waitingTasks);
if (task == timeoutTask)
{
throw new TimeoutException($"Waited too long for the raft command (number {index}) to be executed on any of the relevant nodes to this command.");
}
if (task.IsCompletedSuccessfully)
{
break;
}
waitingTasks.Remove(task);
if (waitingTasks.Count == 1) // only the timeout task is left
{
throw new InvalidDataException($"The database '{database}' was created but is not accessible, because all of the nodes on which this database was supposed to reside on, threw an exception.", task.Exception);
}
}
}
finally
{
cts.Cancel();
foreach (var clusterRequestExecutor in executors)
{
clusterRequestExecutor.Dispose();
}
cts.Dispose();
}
}
internal static async Task WaitAndThrowOnTimeout(this Task task, TimeSpan timeout)
{
var result = await Task.WhenAny(task, TimeoutManager.WaitFor(timeout)).ConfigureAwait(false);
if (result != task)
{
throw new TimeoutException($"Task wasn't completed within {timeout}.");
}
await result.ConfigureAwait(false);
}
public static async Task <bool> WaitWithTimeout(this Task task, TimeSpan?timeout)
{
if (timeout == null)
{
await task.ConfigureAwait(false);
return(true);
}
return(task == await Task.WhenAny(task, TimeoutManager.WaitFor(timeout.Value)).ConfigureAwait(false));
}
public async Task ReplaceClusterCert()
{
var replaceImmediately = GetBoolValueQueryString("replaceImmediately", required: false) ?? false;
ServerStore.EnsureNotPassive();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "replace-cluster-cert"))
{
try
{
var certificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
if (string.IsNullOrWhiteSpace(certificate.Name))
{
certificate.Name = "Cluster-Wide Certificate";
}
// This restriction should be removed when updating to .net core 2.1 when export of collection is fixed in Linux.
// With export, we'll be able to load the certificate and export it without a password, and propogate it through the cluster.
if (string.IsNullOrWhiteSpace(certificate.Password) == false)
{
throw new NotSupportedException("Replacing the cluster certificate with a password protected certificates is currently not supported.");
}
if (string.IsNullOrWhiteSpace(certificate.Certificate))
{
throw new ArgumentException($"{nameof(certificate.Certificate)} is a required field in the certificate definition.");
}
if (IsClusterAdmin() == false)
{
throw new InvalidOperationException("Cannot replace the server certificate. Only a ClusterAdmin can do this.");
}
var timeoutTask = TimeoutManager.WaitFor(TimeSpan.FromSeconds(60), ServerStore.ServerShutdown);
var replicationTask = Server.StartCertificateReplicationAsync(certificate.Certificate, certificate.Name, replaceImmediately);
await Task.WhenAny(replicationTask, timeoutTask);
if (replicationTask.IsCompleted == false)
{
throw new TimeoutException("Timeout when trying to replace the server certificate.");
}
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to replace the server certificate.", e);
}
}
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
private async Task SendHeartBeat()
{
// Todo: this is temporary, we should try using TcpConnection's receive and send timeout properties
var writeAsync = TcpConnection.Stream.WriteAsync(Heartbeat, 0, Heartbeat.Length);
if (writeAsync != await Task.WhenAny(writeAsync, TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(3000))).ConfigureAwait(false))
{
throw new SubscriptionClosedException($"Cannot contact client anymore, closing subscription ({Options?.SubscriptionName})");
}
TcpConnection.RegisterBytesSent(Heartbeat.Length);
}
public async Task <bool> WaitAsync(TimeSpan timeout)
{
var waitAsync = _tcs.Task;
_parent._token.ThrowIfCancellationRequested();
var result = await Task.WhenAny(waitAsync, TimeoutManager.WaitFor(timeout, _parent._token)).ConfigureAwait(false);
if (_parent._token != CancellationToken.None)
{
return(result == waitAsync && !_parent._token.IsCancellationRequested);
}
return(result == waitAsync);
}
protected override async Task Process()
{
_work = true;
while (_work)
{
await FetchOperationStatus().ConfigureAwait(false);
if (_work == false)
{
break;
}
await TimeoutManager.WaitFor(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
}
private async Task IdleFlushTimer()
{
var cancellationToken = _cancellationTokenSource.Token;
while (cancellationToken.IsCancellationRequested == false)
{
if (Disposed)
{
return;
}
if (Options.ManualFlushing)
{
return;
}
try
{
if (await _writeTransactionRunning.WaitAsync(TimeSpan.FromMilliseconds(Options.IdleFlushTimeout)) == false)
{
if (Volatile.Read(ref SizeOfUnflushedTransactionsInJournalFile) != 0)
{
GlobalFlushingBehavior.GlobalFlusher.Value.MaybeFlushEnvironment(this);
}
else if (Journal.Applicator.TotalWrittenButUnsyncedBytes != 0)
{
QueueForSyncDataFile();
}
}
else
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(1000), cancellationToken).ConfigureAwait(false);
}
}
catch (ObjectDisposedException)
{
return;
}
catch (OperationCanceledException)
{
return;
}
}
}
private async Task StartCollectingMetrics()
{
_ioChanges.OnIoChange += OnIOChange;
// 1. First time around, get existing data
var result = IoMetricsUtil.GetIoMetricsResponse(_environments, _performanceMetrics);
_basePath = result.Environments[0].Path;
AddEnvironmentTypeToPathIfNeeded(result.Environments[0]);
foreach (var environment in result.Environments)
{
var folder = environment.Path;
foreach (var file in environment.Files)
{
var fullFilePath = Path.Combine(folder, file.File);
_perEnvironmentsFilesMetrics.TryAdd(fullFilePath, new BlockingCollection <IoMeterBuffer.MeterItem>());
}
}
MetricsQueue.Enqueue(result);
using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(_resourceShutdown, _cts.Token))
{
var token = linkedToken.Token;
// 2. Prepare & put data from the Dictionary into the Queue every 3 seconds
while (token.IsCancellationRequested == false)
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(3000), token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
break;
}
var ioMetricsResponse = PrepareIOMetrics();
if (ioMetricsResponse != null)
{
MetricsQueue.Enqueue(ioMetricsResponse);
}
}
}
}
private void StartThrottling()
{
if (_throttlingStarted)
{
return;
}
if (_throttlingInterval == null)
{
return;
}
if (_timerTask != null)
{
throw new InvalidOperationException("Cannot start throttling timer task because it's already started");
}
_timerCts = new CancellationTokenSource();
_timerTask = Task.Run(async() =>
{
using (var cts = CancellationTokenSource.CreateLinkedTokenSource(_token, _timerCts.Token))
{
try
{
while (cts.IsCancellationRequested == false)
{
await TimeoutManager.WaitFor(_throttlingInterval.Value, cts.Token);
if (_setCalled.Lower())
{
_mre.Set();
}
}
}
catch (OperationCanceledException)
{
// ignore
}
}
});
_throttlingStarted = true;
}
protected async Task WaitOrThrowOperationCanceled(TimeSpan time)
{
try
{
// if cancellation requested then it will throw TaskCancelledException and we stop the work
await TimeoutManager.WaitFor(time, CancellationToken).ConfigureAwait(false);
}
catch (Exception e) when(e is OperationCanceledException == false)
{
// can happen if there is an invalid timespan
if (Logger.IsOperationsEnabled && e is ObjectDisposedException == false)
{
Logger.Operations($"Error in the background worker when {nameof(WaitOrThrowOperationCanceled)} was called", e);
}
throw new OperationCanceledException(); // throw OperationCanceled so we stop the work
}
}
public async Task CollectDatabasesStatusReport()
{
while (_token.IsCancellationRequested == false)
{
try
{
using (_server.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var dbs = CollectDatabaseInformation(ctx);
var djv = new DynamicJsonValue();
foreach (var tuple in dbs)
{
djv[tuple.name] = tuple.report;
}
using (var writer = new BlittableJsonTextWriter(ctx, _tcp.Stream))
{
ctx.Write(writer, djv);
}
}
}
catch (Exception e)
{
if (_tcp.TcpClient?.Connected != true)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("The tcp connection was closed, so we exit the maintenance work.");
}
return;
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Exception occurred while collecting info from {_server.NodeTag}", e);
}
}
finally
{
await TimeoutManager.WaitFor(WorkerSamplePeriod, _token);
}
}
}
private async Task Concurrent_SendAppendEntriesPendingToLeaderAsync(CancellationTokenSource cts, long currentTerm, long lastLogIndex)
{
var timeoutPeriod = _engine.Timeout.TimeoutPeriod / 4;
var timeToWait = TimeSpan.FromMilliseconds(timeoutPeriod);
using (_engine.ContextPool.AllocateOperationContext(out JsonOperationContext timeoutCtx))
{
while (cts.IsCancellationRequested == false)
{
try
{
await TimeoutManager.WaitFor(timeToWait, cts.Token);
if (cts.IsCancellationRequested)
{
break;
}
_engine.Timeout.Defer(_connection.Source);
_connection.Send(timeoutCtx, new AppendEntriesResponse
{
Pending = true,
Success = false,
CurrentTerm = currentTerm,
LastLogIndex = lastLogIndex
});
}
catch (TimeoutException)
{
break;
}
catch (OperationCanceledException)
{
break;
}
catch (ObjectDisposedException)
{
break;
}
}
}
}
public async Task <int> WaitForReplicationAsync(int numberOfReplicasToWaitFor, TimeSpan waitForReplicasTimeout, string lastChangeVector)
{
var sp = Stopwatch.StartNew();
while (true)
{
var internalDestinations = _internalDestinations.Select(x => x.Url).ToHashSet();
var waitForNextReplicationAsync = WaitForNextReplicationAsync();
var past = ReplicatedPastInternalDestinations(internalDestinations, lastChangeVector);
if (past >= numberOfReplicasToWaitFor)
{
return(past);
}
var remaining = waitForReplicasTimeout - sp.Elapsed;
if (remaining < TimeSpan.Zero)
{
return(ReplicatedPastInternalDestinations(internalDestinations, lastChangeVector));
}
var timeout = TimeoutManager.WaitFor(remaining);
try
{
if (await Task.WhenAny(waitForNextReplicationAsync, timeout) == timeout)
{
return(ReplicatedPastInternalDestinations(internalDestinations, lastChangeVector));
}
}
catch (OperationCanceledException e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Get exception while trying to get write assurance on a database with {numberOfReplicasToWaitFor} servers. " +
$"Written so far to {past} servers only. " +
$"LastChangeVector is: {lastChangeVector}.", e);
}
return(ReplicatedPastInternalDestinations(internalDestinations, lastChangeVector));
}
}
}
public async Task <ResponseTimeInformation> ExecuteAllPendingLazyOperationsAsync(CancellationToken token = default(CancellationToken))
{
if (PendingLazyOperations.Count == 0)
{
return(new ResponseTimeInformation());
}
try
{
var sw = Stopwatch.StartNew();
IncrementRequestCount();
var responseTimeDuration = new ResponseTimeInformation();
while (await ExecuteLazyOperationsSingleStep(responseTimeDuration).WithCancellation(token).ConfigureAwait(false))
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(100), token).ConfigureAwait(false);
}
responseTimeDuration.ComputeServerTotal();
foreach (var pendingLazyOperation in PendingLazyOperations)
{
Action <object> value;
if (OnEvaluateLazy.TryGetValue(pendingLazyOperation, out value))
{
value(pendingLazyOperation.Result);
}
}
responseTimeDuration.TotalClientDuration = sw.Elapsed;
return(responseTimeDuration);
}
finally
{
PendingLazyOperations.Clear();
}
}
protected virtual async Task Process()
{
_isProcessing = true;
switch (StatusFetchMode)
{
case OperationStatusFetchMode.ChangesApi:
var changes = await _changes().EnsureConnectedNow().ConfigureAwait(false);
var observable = changes.ForOperationId(_id);
_subscription = observable.Subscribe(this);
await observable.EnsureSubscribedNow().ConfigureAwait(false);
changes.ConnectionStatusChanged += OnConnectionStatusChanged;
// We start the operation before we subscribe,
// so if we subscribe after the operation was already completed we will miss the notification for it.
await FetchOperationStatus().ConfigureAwait(false);
break;
case OperationStatusFetchMode.Polling:
while (_isProcessing)
{
await FetchOperationStatus().ConfigureAwait(false);
if (_isProcessing == false)
{
break;
}
await TimeoutManager.WaitFor(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
}
break;
default:
throw new NotSupportedException($"Invalid operation fetch status mode: '{StatusFetchMode}'");
}
}
private async Task StartCollectingStats()
{
_changes.OnIndexChange += OnIndexChange;
// This is done this way in order to avoid locking _perIndexStats
// for fetching .Values
var stats = _perIndexStats
.Select(x => new IndexPerformanceStats
{
Name = x.Value.Index.Name,
Etag = x.Value.Index.Etag,
Performance = x.Value.Index.GetIndexingPerformance()
})
.ToList();
Stats.Enqueue(stats);
using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(_resourceShutdown, _cts.Token))
{
var token = linkedToken.Token;
while (token.IsCancellationRequested == false)
{
await TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(3000), token).ConfigureAwait(false);
if (token.IsCancellationRequested)
{
break;
}
var performanceStats = PreparePerformanceStats();
if (performanceStats.Count > 0)
{
Stats.Enqueue(performanceStats);
}
}
}
}
public async Task Run(CancellationToken token)
{
// we give some time to populate the stats.
await TimeoutManager.WaitFor(SupervisorSamplePeriod, token);
var prevStats = _maintenance.GetStats();
// wait before collecting the stats again.
await TimeoutManager.WaitFor(SupervisorSamplePeriod, token);
while (_term == _engine.CurrentTerm && token.IsCancellationRequested == false)
{
var delay = TimeoutManager.WaitFor(SupervisorSamplePeriod, token);
try
{
if (Suspended == false)
{
_iteration++;
var newStats = _maintenance.GetStats();
await AnalyzeLatestStats(newStats, prevStats);
prevStats = newStats;
}
}
catch (Exception e)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"An error occurred while analyzing maintenance stats on node {_nodeTag}.", e);
}
}
finally
{
await delay;
}
}
}
public async Task ReplaceClusterCert()
{
var replaceImmediately = GetBoolValueQueryString("replaceImmediately", required: false) ?? false;
ServerStore.EnsureNotPassive();
using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (var certificateJson = ctx.ReadForDisk(RequestBodyStream(), "replace-cluster-cert"))
{
try
{
var certificate = JsonDeserializationServer.CertificateDefinition(certificateJson);
if (string.IsNullOrWhiteSpace(certificate.Name))
{
certificate.Name = "Cluster-Wide Certificate";
}
if (string.IsNullOrWhiteSpace(certificate.Certificate))
{
throw new ArgumentException($"{nameof(certificate.Certificate)} is a required field in the certificate definition.");
}
if (IsClusterAdmin() == false)
{
throw new InvalidOperationException("Cannot replace the server certificate. Only a ClusterAdmin can do this.");
}
byte[] certBytes;
try
{
certBytes = Convert.FromBase64String(certificate.Certificate);
}
catch (Exception e)
{
throw new ArgumentException($"Unable to parse the {nameof(certificate.Certificate)} property, expected a Base64 value", e);
}
X509Certificate2 newCertificate;
try
{
newCertificate = new X509Certificate2(certBytes, certificate.Password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.PersistKeySet);
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to load the new certificate.", e);
}
var timeoutTask = TimeoutManager.WaitFor(TimeSpan.FromSeconds(60), ServerStore.ServerShutdown);
var replicationTask = Server.StartCertificateReplicationAsync(newCertificate, certificate.Name, replaceImmediately);
await Task.WhenAny(replicationTask, timeoutTask);
if (replicationTask.IsCompleted == false)
{
throw new TimeoutException("Timeout when trying to replace the server certificate.");
}
}
catch (Exception e)
{
throw new InvalidOperationException("Failed to replace the server certificate.", e);
}
}
NoContentStatus();
HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
}
private async Task ProcessSubscriptionAsync()
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Starting processing documents for subscription {SubscriptionId} received from {TcpConnection.TcpClient.Client.RemoteEndPoint}");
}
using (DisposeOnDisconnect)
using (TcpConnection.DocumentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext docsContext))
using (RegisterForNotificationOnNewDocuments())
{
var replyFromClientTask = GetReplyFromClientAsync();
string lastChangeVector = null;
string subscriptionChangeVectorBeforeCurrentBatch = SubscriptionState.ChangeVectorForNextBatchStartingPoint;
var startEtag = GetStartEtagForSubscription(docsContext, SubscriptionState);
var patch = SetupFilterScript();
var fetcher = new SubscriptionDocumentsFetcher(TcpConnection.DocumentDatabase, _options.MaxDocsPerBatch, SubscriptionId, TcpConnection.TcpClient.Client.RemoteEndPoint);
while (CancellationTokenSource.IsCancellationRequested == false)
{
bool anyDocumentsSentInCurrentIteration = false;
var sendingCurrentBatchStopwatch = Stopwatch.StartNew();
_buffer.SetLength(0);
var docsToFlush = 0;
using (TcpConnection.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, _buffer))
{
using (docsContext.OpenReadTransaction())
{
foreach (var result in fetcher.GetDataToSend(docsContext, Collection, Revisions, SubscriptionState, patch, startEtag))
{
startEtag = result.Doc.Etag;
lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
? result.Doc.ChangeVector
: ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);
if (result.Doc.Data == null)
{
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
sendingCurrentBatchStopwatch.Restart();
}
continue;
}
anyDocumentsSentInCurrentIteration = true;
writer.WriteStartObject();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(TypeSegment));
writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(DataSegment));
writer.WriteComma();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(DataSegment));
result.Doc.EnsureMetadata();
if (result.Exception != null)
{
var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
writer.WriteValue(BlittableJsonToken.StartObject,
docsContext.ReadObject(new DynamicJsonValue
{
[Client.Constants.Documents.Metadata.Key] = metadata
}, result.Doc.Id)
);
writer.WriteComma();
writer.WritePropertyName(context.GetLazyStringForFieldWithCaching(ExceptionSegment));
writer.WriteValue(BlittableJsonToken.String, context.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
}
else
{
writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
}
writer.WriteEndObject();
docsToFlush++;
// perform flush for current batch after 1000ms of running or 1 MB
if (_buffer.Length > Constants.Size.Megabyte ||
sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
if (docsToFlush > 0)
{
await FlushDocsToClient(writer, docsToFlush);
docsToFlush = 0;
sendingCurrentBatchStopwatch.Restart();
}
else
{
await SendHeartBeat();
}
}
}
}
if (anyDocumentsSentInCurrentIteration)
{
context.Write(writer, new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
});
await FlushDocsToClient(writer, docsToFlush, true);
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
}
}
if (anyDocumentsSentInCurrentIteration == false)
{
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(SubscriptionId,
Options.SubscriptionName,
lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector;
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
}
using (docsContext.OpenReadTransaction())
{
long globalEtag = TcpConnection.DocumentDatabase.DocumentsStorage.GetLastDocumentEtag(docsContext, Collection);
if (globalEtag > startEtag)
{
continue;
}
}
if (await WaitForChangedDocuments(replyFromClientTask))
{
continue;
}
}
SubscriptionConnectionClientMessage clientReply;
while (true)
{
var result = await Task.WhenAny(replyFromClientTask,
TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(5000), CancellationTokenSource.Token)).ConfigureAwait(false);
CancellationTokenSource.Token.ThrowIfCancellationRequested();
if (result == replyFromClientTask)
{
clientReply = await replyFromClientTask;
if (clientReply.Type == SubscriptionConnectionClientMessage.MessageType.DisposedNotification)
{
CancellationTokenSource.Cancel();
break;
}
replyFromClientTask = GetReplyFromClientAsync();
break;
}
await SendHeartBeat();
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
switch (clientReply.Type)
{
case SubscriptionConnectionClientMessage.MessageType.Acknowledge:
await TcpConnection.DocumentDatabase.SubscriptionStorage.AcknowledgeBatchProcessed(
SubscriptionId,
Options.SubscriptionName,
lastChangeVector,
subscriptionChangeVectorBeforeCurrentBatch);
subscriptionChangeVectorBeforeCurrentBatch = lastChangeVector;
Stats.LastAckReceivedAt = DateTime.UtcNow;
Stats.AckRate.Mark();
await WriteJsonAsync(new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.Confirm)
});
break;
//precaution, should not reach this case...
case SubscriptionConnectionClientMessage.MessageType.DisposedNotification:
CancellationTokenSource.Cancel();
break;
default:
throw new ArgumentException("Unknown message type from client " +
clientReply.Type);
}
}
CancellationTokenSource.Token.ThrowIfCancellationRequested();
}
}
private async Task <bool> WaitForChangedDocuments(Task pendingReply)
{
do
{
var hasMoreDocsTask = _waitForMoreDocuments.WaitAsync();
var resultingTask = await Task.WhenAny(hasMoreDocsTask, pendingReply, TimeoutManager.WaitFor(TimeSpan.FromMilliseconds(3000))).ConfigureAwait(false);
if (CancellationTokenSource.IsCancellationRequested)
{
return(false);
}
if (resultingTask == pendingReply)
{
return(false);
}
if (hasMoreDocsTask == resultingTask)
{
_waitForMoreDocuments.Reset();
return(true);
}
await SendHeartBeat();
} while (CancellationTokenSource.IsCancellationRequested == false);
return(false);
}
private async Task DoWork()
{
try
{
await _requestExecutor.GetPreferredNode().ConfigureAwait(false);
}
catch (OperationCanceledException e)
{
NotifyAboutError(e);
return;
}
catch (ChangeProcessingException e)
{
NotifyAboutError(e);
return;
}
catch (Exception e)
{
ConnectionStatusChanged?.Invoke(this, EventArgs.Empty);
NotifyAboutError(e);
return;
}
var url = new Uri($"{_requestExecutor.Url}/databases/{_database}/changes"
.ToLower()
.ToWebSocketPath(), UriKind.Absolute);
while (_cts.IsCancellationRequested == false)
{
try
{
if (Connected == false)
{
await _client.ConnectAsync(url, _cts.Token).ConfigureAwait(false);
Interlocked.Exchange(ref _immediateConnection, 1);
foreach (var counter in _counters)
{
counter.Value.Set(counter.Value.OnConnect());
}
ConnectionStatusChanged?.Invoke(this, EventArgs.Empty);
}
await ProcessChanges().ConfigureAwait(false);
}
catch (OperationCanceledException e)
{
NotifyAboutError(e);
return;
}
catch (ChangeProcessingException)
{
continue;
}
catch (Exception e)
{
//We don't report this error since we can automatically recover from it and we can't
// recover from the OnError accessing the faulty WebSocket.
try
{
ConnectionStatusChanged?.Invoke(this, EventArgs.Empty);
if (ReconnectClient() == false)
{
return;
}
}
catch
{
// we couldn't reconnect
NotifyAboutError(e);
throw;
}
}
finally
{
foreach (var confirmation in _confirmations)
{
confirmation.Value.TrySetCanceled();
}
_confirmations.Clear();
}
try
{
await TimeoutManager.WaitFor(TimeSpan.FromSeconds(1), _cts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
}
}