private RavenConnectionStringOptions GetConnectionOptionsSafe(CounterStorageReplicationDestination destination, out string lastError)
{
try
{
var connectionStringOptions = new RavenConnectionStringOptions
{
Url = destination.ServerUrl,
ApiKey = destination.ApiKey,
};
if (string.IsNullOrEmpty(destination.Username) == false)
{
connectionStringOptions.Credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
lastError = string.Empty;
return connectionStringOptions;
}
catch (Exception e)
{
lastError = e.Message;
Log.ErrorException(string.Format("Ignoring bad replication config!{0}Could not figure out connection options for [Url: {1}]",
Environment.NewLine, destination.ServerUrl), e);
return null;
}
}
private bool ReplicateTo(CounterStorageReplicationDestination destination)
{
var replicationStopwatch = Stopwatch.StartNew();
//todo: here, build url according to :destination.Url + '/counters/' + destination.
try
{
string lastError;
long lastEtag;
bool result = false;
switch (TryReplicate(destination, out lastEtag, out lastError))
{
case ReplicationResult.Success:
DateTime replicationTime = SystemTime.UtcNow;
RecordSuccess(destination.CounterStorageUrl, lastReplicatedEtag: lastEtag, lastReplicatedLastModified: replicationTime);
storage.MetricsCounters.OutgoingReplications.Mark();
result = true;
break;
case ReplicationResult.NotReplicated:
//TODO: Record not replicated
RecordSuccess(destination.CounterStorageUrl, SystemTime.UtcNow);
break;
default:
RecordFailure(destination.CounterStorageUrl, lastError);
storage.MetricsCounters.OutgoingReplications.Mark();
break;
}
return result;
}
catch (Exception ex)
{
Log.ErrorException("Error occured replicating to: " + destination.CounterStorageUrl, ex);
RecordFailure(destination.CounterStorageUrl, ex.Message);
return false;
}
finally
{
replicationStopwatch.Stop();
storage.MetricsCounters.GetReplicationDurationHistogram(destination.CounterStorageUrl).Update((long)replicationStopwatch.Elapsed.TotalMilliseconds);
storage.MetricsCounters.GetReplicationDurationMetric(destination.CounterStorageUrl).Mark((long)replicationStopwatch.Elapsed.TotalMilliseconds);
var holder = activeReplicationTasks.GetOrAdd(destination.CounterStorageUrl, s => new SemaphoreSlim(0, 1));
holder.Release();
}
}
private ReplicationResult TryReplicate(CounterStorageReplicationDestination destination, out long lastEtagSent, out string lastError)
{
long etag = 0;
lastEtagSent = 0;
var connectionStringOptions = GetConnectionOptionsSafe(destination, out lastError);
if (connectionStringOptions != null && GetLastReplicatedEtagFrom(connectionStringOptions, destination.CounterStorageUrl, out etag, out lastError))
{
var replicationData = GetCountersDataSinceEtag(etag, out lastEtagSent);
storage.MetricsCounters.GetReplicationBatchSizeMetric(destination.CounterStorageUrl).Mark(replicationData.Counters.Count);
storage.MetricsCounters.GetReplicationBatchSizeHistogram(destination.CounterStorageUrl).Update(replicationData.Counters.Count);
if (replicationData.Counters.Count > 0)
{
return PerformReplicationToServer(connectionStringOptions, destination.CounterStorageUrl, etag, replicationData, out lastError) ?
ReplicationResult.Success : ReplicationResult.Failure;
}
return ReplicationResult.NotReplicated;
}
return ReplicationResult.Failure;
}
private void ReplicateToDestination(CounterStorageReplicationDestination destination)
{
var dest = destination.CounterStorageUrl;
var holder = activeReplicationTasks.GetOrAdd(dest, s => new SemaphoreSlim(1));
if (holder.Wait(0) == false)
return;
var replicationTask = Task.Factory.StartNew(
() =>
{
//using (LogContext.WithDatabase(storage.Name)) //TODO: log with counter storage contexe
//{
try
{
if (ReplicateTo(destination)) SignalCounterUpdate();
}
catch (Exception e)
{
Log.ErrorException("Could not replicate to " + dest, e);
}
//}
});
activeTasks.Enqueue(replicationTask);
replicationTask.ContinueWith(
_ =>
{
// here we purge all the completed tasks at the head of the queue
Task task;
while (activeTasks.TryPeek(out task))
{
if (!task.IsCompleted && !task.IsCanceled && !task.IsFaulted) break;
activeTasks.TryDequeue(out task); // remove it from end
}
});
}
