private RavenConnectionStringOptions GetConnectionOptionsSafe(CounterReplicationDestination destination, out string lastError)
{
try
{
var connectionStringOptions = new RavenConnectionStringOptions
{
Url = destination.ServerUrl,
ApiKey = destination.ApiKey,
};
if (string.IsNullOrEmpty(destination.Username) == false)
{
connectionStringOptions.Credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
lastError = string.Empty;
connectionStringOptions.DefaultDatabase = destination.CounterStorageName;
return(connectionStringOptions);
}
catch (Exception e)
{
lastError = e.Message;
Log.ErrorException($"Ignoring bad replication config!{Environment.NewLine}Could not figure out connection options for [Url: {destination.ServerUrl}]", e);
return(null);
}
}
private ReplicationResult TryReplicate(CounterReplicationDestination destination, out long lastEtagSent, out string lastError)
{
long etag;
lastEtagSent = 0;
var connectionStringOptions = GetConnectionOptionsSafe(destination, out lastError);
if (connectionStringOptions == null ||
!GetLastReplicatedEtagFrom(connectionStringOptions, destination.CounterStorageUrl, out etag, out lastError))
{
return(ReplicationResult.Failure);
}
var replicationData = GetCountersDataSinceEtag(etag, out lastEtagSent);
storage.MetricsCounters.GetReplicationBatchSizeMetric(destination.CounterStorageUrl).Mark(replicationData.Counters.Count);
storage.MetricsCounters.GetReplicationBatchSizeHistogram(destination.CounterStorageUrl).Update(replicationData.Counters.Count);
if (replicationData.Counters.Count > 0)
{
return(PerformReplicationToServer(connectionStringOptions, destination.CounterStorageUrl, replicationData, out lastError) ?
ReplicationResult.Success : ReplicationResult.Failure);
}
return(ReplicationResult.NotReplicated);
}
private bool ReplicateTo(CounterReplicationDestination destination)
{
var replicationStopwatch = Stopwatch.StartNew();
//todo: here, build url according to :destination.Url + '/counters/' + destination.
try
{
string lastError;
long lastEtag;
bool result = false;
switch (TryReplicate(destination, out lastEtag, out lastError))
{
case ReplicationResult.Success:
DateTime replicationTime = SystemTime.UtcNow;
RecordSuccess(destination.CounterStorageUrl, lastReplicatedEtag: lastEtag, lastReplicatedLastModified: replicationTime);
storage.MetricsCounters.OutgoingReplications.Mark();
result = true;
break;
case ReplicationResult.NotReplicated:
//TODO: Record not replicated
RecordSuccess(destination.CounterStorageUrl, SystemTime.UtcNow);
break;
default:
RecordFailure(destination.CounterStorageUrl, lastError);
storage.MetricsCounters.OutgoingReplications.Mark();
break;
}
return(result);
}
catch (Exception ex)
{
Log.ErrorException("Error occured replicating to: " + destination.CounterStorageUrl, ex);
RecordFailure(destination.CounterStorageUrl, ex.Message);
return(false);
}
finally
{
replicationStopwatch.Stop();
var elapsedMicroseconds = (long)(replicationStopwatch.Elapsed.TotalMilliseconds * SystemTime.MicroSecPerTick);
storage.MetricsCounters.GetReplicationDurationHistogram(destination.CounterStorageUrl).Update(elapsedMicroseconds);
var holder = activeReplicationTasks.GetOrAdd(destination.CounterStorageUrl, s => new SemaphoreSlim(0, 1));
holder.Release();
}
}
private void ReplicateToDestination(CounterReplicationDestination destination)
{
var dest = destination.CounterStorageUrl;
var holder = activeReplicationTasks.GetOrAdd(dest, s => new SemaphoreSlim(1));
if (holder.Wait(0) == false)
{
return;
}
var replicationTask = Task.Factory.StartNew(
() =>
{
using (LogContext.WithResource(storage.Name))
{
try
{
if (ReplicateTo(destination))
{
SignalCounterUpdate();
}
}
catch (Exception e)
{
Log.ErrorException("Could not replicate to " + dest, e);
}
}
});
activeTasks.Enqueue(replicationTask);
replicationTask.ContinueWith(
_ =>
{
// here we purge all the completed tasks at the head of the queue
Task task;
while (activeTasks.TryPeek(out task))
{
if (!task.IsCompleted && !task.IsCanceled && !task.IsFaulted)
{
break;
}
activeTasks.TryDequeue(out task); // remove it from end
}
});
}
private CountersReplicationTopologyDestinationNode HandleDestination(CounterReplicationDestination replicationDestination)
{
RavenConnectionStringOptions connectionStringOptions = new RavenConnectionStringOptions
{
Credentials = replicationDestination.Credentials,
ApiKey = replicationDestination.ApiKey,
Url = replicationDestination.ServerUrl
};
string error;
string targetServerUrl;
// since each server can be addresses using both dns and ips we normalize connection string url by fetching target server url
// it should give us consistent urls
if (FetchTargetServerUrl(replicationDestination.ServerUrl, connectionStringOptions, out targetServerUrl, out error) == false)
{
var offlineNode = CountersReplicationTopologyDestinationNode.Offline(replicationDestination.CounterStorageUrl, counterStorage.ServerId);
if (string.IsNullOrEmpty(error) == false)
{
offlineNode.Errors.Add(error);
}
return(offlineNode);
}
targetServerUrl = targetServerUrl.ForCounter(replicationDestination.CounterStorageName);
if (replicationDestination.Disabled)
{
return(CountersReplicationTopologyDestinationNode.Disabled(targetServerUrl, counterStorage.ServerId));
}
if (from.Contains(targetServerUrl))
{
var state = CheckConnectionState(replicationDestination.CounterStorageUrl, connectionStringOptions);
switch (state)
{
case ReplicatonNodeState.Online:
return(CountersReplicationTopologyDestinationNode.Online(targetServerUrl, counterStorage.ServerId));
case ReplicatonNodeState.Offline:
return(CountersReplicationTopologyDestinationNode.Offline(targetServerUrl, counterStorage.ServerId));
default:
throw new NotSupportedException(state.ToString());
}
}
CountersReplicationTopologyRootNode rootNode;
if (TryGetSchema(targetServerUrl, connectionStringOptions, out rootNode, out error))
{
var node = CountersReplicationTopologyDestinationNode.Online(targetServerUrl, counterStorage.ServerId);
node.Destinations = rootNode.Destinations;
node.Sources = rootNode.Sources;
node.Errors = rootNode.Errors;
return(node);
}
var offline = CountersReplicationTopologyDestinationNode.Offline(targetServerUrl, counterStorage.ServerId);
if (string.IsNullOrEmpty(error) == false)
{
offline.Errors.Add(error);
}
return(offline);
}
