public override void Respond(IHttpContext context)
{
var src = context.Request.QueryString["from"];
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
while (src.EndsWith("/"))
{
src = src.Substring(0, src.Length - 1);// remove last /, because that has special meaning for Raven
}
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
using (ReplicationContext.Enter())
{
var document = Database.Get(ReplicationConstants.RavenReplicationSourcesBasePath + "/" + src, null);
if (document == null)
{
log.DebugFormat("Got replication last etag request from {0}: [{1}]", src, Guid.Empty);
context.WriteJson(new { Etag = Guid.Empty });
}
else
{
var sourceReplicationInformation = document.DataAsJson.JsonDeserialization <SourceReplicationInformation>();
log.DebugFormat("Got replication last etag request from {0}: [{1}]", src, sourceReplicationInformation.LastEtag);
context.WriteJson(new { Etag = sourceReplicationInformation.LastEtag });
}
}
}
public void PrepareUpdate(IEnumerable <ActorStateDataWrapper> actorStateDataWrapperList, long sequenceNumber)
{
if (sequenceNumber == 0) // Invalid LSN
{
return;
}
foreach (ActorStateDataWrapper actorStateDataWrapper in actorStateDataWrapperList)
{
actorStateDataWrapper.UpdateSequenceNumber(sequenceNumber);
}
using (this.rwLock.AcquireWriteLock())
{
var replicationContext = new ReplicationContext();
foreach (ActorStateDataWrapper actorStateDataWrapper in actorStateDataWrapperList)
{
var entry = new ListEntry(actorStateDataWrapper, replicationContext);
this.uncommittedEntriesList.AddLast(entry);
}
this.pendingReplicationContexts.Add(sequenceNumber, replicationContext);
}
}
public override void OnPut(string key, JObject document, JObject metadata, TransactionInformation transactionInformation)
{
if (ReplicationContext.IsInReplicationContext)
{
return;
}
using (ReplicationContext.Enter())
{
metadata.Remove(ReplicationConstants.RavenReplicationConflict);// you can't put conflicts
var oldVersion = Database.Get(key, transactionInformation);
if (oldVersion == null)
{
return;
}
if (oldVersion.Metadata[ReplicationConstants.RavenReplicationConflict] == null)
{
return;
}
// this is a conflict document, holding document keys in the
// values of the properties
foreach (var prop in oldVersion.DataAsJson.Value <JArray>("Conflicts"))
{
Database.Delete(prop.Value <string>(), null, transactionInformation);
}
}
}
public override void Respond(IHttpContext context)
{
var src = context.Request.QueryString["from"];
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
while (src.EndsWith("/"))
{
src = src.Substring(0, src.Length - 1);// remove last /, because that has special meaning for Raven
}
if (string.IsNullOrEmpty(src))
{
context.SetStatusToBadRequest();
return;
}
var array = context.ReadJsonArray();
using (ReplicationContext.Enter())
{
Database.TransactionalStorage.Batch(actions =>
{
string lastEtag = Guid.Empty.ToString();
foreach (JObject document in array)
{
var metadata = document.Value <JObject>("@metadata");
if (metadata[ReplicationConstants.RavenReplicationSource] == null)
{
// not sure why, old document from when the user didn't have replciation
// that we suddenly decided to replicate, choose the source for that
metadata[ReplicationConstants.RavenReplicationSource] = JToken.FromObject(src);
}
lastEtag = metadata.Value <string>("@etag");
var id = metadata.Value <string>("@id");
document.Remove("@metadata");
ReplicateDocument(actions, id, metadata, document, src);
}
var replicationDocKey = ReplicationConstants.RavenReplicationSourcesBasePath + "/" + src;
var replicationDocument = Database.Get(replicationDocKey, null);
var lastAttachmentId = Guid.Empty;
if (replicationDocument != null)
{
lastAttachmentId =
replicationDocument.DataAsJson.JsonDeserialization <SourceReplicationInformation>().
LastAttachmentEtag;
}
Database.Put(replicationDocKey, null,
JObject.FromObject(new SourceReplicationInformation
{
LastDocumentEtag = new Guid(lastEtag),
LastAttachmentEtag = lastAttachmentId
}),
new JObject(), null);
});
}
}
protected override void SetUp()
{
base.SetUp();
replicationStrategy = A.Fake <IReplicationStrategy>();
var simpleFactory = A.Fake <IReplicationStrategyFactory>(x => x.Strict());
simpleFactory
.CallsTo(x => x.StrategyForType(A <Type> .Ignored))
.Returns(replicationStrategy);
replicationContext = new ReplicationContext(simpleFactory);
}
public ListEntry(
ActorStateDataWrapper actorStateDataWrapper,
ReplicationContext replicationContext)
{
this.ActorStateDataWrapper = actorStateDataWrapper;
this.PendingReplicationContext = replicationContext;
if (this.PendingReplicationContext != null)
{
this.PendingReplicationContext.AssociateListEntry();
}
}
private void Execute()
{
var context = docDb.WorkContext;
while (context.DoWork)
{
try
{
using (ReplicationContext.Enter())
{
var destinations = GetReplicationDestinations();
if (destinations.Length == 0)
{
WarnIfNoReplicationTargetsWereFound();
}
else
{
var currentReplicationAttempts = Interlocked.Increment(ref replicationAttempts);
var destinationForReplication = destinations
.Where(dest => IsNotFailing(dest, currentReplicationAttempts));
foreach (var dest in destinationForReplication)
{
var destination = dest;
var holder = activeReplicationTasks.GetOrAdd(destination, new IntHolder());
if (Thread.VolatileRead(ref holder.Value) == 1)
{
continue;
}
Thread.VolatileWrite(ref holder.Value, 1);
Task.Factory.StartNew(() => ReplicateTo(destination), TaskCreationOptions.LongRunning)
.ContinueWith(completedTask =>
{
if (completedTask.Result) // force re-evaluation of replication again
{
docDb.WorkContext.NotifyAboutWork();
}
});
}
}
}
}
catch (Exception e)
{
log.Error("Failed to perform replication", e);
}
context.WaitForWork(TimeSpan.FromMinutes(1), ref workCounter);
}
}
private bool ReplicateTo(string destination)
{
try
{
using (ReplicationContext.Enter())
{
SourceReplicationInformation sourceReplicationInformation;
try
{
sourceReplicationInformation = GetLastReplicatedEtagFrom(destination);
if (sourceReplicationInformation == null)
{
return(false);
}
}
catch (Exception e)
{
log.Warn("Failed to replicate to: " + destination, e);
return(false);
}
bool?replicated = null;
switch (ReplicateDocuments(destination, sourceReplicationInformation))
{
case true:
replicated = true;
break;
case false:
return(false);
}
switch (ReplicateAttachments(destination, sourceReplicationInformation))
{
case true:
replicated = true;
break;
case false:
return(false);
}
return(replicated ?? false);
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination, new IntHolder());
Thread.VolatileWrite(ref holder.Value, 0);
}
}
private void ReplicateTo(string destination)
{
try
{
using (ReplicationContext.Enter())
{
JArray jsonDocuments;
try
{
var etag = GetLastReplicatedEtagFrom(destination);
if (etag == null)
{
return;
}
jsonDocuments = GetJsonDocuments(etag.Value);
if (jsonDocuments == null || jsonDocuments.Count == 0)
{
return;
}
}
catch (Exception e)
{
log.Warn("Failed to replicate to: " + destination, e);
return;
}
if (TryReplicatingData(destination, jsonDocuments) == false)// failed to replicate, start error handling strategy
{
if (IsFirstFailue(destination))
{
log.InfoFormat(
"This is the first failure for {0}, assuming transinet failure and trying again",
destination);
if (TryReplicatingData(destination, jsonDocuments))// second failure!
{
return;
}
}
IncrementFailureCount(destination);
}
}
}
finally
{
var holder = activeReplicationTasks.GetOrAdd(destination, new IntHolder());
Thread.VolatileWrite(ref holder.Value, 0);
}
}
private bool IsSuccessful(ReplicationContext context)
{
try
{
context.Future.get();
return(true);
}
catch (Exception e) when(e is InterruptedException || e is CancellationException)
{
return(false);
}
catch (ExecutionException e)
{
context.Throwable = e.InnerException;
_slaveCommitFailureLogger.error("Slave " + context.Slave.ServerId + ": Replication commit threw" + (context.Throwable is ComException ? " communication" : "") + " exception:", context.Throwable);
return(false);
}
}
public void PrepareUpdate(IEnumerable <ActorStateDataWrapper> actorStateDataWrapperList, long sequenceNumber)
{
if (sequenceNumber == 0L)
{
return;
}
foreach (ActorStateDataWrapper stateDataWrapper in actorStateDataWrapperList)
{
stateDataWrapper.UpdateSequenceNumber(sequenceNumber);
}
using (_rwLock.AcquireWriteLock())
{
ReplicationContext replicationContext = new ReplicationContext();
/*
* public ReplicationContext()
* {
* IsReplicationComplete = false;
* _replicationException = null;
* _pendingCommitTaskSource = new TaskCompletionSource<object>();
* _associatedEntryCount = 0L;
* }
*/
foreach (ActorStateDataWrapper stateDataWrapper in actorStateDataWrapperList)
{
_uncommittedEntriesList.AddLast(new ListEntry(stateDataWrapper, replicationContext));
/*
* public ListEntry(ActorStateDataWrapper actorStateDataWrapper, ReplicationContext replicationContext)
* {
* ActorStateDataWrapper = actorStateDataWrapper;
* PendingReplicationContext = replicationContext;
* PendingReplicationContext?.AssociateListEntry(); // PendingReplicationContext._associatedEntryCount ++;
* }
*/
}
_pendingReplicationContexts.Add(sequenceNumber, replicationContext);
}
}
public async Task CommitUpdateAsync(long sequenceNumber, Exception ex = null)
{
if (sequenceNumber == 0) // Invalid LSN
{
if (ex != null)
{
throw ex;
}
throw new FabricException(FabricErrorCode.SequenceNumberCheckFailed);
}
// This list is used to store the replication contexts that have been commited
// and are then marked as complete outside the read/write lock. Marking as complete
// outside the lock is important because when the replication context is marked as
// complete by calling TaskCompletionSource.SetResult(), the task associated with
// TaskCompletionSource, immediately starts executing synchronously in the same thread
// (while the lock still being held) which then tries to again acquire read/write lock
// causing System.Threading.LockRecursionException.
//
// In .Net 4.6, TaskCompletionSource.SetResult() accepts an additional argument which
// makes the task associated with TaskCompletionSource execute asynchronously on a different
// thread. Till we move to .Net 4.6, we will adopt the above approach.
var committedReplicationContexts = new List <ReplicationContext>();
ReplicationContext replicationContext = null;
using (this.rwLock.AcquireWriteLock())
{
replicationContext = this.pendingReplicationContexts[sequenceNumber];
replicationContext.SetReplicationComplete(ex);
if (sequenceNumber == this.uncommittedEntriesList.First.Value.ActorStateDataWrapper.SequenceNumber)
{
while (
this.uncommittedEntriesList.Count > 0 &&
this.uncommittedEntriesList.First.Value.IsReplicationComplete)
{
var listNode = this.uncommittedEntriesList.First;
this.uncommittedEntriesList.RemoveFirst();
if (!listNode.Value.IsFailed)
{
this.ApplyUpdate_UnderWriteLock(listNode);
}
listNode.Value.CompleteReplication();
var seqNum = listNode.Value.ActorStateDataWrapper.SequenceNumber;
if (this.pendingReplicationContexts[seqNum].IsAllEntriesComplete)
{
committedReplicationContexts.Add(this.pendingReplicationContexts[seqNum]);
this.pendingReplicationContexts.Remove(seqNum);
}
}
replicationContext = null;
}
}
// Mark the committed replication contexts as complete in order of increasing LSN
foreach (var repCtx in committedReplicationContexts)
{
repCtx.MarkAsCompleted();
}
if (replicationContext != null)
{
await replicationContext.WaitForCompletionAsync();
}
}
public ListEntry(ActorStateDataWrapper actorStateDataWrapper, ReplicationContext replicationContext)
{
ActorStateDataWrapper = actorStateDataWrapper;
PendingReplicationContext = replicationContext;
PendingReplicationContext?.AssociateListEntry();
}
