/// <summary>
/// Creates a new node or updates an existing or delete a node.
/// </summary>
/// <param name="kind">Kind of operation.</param>
/// <param name="key">Key of the node.</param>
/// <param name="value">Value of the node.</param>
/// <param name="onCommit">Action performed when the replication is completed.</param>
/// <param name="onFailure">Action performed when the replication is failed.</param>
internal void Write(NodeOperation.OperationKind kind, string key, ArraySegment <byte> value, Action onCommit, Action onFailure)
{
var nodeOp = new NodeOperation(kind, this.lastSequenceNumber, key, value);
var pendingOp = new PendingOperation {
Operation = nodeOp, OnCommit = onCommit, OnFailure = onFailure,
};
this.pendingOperations.Add(pendingOp);
}
private static void WriteBinary(BinaryWriter binaryWriter, NodeOperation nodeOperation)
{
binaryWriter.Write((byte)nodeOperation.Operation);
binaryWriter.Write(nodeOperation.SequenceNumber);
// Nop is only for broadcasting sequence number, no key/value data.
if (nodeOperation.Operation != OperationKind.Nop)
{
binaryWriter.Write(nodeOperation.Key);
binaryWriter.Write(nodeOperation.Value.Count);
if (nodeOperation.Value.Count > 0)
{
binaryWriter.Write(nodeOperation.Value.Array, nodeOperation.Value.Offset, nodeOperation.Value.Count);
}
}
}
public void Serialization()
{
var nodeOp1 = new NodeOperation(
NodeOperation.OperationKind.Create,
10,
"A",
new ArraySegment <byte>(new byte[] { 1, 2, 3, }));
var nodeOp2 = new NodeOperation(
NodeOperation.OperationKind.Update,
11,
"BB",
new ArraySegment <byte>(new byte[] { 2, 3, 4, 5, }));
var nodeOp3 = new NodeOperation(
NodeOperation.OperationKind.Delete,
12,
"CCC",
default);
var opData = NodeOperation.GetOperationData(new[] { nodeOp1, nodeOp2, nodeOp3, });
Assert.AreEqual(1, opData.Count);
var ops = NodeOperation.GetNodeOperationFromBytes(opData.First()).ToArray();
Assert.AreEqual(3, ops.Length);
Assert.AreEqual(NodeOperation.OperationKind.Create, ops[0].Operation);
Assert.AreEqual("A", ops[0].Key);
Assert.AreEqual(3, ops[0].Value.Count);
Assert.AreEqual(NodeOperation.OperationKind.Update, ops[1].Operation);
Assert.AreEqual("BB", ops[1].Key);
Assert.AreEqual(4, ops[1].Value.Count);
Assert.AreEqual(NodeOperation.OperationKind.Delete, ops[2].Operation);
Assert.AreEqual("CCC", ops[2].Key);
Assert.AreEqual(0, ops[2].Value.Count);
Assert.IsNull(ops[2].Value.Array);
Assert.IsNotNull(ops[0].ToString());
Assert.IsNotNull(ops[1].ToString());
Assert.IsNotNull(ops[2].ToString());
}
private Task <bool> ProcessReplicatedOperation(IOperation op)
{
var succeeded = true;
foreach (var data in op.Data)
{
foreach (var nodeOp in NodeOperation.GetNodeOperationFromBytes(data))
{
////ServiceEventSource.Current.ServiceMessage(this.Context, "Received #{0} - {1}", sequenceNumber, nodeOp);
switch (nodeOp.Operation)
{
case NodeOperation.OperationKind.Nop:
// No operation, primary is telling its sequence number to secondaries.
break;
case NodeOperation.OperationKind.Create:
case NodeOperation.OperationKind.Update:
var valueArray = new byte[nodeOp.Value.Count];
Array.Copy(nodeOp.Value.Array, nodeOp.Value.Offset, valueArray, 0, nodeOp.Value.Count);
succeeded &= this.kvsData.AddOrUpdate(nodeOp.Key, nodeOp.SequenceNumber, valueArray);
break;
case NodeOperation.OperationKind.Delete:
succeeded &= this.kvsData.TryRemove(nodeOp.Key, nodeOp.SequenceNumber);
break;
default:
ServiceEventSource.Current.ServiceMessage(this.Context, $"Unknown node operation: {nodeOp}");
break;
}
}
}
ServiceEventSource.Current.ServiceMessage(this.Context, $"Process replicated op, SN={op.SequenceNumber} succeeded={succeeded}");
Interlocked.Exchange(ref this.lastSequenceNumber, op.SequenceNumber - 1);
// The caller is supposed to ack the operation
return(Task.FromResult(succeeded));
}
private async Task ProcessIncomingRequests()
{
CancellationToken cancellation = this.cancellationTokenSource.Token;
var replicationTaskSem = new SemaphoreSlim(128, 128);
try
{
while (!cancellation.IsCancellationRequested)
{
var ops = new List <PendingOperation>();
var op = this.pendingOperations.Take(cancellation);
do
{
ops.Add(op);
}while (this.pendingOperations.TryTake(out op));
var opData = NodeOperation.GetOperationData(ops.Select(o => o.Operation));
try
{
await replicationTaskSem.WaitAsync(cancellation).ConfigureAwait(false);
var replicateTask = this.stateReplicator.ReplicateAsync(opData, cancellation, out long lsn);
var unsed = replicateTask.ContinueWith(
t =>
{
replicationTaskSem.Release();
foreach (var operation in ops)
{
switch (operation.Operation.Operation)
{
case NodeOperation.OperationKind.Nop:
// No operation, primary is telling its sequence number to secondaries.
break;
case NodeOperation.OperationKind.Create:
case NodeOperation.OperationKind.Update:
var data = new byte[operation.Operation.Value.Count];
Array.Copy(operation.Operation.Value.Array, operation.Operation.Value.Offset, data, 0, operation.Operation.Value.Count);
this.kvsData.AddOrUpdate(operation.Operation.Key, operation.Operation.SequenceNumber, data);
break;
case NodeOperation.OperationKind.Delete:
this.kvsData.TryRemove(operation.Operation.Key, operation.Operation.SequenceNumber);
break;
default:
throw new InvalidOperationException();
}
operation.OnCommit();
}
});
// Writes to disk while the replication to secondaries is started.
await this.operationLoggerPrimary.WriteOperations(lsn, opData);
ServiceEventSource.Current.ServiceMessage(this.Context, $"Replicated and persisted on primary, SN={lsn}");
Interlocked.Exchange(ref this.lastSequenceNumber, lsn);
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
foreach (var operation in ops)
{
operation.OnFailure();
}
}
}
SpinWait.SpinUntil(() => replicationTaskSem.CurrentCount == 128);
}
catch (OperationCanceledException)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "ProcessIncomingRequests operation cancelled.");
}
catch (Exception ex)
{
ServiceEventSource.Current.ServiceMessage(this.Context, "ProcessIncomingRequests unhandled exception: {0}", ex);
}
finally
{
ServiceEventSource.Current.ServiceMessage(this.Context, "ProcessIncomingRequests terminated.");
}
}