public void WillSkipSegmentsThatHasItemsInThem()
{
var ranges = Enumerable.Range(0, 500).ToArray();
var assignedSegments = replication.AssignAllEmptySegments(NodeEndpoint.ForTest(1), ReplicationType.Ownership, ranges);
Assert.Equal(ranges.Skip(1).ToArray(), assignedSegments);
}
public void WillAssignAllSegmentsWeAskForImmediately()
{
var ranges = Enumerable.Range(0, 500).ToArray();
var assignedSegments = replication.AssignAllEmptySegments(NodeEndpoint.ForTest(1), ReplicationType.Ownership, ranges);
Assert.Equal(ranges, assignedSegments);
}
public void CanReplicateSegmentWithDataWhileStillServingRequestForSegment()
{
using (var storageProxy = new DistributedHashTableStorageClient(storageHost.Endpoint))
{
var topology = new DistributedHashTableMasterClient(masterUri).GetTopology();
storageProxy.Put(topology.Version, new ExtendedPutRequest
{
Bytes = new byte[] { 1, 2, 3 },
Key = "test",
Segment = 1,
});
var result = storageProxy.ReplicateNextPage(NodeEndpoint.ForTest(13), ReplicationType.Ownership, 1);
Assert.Equal("test", result.PutRequests[0].Key);
storageProxy.Put(topology.Version, new ExtendedPutRequest
{
Bytes = new byte[] { 1, 2, 3 },
Key = "test2",
Segment = 1,
});
result = storageProxy.ReplicateNextPage(NodeEndpoint.ForTest(13), ReplicationType.Ownership, 1);
Assert.Equal("test2", result.PutRequests[0].Key);
}
}
public void AfterBackupsCompleteWillStartReplicationAgain()
{
node.Storage.Stub(x => x.Get(0)).IgnoreArguments().Return(new Value[0][]);
OnlineSegmentReplicationCommand command = null;
executer.Stub(x => x.RegisterForExecution(Arg <OnlineSegmentReplicationCommand> .Is.TypeOf))
.WhenCalled(invocation => command = (OnlineSegmentReplicationCommand)invocation.Arguments[0]);
node.SetTopology(new Topology(new[]
{
new Segment
{
AssignedEndpoint = NodeEndpoint.ForTest(91),
PendingBackups = { endPoint }
},
}, 1));
command.RaiseCompleted();
node.SetTopology(new Topology(new[]
{
new Segment
{
AssignedEndpoint = NodeEndpoint.ForTest(91),
PendingBackups = { endPoint }
},
}, 2));
executer.AssertWasCalled(
x => x.RegisterForExecution(Arg <OnlineSegmentReplicationCommand> .Is.TypeOf),
o => o.Repeat.Twice());
}
public void CanGiveUpOnSegment()
{
var existingEndpoint = new NodeEndpoint
{
Async = new Uri("rhino.queues://other:2202/replication"),
Sync = new Uri("rhino.dht://other:2201")
};
masterProxy.Join(existingEndpoint);
var newEndpoint = new NodeEndpoint
{
Async = new Uri("rhino.queues://localhost:2202/replication"),
Sync = new Uri("rhino.dht://localhost:2201")
};
var segments = masterProxy.Join(newEndpoint);
masterProxy.GaveUp(newEndpoint, ReplicationType.Ownership, segments[0].Index, segments[1].Index);
var topology = masterProxy.GetTopology();
Assert.Equal(existingEndpoint, topology.Segments[segments[0].Index].AssignedEndpoint);
Assert.Equal(existingEndpoint, topology.Segments[segments[1].Index].AssignedEndpoint);
Assert.Null(topology.Segments[segments[0].Index].InProcessOfMovingToEndpoint);
Assert.Null(topology.Segments[segments[1].Index].InProcessOfMovingToEndpoint);
}
public int[] AssignAllEmptySegments(NodeEndpoint replicationEndpoint,
ReplicationType type,
int[] segments)
{
var reservedSegments = new List <int>();
hashTable.Batch(actions =>
{
foreach (var segment in segments)
{
if (actions.HasTag(segment))
{
continue;
}
if (type == ReplicationType.Ownership &&
MarkSegmentAsAssignedToEndpoint(actions, replicationEndpoint, segment) == false)
{
continue;
}
reservedSegments.Add(segment);
}
actions.Commit();
});
return(reservedSegments.ToArray());
}
public IDistributedHashTableStorage Create(NodeEndpoint endpoint)
{
PooledDistributedHashTableStorageClientConnection storage = null;
lock (locker)
{
LinkedList <PooledDistributedHashTableStorageClientConnection> value;
if (pooledConnections.TryGetValue(endpoint, out value) && value.Count > 0)
{
storage = value.First.Value;
value.RemoveFirst();
}
}
if (storage != null)
{
if (storage.Connected == false)
{
log.DebugFormat("Found unconnected connection in the pool for {0}", endpoint.Sync);
try
{
storage.Dispose();
}
catch (Exception e)
{
log.Debug("Error when disposing unconnected connection in the pool", e);
}
}
else
{
return(storage);
}
}
log.DebugFormat("Creating new connection in the pool to {0}", endpoint.Sync);
return(new PooledDistributedHashTableStorageClientConnection(this, endpoint));
}
public void WillNotStartReplicationIfCurrentlyReplicatingBackups()
{
node.Storage.Stub(x => x.Get(0)).IgnoreArguments().Return(new Value[0][]);
node.SetTopology(new Topology(new[]
{
new Segment
{
AssignedEndpoint = NodeEndpoint.ForTest(91),
PendingBackups = { endPoint }
},
}, 1));
node.SetTopology(new Topology(new[]
{
new Segment
{
AssignedEndpoint = NodeEndpoint.ForTest(91),
PendingBackups = { endPoint }
},
}, 2));
executer.AssertWasCalled(
x => x.RegisterForExecution(Arg <OnlineSegmentReplicationCommand> .Is.TypeOf),
o => o.Repeat.Once());
}
private bool[] GetRemovesResults(NodeEndpoint endpoint,
ExtendedRemoveRequest[] removeRequests,
int backupIndex)
{
try
{
using (var client = pool.Create(endpoint))
{
return(client.Remove(topology.Version, removeRequests));
}
}
catch (SeeOtherException soe)
{
return(GetRemovesResults(soe.Endpoint, removeRequests, backupIndex));
}
catch (TopologyVersionDoesNotMatchException)
{
RefreshTopology();
return(RemoveInternal(removeRequests, backupIndex));
}
catch (Exception)
{
try
{
return(RemoveInternal(removeRequests, backupIndex + 1));
}
catch (NoMoreBackupsException)
{
}
throw;
}
}
private PutResult[] GetPutsResults(NodeEndpoint endpoint,
ExtendedPutRequest[] putRequests,
int backupIndex)
{
try
{
using (var client = pool.Create(endpoint))
{
return(client.Put(topology.Version, putRequests));
}
}
catch (SeeOtherException soe)
{
return(GetPutsResults(soe.Endpoint, putRequests, backupIndex));
}
catch (TopologyVersionDoesNotMatchException)
{
RefreshTopology();
return(PutInternal(putRequests, backupIndex));
}
catch (Exception)
{
try
{
return(PutInternal(putRequests, backupIndex + 1));
}
catch (NoMoreBackupsException)
{
}
throw;
}
}
public void WillGiveExistingKeysFromSegment()
{
var result = replication.ReplicateNextPage(NodeEndpoint.ForTest(1), ReplicationType.Ownership, 0);
Assert.Equal(1, result.PutRequests.Length);
Assert.Equal(new byte[] { 1 }, result.PutRequests[0].Bytes);
}
public OnGaveUp()
{
master = new DistributedHashTableMaster();
master.CaughtUp(NodeEndpoint.ForTest(9),
ReplicationType.Ownership,
master.Join(NodeEndpoint.ForTest(9)).Select(x => x.Index).ToArray());
endPoint = NodeEndpoint.ForTest(5);
}
public void AddingNewNodeResultInAllSegmentsHavingAtLeastTwoBackupCopy()
{
var yetAnotherEndPoint = NodeEndpoint.ForTest(7);
var ranges = master.Join(yetAnotherEndPoint);
master.CaughtUp(yetAnotherEndPoint, ReplicationType.Ownership, ranges.Select(x => x.Index).ToArray());
Assert.True(master.Segments.All(x => x.PendingBackups.Count >= 2));
}
public HandlingAssignedSegments()
{
master = MockRepository.GenerateStub <IDistributedHashTableMaster>();
executer = MockRepository.GenerateStub <IExecuter>();
endPoint = NodeEndpoint.ForTest(1);
node = new DistributedHashTableNode(master, executer, new BinaryMessageSerializer(), endPoint, MockRepository.GenerateStub <IQueueManager>(),
MockRepository.GenerateStub <IDistributedHashTableNodeReplicationFactory>());
}
public void DirectlyModifyingTopologyAndThenCallingRefreshEndpointsShouldShowAllEndpoints()
{
Assert.False(master.Endpoints.Any(x => x == NodeEndpoint.ForTest(1)));
master.Topology.Segments[0].AssignedEndpoint = NodeEndpoint.ForTest(1);
master.RefreshEndpoints();
Assert.True(master.Endpoints.Any(x => x == NodeEndpoint.ForTest(1)));
}
public void WillRememberKeysSentDuringPreviousReplication()
{
var result = replication.ReplicateNextPage(NodeEndpoint.ForTest(1), ReplicationType.Ownership, 0);
Assert.Equal(1, result.PutRequests.Length);
result = replication.ReplicateNextPage(NodeEndpoint.ForTest(1), ReplicationType.Ownership, 0);
Assert.Equal(0, result.PutRequests.Length);
}
public OnMasterWithOneExistingNode()
{
master = new DistributedHashTableMaster();
endPoint = NodeEndpoint.ForTest(9);
var existingEndpoint = NodeEndpoint.ForTest(3);
var ranges = master.Join(existingEndpoint);
master.CaughtUp(existingEndpoint, ReplicationType.Ownership, ranges.Select(x => x.Index).ToArray());
}
public WhenFinishedReplicatingSegment()
{
master = MockRepository.GenerateStub <IDistributedHashTableMaster>();
executer = MockRepository.GenerateStub <IExecuter>();
endPoint = NodeEndpoint.ForTest(1);
master.Stub(x => x.Join(Arg.Is(endPoint)))
.Return(new Segment[0]);
node = new DistributedHashTableNode(master, executer, new BinaryMessageSerializer(), endPoint, MockRepository.GenerateStub <IQueueManager>(),
MockRepository.GenerateStub <IDistributedHashTableNodeReplicationFactory>());
}
public void CanReplicateEmptySegments()
{
using (var storageProxy = new DistributedHashTableStorageClient(storageHost.Endpoint))
{
var segments = new[] { 1, 2, 3 };
var assignedSegments = storageProxy.AssignAllEmptySegments(NodeEndpoint.ForTest(13), ReplicationType.Ownership, segments);
Assert.Equal(segments, assignedSegments);
}
}
public void CanJoinToMaster()
{
var endpoint = new NodeEndpoint
{
Async = new Uri("rhino.queues://localhost:2202/replication"),
Sync = new Uri("rhino.dht://localhost:2201")
};
var segments = masterProxy.Join(endpoint);
Assert.Equal(Constants.NumberOfSegments, segments.Length);
Assert.True(segments.All(x => x.AssignedEndpoint.Equals(endpoint)));
}
public OnlineSegmentReplicationCommand(
NodeEndpoint endpoint,
Segment[] segments,
ReplicationType type,
IDistributedHashTableNode node,
IDistributedHashTableNodeReplicationFactory factory)
{
this.endpoint = endpoint;
this.segments = segments;
this.type = type;
this.node = node;
this.factory = factory;
}
public void TopologyContainsPendingBackupsForCurrentNodeWillStartsBackupReplication()
{
node.SetTopology(new Topology(new[]
{
new Segment
{
AssignedEndpoint = NodeEndpoint.ForTest(91),
PendingBackups = { endPoint }
},
}, 1));
executer.AssertWasCalled(x => x.RegisterForExecution(Arg <OnlineSegmentReplicationCommand> .Is.TypeOf));
}
private static bool MarkSegmentAsAssignedToEndpoint(PersistentHashTableActions actions,
NodeEndpoint endpoint,
int segment)
{
var result = actions.Put(new PutRequest
{
Key = Constants.MovedSegment + segment,
OptimisticConcurrency = true,
Bytes = endpoint.ToBytes(),
});
return(result.ConflictExists == false);
}
public void RemovingKeyInSegmentAsisgnedElsewhereShouldThrow()
{
var ranges = Enumerable.Range(0, 500).ToArray();
replication.AssignAllEmptySegments(NodeEndpoint.ForTest(1), ReplicationType.Ownership, ranges);
var exception = Assert.Throws <SeeOtherException>(() => node.Storage.Remove(topologyVersion, new ExtendedRemoveRequest
{
Key = "test",
Segment = 0,
}));
Assert.Equal(NodeEndpoint.ForTest(1), exception.Endpoint);
}
public void WillRegisterSegmentsNotAssignedToMeForReplication()
{
master.Stub(x => x.Join(Arg.Is(endPoint)))
.Return(new[]
{
new Segment {
AssignedEndpoint = NodeEndpoint.ForTest(9)
},
});
node.Start();
executer.AssertWasCalled(x => x.RegisterForExecution(Arg <OnlineSegmentReplicationCommand> .Is.TypeOf));
}
public void WhenNoSegmentIsAssignedToNodeStateWillBeStarting()
{
master.Stub(x => x.Join(Arg.Is(endPoint)))
.Return(new[]
{
new Segment {
AssignedEndpoint = NodeEndpoint.ForTest(9)
},
});
node.Start();
Assert.Equal(NodeState.Starting, node.State);
}
public void WhenItemIsRemovedWillResultInRemovalRequestOnNextReplicationPage()
{
var result = replication.ReplicateNextPage(NodeEndpoint.ForTest(1), ReplicationType.Ownership, 0);
Assert.Equal(1, result.PutRequests.Length);
node.Storage.Remove(node.GetTopologyVersion(), new ExtendedRemoveRequest
{
Key = "test",
SpecificVersion = putResult.Version,
Segment = 0,
});
result = replication.ReplicateNextPage(NodeEndpoint.ForTest(1), ReplicationType.Ownership, 0);
Assert.Equal(1, result.RemoveRequests.Length);
}
public void WhenDoneGettingAllKeysWillAssignSegmentToEndpoint()
{
var result = replication.ReplicateNextPage(NodeEndpoint.ForTest(1), ReplicationType.Ownership, 0);
Assert.Equal(1, result.PutRequests.Length);
result = replication.ReplicateNextPage(NodeEndpoint.ForTest(1), ReplicationType.Ownership, 0);
Assert.Equal(0, result.PutRequests.Length);
var exception = Assert.Throws <SeeOtherException>(() => node.Storage.Remove(topologyVersion, new ExtendedRemoveRequest
{
Key = "test",
Segment = 0,
}));
Assert.Equal(NodeEndpoint.ForTest(1), exception.Endpoint);
}
public void WhenFinishedReplicatingWillTellTheReplicatorSo()
{
using (var storageProxy = new DistributedHashTableStorageClient(storageHost.Endpoint))
{
var topology = new DistributedHashTableMasterClient(masterUri).GetTopology();
storageProxy.Put(topology.Version, new ExtendedPutRequest
{
Bytes = new byte[] { 1, 2, 3 },
Key = "test",
Segment = 1,
});
var result = storageProxy.ReplicateNextPage(NodeEndpoint.ForTest(13), ReplicationType.Ownership, 1);
Assert.Equal("test", result.PutRequests[0].Key);
result = storageProxy.ReplicateNextPage(NodeEndpoint.ForTest(13), ReplicationType.Ownership, 1);
Assert.True(result.Done);
}
}
public void WhenReplicatingEmptySegmentsWillNotReplicateSegmentsThatHasValues()
{
using (var storageProxy = new DistributedHashTableStorageClient(storageHost.Endpoint))
{
var topology = new DistributedHashTableMasterClient(masterUri).GetTopology();
storageProxy.Put(topology.Version, new ExtendedPutRequest
{
Bytes = new byte[] { 1, 2, 3 },
Key = "test",
Segment = 1,
});
var segments = new[] { 1, 2, 3 };
var assignedSegments = storageProxy.AssignAllEmptySegments(NodeEndpoint.ForTest(13), ReplicationType.Ownership, segments);
Assert.Equal(new[] { 2, 3 }, assignedSegments);
}
}
