private void ClientOnReceived(ClientController sender, IRecievedMessage message)
{
var entry = new GatewayEntry {
ClientName = sender.Name, Message = message
};
foreach (var client in Clients.Where(client => client != sender))
{
var messageId = client.Send(message);
if (messageId == null)
{
throw new Exception("Message not sent"); // TODO MKA logging
}
entry.MessageIds.Add(new KeyValuePair <string, ulong>(client.Name,
messageId.Value));
}
_entries.Add(entry);
}
internal async Task<bool> TryUpdateGatewayEntryAsync(GatewayEntry entry, GossipTableEntry row, string eTag)
{
row.Status = entry.Status.ToString();
row.GossipTimestamp = entry.HeartbeatTimestamp;
return (await TryUpdateTableEntryAsync(row, eTag).ConfigureAwait(false));
}
internal async Task<bool> TryCreateGatewayEntryAsync(GatewayEntry entry)
{
var row = new GossipTableEntry()
{
PartitionKey = GlobalServiceId,
RowKey = GossipTableEntry.ConstructRowKey(entry.SiloAddress, entry.ClusterId),
Status = entry.Status.ToString(),
GossipTimestamp = entry.HeartbeatTimestamp
};
return (await TryCreateTableEntryAsync(row).ConfigureAwait(false));
}
internal async Task<GossipTableEntry> ReadGatewayEntryAsync(GatewayEntry gateway)
{
var result = await storage.ReadSingleTableEntryAsync(this.GlobalServiceId, GossipTableEntry.ConstructRowKey(gateway.SiloAddress, gateway.ClusterId)).ConfigureAwait(false);
if (result != null)
{
var tableEntry = result.Item1;
try
{
tableEntry.ParseSiloAddressFromRowKey();
return tableEntry;
}
catch (Exception exc)
{
logger.Error(
ErrorCode.AzureTable_61,
string.Format("Intermediate error parsing GossipTableEntry: {0}. Ignoring this entry.", tableEntry),
exc);
}
}
return null;
}
public async Task AzureGossip_GatewayGossip()
{
// start clean
await this.gossipTable.DeleteAllEntries();
var ts1 = DateTime.UtcNow;
var ts2 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 1);
var ts3 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 2);
var G1 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts1,
Status = GatewayStatus.Active
};
var G2 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var H1 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts2,
Status = GatewayStatus.Active
};
var H2 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
// push G1
await this.gossipTable.Publish(new MultiClusterData(G1));
// push H1, retrieve G1
var answer = await this.gossipTable.Synchronize(new MultiClusterData(H1));
Assert.Equal(1, answer.Gateways.Count);
Assert.True(answer.Gateways.ContainsKey(this.siloAddress1));
Assert.Equal(G1, answer.Gateways[this.siloAddress1]);
// push G2, retrieve H1
answer = await this.gossipTable.Synchronize(new MultiClusterData(G2));
Assert.Equal(1, answer.Gateways.Count);
Assert.True(answer.Gateways.ContainsKey(this.siloAddress2));
Assert.Equal(H1, answer.Gateways[this.siloAddress2]);
// gossip stable
await this.gossipTable.Publish(new MultiClusterData(H1));
await this.gossipTable.Publish(new MultiClusterData(G1));
answer = await this.gossipTable.Synchronize(new MultiClusterData(new GatewayEntry[] { H1, G2 }));
Assert.True(answer.IsEmpty);
// retrieve
answer = await this.gossipTable.Synchronize(new MultiClusterData(new GatewayEntry[] { H1, G2 }));
Assert.True(answer.IsEmpty);
// push H2
await this.gossipTable.Publish(new MultiClusterData(H2));
// retrieve all
answer = await this.gossipTable.Synchronize(new MultiClusterData(new GatewayEntry[] { G1, H1 }));
Assert.Equal(2, answer.Gateways.Count);
Assert.True(answer.Gateways.ContainsKey(this.siloAddress1));
Assert.True(answer.Gateways.ContainsKey(this.siloAddress2));
Assert.Equal(G2, answer.Gateways[this.siloAddress1]);
Assert.Equal(H2, answer.Gateways[this.siloAddress2]);
}
public async Task AzureGossip_GatewayGossip()
{
// start clean
await gossipTable.DeleteAllEntries();
var ts1 = DateTime.UtcNow;
var ts2 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 1);
var ts3 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 2);
var G1 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts1,
Status = GatewayStatus.Active
};
var G2 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var H1 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts2,
Status = GatewayStatus.Active
};
var H2 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
// push G1
await gossipTable.Publish(new MultiClusterData(G1));
// push H1, retrieve G1
var answer = await gossipTable.Synchronize(new MultiClusterData(H1));
Assert.Equal(1, answer.Gateways.Count);
Assert.True(answer.Gateways.ContainsKey(siloAddress1));
Assert.Equal(G1, answer.Gateways[siloAddress1]);
// push G2, retrieve H1
answer = await gossipTable.Synchronize(new MultiClusterData(G2));
Assert.Equal(1, answer.Gateways.Count);
Assert.True(answer.Gateways.ContainsKey(siloAddress2));
Assert.Equal(H1, answer.Gateways[siloAddress2]);
// gossip stable
await gossipTable.Publish(new MultiClusterData(H1));
await gossipTable.Publish(new MultiClusterData(G1));
answer = await gossipTable.Synchronize(new MultiClusterData(new GatewayEntry[] { H1, G2 }));
Assert.True(answer.IsEmpty);
// retrieve
answer = await gossipTable.Synchronize(new MultiClusterData(new GatewayEntry[] { H1, G2 }));
Assert.True(answer.IsEmpty);
// push H2
await gossipTable.Publish(new MultiClusterData(H2));
// retrieve all
answer = await gossipTable.Synchronize(new MultiClusterData(new GatewayEntry[] { G1, H1 }));
Assert.Equal(2, answer.Gateways.Count);
Assert.True(answer.Gateways.ContainsKey(siloAddress1));
Assert.True(answer.Gateways.ContainsKey(siloAddress2));
Assert.Equal(G2, answer.Gateways[siloAddress1]);
Assert.Equal(H2, answer.Gateways[siloAddress2]);
}
// compare gatewayInfo with gatewayInfoInStorage, and
// - write gatewayInfo to storage if it is newer (or do nothing on etag conflict)
// - remove expired gateway info from storage
// - return gatewayInfoInStorage if it is newer
internal async Task<GatewayEntry> DiffAndWriteBackGatewayInfoAsync(GatewayEntry gatewayInfo, GossipTableEntry gatewayInfoInStorage)
{
if ((gatewayInfo != null && !gatewayInfo.Expired)
&& (gatewayInfoInStorage == null || gatewayInfoInStorage.GossipTimestamp < gatewayInfo.HeartbeatTimestamp))
{
// push the more recent gateway info to storage
if (gatewayInfoInStorage == null)
{
await tableManager.TryCreateGatewayEntryAsync(gatewayInfo);
}
else
{
await tableManager.TryUpdateGatewayEntryAsync(gatewayInfo, gatewayInfoInStorage, gatewayInfoInStorage.ETag);
}
}
else if (gatewayInfoInStorage != null &&
(gatewayInfo == null || gatewayInfo.HeartbeatTimestamp < gatewayInfoInStorage.GossipTimestamp))
{
var fromstorage = gatewayInfoInStorage.ToGatewayEntry();
if (fromstorage.Expired)
{
// remove gateway info from storage
await tableManager.TryDeleteGatewayEntryAsync(gatewayInfoInStorage, gatewayInfoInStorage.ETag);
}
else
{
// pull the more recent info from storage
return fromstorage;
}
}
return null;
}
public void MultiClusterData_Gateways()
{
var ts1 = DateTime.UtcNow;
var ts2 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 1);
var ts3 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 2);
IPAddress ip;
Assert.True(IPAddress.TryParse("127.0.0.1", out ip));
IPEndPoint ep1 = new IPEndPoint(ip, 21111);
var siloAddress1 = SiloAddress.New(ep1, 0);
IPEndPoint ep2 = new IPEndPoint(ip, 21112);
var siloAddress2 = SiloAddress.New(ep2, 0);
var G1 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts1,
Status = GatewayStatus.Active
};
var G2 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var H1 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts2,
Status = GatewayStatus.Active
};
var H2 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var gd1 = new MultiClusterData();
var gd2 = new MultiClusterData(G1);
var gd3 = new MultiClusterData(G2);
TestAlgebraicProperties(gd1, gd1);
TestAlgebraicProperties(gd2, gd2);
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd3, gd1);
TestAlgebraicProperties(gd2, gd3);
TestAlgebraicProperties(gd3, gd2);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H2, G1 });
TestAlgebraicProperties(gd1, gd1);
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H1, G1 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { G1 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H2 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
}
public void MultiClusterData_Gateways()
{
var ts1 = DateTime.UtcNow;
var ts2 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 1);
var ts3 = ts1 + new TimeSpan(hours: 0, minutes: 0, seconds: 2);
IPAddress ip;
Assert.True(IPAddress.TryParse("127.0.0.1", out ip));
IPEndPoint ep1 = new IPEndPoint(ip, 21111);
var siloAddress1 = SiloAddress.New(ep1, 0);
IPEndPoint ep2 = new IPEndPoint(ip, 21112);
var siloAddress2 = SiloAddress.New(ep2, 0);
var G1 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts1,
Status = GatewayStatus.Active
};
var G2 = new GatewayEntry()
{
SiloAddress = siloAddress1,
ClusterId = "1",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var H1 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts2,
Status = GatewayStatus.Active
};
var H2 = new GatewayEntry()
{
SiloAddress = siloAddress2,
ClusterId = "2",
HeartbeatTimestamp = ts3,
Status = GatewayStatus.Inactive
};
var gd1 = new MultiClusterData();
var gd2 = new MultiClusterData(G1);
var gd3 = new MultiClusterData(G2);
TestAlgebraicProperties(gd1, gd1);
TestAlgebraicProperties(gd2, gd2);
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd3, gd1);
TestAlgebraicProperties(gd2, gd3);
TestAlgebraicProperties(gd3, gd2);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H2, G1 });
TestAlgebraicProperties(gd1, gd1);
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H1, G1 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { G1 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
gd1 = new MultiClusterData(new GatewayEntry[] { H1, G2 });
gd2 = new MultiClusterData(new GatewayEntry[] { H2 });
TestAlgebraicProperties(gd1, gd2);
TestAlgebraicProperties(gd2, gd1);
}