// Assumes partitions_ is not null and the data structures are already locked
private static void RemovePartitionFromService(Guid partitionId)
{
EnvoyDefaults.LogMessage(String.Format("Removed: {0}", partitionId));
foreach (var service in services_)
{
service.Value.Partitions.RemoveAll(x => x == partitionId);
}
foreach (var service in services_.Where(x => x.Value.Partitions.Count == 0).ToList())
{
services_.Remove(service.Key);
}
}
private static void AddPartitionToService(Guid partitionId, SF_Partition partitionInfo)
{
EnvoyDefaults.LogMessage(String.Format("Added: {0}={1}", partitionId,
JsonConvert.SerializeObject(partitionInfo)));
for (int listenerIndex = 0; listenerIndex < partitionInfo.listeners_.Count; listenerIndex++)
{
string serviceName = CalculateNameForService(partitionInfo, listenerIndex);
if (!services_.ContainsKey(serviceName))
{
services_[serviceName] = new ServicePartitions
{
EndpointIndex = listenerIndex,
StatefulService = (partitionInfo.serviceKind_ == ServiceKind.Stateful),
Partitions = new List <Guid>()
};
}
var entry = services_[serviceName].Partitions.Find(x => x == partitionId);
if (entry == Guid.Empty)
{
services_[serviceName].Partitions.Add(partitionId);
}
}
}
/// <summary>
/// This function gathers the state of the cluster on startup and caches the information
/// Changes to cluster state are handled through notifications.
///
/// Capture information for each replica for every service running in the cluster.
/// </summary>
/// <returns></returns>
public static async Task InitializePartitionData()
{
EnvoyDefaults.LogMessage("InitializePartitionData started");
// Populate data locally
Dictionary <Guid, SF_Partition> partitionData = new Dictionary <Guid, SF_Partition>();
var queryManager = client.QueryManager;
ApplicationList applications = null;
try
{
applications = await queryManager.GetApplicationListAsync();
}
catch (Exception e)
{
EnvoyDefaults.LogMessage("GetApplicationListAsync failed");
EnvoyDefaults.LogMessage(String.Format("Error={0}", e.Message));
EnvoyDefaults.LogMessage(String.Format("Error={0}", e.StackTrace));
Environment.FailFast("GetApplicationListAsync failed");
}
EnvoyDefaults.LogMessage("GetApplicationListAsync Succeeded");
foreach (var application in applications)
{
var services = await queryManager.GetServiceListAsync(application.ApplicationName);
foreach (var service in services)
{
var partitions = await queryManager.GetPartitionListAsync(service.ServiceName);
foreach (var partition in partitions)
{
List <SF_Endpoint> listeners = new List <SF_Endpoint>();
var replicas = await queryManager.GetReplicaListAsync(partition.PartitionInformation.Id);
foreach (var replica in replicas)
{
if (replica.ReplicaAddress.Length == 0)
{
continue;
}
JObject addresses;
try
{
addresses = JObject.Parse(replica.ReplicaAddress);
}
catch
{
continue;
}
var replicaListeners = addresses["Endpoints"].Value <JObject>();
foreach (var replicaListener in replicaListeners)
{
var role = ServiceEndpointRole.Stateless;
if (partition.ServiceKind == ServiceKind.Stateful)
{
var statefulRole = ((StatefulServiceReplica)replica).ReplicaRole;
switch (statefulRole)
{
case ReplicaRole.Primary:
role = ServiceEndpointRole.StatefulPrimary;
break;
case ReplicaRole.ActiveSecondary:
role = ServiceEndpointRole.StatefulSecondary;
break;
default:
role = ServiceEndpointRole.Invalid;
break;
}
}
int listenerIndex = listeners.FindIndex(x => x.Name == replicaListener.Key);
if (listenerIndex == -1)
{
listeners.Add(new SF_Endpoint(replicaListener.Key));
listenerIndex = listeners.Count - 1;
}
try
{
var listenerAddressString = replicaListener.Value.ToString();
if (!listenerAddressString.StartsWith("http") &&
!listenerAddressString.StartsWith("https"))
{
continue;
}
var listenerAddress = new Uri(replicaListener.Value.ToString());
if (listenerAddress.HostNameType == UriHostNameType.Dns)
{
var ipaddrs = Dns.GetHostAddresses(listenerAddress.Host);
foreach (var ipaddr in ipaddrs)
{
if (ipaddr.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)
{
var saddrstring = ipaddr.ToString();
if (saddrstring.StartsWith("172"))
{
listenerAddress = new Uri(listenerAddress.Scheme + "://" +
saddrstring +
":" + listenerAddress.Port + listenerAddress.PathAndQuery);
break;
}
}
}
}
listeners[listenerIndex].AddInstance(role, listenerAddress);
}
catch (System.Exception e)
{
EnvoyDefaults.LogMessage(String.Format("Error={0}", e));
}
}
}
// Remove any listeners without active endpoints
listeners.RemoveAll(x => x.InstanceCount() == 0);
if (listeners.Count == 0)
{
continue;
}
var partitionInfo = new SF_Partition(service.ServiceName,
service.ServiceKind,
partition.PartitionInformation,
null,
listeners);
partitionData[partition.PartitionInformation.Id] = partitionInfo;
}
}
}
// Process changes received through notifications
lock (lock_)
{
foreach (var partition in partitionsAdd_)
{
partitionData[partition.Key] = partition.Value;
}
foreach (var partition in partitionsRemove_)
{
partitionData.Remove(partition.Key);
}
// Finally update global state, populate Service List and log details
partitions_ = partitionData;
services_ = new Dictionary <string, ServicePartitions>();
foreach (var partition in partitionData)
{
AddPartitionToService(partition.Key, partition.Value);
EnvoyDefaults.LogMessage(String.Format("Added: {0}={1}", partition.Key,
JsonConvert.SerializeObject(partition.Value)));
}
partitionsRemove_ = null;
partitionsAdd_ = null;
}
}
private static void Handler(Object sender, EventArgs eargs)
{
try
{
var notification = ((FabricClient.ServiceManagementClient.ServiceNotificationEventArgs)eargs).Notification;
if (notification.Endpoints.Count == 0)
{
//remove
lock (lock_)
{
if (partitions_ != null)
{
partitions_.Remove(notification.PartitionId);
RemovePartitionFromService(notification.PartitionId);
}
else
{
partitionsAdd_.Remove(notification.PartitionId);
partitionsRemove_[notification.PartitionId] = null;
}
EnvoyDefaults.LogMessage(String.Format("Removed: {0}", notification.PartitionId));
return;
}
}
List <SF_Endpoint> listeners = new List <SF_Endpoint>();
ServiceEndpointRole role = ServiceEndpointRole.Invalid;
foreach (var notificationEndpoint in notification.Endpoints)
{
if (notificationEndpoint.Address.Length == 0)
{
continue;
}
JObject addresses;
try
{
addresses = JObject.Parse(notificationEndpoint.Address);
}
catch
{
continue;
}
var notificationListeners = addresses["Endpoints"].Value <JObject>();
foreach (var notificationListener in notificationListeners)
{
int listenerIndex = listeners.FindIndex(x => x.Name == notificationListener.Key);
if (listenerIndex == -1)
{
listeners.Add(new SF_Endpoint(notificationListener.Key));
listenerIndex = listeners.Count - 1;
}
try
{
var listenerAddressString = notificationListener.Value.ToString();
if (!listenerAddressString.StartsWith("http") &&
!listenerAddressString.StartsWith("https"))
{
continue;
}
var listenerAddress = new Uri(listenerAddressString);
if (listenerAddress.HostNameType == UriHostNameType.Dns)
{
var ipaddrs = Dns.GetHostAddresses(listenerAddress.Host);
foreach (var ipaddr in ipaddrs)
{
if (ipaddr.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)
{
var saddrstring = ipaddr.ToString();
if (saddrstring.StartsWith("172"))
{
listenerAddress = new Uri(listenerAddress.Scheme + "://" +
saddrstring +
":" + listenerAddress.Port + listenerAddress.PathAndQuery);
break;
}
}
}
}
listeners[listenerIndex].AddInstance(notificationEndpoint.Role, listenerAddress);
}
catch (System.Exception e)
{
EnvoyDefaults.LogMessage(String.Format("Error={0}", e));
}
}
if (role == ServiceEndpointRole.Invalid)
{
role = notificationEndpoint.Role;
}
}
// Remove any listeners without active endpoints
listeners.RemoveAll(x => x.InstanceCount() == 0);
if (listeners.Count == 0)
{
return;
}
var partitionInfo = new SF_Partition(notification.ServiceName,
(role == ServiceEndpointRole.Stateless) ? ServiceKind.Stateless : ServiceKind.Stateful,
notification.PartitionInfo,
notification.Version,
listeners
);
lock (lock_)
{
if (partitions_ != null)
{
partitions_[notification.PartitionId] = partitionInfo;
AddPartitionToService(notification.PartitionId, partitionInfo);
}
else
{
partitionsRemove_.Remove(notification.PartitionId);
partitionsAdd_[notification.PartitionId] = partitionInfo;
}
EnvoyDefaults.LogMessage(String.Format("Added: {0}={1}", notification.PartitionId,
JsonConvert.SerializeObject(partitionInfo)));
}
}
catch (Exception e)
{
EnvoyDefaults.LogMessage(String.Format("Error={0}", e.Message));
EnvoyDefaults.LogMessage(String.Format("Error={0}", e.StackTrace));
}
}
static SF_Services()
{
try
{
partitions_ = null;
if (EnvoyDefaults.client_cert_subject_name != null)
{
X509Credentials creds = new X509Credentials();
creds.FindType = X509FindType.FindBySubjectName;
creds.FindValue = EnvoyDefaults.client_cert_subject_name;
if (EnvoyDefaults.client_cert_issuer_thumbprints != null)
{
foreach (var issuer in EnvoyDefaults.client_cert_issuer_thumbprints)
{
creds.IssuerThumbprints.Add(issuer);
}
}
if (EnvoyDefaults.server_cert_common_names != null)
{
foreach (var commonName in EnvoyDefaults.server_cert_common_names)
{
creds.RemoteCommonNames.Add(commonName);
}
}
else
{
creds.RemoteCommonNames.Add(EnvoyDefaults.client_cert_subject_name);
}
if (EnvoyDefaults.server_cert_issuer_thumbprints != null)
{
foreach (var issuer in EnvoyDefaults.server_cert_issuer_thumbprints)
{
creds.RemoteCertThumbprints.Add(issuer);
}
}
else if (EnvoyDefaults.client_cert_issuer_thumbprints != null)
{
foreach (var issuer in EnvoyDefaults.client_cert_issuer_thumbprints)
{
creds.RemoteCertThumbprints.Add(issuer);
}
}
creds.StoreLocation = StoreLocation.LocalMachine;
creds.StoreName = "/app/sfcerts";
client = new FabricClient(creds, EnvoyDefaults.fabricUri);
}
else
{
client = new FabricClient(EnvoyDefaults.fabricUri);
}
EnvoyDefaults.LogMessage("Client sucessfully created");
EnableResolveNotifications.RegisterNotificationFilter("fabric:", client, Handler);
EnvoyDefaults.LogMessage("Notification handler sucessfully set");
}
catch (Exception e)
{
EnvoyDefaults.LogMessage(String.Format("Error={0}", e));
}
}