private static object ExtractPartitionEntityFromReconfigEvent(
IClusterQuery clusterQuery,
ReconfigurationCompletedTraceRecord reconfigEvent,
CancellationToken cancelToken)
{
return(clusterQuery.GetPartitionAsync(reconfigEvent.PartitionId, cancelToken).GetAwaiter().GetResult());
}
/// <inheritdoc />
protected TimeTriggeredAgent(
ITaskRunner taskRunner,
ILogger logger,
IStoreProvider storeProvider,
ITraceStoreReader eventStoreReader,
ITraceStoreReader queryStoreReader,
IClusterQuery clusterQuery,
CancellationToken token) : base(taskRunner, logger, storeProvider, eventStoreReader, queryStoreReader, clusterQuery, token)
{
}
// TODO: Make Async
public static BaseEntity ExtractClusterEntity(ILogger logger, IClusterQuery clusterQuery, TraceRecord traceRecord, CancellationToken cancelToken)
{
Assert.IsNotNull(clusterQuery);
Assert.IsNotNull(traceRecord);
if (traceRecord is ReconfigurationCompletedTraceRecord)
{
return((BaseEntity)ExtractPartitionEntityFromReconfigEvent(clusterQuery, (ReconfigurationCompletedTraceRecord)traceRecord, cancelToken));
}
throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, "Type '{0}' Event not supported", traceRecord.GetType()));
}
/// <inheritdoc />
internal ReconfigAnalysisAgent(
Config config,
ITaskRunner taskRunner,
ILogger logger,
IStoreProvider storeProvider,
ITraceStoreReader eventStoreReader,
ITraceStoreReader queryStoreReader,
IClusterQuery clusterQuery,
CancellationToken token) : base(taskRunner, logger, storeProvider, eventStoreReader, queryStoreReader, clusterQuery, token)
{
Assert.IsNotNull(config, "Config can't be null");
this.currentConfig = config.NamedConfigManager.GetConfiguration <ReconfigAnalysisAgentConfig>();
this.Logger.LogMessage("Init with Config: {0}", this.currentConfig);
}
/// <inheritdoc />
internal PrimaryMoveAnalysisAgent(
Config config,
ITaskRunner taskRunner,
ILogger logger,
IStoreProvider storeProvider,
ITraceStoreReader eventStoreReader,
ITraceStoreReader queryStoreReader,
IClusterQuery clusterQuery,
CancellationToken token) : base(taskRunner, logger, storeProvider, eventStoreReader, queryStoreReader, clusterQuery, token)
{
// This is to keep track of the previous location of the primary
this.primaryReplicaContextStore = new PrimaryReplicaContextPersistentStoreWrapper(this.Logger, storeProvider, this.CancelToken);
this.primaryMoveAnalysisQueryStoreReader = new PrimaryMoveAnalysisQueryStoreReaderWrapper(this.Logger, this.QueryStoreReader, this.CancelToken);
this.primaryMoveAnalysisEventStoreReader = new PrimaryMoveAnalysisEventStoreReaderWrapper(this.Logger, this.EventStoreReader, this.CancelToken);
}
public static void InitializeSingleInstance(
Config config,
ILogProvider logProvider,
IStoreProvider storeProvider,
ITaskRunner taskRunner,
TraceStoreConnection traceStoreConnection,
IClusterQuery clusterQuery,
CancellationToken token)
{
if (SingleInstance != null)
{
return;
}
lock (singleAccessLock)
{
if (SingleInstance == null)
{
SingleInstance = new AgentDirectory(config, logProvider, storeProvider, taskRunner, traceStoreConnection, clusterQuery, token);
}
}
}
public static bool TryExtractClusterEntity(
ILogger logger,
IClusterQuery clusterQuery,
TraceRecord traceRecord,
CancellationToken cancelToken,
out BaseEntity extractedEntity)
{
Assert.IsNotNull(clusterQuery);
Assert.IsNotNull(traceRecord);
extractedEntity = null;
try
{
extractedEntity = ExtractClusterEntity(logger, clusterQuery, traceRecord, cancelToken);
}
catch (Exception exp)
{
logger.LogWarning("TryExtractClusterEntity:: Couldn't Extract Entity for Event: '{0}'. Exp: '{1}'", traceRecord, exp);
return(false);
}
return(true);
}
protected Agent(
ITaskRunner taskRunner,
ILogger logger,
IStoreProvider storeProvider,
ITraceStoreReader eventStoreReader,
ITraceStoreReader queryStoreReader,
IClusterQuery clusterQuery,
CancellationToken token)
{
Common.Util.Assert.IsNotNull(storeProvider, "Store Provider can't be null");
Common.Util.Assert.IsNotNull(logger, "Loger can't be null");
Common.Util.Assert.IsNotNull(eventStoreReader, "Query Store reader can't be null");
Common.Util.Assert.IsNotNull(taskRunner, "Task Runner can't be null");
Common.Util.Assert.IsNotNull(queryStoreReader, "Query Store reader can't be null");
Common.Util.Assert.IsNotNull(clusterQuery, "Cluster Query can't be null");
this.TaskRunner = taskRunner;
this.Logger = logger;
this.StoreProvider = storeProvider;
this.EventStoreReader = eventStoreReader;
this.QueryStoreReader = queryStoreReader;
this.ClusterQueryInstance = clusterQuery;
this.CancelToken = token;
}
private AgentDirectory(
Config configuration,
ILogProvider logProvider,
IStoreProvider storeProvider,
ITaskRunner taskRunner,
TraceStoreConnection traceStoreConnection,
IClusterQuery clusterQuery,
CancellationToken token)
{
Assert.IsNotNull(configuration, "Config can't be null");
Assert.IsNotNull(logProvider, "Log Provider can't be null");
Assert.IsNotNull(storeProvider, "Store Provider can't be null");
Assert.IsNotNull(taskRunner, "Task Runner can't be null");
Assert.IsNotNull(traceStoreConnection, "Trace store connection can't be null");
Assert.IsNotNull(clusterQuery, "Cluster Query can't be null");
this.taskRunner = taskRunner;
this.logProvider = logProvider;
this.clusterQuery = clusterQuery;
this.storeConnection = traceStoreConnection;
this.storeProvider = storeProvider;
this.token = token;
this.configuration = configuration;
this.agentMap = new ConcurrentDictionary <AgentIdentifier, Agent>();
}
/// <summary>
/// Best Effort Today. TODO: Need to improve this but require the incoming event to have more information
/// attached to it (like Service Manifest Name etc.). This would increase the efficiency as well.
/// </summary>
/// <remarks>
/// TODO: Look at moving all out the special casing complexity into Cluster query object itself.
/// </remarks>
/// <param name="logger"></param>
/// <param name="clusterQuery"></param>
/// <param name="processTerminationEvent"></param>
/// <param name="cancelToken"></param>
/// <returns></returns>
private static object ExtractDeployedCodePackageEntityFromProcessCrashEvent(
ILogger logger,
IClusterQuery clusterQuery,
ProcessUnexpectedTerminationTraceRecord processTerminationEvent,
CancellationToken cancelToken)
{
var nodeObject = clusterQuery.GetNodeListAsync(CancellationToken.None).GetAwaiter().GetResult().Single(node => node.NodeId == processTerminationEvent.NodeId);
var allCps = clusterQuery.GetDeployedCodePackageAsync(nodeObject.NodeName, cancelToken).GetAwaiter().GetResult();
logger.LogMessage(
"ExtractDeployedCodePackageEntityFromProcessCrashEvent:: Found '{0}' code packages on Node '{1}'",
allCps.Count(),
nodeObject.NodeName);
foreach (var codePackage in allCps)
{
if ((codePackage.SetupPointLocation != null && codePackage.SetupPointLocation.Contains(processTerminationEvent.ProcessName)) ||
codePackage.EntryPointLocation.Contains(processTerminationEvent.ProcessName))
{
return(codePackage);
}
}
// There are bunch of system processes which SF tracks and traces but they don't have a Code package for them (e.g. FabricDca.exe)
// For such processes, we wrap them up in a Dummy System Code Package.
if (SystemProcessesWithParentCodePackage.Contains(processTerminationEvent.ProcessName, StringComparer.InvariantCultureIgnoreCase))
{
var defaultSystemProcessApp = new ApplicationEntity()
{
ApplicationName = ApplicationEntity.DefaultAppForSystemProcesses,
ApplicationTypeName = ApplicationEntity.DefaultAppTypeNameForSystemProcesses,
ApplicationTypeVersion = nodeObject.CodeVersion
};
return(new DeployedCodePackageEntity
{
NodeName = nodeObject.NodeName,
Application = defaultSystemProcessApp,
CodePackageName = DeployedCodePackageEntity.SystemCodePackageName,
CodePackageVersion = nodeObject.CodeVersion,
ServiceManifestName = DeployedCodePackageEntity.SystemServiceManifestName,
EntryPointLocation = processTerminationEvent.ProcessName
});
}
var placeholderApplicationEntity = new ApplicationEntity()
{
ApplicationName = ApplicationEntity.PlaceholderApplicationName,
ApplicationTypeName = ApplicationEntity.PlaceholderApplicationTypeName,
ApplicationTypeVersion = nodeObject.CodeVersion
};
// Today this calculation is best effort. We continue to work toward encriching trace information so that we can get to cluster entity with more confidence
// With the information we have today, there are few scenario where we may run into below return. There is a time gap between process crashing and we kicking
// off analysis. In that meantime, the replica may have been moved of the node (for example by PLB or let's say Chaos). In such cases, we don't want to stop
// analyzing the process crash and hence we place this process into a Place holder Code Package Object.
return(new DeployedCodePackageEntity
{
NodeName = nodeObject.NodeName,
Application = placeholderApplicationEntity,
CodePackageName = DeployedCodePackageEntity.PlaceholderCodePackageName,
CodePackageVersion = nodeObject.CodeVersion,
ServiceManifestName = DeployedCodePackageEntity.PlaceholderServiceManifestName,
EntryPointLocation = processTerminationEvent.ProcessName
});
}
public HealthQueryHelper(IClusterQuery query, FabricClient fabricClient, ILogProvider logProvider)
{
this.logger = logProvider.CreateLoggerInstance("HealthQuery");
this.clusterQuery = query;
this.fabricClient = fabricClient;
}
