private async Task<ILongRunningTaskGrain<bool>> GetGrainOnTargetSilo(SiloHandle siloHandle)
{
const int maxRetry = 5;
for (int i = 0; i < maxRetry; i++)
{
var grain = HostedCluster.GrainFactory.GetGrain<ILongRunningTaskGrain<bool>>(Guid.NewGuid());
var instanceId = await grain.GetRuntimeInstanceId();
if (instanceId.Contains(siloHandle.SiloAddress.Endpoint.ToString()))
{
return grain;
}
}
return null;
}
/// <summary>
/// Imports assemblies generated by runtime code generation from the provided silo.
/// </summary>
/// <param name="siloHandle">The silo.</param>
private void ImportGeneratedAssemblies(SiloHandle siloHandle)
{
var generatedAssemblies = TryGetGeneratedAssemblies(siloHandle);
if (generatedAssemblies != null)
{
foreach (var assembly in generatedAssemblies)
{
// If we have never seen generated code for this assembly before, or generated code might be
// newer, store it for later silo creation.
byte[] existing;
if (!this.additionalAssemblies.TryGetValue(assembly.Key, out existing) || assembly.Value != null)
{
this.additionalAssemblies[assembly.Key] = assembly.Value;
}
}
}
}
/// <summary>
/// Restart the default Primary and Secondary silos.
/// </summary>
public void RestartDefaultSilos()
{
TestingSiloOptions primarySiloOptions = Primary.Options;
TestingSiloOptions secondarySiloOptions = Secondary.Options;
// Restart as the same deployment
string deploymentId = DeploymentId;
StopDefaultSilos();
DeploymentId = deploymentId;
primarySiloOptions.PickNewDeploymentId = false;
secondarySiloOptions.PickNewDeploymentId = false;
Primary = StartOrleansSilo(Silo.SiloType.Primary, primarySiloOptions, InstanceCounter++);
Secondary = StartOrleansSilo(Silo.SiloType.Secondary, secondarySiloOptions, InstanceCounter++);
WaitForLivenessToStabilizeAsync().Wait();
GrainClient.Initialize();
}
/// <summary>
/// Do a semi-graceful Stop of the specified silo.
/// </summary>
/// <param name="instance">Silo to be stopped.</param>
public async Task StopSiloAsync(SiloHandle instance)
{
if (instance != null)
{
await StopSiloAsync(instance, true);
if (Primary == instance)
{
Primary = null;
}
else
{
lock (additionalSilos)
{
additionalSilos.Remove(instance);
}
}
}
}
/// <summary>
/// Imports assemblies generated by runtime code generation from the provided silo.
/// </summary>
/// <param name="siloHandle">The silo.</param>
private static void ImportGeneratedAssemblies(SiloHandle siloHandle)
{
var silo = siloHandle.Silo;
if (silo != null && silo.TestHookup != null)
{
var generatedAssemblies = new Silo.TestHookups.GeneratedAssemblies();
silo.TestHookup.UpdateGeneratedAssemblies(generatedAssemblies);
foreach (var assembly in generatedAssemblies.Assemblies)
{
// If we have never seen generated code for this assembly before, or generated code might be
// newer, store it for later silo creation.
byte[] existing;
if (!additionalAssemblies.TryGetValue(assembly.Key, out existing) || assembly.Value != null)
{
additionalAssemblies[assembly.Key] = assembly.Value;
}
}
}
}
/// <summary>
/// Do an immediate Kill of the specified silo.
/// </summary>
/// <param name="instance">Silo to be killed.</param>
public async Task KillSiloAsync(SiloHandle instance)
{
if (instance != null)
{
// do NOT stop, just kill directly, to simulate crash.
await StopSiloAsync(instance, false);
if (Primary == instance)
{
Primary = null;
}
else
{
lock (additionalSilos)
{
additionalSilos.Remove(instance);
}
}
}
}
private static void StopOrleansSilo(SiloHandle instance, bool stopGracefully)
{
var silo = instance.Silo;
if (stopGracefully)
{
try
{
if (silo != null)
{
silo.Shutdown();
}
}
catch (RemotingException re)
{
Console.WriteLine(re); /* Ignore error */
}
catch (Exception exc)
{
Console.WriteLine(exc);
throw;
}
}
ImportGeneratedAssemblies(instance);
try
{
UnloadSiloInAppDomain(instance.AppDomain);
}
catch (Exception exc)
{
Console.WriteLine(exc);
throw;
}
instance.AppDomain = null;
instance.Silo = null;
instance.Process = null;
}
/// <summary>
/// Start an additional silo, so that it joins the existing cluster.
/// </summary>
/// <returns>SiloHandle for the newly started silo.</returns>
public SiloHandle StartAdditionalSilo()
{
var clusterConfig = this.ClusterConfiguration;
short instanceNumber = (short)clusterConfig.Overrides.Count;
var defaultNode = clusterConfig.Defaults;
int baseSiloPort = defaultNode.Port;
int baseGatewayPort = defaultNode.ProxyGatewayEndpoint.Port;
var nodeConfig = TestClusterOptions.AddNodeConfiguration(
this.ClusterConfiguration,
Silo.SiloType.Secondary,
instanceNumber,
baseSiloPort,
baseGatewayPort);
SiloHandle instance = StartOrleansSilo(
Silo.SiloType.Secondary,
this.ClusterConfiguration,
nodeConfig);
additionalSilos.Add(instance);
return(instance);
}
private static void StopOrleansSilo(SiloHandle instance, bool stopGracefully)
{
if (stopGracefully)
{
try { if (instance.Silo != null)
{
instance.Silo.Stop();
}
}
catch (RemotingException re) { Console.WriteLine(re); /* Ignore error */ }
catch (Exception exc) { Console.WriteLine(exc); throw; }
}
try
{
UnloadSiloInAppDomain(instance.AppDomain);
}
catch (Exception exc) { Console.WriteLine(exc); throw; }
instance.AppDomain = null;
instance.Silo = null;
instance.Process = null;
}
/// <summary>
/// Do a Stop or Kill of the specified silo, followed by a restart.
/// </summary>
/// <param name="instance">Silo to be restarted.</param>
public SiloHandle RestartSilo(SiloHandle instance)
{
if (instance != null)
{
var type = instance.Silo.Type;
var siloName = instance.Name;
StopSilo(instance);
var newInstance = StartOrleansSilo(type, this.ClusterConfiguration, this.ClusterConfiguration.Overrides[siloName]);
if (type == Silo.SiloType.Primary && siloName == Silo.PrimarySiloName)
{
Primary = newInstance;
}
else
{
additionalSilos.Add(newInstance);
}
return(newInstance);
}
return(null);
}
/// <summary>
/// Do a Stop or Kill of the specified silo, followed by a restart.
/// </summary>
/// <param name="instance">Silo to be restarted.</param>
public SiloHandle RestartSilo(SiloHandle instance)
{
if (instance != null)
{
var instanceNumber = instance.InstanceNumber;
var siloName = instance.Name;
StopSilo(instance);
var newInstance = StartOrleansSilo(instanceNumber, this.options);
if (siloName == Silo.PrimarySiloName)
{
Primary = newInstance;
}
else
{
additionalSilos.Add(newInstance);
}
return(newInstance);
}
return(null);
}
/// <summary>
/// Do a Stop or Kill of the specified silo, followed by a restart.
/// </summary>
/// <param name="instance">Silo to be restarted.</param>
public async Task <SiloHandle> RestartSiloAsync(SiloHandle instance)
{
if (instance != null)
{
var instanceNumber = instance.InstanceNumber;
var siloName = instance.Name;
await StopSiloAsync(instance);
var newInstance = await StartSiloAsync(instanceNumber, this.options);
if (siloName == Silo.PrimarySiloName)
{
Primary = newInstance;
}
else
{
additionalSilos.Add(newInstance);
}
return(newInstance);
}
return(null);
}
/// <summary>
/// Start a Secondary silo with a given instanceCounter
/// (allows to set the port number as before or new, depending on the scenario).
/// </summary>
public void StartSecondarySilo(TestingSiloOptions secondarySiloOptions, int instanceCounter)
{
secondarySiloOptions.PickNewDeploymentId = false;
Secondary = StartOrleansSilo(Silo.SiloType.Secondary, secondarySiloOptions, instanceCounter);
}
/// <summary>
/// Do a Stop or Kill of the specified silo, followed by a restart.
/// </summary>
/// <param name="instance">Silo to be restarted.</param>
public SiloHandle RestartSilo(SiloHandle instance)
{
if (instance != null)
{
var options = instance.Options;
var type = instance.Silo.Type;
StopOrleansSilo(instance, true);
var newInstance = StartOrleansSilo(type, options, InstanceCounter++);
if (Primary == instance)
{
Primary = newInstance;
}
else if (Secondary == instance)
{
Secondary = newInstance;
}
else
{
additionalSilos.Remove(instance);
additionalSilos.Add(newInstance);
}
return newInstance;
}
return null;
}
private void StopOrleansSilo(SiloHandle instance, bool stopGracefully)
{
instance.StopSilo(stopGracefully);
instance.Dispose();
}
/// <summary>
/// Returns test hooks for the specified silo.
/// </summary>
/// <param name="client">The client.</param>
/// <param name="silo">The silo.</param>
/// <returns>Test hooks for the specified silo.</returns>
public static ITestHooks GetTestHooks(this IClusterClient client, SiloHandle silo)
{
var internalClient = (IInternalClusterClient)client;
return(internalClient.GetSystemTarget <ITestHooksSystemTarget>(Constants.TestHooksSystemTargetId, silo.ProxyAddress));
}
/// <summary>
/// Start a Secondary silo with a given instanceCounter
/// (allows to set the port number as before or new, depending on the scenario).
/// </summary>
public void StartSecondarySilo(TestingSiloOptions secondarySiloOptions, int instanceCounter)
{
secondarySiloOptions.PickNewDeploymentId = false;
Secondary = StartOrleansSilo(Silo.SiloType.Secondary, secondarySiloOptions, instanceCounter);
}
// This is a static version that can be called without a TestingSiloHost object (host = null)
public static SiloHandle StartOrleansSilo(TestingSiloHost host, Silo.SiloType type, TestingSiloOptions options, int instanceCount, AppDomain shared = null)
{
// Load initial config settings, then apply some overrides below.
ClusterConfiguration config = new ClusterConfiguration();
if (options.SiloConfigFile == null)
{
config.StandardLoad();
}
else
{
config.LoadFromFile(options.SiloConfigFile.FullName);
}
int basePort = options.BasePort < 0 ? BasePort : options.BasePort;
if (config.Globals.SeedNodes.Count > 0 && options.BasePort < 0)
{
config.PrimaryNode = config.Globals.SeedNodes[0];
}
else
{
config.PrimaryNode = new IPEndPoint(IPAddress.Loopback, basePort);
}
config.Globals.SeedNodes.Clear();
config.Globals.SeedNodes.Add(config.PrimaryNode);
if (!String.IsNullOrEmpty(DeploymentId))
{
config.Globals.DeploymentId = DeploymentId;
}
config.Defaults.PropagateActivityId = options.PropagateActivityId;
if (options.LargeMessageWarningThreshold > 0)
{
config.Defaults.LargeMessageWarningThreshold = options.LargeMessageWarningThreshold;
}
config.Globals.LivenessType = options.LivenessType;
config.Globals.ReminderServiceType = options.ReminderServiceType;
if (!String.IsNullOrEmpty(options.DataConnectionString))
{
config.Globals.DataConnectionString = options.DataConnectionString;
}
_livenessStabilizationTime = GetLivenessStabilizationTime(config.Globals);
if (host != null)
{
host.Globals = config.Globals;
}
string siloName;
switch (type)
{
case Silo.SiloType.Primary:
siloName = "Primary";
break;
default:
siloName = "Secondary_" + instanceCount.ToString(CultureInfo.InvariantCulture);
break;
}
NodeConfiguration nodeConfig = config.GetConfigurationForNode(siloName);
nodeConfig.HostNameOrIPAddress = "loopback";
nodeConfig.Port = basePort + instanceCount;
nodeConfig.DefaultTraceLevel = config.Defaults.DefaultTraceLevel;
nodeConfig.PropagateActivityId = config.Defaults.PropagateActivityId;
nodeConfig.BulkMessageLimit = config.Defaults.BulkMessageLimit;
if (nodeConfig.ProxyGatewayEndpoint != null && nodeConfig.ProxyGatewayEndpoint.Address != null)
{
int proxyBasePort = options.ProxyBasePort < 0 ? ProxyBasePort : options.ProxyBasePort;
nodeConfig.ProxyGatewayEndpoint = new IPEndPoint(nodeConfig.ProxyGatewayEndpoint.Address, proxyBasePort + instanceCount);
}
config.Globals.ExpectedClusterSize = 2;
config.Overrides[siloName] = nodeConfig;
AdjustForTest(config, options);
WriteLog("Starting a new silo in app domain {0} with config {1}", siloName, config.ToString(siloName));
AppDomain appDomain;
Silo silo = LoadSiloInNewAppDomain(siloName, type, config, out appDomain);
silo.Start();
SiloHandle retValue = new SiloHandle
{
Name = siloName,
Silo = silo,
Options = options,
Endpoint = silo.SiloAddress.Endpoint,
AppDomain = appDomain,
};
ImportGeneratedAssemblies(retValue);
return retValue;
}
/// <summary>
/// Do a Stop or Kill of the specified silo, followed by a restart.
/// </summary>
/// <param name="instance">Silo to be restarted.</param>
public SiloHandle RestartSilo(SiloHandle instance)
{
if (instance != null)
{
var options = instance.Options;
var type = instance.Silo.Type;
StopOrleansSilo(instance, true);
instance = StartOrleansSilo(type, options, InstanceCounter++);
return instance;
}
return null;
}
private static Dictionary <string, GeneratedAssembly> TryGetGeneratedAssemblies(SiloHandle siloHandle)
{
var tryToRetrieveGeneratedAssemblies = Task.Run(() =>
{
try
{
var silo = siloHandle.Silo;
if (silo != null && silo.TestHook != null)
{
var generatedAssemblies = new Silo.TestHooks.GeneratedAssemblies();
silo.TestHook.UpdateGeneratedAssemblies(generatedAssemblies);
return(generatedAssemblies.Assemblies);
}
}
catch (Exception exc)
{
WriteLog("UpdateGeneratedAssemblies threw an exception. Ignoring it. Exception: {0}", exc);
}
return(null);
});
// best effort to try to import generated assemblies, otherwise move on.
if (tryToRetrieveGeneratedAssemblies.Wait(TimeSpan.FromSeconds(3)))
{
return(tryToRetrieveGeneratedAssemblies.Result);
}
return(null);
}
private async Task InitializeAsync(TestingSiloOptions options, TestingClientOptions clientOptions)
{
bool doStartPrimary = false;
bool doStartSecondary = false;
if (options.StartFreshOrleans)
{
// the previous test was !startFresh, so we need to cleanup after it.
if (Primary != null || Secondary != null || GrainClient.IsInitialized)
{
StopDefaultSilos();
}
StopAdditionalSilos();
if (options.StartPrimary)
{
doStartPrimary = true;
}
if (options.StartSecondary)
{
doStartSecondary = true;
}
}
else
{
if (options.StartPrimary && Primary == null)
{
// first time.
doStartPrimary = true;
}
if (options.StartSecondary && Secondary == null)
{
doStartSecondary = true;
}
}
if (options.PickNewDeploymentId && String.IsNullOrEmpty(DeploymentId))
{
DeploymentId = GetDeploymentId();
}
if (options.ParallelStart)
{
var handles = new List<Task<SiloHandle>>();
if (doStartPrimary)
{
int instanceCount = InstanceCounter++;
handles.Add(Task.Run(() => StartOrleansSilo(Silo.SiloType.Primary, options, instanceCount)));
}
if (doStartSecondary)
{
int instanceCount = InstanceCounter++;
handles.Add(Task.Run(() => StartOrleansSilo(Silo.SiloType.Secondary, options, instanceCount)));
}
await Task.WhenAll(handles.ToArray());
if (doStartPrimary)
{
Primary = await handles[0];
}
if (doStartSecondary)
{
Secondary = await handles[1];
}
}else
{
if (doStartPrimary)
{
Primary = StartOrleansSilo(Silo.SiloType.Primary, options, InstanceCounter++);
}
if (doStartSecondary)
{
Secondary = StartOrleansSilo(Silo.SiloType.Secondary, options, InstanceCounter++);
}
}
if (!GrainClient.IsInitialized && options.StartClient)
{
ClientConfiguration clientConfig;
if (clientOptions.ClientConfigFile != null)
{
clientConfig = ClientConfiguration.LoadFromFile(clientOptions.ClientConfigFile.FullName);
}
else
{
clientConfig = ClientConfiguration.StandardLoad();
}
if (clientOptions.ProxiedGateway && clientOptions.Gateways != null)
{
clientConfig.Gateways = clientOptions.Gateways;
if (clientOptions.PreferedGatewayIndex >= 0)
clientConfig.PreferedGatewayIndex = clientOptions.PreferedGatewayIndex;
}
if (clientOptions.PropagateActivityId)
{
clientConfig.PropagateActivityId = clientOptions.PropagateActivityId;
}
if (!String.IsNullOrEmpty(DeploymentId))
{
clientConfig.DeploymentId = DeploymentId;
}
if (Debugger.IsAttached)
{
// Test is running inside debugger - Make timeout ~= infinite
clientConfig.ResponseTimeout = TimeSpan.FromMilliseconds(1000000);
}
else if (clientOptions.ResponseTimeout > TimeSpan.Zero)
{
clientConfig.ResponseTimeout = clientOptions.ResponseTimeout;
}
if (options.LargeMessageWarningThreshold > 0)
{
clientConfig.LargeMessageWarningThreshold = options.LargeMessageWarningThreshold;
}
clientConfig.AdjustForTestEnvironment();
GrainClient.Initialize(clientConfig);
GrainFactory = GrainClient.GrainFactory;
}
}
/// <summary>
/// Restart the default Primary and Secondary silos.
/// </summary>
public void RestartDefaultSilos()
{
TestingSiloOptions primarySiloOptions = Primary.Options;
TestingSiloOptions secondarySiloOptions = Secondary.Options;
// Restart as the same deployment
string deploymentId = DeploymentId;
StopDefaultSilos();
DeploymentId = deploymentId;
primarySiloOptions.PickNewDeploymentId = false;
secondarySiloOptions.PickNewDeploymentId = false;
Primary = StartOrleansSilo(Silo.SiloType.Primary, primarySiloOptions, InstanceCounter++);
Secondary = StartOrleansSilo(Silo.SiloType.Secondary, secondarySiloOptions, InstanceCounter++);
WaitForLivenessToStabilizeAsync().Wait();
GrainClient.Initialize();
}
/// <summary>
/// Stop the default Primary and Secondary silos.
/// </summary>
public static void StopDefaultSilos()
{
try
{
GrainClient.Uninitialize();
}
catch (Exception exc) { WriteLog("Exception Uninitializing grain client: {0}", exc); }
StopSilo(Secondary);
StopSilo(Primary);
Secondary = null;
Primary = null;
InstanceCounter = 0;
DeploymentId = null;
}
private async Task InitializeAsync(TestingSiloOptions options, TestingClientOptions clientOptions)
{
bool doStartPrimary = false;
bool doStartSecondary = false;
if (options.StartFreshOrleans)
{
// the previous test was !startFresh, so we need to cleanup after it.
if (Primary != null || Secondary != null || GrainClient.IsInitialized)
{
StopDefaultSilos();
}
StopAdditionalSilos();
if (options.StartPrimary)
{
doStartPrimary = true;
}
if (options.StartSecondary)
{
doStartSecondary = true;
}
}
else
{
if (options.StartPrimary && Primary == null)
{
// first time.
doStartPrimary = true;
}
if (options.StartSecondary && Secondary == null)
{
doStartSecondary = true;
}
}
if (options.PickNewDeploymentId && String.IsNullOrEmpty(DeploymentId))
{
DeploymentId = GetDeploymentId();
}
if (options.ParallelStart)
{
var handles = new List <Task <SiloHandle> >();
if (doStartPrimary)
{
int instanceCount = InstanceCounter++;
handles.Add(Task.Run(() => StartOrleansSilo(Silo.SiloType.Primary, options, instanceCount)));
}
if (doStartSecondary)
{
int instanceCount = InstanceCounter++;
handles.Add(Task.Run(() => StartOrleansSilo(Silo.SiloType.Secondary, options, instanceCount)));
}
await Task.WhenAll(handles.ToArray());
if (doStartPrimary)
{
Primary = await handles[0];
}
if (doStartSecondary)
{
Secondary = await handles[1];
}
}
else
{
if (doStartPrimary)
{
Primary = StartOrleansSilo(Silo.SiloType.Primary, options, InstanceCounter++);
}
if (doStartSecondary)
{
Secondary = StartOrleansSilo(Silo.SiloType.Secondary, options, InstanceCounter++);
}
}
if (!GrainClient.IsInitialized && options.StartClient)
{
ClientConfiguration clientConfig;
if (clientOptions.ClientConfigFile != null)
{
clientConfig = ClientConfiguration.LoadFromFile(clientOptions.ClientConfigFile.FullName);
}
else
{
clientConfig = ClientConfiguration.StandardLoad();
}
if (clientOptions.ProxiedGateway && clientOptions.Gateways != null)
{
clientConfig.Gateways = clientOptions.Gateways;
if (clientOptions.PreferedGatewayIndex >= 0)
{
clientConfig.PreferedGatewayIndex = clientOptions.PreferedGatewayIndex;
}
}
if (clientOptions.PropagateActivityId)
{
clientConfig.PropagateActivityId = clientOptions.PropagateActivityId;
}
if (!String.IsNullOrEmpty(DeploymentId))
{
clientConfig.DeploymentId = DeploymentId;
}
if (Debugger.IsAttached)
{
// Test is running inside debugger - Make timeout ~= infinite
clientConfig.ResponseTimeout = TimeSpan.FromMilliseconds(1000000);
}
else if (clientOptions.ResponseTimeout > TimeSpan.Zero)
{
clientConfig.ResponseTimeout = clientOptions.ResponseTimeout;
}
if (options.LargeMessageWarningThreshold > 0)
{
clientConfig.LargeMessageWarningThreshold = options.LargeMessageWarningThreshold;
}
clientConfig.AdjustForTestEnvironment();
GrainClient.Initialize(clientConfig);
GrainFactory = GrainClient.GrainFactory;
}
}
/// <summary>
/// Do a semi-graceful Stop of the specified silo.
/// </summary>
/// <param name="instance">Silo to be stopped.</param>
public static void StopSilo(SiloHandle instance)
{
if (instance != null)
{
StopOrleansSilo(instance, true);
}
}
/// <summary>
/// Imports assemblies generated by runtime code generation from the provided silo.
/// </summary>
/// <param name="siloHandle">The silo.</param>
private static void ImportGeneratedAssemblies(SiloHandle siloHandle)
{
try
{
var silo = siloHandle.Silo;
if (silo != null && silo.TestHook != null)
{
var generatedAssemblies = new Silo.TestHooks.GeneratedAssemblies();
silo.TestHook.UpdateGeneratedAssemblies(generatedAssemblies);
foreach (var assembly in generatedAssemblies.Assemblies)
{
// If we have never seen generated code for this assembly before, or generated code might be
// newer, store it for later silo creation.
byte[] existing;
if (!additionalAssemblies.TryGetValue(assembly.Key, out existing) || assembly.Value != null)
{
additionalAssemblies[assembly.Key] = assembly.Value;
}
}
}
}
catch (Exception exc)
{
Console.WriteLine("UpdateGeneratedAssemblies threw an exception. Ignoring it. Exception: {0}", exc);
return;
}
}
/// <summary>
/// Do an immediate Kill of the specified silo.
/// </summary>
/// <param name="instance">Silo to be killed.</param>
public static void KillSilo(SiloHandle instance)
{
if (instance != null)
{
// do NOT stop, just kill directly, to simulate crash.
StopOrleansSilo(instance, false);
}
}
/// <summary>Gets a detailed grain report from a specified silo</summary>
/// <param name="grainId">The grain id we are requesting information from</param>
/// <param name="siloHandle">The target silo that should provide this information from it's cache</param>
internal static Task<DetailedGrainReport> GetDetailedGrainReport(GrainId grainId, SiloHandle siloHandle)
{
var siloControl = GrainClient.InternalGrainFactory.GetSystemTarget<ISiloControl>(Constants.SiloControlId, siloHandle.ProxyAddress);
return siloControl.GetDetailedGrainReport(grainId);
}
/// <summary>
/// Imports assemblies generated by runtime code generation from the provided silo.
/// </summary>
/// <param name="siloHandle">The silo.</param>
private static void ImportGeneratedAssemblies(SiloHandle siloHandle)
{
var silo = siloHandle.Silo;
if (silo != null && silo.TestHookup != null)
{
var generatedAssemblies = new Silo.TestHookups.GeneratedAssemblies();
silo.TestHookup.UpdateGeneratedAssemblies(generatedAssemblies);
foreach (var assembly in generatedAssemblies.Assemblies)
{
// If we have never seen generated code for this assembly before, or generated code might be
// newer, store it for later silo creation.
byte[] existing;
if (!additionalAssemblies.TryGetValue(assembly.Key, out existing) || assembly.Value != null)
{
additionalAssemblies[assembly.Key] = assembly.Value;
}
}
}
}
/// <summary>
/// Do a Stop or Kill of the specified silo, followed by a restart.
/// </summary>
/// <param name="instance">Silo to be restarted.</param>
public SiloHandle RestartSilo(SiloHandle instance)
{
if (instance != null)
{
var type = instance.Silo.Type;
var siloName = instance.Name;
StopSilo(instance);
var newInstance = StartOrleansSilo(type, this.ClusterConfiguration, this.ClusterConfiguration.Overrides[siloName]);
if (type == Silo.SiloType.Primary && siloName == Silo.PrimarySiloName)
{
Primary = newInstance;
}
else
{
additionalSilos.Add(newInstance);
}
return newInstance;
}
return null;
}
private static void StopOrleansSilo(SiloHandle instance, bool stopGracefully)
{
var silo = instance.Silo;
if (stopGracefully)
{
try
{
if (silo != null)
{
silo.Shutdown();
}
}
catch (RemotingException re)
{
Console.WriteLine(re); /* Ignore error */
}
catch (Exception exc)
{
Console.WriteLine(exc);
throw;
}
}
ImportGeneratedAssemblies(instance);
try
{
UnloadSiloInAppDomain(instance.AppDomain);
}
catch (Exception exc)
{
Console.WriteLine(exc);
throw;
}
instance.AppDomain = null;
instance.Silo = null;
instance.Process = null;
}
/// <summary>
/// Imports assemblies generated by runtime code generation from the provided silo.
/// </summary>
/// <param name="siloHandle">The silo.</param>
private void ImportGeneratedAssemblies(SiloHandle siloHandle)
{
var generatedAssemblies = TryGetGeneratedAssemblies(siloHandle);
if (generatedAssemblies != null)
{
foreach (var assembly in generatedAssemblies)
{
// If we have never seen generated code for this assembly before, or generated code might be
// newer, store it for later silo creation.
byte[] existing;
if (!this.additionalAssemblies.TryGetValue(assembly.Key, out existing) || assembly.Value != null)
{
this.additionalAssemblies[assembly.Key] = assembly.Value;
}
}
}
}
private SiloHandle StartOrleansSilo(Silo.SiloType type, TestingSiloOptions options, int instanceCount, AppDomain shared = null)
{
// Load initial config settings, then apply some overrides below.
ClusterConfiguration config = new ClusterConfiguration();
if (options.SiloConfigFile == null)
{
config.StandardLoad();
}
else
{
config.LoadFromFile(options.SiloConfigFile.FullName);
}
int basePort = options.BasePort < 0 ? BasePort : options.BasePort;
if (config.Globals.SeedNodes.Count > 0 && options.BasePort < 0)
{
config.PrimaryNode = config.Globals.SeedNodes[0];
}
else
{
config.PrimaryNode = new IPEndPoint(IPAddress.Loopback, basePort);
}
config.Globals.SeedNodes.Clear();
config.Globals.SeedNodes.Add(config.PrimaryNode);
if (!String.IsNullOrEmpty(DeploymentId))
{
config.Globals.DeploymentId = DeploymentId;
}
config.Defaults.PropagateActivityId = options.PropagateActivityId;
if (options.LargeMessageWarningThreshold > 0)
{
config.Defaults.LargeMessageWarningThreshold = options.LargeMessageWarningThreshold;
}
config.Globals.LivenessType = options.LivenessType;
config.Globals.ReminderServiceType = options.ReminderServiceType;
if (!String.IsNullOrEmpty(options.DataConnectionString))
{
config.Globals.DataConnectionString = options.DataConnectionString;
}
_livenessStabilizationTime = GetLivenessStabilizationTime(config.Globals);
string siloName;
switch (type)
{
case Silo.SiloType.Primary:
siloName = "Primary";
break;
default:
siloName = "Secondary_" + instanceCount.ToString(CultureInfo.InvariantCulture);
break;
}
NodeConfiguration nodeConfig = config.GetConfigurationForNode(siloName);
nodeConfig.HostNameOrIPAddress = "loopback";
nodeConfig.Port = basePort + instanceCount;
nodeConfig.DefaultTraceLevel = config.Defaults.DefaultTraceLevel;
nodeConfig.PropagateActivityId = config.Defaults.PropagateActivityId;
nodeConfig.BulkMessageLimit = config.Defaults.BulkMessageLimit;
if (nodeConfig.ProxyGatewayEndpoint != null && nodeConfig.ProxyGatewayEndpoint.Address != null)
{
nodeConfig.ProxyGatewayEndpoint = new IPEndPoint(nodeConfig.ProxyGatewayEndpoint.Address, ProxyBasePort + instanceCount);
}
config.Globals.ExpectedClusterSize = 2;
config.Overrides[siloName] = nodeConfig;
AdjustForTest(config, options);
WriteLog("Starting a new silo in app domain {0} with config {1}", siloName, config.ToString(siloName));
AppDomain appDomain;
Silo silo = LoadSiloInNewAppDomain(siloName, type, config, out appDomain);
silo.Start();
SiloHandle retValue = new SiloHandle
{
Name = siloName,
Silo = silo,
Options = options,
Endpoint = silo.SiloAddress.Endpoint,
AppDomain = appDomain,
};
ImportGeneratedAssemblies(retValue);
return(retValue);
}
private Dictionary<string, byte[]> TryGetGeneratedAssemblies(SiloHandle siloHandle)
{
var tryToRetrieveGeneratedAssemblies = Task.Run(() =>
{
try
{
var silo = siloHandle.Silo;
if (silo?.TestHook != null)
{
var generatedAssemblies = new Silo.TestHooks.GeneratedAssemblies();
silo.TestHook.UpdateGeneratedAssemblies(generatedAssemblies);
return generatedAssemblies.Assemblies;
}
}
catch (Exception exc)
{
WriteLog("UpdateGeneratedAssemblies threw an exception. Ignoring it. Exception: {0}", exc);
}
return null;
});
// best effort to try to import generated assemblies, otherwise move on.
if (tryToRetrieveGeneratedAssemblies.Wait(TimeSpan.FromSeconds(3)))
{
return tryToRetrieveGeneratedAssemblies.Result;
}
return null;
}
private void StopOrleansSilo(SiloHandle instance, bool stopGracefully)
{
instance.StopSilo(stopGracefully);
instance.Dispose();
}
public static SiloHandle StartOrleansSilo(TestCluster cluster, Silo.SiloType type, ClusterConfiguration clusterConfig, NodeConfiguration nodeConfig)
{
if (cluster == null) throw new ArgumentNullException(nameof(cluster));
var siloName = nodeConfig.SiloName;
cluster.WriteLog("Starting a new silo in app domain {0} with config {1}", siloName, clusterConfig.ToString(siloName));
AppDomain appDomain;
Silo silo = cluster.LoadSiloInNewAppDomain(siloName, type, clusterConfig, out appDomain);
silo.Start();
SiloHandle retValue = new SiloHandle
{
Name = siloName,
Silo = silo,
NodeConfiguration = nodeConfig,
Endpoint = silo.SiloAddress.Endpoint,
AppDomain = appDomain,
};
cluster.ImportGeneratedAssemblies(retValue);
return retValue;
}
public static SiloHandle StartOrleansSilo(TestingSiloHost host, Silo.SiloType type, TestingSiloOptions options, int instanceCount, AppDomain shared = null)
{
if (host == null)
{
throw new ArgumentNullException("host");
}
// Load initial config settings, then apply some overrides below.
ClusterConfiguration config = new ClusterConfiguration();
try
{
if (options.SiloConfigFile == null)
{
config.StandardLoad();
}
else
{
config.LoadFromFile(options.SiloConfigFile.FullName);
}
}
catch (FileNotFoundException)
{
if (options.SiloConfigFile != null &&
!string.Equals(options.SiloConfigFile.Name, TestingSiloOptions.DEFAULT_SILO_CONFIG_FILE, StringComparison.InvariantCultureIgnoreCase))
{
// if the user is not using the defaults, then throw because the file was legitimally not found
throw;
}
config = ClusterConfiguration.LocalhostPrimarySilo();
config.AddMemoryStorageProvider("Default");
config.AddMemoryStorageProvider("MemoryStore");
}
int basePort = options.BasePort < 0 ? BasePort : options.BasePort;
if (config.Globals.SeedNodes.Count > 0 && options.BasePort < 0)
{
config.PrimaryNode = config.Globals.SeedNodes[0];
}
else
{
config.PrimaryNode = new IPEndPoint(IPAddress.Loopback, basePort);
}
config.Globals.SeedNodes.Clear();
config.Globals.SeedNodes.Add(config.PrimaryNode);
if (!String.IsNullOrEmpty(host.DeploymentId))
{
config.Globals.DeploymentId = host.DeploymentId;
}
config.Defaults.PropagateActivityId = options.PropagateActivityId;
if (options.LargeMessageWarningThreshold > 0)
{
config.Defaults.LargeMessageWarningThreshold = options.LargeMessageWarningThreshold;
}
config.Globals.LivenessType = options.LivenessType;
config.Globals.ReminderServiceType = options.ReminderServiceType;
if (!String.IsNullOrEmpty(options.DataConnectionString))
{
config.Globals.DataConnectionString = options.DataConnectionString;
}
host.Globals = config.Globals;
string siloName;
switch (type)
{
case Silo.SiloType.Primary:
siloName = "Primary";
break;
default:
siloName = "Secondary_" + instanceCount.ToString(CultureInfo.InvariantCulture);
break;
}
NodeConfiguration nodeConfig = config.GetOrCreateNodeConfigurationForSilo(siloName);
nodeConfig.HostNameOrIPAddress = "loopback";
nodeConfig.Port = basePort + instanceCount;
nodeConfig.DefaultTraceLevel = config.Defaults.DefaultTraceLevel;
nodeConfig.PropagateActivityId = config.Defaults.PropagateActivityId;
nodeConfig.BulkMessageLimit = config.Defaults.BulkMessageLimit;
int?gatewayport = null;
if (nodeConfig.ProxyGatewayEndpoint != null && nodeConfig.ProxyGatewayEndpoint.Address != null)
{
gatewayport = (options.ProxyBasePort < 0 ? ProxyBasePort : options.ProxyBasePort) + instanceCount;
nodeConfig.ProxyGatewayEndpoint = new IPEndPoint(nodeConfig.ProxyGatewayEndpoint.Address, gatewayport.Value);
}
config.Globals.ExpectedClusterSize = 2;
config.Overrides[siloName] = nodeConfig;
AdjustForTest(config, options);
WriteLog("Starting a new silo in app domain {0} with config {1}", siloName, config.ToString(siloName));
AppDomain appDomain;
Silo silo = host.LoadSiloInNewAppDomain(siloName, type, config, out appDomain);
silo.Start();
SiloHandle retValue = new SiloHandle
{
Name = siloName,
Silo = silo,
Options = options,
Endpoint = silo.SiloAddress.Endpoint,
GatewayPort = gatewayport,
AppDomain = appDomain,
};
host.ImportGeneratedAssemblies(retValue);
return(retValue);
}
/// <summary>
/// Start a new silo in the target cluster
/// </summary>
/// <param name="host">The target cluster</param>
/// <param name="type">The type of the silo to deploy</param>
/// <param name="options">The options to use for the silo</param>
/// <param name="instanceCount">The instance count of the silo</param>
/// <param name="shared">The shared AppDomain to use</param>
/// <returns>A handle to the deployed silo</returns>
public static SiloHandle StartOrleansSilo(TestingSiloHost host, Silo.SiloType type, TestingSiloOptions options, int instanceCount, AppDomain shared = null)
{
if (host == null) throw new ArgumentNullException("host");
// Load initial config settings, then apply some overrides below.
ClusterConfiguration config = new ClusterConfiguration();
try
{
if (options.SiloConfigFile == null)
{
config.StandardLoad();
}
else
{
config.LoadFromFile(options.SiloConfigFile.FullName);
}
}
catch (FileNotFoundException)
{
if (options.SiloConfigFile != null
&& !string.Equals(options.SiloConfigFile.Name, TestingSiloOptions.DEFAULT_SILO_CONFIG_FILE, StringComparison.InvariantCultureIgnoreCase))
{
// if the user is not using the defaults, then throw because the file was legitimally not found
throw;
}
config = ClusterConfiguration.LocalhostPrimarySilo();
config.AddMemoryStorageProvider("Default");
config.AddMemoryStorageProvider("MemoryStore");
}
int basePort = options.BasePort < 0 ? BasePort : options.BasePort;
if (config.Globals.SeedNodes.Count > 0 && options.BasePort < 0)
{
config.PrimaryNode = config.Globals.SeedNodes[0];
}
else
{
config.PrimaryNode = new IPEndPoint(IPAddress.Loopback, basePort);
}
config.Globals.SeedNodes.Clear();
config.Globals.SeedNodes.Add(config.PrimaryNode);
if (!String.IsNullOrEmpty(host.DeploymentId))
{
config.Globals.DeploymentId = host.DeploymentId;
}
config.Defaults.PropagateActivityId = options.PropagateActivityId;
if (options.LargeMessageWarningThreshold > 0)
{
config.Defaults.LargeMessageWarningThreshold = options.LargeMessageWarningThreshold;
}
config.Globals.LivenessType = options.LivenessType;
config.Globals.ReminderServiceType = options.ReminderServiceType;
if (!String.IsNullOrEmpty(options.DataConnectionString))
{
config.Globals.DataConnectionString = options.DataConnectionString;
}
host.Globals = config.Globals;
string siloName;
switch (type)
{
case Silo.SiloType.Primary:
siloName = "Primary";
break;
default:
siloName = "Secondary_" + instanceCount.ToString(CultureInfo.InvariantCulture);
break;
}
NodeConfiguration nodeConfig = config.GetOrCreateNodeConfigurationForSilo(siloName);
nodeConfig.HostNameOrIPAddress = "loopback";
nodeConfig.Port = basePort + instanceCount;
nodeConfig.DefaultTraceLevel = config.Defaults.DefaultTraceLevel;
nodeConfig.PropagateActivityId = config.Defaults.PropagateActivityId;
nodeConfig.BulkMessageLimit = config.Defaults.BulkMessageLimit;
int? gatewayport = null;
if (nodeConfig.ProxyGatewayEndpoint != null && nodeConfig.ProxyGatewayEndpoint.Address != null)
{
gatewayport = (options.ProxyBasePort < 0 ? ProxyBasePort : options.ProxyBasePort) + instanceCount;
nodeConfig.ProxyGatewayEndpoint = new IPEndPoint(nodeConfig.ProxyGatewayEndpoint.Address, gatewayport.Value);
}
config.Globals.ExpectedClusterSize = 2;
config.Overrides[siloName] = nodeConfig;
AdjustForTest(config, options);
WriteLog("Starting a new silo in app domain {0} with config {1}", siloName, config.ToString(siloName));
AppDomain appDomain;
Silo silo = host.LoadSiloInNewAppDomain(siloName, type, config, out appDomain);
silo.Start();
SiloHandle retValue = new SiloHandle
{
Name = siloName,
Silo = silo,
Options = options,
Endpoint = silo.SiloAddress.Endpoint,
GatewayPort = gatewayport,
AppDomain = appDomain,
};
host.ImportGeneratedAssemblies(retValue);
return retValue;
}
/// <summary>
/// Do a semi-graceful Stop of the specified silo.
/// </summary>
/// <param name="instance">Silo to be stopped.</param>
public void StopSilo(SiloHandle instance)
{
if (instance != null)
{
StopOrleansSilo(instance, true);
if (Primary == instance)
{
Primary = null;
}
else
{
additionalSilos.Remove(instance);
}
}
}
/// <summary>Gets a detailed grain report from a specified silo</summary>
/// <param name="grainId">The grain id we are requesting information from</param>
/// <param name="siloHandle">The target silo that should provide this information from it's cache</param>
internal static Task<DetailedGrainReport> GetDetailedGrainReport(GrainId grainId, SiloHandle siloHandle)
{
var proxyAddress = SiloAddress.New(siloHandle.NodeConfiguration.ProxyGatewayEndpoint, 0);
var siloControl = GrainClient.InternalGrainFactory.GetSystemTarget<ISiloControl>(Constants.SiloControlId, proxyAddress);
return siloControl.GetDetailedGrainReport(grainId);
}