public ScriptedEventProcessorHost(
string eventHubPath,
string consumerGroupName,
string eventHubConnectionString,
string storageConnectionString,
string leaseContainerName,
TransportAbstraction.IHost host,
TransportAbstraction.ISender sender,
EventHubsConnections connections,
TaskhubParameters parameters,
NetheriteOrchestrationServiceSettings settings,
EventHubsTraceHelper logger,
string workerId)
{
this.eventHubPath = eventHubPath;
this.consumerGroupName = consumerGroupName;
this.eventHubConnectionString = eventHubConnectionString;
this.storageConnectionString = storageConnectionString;
this.leaseContainerName = leaseContainerName;
this.host = host;
this.sender = sender;
this.connections = connections;
this.parameters = parameters;
this.taskHubGuid = parameters.TaskhubGuid.ToByteArray();
this.settings = settings;
this.logger = logger;
this.workerId = workerId;
}
public void StartEventProcessing(NetheriteOrchestrationServiceSettings settings, CloudBlockBlob partitionScript)
{
if (!partitionScript.Exists())
{
this.logger.LogInformation("ScriptedEventProcessorHost workerId={workerId} is waiting for script", this.workerId);
while (!partitionScript.Exists())
{
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
// we use the UTC modification timestamp on the script as the scenario start time
DateTime scenarioStartTimeUtc = partitionScript.Properties.LastModified.Value.UtcDateTime;
// the number of partitions matters only if the script contains wildcards
this.numberOfPartitions = this.parameters.StartPositions.Length;
for (var partitionIndex = 0; partitionIndex < this.numberOfPartitions; partitionIndex++)
{
this.partitionInstances.Add(null);
}
List <PartitionScript.ProcessorHostEvent> timesteps = new List <PartitionScript.ProcessorHostEvent>();;
try
{
using (var memoryStream = new System.IO.MemoryStream())
{
partitionScript.DownloadRangeToStream(memoryStream, null, null);
memoryStream.Seek(0, System.IO.SeekOrigin.Begin);
timesteps.AddRange(PartitionScript.ParseEvents(scenarioStartTimeUtc, settings.WorkerId, this.numberOfPartitions, memoryStream));
}
this.logger.LogInformation("ScriptedEventProcessorHost workerId={workerId} started.", this.workerId);
}
catch (Exception e)
{
this.logger.LogError($"ScriptedEventProcessorHost workerId={this.workerId} failed to parse partitionscript: {e}");
}
int nextTime = 0;
List <PartitionScript.ProcessorHostEvent> nextGroup = new List <PartitionScript.ProcessorHostEvent>();
foreach (var timestep in timesteps)
{
if (nextTime == timestep.TimeSeconds)
{
nextGroup.Add(timestep);
}
else
{
this.Process(nextGroup);
nextGroup.Clear();
nextGroup.Add(timestep);
nextTime = timestep.TimeSeconds;
}
}
this.Process(nextGroup);
}
public NetheriteProvider(
NetheriteOrchestrationService service,
NetheriteOrchestrationServiceSettings settings)
: base("Netherite", service, service, settings.ResolvedStorageConnectionString)
{
this.Service = service;
this.Settings = settings;
}
public MemoryTransport(TransportAbstraction.IHost host, NetheriteOrchestrationServiceSettings settings, ILogger logger)
{
this.host = host;
this.settings = settings;
TransportConnectionString.Parse(settings.ResolvedTransportConnectionString, out _, out _);
this.numberPartitions = (uint)settings.PartitionCount;
this.logger = logger;
}
public EventHubsTransport(TransportAbstraction.IHost host, NetheriteOrchestrationServiceSettings settings, ILoggerFactory loggerFactory)
{
this.host = host;
this.settings = settings;
this.cloudStorageAccount = CloudStorageAccount.Parse(this.settings.ResolvedStorageConnectionString);
string namespaceName = TransportConnectionString.EventHubsNamespaceName(settings.ResolvedTransportConnectionString);
this.traceHelper = new EventHubsTraceHelper(loggerFactory, settings.TransportLogLevelLimit, this.cloudStorageAccount.Credentials.AccountName, settings.HubName, namespaceName);
this.ClientId = Guid.NewGuid();
var blobContainerName = GetContainerName(settings.HubName);
var cloudBlobClient = this.cloudStorageAccount.CreateCloudBlobClient();
this.cloudBlobContainer = cloudBlobClient.GetContainerReference(blobContainerName);
this.taskhubParameters = this.cloudBlobContainer.GetBlockBlobReference("taskhubparameters.json");
this.partitionScript = this.cloudBlobContainer.GetBlockBlobReference("partitionscript.json");
}
public TestOrchestrationHost(NetheriteOrchestrationServiceSettings settings, ILoggerFactory loggerFactory)
{
this.orchestrationService = new Netherite.NetheriteOrchestrationService(settings, loggerFactory);
if (TestConstants.DeleteStorageBeforeRunningTests)
{
this.orchestrationService.DeleteAsync().GetAwaiter().GetResult();
}
this.orchestrationService.CreateAsync(false).GetAwaiter().GetResult();
this.settings = settings;
this.worker = new TaskHubWorker(this.orchestrationService);
this.client = new TaskHubClient(this.orchestrationService);
this.addedOrchestrationTypes = new HashSet <Type>();
this.addedActivityTypes = new HashSet <Type>();
}
public FasterLog(BlobManager blobManager, NetheriteOrchestrationServiceSettings settings)
{
this.log = new FASTER.core.FasterLog(blobManager.EventLogSettings(settings.UsePremiumStorage));
this.terminationToken = blobManager.PartitionErrorHandler.Token;
var _ = this.terminationToken.Register(
() => {
try
{
this.log.Dispose();
blobManager.EventLogDevice.Dispose();
}
catch (Exception e)
{
blobManager.TraceHelper.FasterStorageError("Disposing FasterLog", e);
}
},
useSynchronizationContext: false);
}
NetheriteOrchestrationServiceSettings GetNetheriteOrchestrationServiceSettings(string taskHubNameOverride = null)
{
var eventSourcedSettings = new NetheriteOrchestrationServiceSettings();
// override DTFx defaults to the defaults we want to use in DF
eventSourcedSettings.ThrowExceptionOnInvalidDedupeStatus = true;
// copy all applicable fields from both the options and the storageProvider options
JsonConvert.PopulateObject(JsonConvert.SerializeObject(this.extensionOptions), eventSourcedSettings);
JsonConvert.PopulateObject(JsonConvert.SerializeObject(this.extensionOptions.StorageProvider), eventSourcedSettings);
// if worker id is specified in environment, it overrides the configured setting
string workerId = Environment.GetEnvironmentVariable("WorkerId");
if (!string.IsNullOrEmpty(workerId))
{
if (workerId == "HostId")
{
workerId = this.hostIdProvider.GetHostIdAsync(CancellationToken.None).GetAwaiter().GetResult();
}
eventSourcedSettings.WorkerId = workerId;
}
eventSourcedSettings.HubName = this.extensionOptions.HubName;
if (taskHubNameOverride != null)
{
eventSourcedSettings.HubName = taskHubNameOverride;
}
eventSourcedSettings.Validate((name) => this.connectionStringResolver.Resolve(name));
if (this.TraceToConsole || this.TraceToBlob)
{
// capture trace events generated in the backend and redirect them to additional sinks
this.loggerFactory = new LoggerFactoryWrapper(this.loggerFactory, eventSourcedSettings.HubName, eventSourcedSettings.WorkerId, this);
}
return(eventSourcedSettings);
}
public static NetheriteOrchestrationServiceSettings GetNetheriteOrchestrationServiceSettings()
{
var settings = new NetheriteOrchestrationServiceSettings
{
StorageConnectionName = StorageConnectionName,
EventHubsConnectionName = EventHubsConnectionName,
HubName = TaskHubName,
TransportLogLevelLimit = LogLevel.Trace,
StorageLogLevelLimit = LogLevel.Trace,
LogLevelLimit = LogLevel.Trace,
EventLogLevelLimit = LogLevel.Trace,
WorkItemLogLevelLimit = LogLevel.Trace,
TakeStateCheckpointWhenStoppingPartition = true, // set to false for testing recovery from log
UseAlternateObjectStore = false, // set to true to bypass FasterKV; default is false
MaxTimeMsBetweenCheckpoints = 1000000000, // set this low for testing frequent checkpointing
//MaxNumberBytesBetweenCheckpoints = 10000000, // set this low for testing frequent checkpointing
//MaxNumberEventsBetweenCheckpoints = 10, // set this low for testing frequent checkpointing
};
settings.Validate((name) => Environment.GetEnvironmentVariable(name));
return(settings);
}
