static void Main(string[] args)
{
ServicePointManager.DefaultConnectionLimit = 1000;
ThreadPool.SetMinThreads(100, 100);
var container = new WindsorContainer();
var serviceLocator = new WindsorServiceLocator(container);
_configurationValueProvider = new AppSettingsConfigurationValueProvider();
var storageConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.TableStorageConnectionString);
var servicebusConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.ServiceBusConnectionString);
container.Register(
Component.For <Orchestrator>()
.ImplementedBy <Orchestrator>()
.LifestyleSingleton(),
Component.For <MasterScheduler>()
.ImplementedBy <MasterScheduler>()
.LifestyleSingleton(),
Component.For <IConfigurationValueProvider>()
.Instance(_configurationValueProvider),
Component.For <IServiceLocator>()
.Instance(serviceLocator),
Component.For <IActorConfiguration>()
.Instance(
ActorDescriptors.FromAssemblyContaining <PeckSourceScheduled>()
.ToConfiguration()),
Component.For <IFactoryActor>()
.ImplementedBy <FactoryActor>()
.LifestyleTransient(),
Component.For <PeckSourceProcessor>()
.ImplementedBy <PeckSourceProcessor>()
.LifestyleTransient(),
Component.For <ILockStore>()
.Instance(new AzureLockStore(new BlobSource()
{
ConnectionString = storageConnectionString,
ContainerName = "locks",
Path = "woodpecker/locks/master_Keys/"
})),
Component.For <IEventQueueOperator>()
.Instance(new ServiceBusOperator(servicebusConnectionString))
);
_orchestrator = container.Resolve <Orchestrator>();
_scheduler = container.Resolve <MasterScheduler>();
Task.Run(() => _orchestrator.SetupAsync()).Wait();
_orchestrator.Start();
Console.WriteLine("Working ...");
Work();
_orchestrator.Stop();
}
protected static void Setup()
{
var container = new WindsorContainer();
var serviceLocator = new WindsorServiceLocator(container);
_configurationValueProvider = new AppSettingsConfigProvider();
var storageConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.StorageConnectionString);
var servicebusConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.ServiceBusConnectionString);
var headersText = _configurationValueProvider.GetValue(ConfigurationKeys.TabSeparatedCustomEsHttpHeaders);
var headers = new List <KeyValuePair <string, string> >();
if (headersText != null)
{
foreach (var header in headersText.Split('\t'))
{
var strings = header.Split(new[] { ": " }, StringSplitOptions.RemoveEmptyEntries);
if (strings.Length == 2)
{
headers.Add(new KeyValuePair <string, string>(strings[0], strings[1]));
}
}
}
container.Register(
Component.For <IElasticsearchClient>()
.ImplementedBy <ElasticsearchClient>()
.LifestyleSingleton(),
Component.For <Orchestrator>()
.ImplementedBy <Orchestrator>()
.LifestyleSingleton(),
Component.For <MasterScheduler>()
.ImplementedBy <MasterScheduler>()
.LifestyleSingleton(),
Component.For <IConfigurationValueProvider>()
.Instance(_configurationValueProvider),
Component.For <ISourceConfiguration>()
.ImplementedBy <TableStorageConfigurationSource>(),
Component.For <IServiceLocator>()
.Instance(serviceLocator),
Component.For <IActorConfiguration>()
.Instance(
ActorDescriptors.FromAssemblyContaining <ShardRangeActor>()
.ToConfiguration()),
Component.For <IFactoryActor>()
.ImplementedBy <FactoryActor>()
.LifestyleTransient(),
Component.For <IHttpClient>()
.ImplementedBy <DefaultHttpClient>()
.LifestyleSingleton()
.DependsOn(Dependency.OnValue("defaultHeaders", headers)),
Component.For <ShardKeyActor>()
.ImplementedBy <ShardKeyActor>()
.LifestyleTransient(),
Component.For <ShardRangeActor>()
.ImplementedBy <ShardRangeActor>()
.LifestyleTransient(),
Component.For <BlobFileActor>()
.ImplementedBy <BlobFileActor>()
.LifestyleTransient(),
Component.For <BlobFileConventionActor>()
.ImplementedBy <BlobFileConventionActor>()
.LifestyleTransient(),
Component.For <ITempDownloadLocationProvider>()
.ImplementedBy <TempDownloadLocationProvider>()
.LifestyleSingleton(),
Component.For <IisBlobScheduler>()
.ImplementedBy <IisBlobScheduler>()
.LifestyleTransient(),
Component.For <IisBlobConventionScheduler>()
.ImplementedBy <IisBlobConventionScheduler>()
.LifestyleTransient(),
Component.For <RangeShardKeyScheduler>()
.ImplementedBy <RangeShardKeyScheduler>()
.LifestyleTransient(),
Component.For <MinuteTableShardScheduler>()
.ImplementedBy <MinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <ReverseTimestampMinuteTableShardScheduler>()
.ImplementedBy <ReverseTimestampMinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <IisLogParser>()
.ImplementedBy <IisLogParser>()
.LifestyleTransient(),
Component.For <IElasticsearchBatchPusher>()
.ImplementedBy <ElasticsearchBatchPusher>()
.LifestyleTransient()
.DependsOn(Dependency.OnValue("esUrl", _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl))),
Component.For <ILockStore>()
.Instance(new AzureLockStore(new BlobSource()
{
ConnectionString = storageConnectionString,
ContainerName = "locks",
Path = "conveyor_belt/locks/master_Keys/"
})),
Component.For <IEventQueueOperator>()
.Instance(new ServiceBusOperator(servicebusConnectionString))
);
_orchestrator = container.Resolve <Orchestrator>();
_scheduler = container.Resolve <MasterScheduler>();
}
public WorkerRole()
{
var container = new WindsorContainer();
var serviceLocator = new WindsorServiceLocator(container);
_configurationValueProvider = new AzureConfigurationValueProvider();
var storageConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.StorageConnectionString);
var clusterLockContainer = _configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockContainer);
var clusterLockRootPath = _configurationValueProvider.GetValue(ConfigurationKeys.ClusterLockRootPath);
var headersText = _configurationValueProvider.GetValue(ConfigurationKeys.TabSeparatedCustomEsHttpHeaders);
var headers = new List <KeyValuePair <string, string> >();
if (headersText != null)
{
foreach (var header in headersText.Split('\t'))
{
var strings = header.Split(new [] { ": " }, StringSplitOptions.RemoveEmptyEntries);
if (strings.Length == 2)
{
headers.Add(new KeyValuePair <string, string>(strings[0], strings[1]));
}
}
}
int bulkBatchSize = 100;
var bulkBatchSizeString = _configurationValueProvider.GetValue(ConfigurationKeys.BulkBatchSize);
int.TryParse(bulkBatchSizeString, out bulkBatchSize);
var servicebusConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.ServiceBusConnectionString);
container.Register(
Component.For <IElasticsearchClient>()
.ImplementedBy <ElasticsearchClient>()
.LifestyleSingleton(),
Component.For <Orchestrator>()
.ImplementedBy <Orchestrator>()
.LifestyleSingleton(),
Component.For <MasterScheduler>()
.ImplementedBy <MasterScheduler>()
.LifestyleSingleton(),
Component.For <IConfigurationValueProvider>()
.Instance(_configurationValueProvider),
Component.For <IServiceLocator>()
.Instance(serviceLocator),
Component.For <IActorConfiguration>()
.Instance(
ActorDescriptors.FromAssemblyContaining <ShardRangeActor>()
.ToConfiguration().UpdateParallelism(_configurationValueProvider)),
Component.For <ISourceConfiguration>()
.ImplementedBy <TableStorageConfigurationSource>(),
Component.For <IFactoryActor>()
.ImplementedBy <FactoryActor>()
.LifestyleTransient(),
Component.For <ShardKeyActor>()
.ImplementedBy <ShardKeyActor>()
.LifestyleTransient(),
Component.For <ShardRangeActor>()
.ImplementedBy <ShardRangeActor>()
.LifestyleTransient(),
Component.For <BlobFileActor>()
.ImplementedBy <BlobFileActor>()
.LifestyleTransient(),
Component.For <BlobFileConventionActor>()
.ImplementedBy <BlobFileConventionActor>()
.LifestyleTransient(),
Component.For <IisBlobScheduler>()
.ImplementedBy <IisBlobScheduler>()
.LifestyleTransient(),
Component.For <IisBlobConventionScheduler>()
.ImplementedBy <IisBlobConventionScheduler>()
.LifestyleTransient(),
Component.For <RangeShardKeyScheduler>()
.ImplementedBy <RangeShardKeyScheduler>()
.LifestyleTransient(),
Component.For <SimpleBlobScheduler>()
.ImplementedBy <SimpleBlobScheduler>()
.LifestyleTransient(),
Component.For <MinuteTableShardScheduler>()
.ImplementedBy <MinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <D18MinuteTableShardScheduler>()
.ImplementedBy <D18MinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <Modulo10MinuteTableShardScheduler>()
.ImplementedBy <Modulo10MinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <ReverseTimestampMinuteTableShardScheduler>()
.ImplementedBy <ReverseTimestampMinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <IisLogParser>()
.ImplementedBy <IisLogParser>()
.LifestyleTransient(),
Component.For <AkamaiLogParser>()
.ImplementedBy <AkamaiLogParser>()
.LifestyleTransient(),
Component.For <IIndexNamer>()
.ImplementedBy <IndexNamer>()
.LifestyleSingleton(),
Component.For <IHttpClient>()
.ImplementedBy <DefaultHttpClient>()
.LifestyleSingleton()
.DependsOn(Dependency.OnValue("defaultHeaders", headers)),
Component.For <ITempDownloadLocationProvider>()
.ImplementedBy <AzureTempDownloadLocationProvider>()
.LifestyleSingleton(),
Component.For <IElasticsearchBatchPusher>()
.ImplementedBy <ElasticsearchBatchPusher>()
.LifestyleTransient()
.DependsOn(Dependency.OnValue("esUrl", _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl)))
.DependsOn(Dependency.OnValue("batchSize", bulkBatchSize)),
Component.For <ILockStore>()
.Instance(new AzureLockStore(new BlobSource()
{
ConnectionString = storageConnectionString,
ContainerName = clusterLockContainer,
Path = clusterLockRootPath
})),
Component.For <IKeyValueStore>()
.Instance(new AzureKeyValueStore(storageConnectionString, clusterLockContainer)),
Component.For <IEventQueueOperator>()
.Instance(new ServiceBusOperator(servicebusConnectionString)),
Component.For <ITelemetryProvider>()
.ImplementedBy <TelemetryProvider>()
.LifestyleSingleton()
);
_orchestrator = container.Resolve <Orchestrator>();
_scheduler = container.Resolve <MasterScheduler>();
_keyValueStore = container.Resolve <IKeyValueStore>();
ServicePointHelper.ApplyStandardSettings(_configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl));
}
protected static void Setup()
{
var container = new WindsorContainer();
var serviceLocator = new WindsorServiceLocator(container);
_configurationValueProvider = new AppSettingsConfigProvider();
var storageConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.StorageConnectionString);
var servicebusConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.ServiceBusConnectionString);
container.Register(
Component.For <IElasticsearchClient>()
.ImplementedBy <ElasticsearchClient>()
.LifestyleSingleton(),
Component.For <Orchestrator>()
.ImplementedBy <Orchestrator>()
.LifestyleSingleton(),
Component.For <MasterScheduler>()
.ImplementedBy <MasterScheduler>()
.LifestyleSingleton(),
Component.For <IConfigurationValueProvider>()
.Instance(_configurationValueProvider),
Component.For <ISourceConfiguration>()
.ImplementedBy <TableStorageConfigurationSource>(),
Component.For <IServiceLocator>()
.Instance(serviceLocator),
Component.For <IActorConfiguration>()
.Instance(
ActorDescriptors.FromAssemblyContaining <ShardRangeActor>()
.ToConfiguration()),
Component.For <IFactoryActor>()
.ImplementedBy <FactoryActor>()
.LifestyleTransient(),
Component.For <IHttpClient>()
.ImplementedBy <DefaultHttpClient>()
.LifestyleSingleton(),
Component.For <ShardKeyActor>()
.ImplementedBy <ShardKeyActor>()
.LifestyleTransient(),
Component.For <ShardRangeActor>()
.ImplementedBy <ShardRangeActor>()
.LifestyleTransient(),
Component.For <BlobFileActor>()
.ImplementedBy <BlobFileActor>()
.LifestyleTransient(),
Component.For <IisBlobScheduler>()
.ImplementedBy <IisBlobScheduler>()
.LifestyleTransient(),
Component.For <RangeShardKeyScheduler>()
.ImplementedBy <RangeShardKeyScheduler>()
.LifestyleTransient(),
Component.For <MinuteTableShardScheduler>()
.ImplementedBy <MinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <ReverseTimestampMinuteTableShardScheduler>()
.ImplementedBy <ReverseTimestampMinuteTableShardScheduler>()
.LifestyleTransient(),
Component.For <IisLogParser>()
.ImplementedBy <IisLogParser>()
.LifestyleTransient(),
Component.For <IElasticsearchBatchPusher>()
.ImplementedBy <ElasticsearchBatchPusher>()
.LifestyleTransient()
.DependsOn(Dependency.OnValue("esUrl", _configurationValueProvider.GetValue(ConfigurationKeys.ElasticSearchUrl))),
Component.For <ILockStore>()
.Instance(new AzureLockStore(new BlobSource()
{
ConnectionString = storageConnectionString,
ContainerName = "locks",
Path = "conveyor_belt/locks/master_Keys/"
})),
Component.For <IEventQueueOperator>()
.Instance(new ServiceBusOperator(servicebusConnectionString))
);
_orchestrator = container.Resolve <Orchestrator>();
_scheduler = container.Resolve <MasterScheduler>();
}
public override bool OnStart()
{
ServicePointManager.DefaultConnectionLimit = 1000;
ThreadPool.SetMinThreads(100, 100);
var container = new WindsorContainer();
var serviceLocator = new WindsorServiceLocator(container);
_configurationValueProvider = new AzureConfigurationValueProvider();
var storageConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.TableStorageConnectionString);
var servicebusConnectionString = _configurationValueProvider.GetValue(ConfigurationKeys.ServiceBusConnectionString);
container.Register(
Component.For <Orchestrator>()
.ImplementedBy <Orchestrator>()
.LifestyleSingleton(),
Component.For <MasterScheduler>()
.ImplementedBy <MasterScheduler>()
.LifestyleSingleton(),
Component.For <IConfigurationValueProvider>()
.Instance(_configurationValueProvider),
Component.For <IServiceLocator>()
.Instance(serviceLocator),
Component.For <IActorConfiguration>()
.Instance(
ActorDescriptors.FromAssemblyContaining <PeckSourceScheduled>()
.ToConfiguration()),
Component.For <IFactoryActor>()
.ImplementedBy <FactoryActor>()
.LifestyleTransient(),
Component.For <PeckSourceProcessor>()
.ImplementedBy <PeckSourceProcessor>()
.LifestyleTransient(),
Component.For <ILockStore>()
.Instance(new AzureLockStore(new BlobSource()
{
ConnectionString = storageConnectionString,
ContainerName = "locks",
Path = "woodpecker/locks/master_Keys/"
})),
Component.For <IEventQueueOperator>()
.Instance(new ServiceBusOperator(servicebusConnectionString))
);
_orchestrator = container.Resolve <Orchestrator>();
_scheduler = container.Resolve <MasterScheduler>();
Task.Run(() => _orchestrator.SetupAsync()).Wait();
_orchestrator.Start();
Console.WriteLine("Working ...");
// For information on handling configuration changes
// see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357.
bool result = base.OnStart();
Trace.TraceInformation("Woodpecker.Worker.Role has been started");
return(result);
}