protected override async Task <JobResult> RunInternalAsync(CancellationToken token)
{
if (!ConfigurationManager.AppSettings.GetBool("Migration:CanResetData").GetValueOrDefault())
{
return(JobResult.FailedWithMessage("Migration:CanResetData was not set in the app.config."));
}
Log.Info().Message("Flushing redis cache").Write();
_cacheClient.FlushAll();
Log.Info().Message("Resetting elastic search").Write();
ElasticSearchConfiguration.ConfigureMapping(_elasticClient, true);
foreach (var collectionName in _mongoDatabase.GetCollectionNames().Where(name => !name.StartsWith("system")))
{
Log.Info().Message("Dropping collection: {0}", collectionName).Write();
_mongoDatabase.DropCollection(collectionName);
}
Log.Info().Message("Creating indexes...").Write();
new ApplicationRepository(_mongoDatabase);
new OrganizationRepository(_mongoDatabase);
new ProjectRepository(_mongoDatabase);
new TokenRepository(_mongoDatabase);
new WebHookRepository(_mongoDatabase);
new UserRepository(_mongoDatabase);
Log.Info().Message("Finished creating indexes...").Write();
return(JobResult.Success);
}
public static void AddElasticSearch(this IServiceCollection services, ElasticSearchConfiguration configuration)
{
var settings = new ConnectionSettings(new Uri(configuration.ConnectionString));
services.AddSingleton(settings);
services.AddTransient <IElasticClient, ElasticClient>(e => new ElasticClient(settings));
}
public ElasticSearchRepository(
ElasticSearchConfiguration configuration,
ILogger <ElasticSearchRepository <T> > logger = null)
{
_configuration = configuration;
_logger = (ILogger)logger ?? NullLogger.Instance;
}
public static IServiceCollection AddElasticSearch(this IServiceCollection services, Action <ElasticSearchConfiguration> configureOptions)
{
var options = new ElasticSearchConfiguration();
configureOptions?.Invoke(options);
services.TryAddSingleton(options);
return(services);
}
public IisFlow()
{
var elasticConfiguration = new ElasticSearchConfiguration();
elasticConfiguration.IndexNameFormat = @"\i\i\s\l\o\g\s\-yyyyMM";
CreateProcess()
.FromInput(new FileInput(@"C:\inetpub\logs\LogFiles\W3SVC1\*.log", System.Text.Encoding.UTF8, true))
.Then(new IISLogProcessor())
.Then(new InfluxDbOutput())
.Then(new ElasticSearchOutput(elasticConfiguration));
}
public FruitSearcherFlow ()
{
var elasticConfiguration = new ElasticSearchConfiguration();
elasticConfiguration.IndexNameFormat = @"\a\p\p\l\o\g\s\-yyyyMM";
CreateProcess()
.FromInput(new FileInput(@"C:\Repo\deliberate-logging\code\FruitSearcher\logs\*.log", System.Text.Encoding.UTF8, true))
.Then(new LogJsonParser())
.Then(new InfluxDbOutputForFruitSearcher())
.Then(new ElasticSearchOutput(elasticConfiguration));
}
public static void AddElasticsearch(
this IServiceCollection services, IConfiguration configuration)
{
var elasticSearchConfiguration = new ElasticSearchConfiguration();
configuration.GetSection(nameof(ElasticSearchConfiguration)).Bind(elasticSearchConfiguration);
var settings = new ConnectionSettings(new Uri(elasticSearchConfiguration.Uri))
.DefaultIndex(elasticSearchConfiguration.DefaultIndex);
var client = new ElasticClient(settings);
services.AddSingleton <IElasticClient>(client);
}
public RandomDataFlow()
{
var elasticConfiguration = new ElasticSearchConfiguration();
elasticConfiguration.Host = "localhost";
elasticConfiguration.Port = 9200;
elasticConfiguration.IndexNameFormat = @"\b\a\c\o\n\l\o\g\-yyyyMMdd";
elasticConfiguration.Mappings = mappings =>
{
mappings.String(m => m.Name("Message").Index(FieldIndexOption.not_analyzed));
};
CreateProcess().FromInput(new RandomBaconInput())
.ToOutput(new ElasticSearchOutput(elasticConfiguration));
}
//[Theory]
//[InlineData("\"data.load time-s\":\"262", 1)]
//public void GetByCustomData(string filter, int count) {
// var result = GetByFilter(filter);
// Assert.NotNull(result);
// Assert.Equal(count, result.Count);
//}
private void CreateEvents()
{
ElasticSearchConfiguration.ConfigureMapping(_client, true);
var parserPluginManager = IoC.GetInstance <EventParserPluginManager>();
foreach (var file in Directory.GetFiles(@"..\..\Search\Data\", "event*.json", SearchOption.AllDirectories))
{
var events = parserPluginManager.ParseEvents(File.ReadAllText(file), 2, "exceptionless/2.0.0.0");
Assert.NotNull(events);
Assert.True(events.Count > 0);
_repository.Add(events);
}
_client.Refresh();
}
public static IServiceCollection AddElasticSearch(this IServiceCollection services,
Action <IElasticSearchConfiguration> options)
{
var esConfiguration = new ElasticSearchConfiguration();
options(esConfiguration);
if (esConfiguration.UseAuditingLog && string.IsNullOrWhiteSpace(esConfiguration.AuditingLogIndexName))
{
throw new ArgumentNullException(nameof(esConfiguration.AuditingLogIndexName));
}
services.AddSingleton <IElasticSearchConfiguration, ElasticSearchConfiguration>(service => esConfiguration);
return(services);
}
private void CreateStacks()
{
ElasticSearchConfiguration.ConfigureMapping(_client, true);
var serializer = JsonSerializer.CreateDefault(new JsonSerializerSettings {
ContractResolver = new LowerCaseUnderscorePropertyNamesContractResolver()
});
foreach (var file in Directory.GetFiles(@"..\..\Search\Data\", "stack*.json", SearchOption.AllDirectories))
{
using (var stream = new FileStream(file, FileMode.Open)) {
using (var streamReader = new StreamReader(stream)) {
var stack = serializer.Deserialize(streamReader, typeof(Stack)) as Stack;
Assert.NotNull(stack);
_repository.Add(stack);
}
}
}
_client.Refresh();
}
private void CreateStacks()
{
ElasticSearchConfiguration.ConfigureMapping(_client, true);
var serializer = IoC.GetInstance <JsonSerializer>();
foreach (var file in Directory.GetFiles(@"..\..\Search\Data\", "stack*.json", SearchOption.AllDirectories))
{
if (file.EndsWith("summary.json"))
{
continue;
}
using (var stream = new FileStream(file, FileMode.Open)) {
using (var streamReader = new StreamReader(stream)) {
var stack = serializer.Deserialize(streamReader, typeof(Stack)) as Stack;
Assert.NotNull(stack);
_repository.Add(stack);
}
}
}
_client.Refresh();
}
public void RegisterServices(Container container)
{
container.RegisterSingle <IDependencyResolver>(() => new SimpleInjectorCoreDependencyResolver(container));
if (Settings.Current.EnableAppStats)
{
container.RegisterSingle <IAppStatsClient>(() => new AppStatsClient(Settings.Current.AppStatsServerName, Settings.Current.AppStatsServerPort));
}
else
{
container.RegisterSingle <IAppStatsClient, InMemoryAppStatsClient>();
}
container.RegisterSingle <IDependencyResolver>(() => new SimpleInjectorCoreDependencyResolver(container));
container.RegisterSingle <MongoDatabase>(() => {
if (String.IsNullOrEmpty(Settings.Current.MongoConnectionString))
{
throw new ConfigurationErrorsException("MongoConnectionString was not found in the Web.config.");
}
MongoDefaults.MaxConnectionIdleTime = TimeSpan.FromMinutes(1);
var url = new MongoUrl(Settings.Current.MongoConnectionString);
string databaseName = url.DatabaseName;
if (Settings.Current.AppendMachineNameToDatabase)
{
databaseName += String.Concat("-", Environment.MachineName.ToLower());
}
MongoServer server = new MongoClient(url).GetServer();
return(server.GetDatabase(databaseName));
});
container.RegisterSingle <IElasticClient>(() => ElasticSearchConfiguration.GetElasticClient(Settings.Current.ElasticSearchConnectionString.Split(',').Select(url => new Uri(url))));
if (Settings.Current.EnableRedis)
{
var muxer = ConnectionMultiplexer.Connect(Settings.Current.RedisConnectionString);
container.RegisterSingle(muxer);
container.Register <ICacheClient, HybridCacheClient>();
container.RegisterSingle <IQueue <EventPost> >(() => new RedisQueue <EventPost>(muxer, statName: StatNames.PostsQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <EventUserDescription> >(() => new RedisQueue <EventUserDescription>(muxer, statName: StatNames.EventsUserDescriptionQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <EventNotification> >(() => new RedisQueue <EventNotification>(muxer, statName: StatNames.EventNotificationQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <WebHookNotification> >(() => new RedisQueue <WebHookNotification>(muxer, statName: StatNames.WebHookQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <MailMessage> >(() => new RedisQueue <MailMessage>(muxer, statName: StatNames.EmailsQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <RedisMessageBus>(() => new RedisMessageBus(muxer.GetSubscriber()));
container.Register <IMessagePublisher>(container.GetInstance <RedisMessageBus>);
container.Register <IMessageSubscriber>(container.GetInstance <RedisMessageBus>);
}
else
{
container.RegisterSingle <ICacheClient, InMemoryCacheClient>();
container.RegisterSingle <IQueue <EventPost> >(() => new InMemoryQueue <EventPost>(statName: StatNames.PostsQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <EventUserDescription> >(() => new InMemoryQueue <EventUserDescription>(statName: StatNames.EventsUserDescriptionQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <EventNotification> >(() => new InMemoryQueue <EventNotification>(statName: StatNames.EventNotificationQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <WebHookNotification> >(() => new InMemoryQueue <WebHookNotification>(statName: StatNames.WebHookQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <IQueue <MailMessage> >(() => new InMemoryQueue <MailMessage>(statName: StatNames.EmailsQueueSize, stats: container.GetInstance <IAppStatsClient>()));
container.RegisterSingle <InMemoryMessageBus>();
container.Register <IMessagePublisher>(container.GetInstance <InMemoryMessageBus>);
container.Register <IMessageSubscriber>(container.GetInstance <InMemoryMessageBus>);
}
if (Settings.Current.EnableAzureStorage)
{
container.RegisterSingle <IFileStorage>(new AzureFileStorage(Settings.Current.AzureStorageConnectionString));
}
else if (!String.IsNullOrEmpty(Settings.Current.StorageFolder))
{
container.RegisterSingle <IFileStorage>(new FolderFileStorage(Settings.Current.StorageFolder));
}
else
{
container.RegisterSingle <IFileStorage>(new InMemoryFileStorage());
}
container.RegisterSingle <IStackRepository, StackRepository>();
container.RegisterSingle <IEventRepository, EventRepository>();
container.RegisterSingle <IOrganizationRepository, OrganizationRepository>();
container.RegisterSingle <IProjectRepository, ProjectRepository>();
container.RegisterSingle <IUserRepository, UserRepository>();
container.RegisterSingle <IWebHookRepository, WebHookRepository>();
container.RegisterSingle <ITokenRepository, TokenRepository>();
container.RegisterSingle <IApplicationRepository, ApplicationRepository>();
container.RegisterSingle <IGeoIPResolver, MindMaxGeoIPResolver>();
container.RegisterSingle <IValidator <Application>, ApplicationValidator>();
container.RegisterSingle <IValidator <Organization>, OrganizationValidator>();
container.RegisterSingle <IValidator <PersistentEvent>, PersistentEventValidator>();
container.RegisterSingle <IValidator <Project>, ProjectValidator>();
container.RegisterSingle <IValidator <Stack>, StackValidator>();
container.RegisterSingle <IValidator <Token>, TokenValidator>();
container.RegisterSingle <IValidator <UserDescription>, UserDescriptionValidator>();
container.RegisterSingle <IValidator <User>, UserValidator>();
container.RegisterSingle <IValidator <WebHook>, WebHookValidator>();
container.RegisterSingle <IEmailGenerator>(() => new RazorEmailGenerator(@"Mail\Templates"));
container.RegisterSingle <IMailer, Mailer>();
if (Settings.Current.WebsiteMode != WebsiteMode.Dev)
{
container.RegisterSingle <IMailSender, SmtpMailSender>();
}
else
{
container.RegisterSingle <IMailSender>(() => new InMemoryMailSender());
}
container.Register <ILockProvider, CacheLockProvider>();
container.Register <StripeEventHandler>();
container.RegisterSingle <BillingManager>();
container.RegisterSingle <DataHelper>();
container.RegisterSingle <EventStats>();
container.RegisterSingle <EventPipeline>();
container.RegisterSingle <EventPluginManager>();
container.RegisterSingle <FormattingPluginManager>();
}
public void Deconstruct(
out ConsoleConfiguration console,
out FileConfiguration file,
out ElasticSearchConfiguration elasticSearch,
out TracingConfiguration tracing) => (console, file, elasticSearch, tracing) = (
this.Console ?? new ConsoleConfiguration(),
public BookSearchAppService(ElasticSearchConfiguration config)
: base(config)
{
}
public BookChangedEventHandler(ElasticSearchConfiguration config)
: base(config)
{
}
public void RegisterServices(Container container)
{
container.RegisterSingle <IDependencyResolver>(() => new SimpleInjectorCoreDependencyResolver(container));
var metricsClient = new InMemoryMetricsClient();
metricsClient.StartDisplayingStats();
container.RegisterSingle <IMetricsClient>(metricsClient);
container.RegisterSingle <MongoDatabase>(() => {
if (String.IsNullOrEmpty(Settings.Current.MongoConnectionString))
{
throw new ConfigurationErrorsException("MongoConnectionString was not found in the Web.config.");
}
MongoDefaults.MaxConnectionIdleTime = TimeSpan.FromMinutes(1);
MongoServer server = new MongoClient(new MongoUrl(Settings.Current.MongoConnectionString)).GetServer();
return(server.GetDatabase(Settings.Current.MongoDatabaseName));
});
container.RegisterSingle <IElasticClient>(() => ElasticSearchConfiguration.GetElasticClient(Settings.Current.ElasticSearchConnectionString.Split(',').Select(url => new Uri(url))));
container.RegisterSingle <ICacheClient, InMemoryCacheClient>();
container.RegisterSingle <IQueue <EventPost> >(() => new InMemoryQueue <EventPost>(statName: MetricNames.PostsQueueSize, metrics: container.GetInstance <IMetricsClient>()));
container.RegisterSingle <IQueue <EventUserDescription> >(() => new InMemoryQueue <EventUserDescription>(statName: MetricNames.EventsUserDescriptionQueueSize, metrics: container.GetInstance <IMetricsClient>()));
container.RegisterSingle <IQueue <EventNotificationWorkItem> >(() => new InMemoryQueue <EventNotificationWorkItem>(statName: MetricNames.EventNotificationQueueSize, metrics: container.GetInstance <IMetricsClient>()));
container.RegisterSingle <IQueue <WebHookNotification> >(() => new InMemoryQueue <WebHookNotification>(statName: MetricNames.WebHookQueueSize, metrics: container.GetInstance <IMetricsClient>()));
container.RegisterSingle <IQueue <MailMessage> >(() => new InMemoryQueue <MailMessage>(statName: MetricNames.EmailsQueueSize, metrics: container.GetInstance <IMetricsClient>()));
container.RegisterSingle <IQueue <StatusMessage> >(() => new InMemoryQueue <StatusMessage>());
container.RegisterSingle <IMessageBus, InMemoryMessageBus>();
container.RegisterSingle <IMessagePublisher>(container.GetInstance <IMessageBus>);
container.RegisterSingle <IMessageSubscriber>(container.GetInstance <IMessageBus>);
if (!String.IsNullOrEmpty(Settings.Current.StorageFolder))
{
container.RegisterSingle <IFileStorage>(new FolderFileStorage(Settings.Current.StorageFolder));
}
else
{
container.RegisterSingle <IFileStorage>(new InMemoryFileStorage());
}
container.RegisterSingle <IStackRepository, StackRepository>();
container.RegisterSingle <IEventRepository, EventRepository>();
container.RegisterSingle <IOrganizationRepository, OrganizationRepository>();
container.RegisterSingle <IProjectRepository, ProjectRepository>();
container.RegisterSingle <IUserRepository, UserRepository>();
container.RegisterSingle <IWebHookRepository, WebHookRepository>();
container.RegisterSingle <ITokenRepository, TokenRepository>();
container.RegisterSingle <IApplicationRepository, ApplicationRepository>();
container.RegisterSingle <IGeoIPResolver, MindMaxGeoIPResolver>();
container.RegisterSingle <IValidator <Application>, ApplicationValidator>();
container.RegisterSingle <IValidator <Organization>, OrganizationValidator>();
container.RegisterSingle <IValidator <PersistentEvent>, PersistentEventValidator>();
container.RegisterSingle <IValidator <Project>, ProjectValidator>();
container.RegisterSingle <IValidator <Stack>, StackValidator>();
container.RegisterSingle <IValidator <Models.Admin.Token>, TokenValidator>();
container.RegisterSingle <IValidator <UserDescription>, UserDescriptionValidator>();
container.RegisterSingle <IValidator <User>, UserValidator>();
container.RegisterSingle <IValidator <WebHook>, WebHookValidator>();
container.RegisterSingle <IEmailGenerator>(() => new RazorEmailGenerator(@"Mail\Templates"));
container.RegisterSingle <IMailer, Mailer>();
if (Settings.Current.WebsiteMode != WebsiteMode.Dev)
{
container.RegisterSingle <IMailSender, SmtpMailSender>();
}
else
{
container.RegisterSingle <IMailSender>(() => new InMemoryMailSender());
}
container.Register <ILockProvider, CacheLockProvider>();
container.Register <StripeEventHandler>();
container.RegisterSingle <BillingManager>();
container.RegisterSingle <DataHelper>();
container.RegisterSingle <EventStats>();
container.RegisterSingle <EventPipeline>();
container.RegisterSingle <EventPluginManager>();
container.RegisterSingle <FormattingPluginManager>();
container.RegisterSingle <ICoreLastReferenceIdManager, NullCoreLastReferenceIdManager>();
}
protected ContentChangedEventHandlerBase(ElasticSearchConfiguration config)
{
CurrentIndexName = config.DefaultIndexName;
Client = config.Client;
}
public static IHealthChecksBuilder AddElasticSearchHealthCheck(this IHealthChecksBuilder builder, ElasticSearchConfiguration configuration)
{
//if (configuration.Enabled)
//{
// var elasticUrl = configuration.ElasticUrl;
// builder.AddElasticsearch(elasticUrl, name: "ElasticSearch Client", failureStatus: HealthStatus.Degraded,
// tags: new[] { HealthCheckType.Infrastructure.ToString(), HealthCheckType.Logging.ToString(), "Port:9200" }, timeout: new TimeSpan(0,1,0));
//}
return(builder);
}
protected SearchAppServiceBase(ElasticSearchConfiguration config)
{
Client = config.Client;
CurrentIndexName = config.DefaultIndexName;
}
