private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient) { var searcher = searcherManager.Get(); try { // Track number of documents in index searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs()); // Track time between Lucene commit and reopen string temp; if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp)) { var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal); searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag, (searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds, new Dictionary <string, string>() { { SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") }, { SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") } }); } } finally { searcherManager.Release(searcher); } }
public void Configuration(IAppBuilder app, IConfiguration configuration, Directory directory, ILoader loader) { // Configure Logging.ApplicationInsights.Initialize(configuration.Get("serilog:ApplicationInsightsInstrumentationKey")); // Create telemetry sink _searchTelemetryClient = new SearchTelemetryClient(); // Create an ILoggerFactory var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: false) .Enrich.With<HttpRequestIdEnricher>() .Enrich.With<HttpRequestTraceIdEnricher>() .Enrich.With<HttpRequestTypeEnricher>() .Enrich.With<HttpRequestUrlReferrerEnricher>() .Enrich.With<HttpRequestUserAgentEnricher>() .Enrich.With<HttpRequestRawUrlEnricher>(); // Customize Serilog web logging - https://github.com/serilog-web/classic ApplicationLifecycleModule.RequestLoggingLevel = LogEventLevel.Warning; ApplicationLifecycleModule.LogPostedFormData = LogPostedFormDataOption.OnlyOnError; var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration); // Create a logger that is scoped to this class (only) _logger = loggerFactory.CreateLogger<Startup>(); _logger.LogInformation(LogMessages.AppStartup); // Correlate requests app.Use(typeof(CorrelationIdMiddleware)); // Add Application Insights app.Use(typeof(RequestTrackingMiddleware)); // Enable HSTS app.Use(async (context, next) => { context.Response.Headers.Add("Strict-Transport-Security", new string[] { "max-age=31536000; includeSubDomains" }); await next.Invoke(); }); // Enable CORS var corsPolicy = new CorsPolicy { Methods = { "GET", "HEAD", "OPTIONS" }, Headers = { "Content-Type", "If-Match", "If-Modified-Since", "If-None-Match", "If-Unmodified-Since", "Accept-Encoding" }, ExposedHeaders = { "Content-Type", "Content-Length", "Last-Modified", "Transfer-Encoding", "ETag", "Date", "Vary", "Server", "X-Hit", "X-CorrelationId" }, AllowAnyOrigin = true, PreflightMaxAge = 3600 }; app.UseCors(new CorsOptions { PolicyProvider = new CorsPolicyProvider { PolicyResolver = context => Task.FromResult(corsPolicy) } }); // Search test console app.Use(typeof(SearchConsoleMiddleware)); app.UseStaticFiles(new StaticFileOptions(new SharedOptions { RequestPath = new PathString("/console"), FileSystem = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console") })); // Start the service running - the Lucene index needs to be reopened regularly on a background thread var searchIndexRefresh = configuration.Get("Search.IndexRefresh") ?? "300"; int seconds; if (!int.TryParse(searchIndexRefresh, out seconds)) { seconds = 120; } _logger.LogInformation(LogMessages.SearchIndexRefreshConfiguration, seconds); if (InitializeSearcherManager(configuration, directory, loader, loggerFactory)) { var intervalInMs = seconds * 1000; _gate = 0; _indexReloadTimer = new Timer(ReopenCallback, 0, intervalInMs, intervalInMs); } _responseWriter = new ResponseWriter(); app.Run(InvokeAsync); }
private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient) { var searcher = searcherManager.Get(); try { // Track number of documents in index searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs()); // Track time between Lucene commit and reopen string temp; if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp)) { var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal); searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag, (searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds, new Dictionary<string, string>() { { SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") }, { SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") } }); } } finally { searcherManager.Release(searcher); } }
public void Configuration(IAppBuilder app, IConfigurationFactory configFactory, Directory directory, ILoader loader) { _configFactory = configFactory; var config = _configFactory.Get <BasicSearchConfiguration>().Result; // Configure if (!string.IsNullOrEmpty(config.ApplicationInsightsInstrumentationKey)) { TelemetryConfiguration.Active.InstrumentationKey = config.ApplicationInsightsInstrumentationKey; } // Add telemetry initializers TelemetryConfiguration.Active.TelemetryInitializers.Add(new MachineNameTelemetryInitializer()); TelemetryConfiguration.Active.TelemetryInitializers.Add(new DeploymentIdTelemetryInitializer()); // Create telemetry sink _searchTelemetryClient = new SearchTelemetryClient(); // Add telemetry processors var processorChain = TelemetryConfiguration.Active.TelemetryProcessorChainBuilder; processorChain.Use(next => { var processor = new RequestTelemetryProcessor(next); processor.SuccessfulResponseCodes.Add(400); processor.SuccessfulResponseCodes.Add(404); return(processor); }); processorChain.Use(next => new ExceptionTelemetryProcessor(next, _searchTelemetryClient.TelemetryClient)); processorChain.Build(); // Create an ILoggerFactory var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: false) .Enrich.With <HttpRequestIdEnricher>() .Enrich.With <HttpRequestTraceIdEnricher>() .Enrich.With <HttpRequestTypeEnricher>() .Enrich.With <HttpRequestUrlReferrerEnricher>() .Enrich.With <HttpRequestUserAgentEnricher>() .Enrich.With <HttpRequestRawUrlEnricher>(); var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration); // Create a logger that is scoped to this class (only) _logger = loggerFactory.CreateLogger <Startup>(); _logger.LogInformation(LogMessages.AppStartup); // Overwrite the index's Azure Directory cache path if configured ot use an Azure Local Storage resource. if (!string.IsNullOrEmpty(config.AzureDirectoryCacheLocalResourceName)) { if (SafeRoleEnvironment.TryGetLocalResourceRootPath(config.AzureDirectoryCacheLocalResourceName, out var path)) { config.AzureDirectoryCachePath = path; _logger.LogInformation( "Set Azure Directory cache path to Azure Local Resource = {LocalResourceName}, Path = {LocalResourcePath}", config.AzureDirectoryCacheLocalResourceName, config.AzureDirectoryCachePath); } else { _logger.LogWarning( "Cannot use Azure Local Resource {LocalResourceName} for caching when the RoleEnvironment is not available", config.AzureDirectoryCacheLocalResourceName); } } // redirect all HTTP requests to HTTPS if (config.RequireSsl) { if (string.IsNullOrWhiteSpace(config.ForceSslExclusion)) { app.UseForceSsl(config.SslPort); } else { app.UseForceSsl(config.SslPort, new[] { config.ForceSslExclusion }); } } // Correlate requests app.Use(typeof(CorrelationIdMiddleware)); // Add Application Insights app.Use(typeof(RequestTrackingMiddleware)); // Set up exception logging app.Use(typeof(ExceptionTrackingMiddleware)); // Enable HSTS app.Use(async(context, next) => { context.Response.Headers.Add("Strict-Transport-Security", new string[] { "max-age=31536000; includeSubDomains" }); await next.Invoke(); }); // Disable content type sniffing app.Use(async(context, next) => { context.Response.Headers.Add("X-Content-Type-Options", new[] { "nosniff" }); await next.Invoke(); }); // Enable CORS var corsPolicy = new CorsPolicy { Methods = { "GET", "HEAD", "OPTIONS" }, Headers = { "Content-Type", "If-Match", "If-Modified-Since", "If-None-Match", "If-Unmodified-Since", "Accept-Encoding" }, ExposedHeaders = { "Content-Type", "Content-Length", "Last-Modified", "Transfer-Encoding", "ETag", "Date", "Vary", "Server", "X-Hit", "X-CorrelationId" }, AllowAnyOrigin = true, PreflightMaxAge = 3600 }; app.UseCors(new CorsOptions { PolicyProvider = new CorsPolicyProvider { PolicyResolver = context => Task.FromResult(corsPolicy) } }); // Search test console app.Use(typeof(SearchConsoleMiddleware)); app.UseStaticFiles(new StaticFileOptions(new SharedOptions { RequestPath = new PathString("/console"), FileSystem = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console") })); // Start the service running - the Lucene index needs to be reopened regularly on a background thread var intervalSec = config.IndexRefreshSec; _logger.LogInformation(LogMessages.SearchIndexRefreshConfiguration, intervalSec); if (InitializeSearcherManager(config, directory, loader, loggerFactory)) { var intervalMs = intervalSec * 1000; _gate = 0; _indexReloadTimer = new Timer(ReopenCallback, 0, intervalMs, intervalMs); } _responseWriter = new ResponseWriter(); app.Run(InvokeAsync); }