private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient)
{
var searcher = searcherManager.Get();
try
{
// Track number of documents in index
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs());
// Track time between Lucene commit and reopen
string temp;
if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp))
{
var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal);
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag,
(searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds,
new Dictionary <string, string>()
{
{ SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") },
{ SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") }
});
}
}
finally
{
searcherManager.Release(searcher);
}
}
public void Configuration(IAppBuilder app, IConfiguration configuration, Directory directory, ILoader loader)
{
// Configure
Logging.ApplicationInsights.Initialize(configuration.Get("serilog:ApplicationInsightsInstrumentationKey"));
// Create telemetry sink
_searchTelemetryClient = new SearchTelemetryClient();
// Create an ILoggerFactory
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: false)
.Enrich.With<HttpRequestIdEnricher>()
.Enrich.With<HttpRequestTraceIdEnricher>()
.Enrich.With<HttpRequestTypeEnricher>()
.Enrich.With<HttpRequestUrlReferrerEnricher>()
.Enrich.With<HttpRequestUserAgentEnricher>()
.Enrich.With<HttpRequestRawUrlEnricher>();
// Customize Serilog web logging - https://github.com/serilog-web/classic
ApplicationLifecycleModule.RequestLoggingLevel = LogEventLevel.Warning;
ApplicationLifecycleModule.LogPostedFormData = LogPostedFormDataOption.OnlyOnError;
var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration);
// Create a logger that is scoped to this class (only)
_logger = loggerFactory.CreateLogger<Startup>();
_logger.LogInformation(LogMessages.AppStartup);
// Correlate requests
app.Use(typeof(CorrelationIdMiddleware));
// Add Application Insights
app.Use(typeof(RequestTrackingMiddleware));
// Enable HSTS
app.Use(async (context, next) =>
{
context.Response.Headers.Add("Strict-Transport-Security", new string[] { "max-age=31536000; includeSubDomains" });
await next.Invoke();
});
// Enable CORS
var corsPolicy = new CorsPolicy
{
Methods = { "GET", "HEAD", "OPTIONS" },
Headers = { "Content-Type", "If-Match", "If-Modified-Since", "If-None-Match", "If-Unmodified-Since", "Accept-Encoding" },
ExposedHeaders = { "Content-Type", "Content-Length", "Last-Modified", "Transfer-Encoding", "ETag", "Date", "Vary", "Server", "X-Hit", "X-CorrelationId" },
AllowAnyOrigin = true,
PreflightMaxAge = 3600
};
app.UseCors(new CorsOptions
{
PolicyProvider = new CorsPolicyProvider
{
PolicyResolver = context => Task.FromResult(corsPolicy)
}
});
// Search test console
app.Use(typeof(SearchConsoleMiddleware));
app.UseStaticFiles(new StaticFileOptions(new SharedOptions
{
RequestPath = new PathString("/console"),
FileSystem = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console")
}));
// Start the service running - the Lucene index needs to be reopened regularly on a background thread
var searchIndexRefresh = configuration.Get("Search.IndexRefresh") ?? "300";
int seconds;
if (!int.TryParse(searchIndexRefresh, out seconds))
{
seconds = 120;
}
_logger.LogInformation(LogMessages.SearchIndexRefreshConfiguration, seconds);
if (InitializeSearcherManager(configuration, directory, loader, loggerFactory))
{
var intervalInMs = seconds * 1000;
_gate = 0;
_indexReloadTimer = new Timer(ReopenCallback, 0, intervalInMs, intervalInMs);
}
_responseWriter = new ResponseWriter();
app.Run(InvokeAsync);
}
private void TrackIndexMetrics(NuGetSearcherManager searcherManager, SearchTelemetryClient searchTelemetryClient)
{
var searcher = searcherManager.Get();
try
{
// Track number of documents in index
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneNumDocs, searcher.IndexReader.NumDocs());
// Track time between Lucene commit and reopen
string temp;
if (searcher.CommitUserData.TryGetValue("commitTimeStamp", out temp))
{
var commitTimestamp = DateTimeOffset.Parse(temp, null, DateTimeStyles.AssumeUniversal);
searchTelemetryClient.TrackMetric(SearchTelemetryClient.MetricName.LuceneLoadLag,
(searcher.LastReopen - commitTimestamp.UtcDateTime).TotalSeconds,
new Dictionary<string, string>()
{
{ SearchTelemetryClient.MetricName.LuceneLastReopen, searcher.LastReopen.ToString("o") },
{ SearchTelemetryClient.MetricName.LuceneCommitTimestamp, commitTimestamp.UtcDateTime.ToString("o") }
});
}
}
finally
{
searcherManager.Release(searcher);
}
}
public void Configuration(IAppBuilder app, IConfigurationFactory configFactory, Directory directory,
ILoader loader)
{
_configFactory = configFactory;
var config = _configFactory.Get <BasicSearchConfiguration>().Result;
// Configure
if (!string.IsNullOrEmpty(config.ApplicationInsightsInstrumentationKey))
{
TelemetryConfiguration.Active.InstrumentationKey = config.ApplicationInsightsInstrumentationKey;
}
// Add telemetry initializers
TelemetryConfiguration.Active.TelemetryInitializers.Add(new MachineNameTelemetryInitializer());
TelemetryConfiguration.Active.TelemetryInitializers.Add(new DeploymentIdTelemetryInitializer());
// Create telemetry sink
_searchTelemetryClient = new SearchTelemetryClient();
// Add telemetry processors
var processorChain = TelemetryConfiguration.Active.TelemetryProcessorChainBuilder;
processorChain.Use(next =>
{
var processor = new RequestTelemetryProcessor(next);
processor.SuccessfulResponseCodes.Add(400);
processor.SuccessfulResponseCodes.Add(404);
return(processor);
});
processorChain.Use(next => new ExceptionTelemetryProcessor(next, _searchTelemetryClient.TelemetryClient));
processorChain.Build();
// Create an ILoggerFactory
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: false)
.Enrich.With <HttpRequestIdEnricher>()
.Enrich.With <HttpRequestTraceIdEnricher>()
.Enrich.With <HttpRequestTypeEnricher>()
.Enrich.With <HttpRequestUrlReferrerEnricher>()
.Enrich.With <HttpRequestUserAgentEnricher>()
.Enrich.With <HttpRequestRawUrlEnricher>();
var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration);
// Create a logger that is scoped to this class (only)
_logger = loggerFactory.CreateLogger <Startup>();
_logger.LogInformation(LogMessages.AppStartup);
// Overwrite the index's Azure Directory cache path if configured ot use an Azure Local Storage resource.
if (!string.IsNullOrEmpty(config.AzureDirectoryCacheLocalResourceName))
{
if (SafeRoleEnvironment.TryGetLocalResourceRootPath(config.AzureDirectoryCacheLocalResourceName, out var path))
{
config.AzureDirectoryCachePath = path;
_logger.LogInformation(
"Set Azure Directory cache path to Azure Local Resource = {LocalResourceName}, Path = {LocalResourcePath}",
config.AzureDirectoryCacheLocalResourceName,
config.AzureDirectoryCachePath);
}
else
{
_logger.LogWarning(
"Cannot use Azure Local Resource {LocalResourceName} for caching when the RoleEnvironment is not available",
config.AzureDirectoryCacheLocalResourceName);
}
}
// redirect all HTTP requests to HTTPS
if (config.RequireSsl)
{
if (string.IsNullOrWhiteSpace(config.ForceSslExclusion))
{
app.UseForceSsl(config.SslPort);
}
else
{
app.UseForceSsl(config.SslPort, new[] { config.ForceSslExclusion });
}
}
// Correlate requests
app.Use(typeof(CorrelationIdMiddleware));
// Add Application Insights
app.Use(typeof(RequestTrackingMiddleware));
// Set up exception logging
app.Use(typeof(ExceptionTrackingMiddleware));
// Enable HSTS
app.Use(async(context, next) =>
{
context.Response.Headers.Add("Strict-Transport-Security", new string[] { "max-age=31536000; includeSubDomains" });
await next.Invoke();
});
// Disable content type sniffing
app.Use(async(context, next) =>
{
context.Response.Headers.Add("X-Content-Type-Options", new[] { "nosniff" });
await next.Invoke();
});
// Enable CORS
var corsPolicy = new CorsPolicy
{
Methods = { "GET", "HEAD", "OPTIONS" },
Headers = { "Content-Type", "If-Match", "If-Modified-Since", "If-None-Match", "If-Unmodified-Since", "Accept-Encoding" },
ExposedHeaders = { "Content-Type", "Content-Length", "Last-Modified", "Transfer-Encoding", "ETag", "Date", "Vary", "Server", "X-Hit", "X-CorrelationId" },
AllowAnyOrigin = true,
PreflightMaxAge = 3600
};
app.UseCors(new CorsOptions
{
PolicyProvider = new CorsPolicyProvider
{
PolicyResolver = context => Task.FromResult(corsPolicy)
}
});
// Search test console
app.Use(typeof(SearchConsoleMiddleware));
app.UseStaticFiles(new StaticFileOptions(new SharedOptions
{
RequestPath = new PathString("/console"),
FileSystem = new EmbeddedResourceFileSystem(typeof(Startup).Assembly, "NuGet.Services.BasicSearch.Console")
}));
// Start the service running - the Lucene index needs to be reopened regularly on a background thread
var intervalSec = config.IndexRefreshSec;
_logger.LogInformation(LogMessages.SearchIndexRefreshConfiguration, intervalSec);
if (InitializeSearcherManager(config, directory, loader, loggerFactory))
{
var intervalMs = intervalSec * 1000;
_gate = 0;
_indexReloadTimer = new Timer(ReopenCallback, 0, intervalMs, intervalMs);
}
_responseWriter = new ResponseWriter();
app.Run(InvokeAsync);
}
