public static async Task MainAsync(string[] args) { if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase)) { args = args.Skip(1).ToArray(); Debugger.Launch(); } NgJob job = null; try { // Get arguments var arguments = CommandHelpers.GetArguments(args, 1); // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled. ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; // Determine the job name if (args.Length == 0) { throw new ArgumentException("Missing job name argument."); } var jobName = args[0]; TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName)); // Configure ApplicationInsights ApplicationInsights.Initialize(arguments.GetOrDefault <string>(Arguments.InstrumentationKey)); // Create an ILoggerFactory var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true); loggerConfiguration.WriteTo.File("Log.txt", retainedFileCountLimit: 3, fileSizeLimitBytes: 1000000, rollOnFileSizeLimit: true); var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Debug); // Create a logger that is scoped to this class (only) _logger = loggerFactory.CreateLogger <Program>(); var cancellationTokenSource = new CancellationTokenSource(); // Create an ITelemetryService var telemetryService = new TelemetryService(new TelemetryClient()); // Allow jobs to set global custom dimensions TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobPropertiesTelemetryInitializer(telemetryService)); job = NgJobFactory.GetJob(jobName, telemetryService, loggerFactory); await job.RunAsync(arguments, cancellationTokenSource.Token); } catch (ArgumentException ae) { _logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae); Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase()); } catch (Exception e) { _logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e); } Trace.Close(); TelemetryConfiguration.Active.TelemetryChannel.Flush(); }
private static async Task Run(JobBase job, string[] commandLineArgs, bool?runContinuously) { if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase)) { commandLineArgs = commandLineArgs.Skip(1).ToArray(); Debugger.Launch(); } // Configure logging before Application Insights is enabled. // This is done so, in case Application Insights fails to initialize, we still see output. var loggerFactory = ConfigureLogging(job); try { _logger.LogInformation("Started..."); // Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue> var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary( ServiceContainer, loggerFactory.CreateLogger(typeof(JobConfigurationManager)), commandLineArgs); // Determine job and instance name, for logging. var jobName = job.GetType().Assembly.GetName().Name; var instanceName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstanceName) ?? jobName; TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName, instanceName)); // Setup logging if (!ApplicationInsights.Initialized) { string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); if (!string.IsNullOrWhiteSpace(instrumentationKey)) { ApplicationInsights.Initialize(instrumentationKey); } } // Configure our logging again with Application Insights initialized. loggerFactory = ConfigureLogging(job); var hasOnceArgument = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once); if (runContinuously.HasValue && hasOnceArgument) { _logger.LogWarning( $"This job is designed to {(runContinuously.Value ? "run continuously" : "run once")} so " + $"the -{JobArgumentNames.Once} argument is {(runContinuously.Value ? "ignored" : "redundant")}."); } runContinuously = runContinuously ?? !hasOnceArgument; var reinitializeAfterSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.ReinitializeAfterSeconds); var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds if (!sleepDuration.HasValue) { sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval); if (sleepDuration.HasValue) { sleepDuration = sleepDuration.Value * 1000; // interval is in seconds } } if (!sleepDuration.HasValue) { if (runContinuously.Value) { _logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms..."); } sleepDuration = 5000; } else if (!runContinuously.Value) { _logger.LogWarning( $"The job is designed to run once so the -{JobArgumentNames.Sleep} and " + $"-{JobArgumentNames.Interval} arguments are ignored."); } if (!reinitializeAfterSeconds.HasValue) { _logger.LogInformation( $"{JobArgumentNames.ReinitializeAfterSeconds} command line argument is not provided or is not a valid integer. " + "The job will reinitialize on every iteration"); } else if (!runContinuously.Value) { _logger.LogWarning( $"The job is designed to run once so the -{JobArgumentNames.ReinitializeAfterSeconds} " + $"argument is ignored."); } // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled. ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; // Run the job loop await JobLoop(job, runContinuously.Value, sleepDuration.Value, reinitializeAfterSeconds, jobArgsDictionary); } catch (Exception ex) { _logger.LogError("Job runner threw an exception: {Exception}", ex); } Trace.Close(); TelemetryConfiguration.Active.TelemetryChannel.Flush(); }
public static void Init(){ var forceLoad = new ApplicationInsights (); }
private async Task <IReadOnlyCollection <T> > GetDimension <T>(string dimension, string logFileName, Func <SqlConnection, Task <IReadOnlyCollection <T> > > retrieve) { var stopwatch = Stopwatch.StartNew(); var count = _maxRetryCount; while (count > 0) { try { _jobEventSource.BeginningRetrieveDimension(dimension); IReadOnlyCollection <T> dimensions; using (var connection = await _targetDatabase.ConnectTo()) { dimensions = await retrieve(connection); } stopwatch.Stop(); _jobEventSource.FinishedRetrieveDimension(dimension, stopwatch.ElapsedMilliseconds); ApplicationInsights.TrackRetrieveDimensionDuration(dimension, stopwatch.ElapsedMilliseconds, logFileName); return(dimensions); } catch (SqlException e) { --count; if (count <= 0) { throw; } if (e.Number == 1205) { Trace.TraceWarning("Deadlock, retrying..."); ApplicationInsights.TrackSqlException("SQL Deadlock", e, logFileName, dimension); } else if (e.Number == -2) { Trace.TraceWarning("Timeout, retrying..."); ApplicationInsights.TrackSqlException("SQL Timeout", e, logFileName, dimension); } else if (e.Number == 2601) { Trace.TraceWarning("Duplicate key, retrying..."); ApplicationInsights.TrackSqlException("SQL Duplicate Key", e, logFileName, dimension); } else { throw; } Task.Delay(_retryDelay).Wait(); } catch (Exception exception) { _jobEventSource.FailedRetrieveDimension(dimension); ApplicationInsights.TrackException(exception, logFileName); if (stopwatch.IsRunning) { stopwatch.Stop(); } throw; } } return(Enumerable.Empty <T>().ToList()); }
public async Task <DataTable> CreateAsync(IReadOnlyCollection <PackageStatistics> sourceData, string logFileName) { var stopwatch = Stopwatch.StartNew(); // insert any new dimension data first if (_times == null) { // this call is only needed once in the lifetime of the service _times = await GetDimension("time", logFileName, connection => RetrieveTimeDimensions(connection)); } var packagesTask = GetDimension("package", logFileName, connection => RetrievePackageDimensions(sourceData, connection)); var operationsTask = GetDimension("operation", logFileName, connection => RetrieveOperationDimensions(sourceData, connection)); var projectTypesTask = GetDimension("project type", logFileName, connection => RetrieveProjectTypeDimensions(sourceData, connection)); var clientsTask = GetDimension("client", logFileName, connection => RetrieveClientDimensions(sourceData, connection)); var platformsTask = GetDimension("platform", logFileName, connection => RetrievePlatformDimensions(sourceData, connection)); var datesTask = GetDimension("date", logFileName, connection => RetrieveDateDimensions(connection, sourceData.Min(e => e.EdgeServerTimeDelivered), sourceData.Max(e => e.EdgeServerTimeDelivered))); var packageTranslationsTask = GetDimension("package translations", logFileName, connection => RetrievePackageTranslations(sourceData, connection)); await Task.WhenAll(operationsTask, projectTypesTask, clientsTask, platformsTask, datesTask, packagesTask, packageTranslationsTask); var operations = operationsTask.Result; var projectTypes = projectTypesTask.Result; var clients = clientsTask.Result; var platforms = platformsTask.Result; var dates = datesTask.Result; var packages = packagesTask.Result; var packageTranslations = packageTranslationsTask.Result; // create facts data rows by linking source data with dimensions var dataImporter = new DataImporter(_targetDatabase); var dataTable = await dataImporter.GetDataTableAsync("Fact_Download"); // ensure all dimension IDs are set to the Unknown equivalent if no dimension data is available int?operationId = !operations.Any() ? DimensionId.Unknown : (int?)null; int?projectTypeId = !projectTypes.Any() ? DimensionId.Unknown : (int?)null; int?clientId = !clients.Any() ? DimensionId.Unknown : (int?)null; int?platformId = !platforms.Any() ? DimensionId.Unknown : (int?)null; Trace.WriteLine("Creating facts..."); foreach (var groupedByPackageId in sourceData.GroupBy(e => e.PackageId, StringComparer.OrdinalIgnoreCase)) { var packagesForId = packages.Where(e => string.Equals(e.PackageId, groupedByPackageId.Key, StringComparison.OrdinalIgnoreCase)).ToList(); foreach (var groupedByPackageIdAndVersion in groupedByPackageId.GroupBy(e => e.PackageVersion, StringComparer.OrdinalIgnoreCase)) { int packageId; var package = packagesForId.FirstOrDefault(e => string.Equals(e.PackageVersion, groupedByPackageIdAndVersion.Key, StringComparison.OrdinalIgnoreCase)); if (package == null) { // This package id and version could not be 100% accurately parsed from the CDN Request URL, // likely due to weird package ID which could be interpreted as a version string. // Look for a mapping in the support table in an attempt to auto-correct this entry. var packageTranslation = packageTranslations.FirstOrDefault(t => t.IncorrectPackageId == groupedByPackageId.Key && t.IncorrectPackageVersion == groupedByPackageIdAndVersion.Key); if (packageTranslation != null) { // there seems to be a mapping packageId = packageTranslation.CorrectedPackageId; } else { // Track it in Application Insights. ApplicationInsights.TrackPackageNotFound(groupedByPackageId.Key, groupedByPackageIdAndVersion.Key, logFileName); continue; } } else { packageId = package.Id; } foreach (var element in groupedByPackageIdAndVersion) { // required dimensions var dateId = dates.First(e => e.Date.Equals(element.EdgeServerTimeDelivered.Date)).Id; var timeId = _times.First(e => e.HourOfDay == element.EdgeServerTimeDelivered.Hour).Id; // dimensions that could be "(unknown)" if (!operationId.HasValue) { if (!operations.ContainsKey(element.Operation)) { operationId = DimensionId.Unknown; } else { operationId = operations[element.Operation]; } } if (!platformId.HasValue) { if (!platforms.ContainsKey(element.UserAgent)) { platformId = DimensionId.Unknown; } else { platformId = platforms[element.UserAgent]; } } if (!clientId.HasValue) { if (!clients.ContainsKey(element.UserAgent)) { clientId = DimensionId.Unknown; } else { clientId = clients[element.UserAgent]; } } if (!projectTypeId.HasValue) { // foreach project type foreach (var projectGuid in element.ProjectGuids.Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries)) { projectTypeId = projectTypes[projectGuid]; var dataRow = dataTable.NewRow(); FillDataRow(dataRow, dateId, timeId, packageId, operationId.Value, platformId.Value, projectTypeId.Value, clientId.Value, logFileName, element.UserAgent); dataTable.Rows.Add(dataRow); } } else { var dataRow = dataTable.NewRow(); FillDataRow(dataRow, dateId, timeId, packageId, operationId.Value, platformId.Value, projectTypeId.Value, clientId.Value, logFileName, element.UserAgent); dataTable.Rows.Add(dataRow); } } } } stopwatch.Stop(); Trace.Write(" DONE (" + dataTable.Rows.Count + " records, " + stopwatch.ElapsedMilliseconds + "ms)"); ApplicationInsights.TrackMetric("Blob record count", dataTable.Rows.Count, logFileName); return(dataTable); }
public UserProfile() { Roles = new List <string>(); ApplicationInsights = new ApplicationInsights(); }
//internal static ClientContext GetCachedContext(string url) //{ // return ContextCache.FirstOrDefault(c => System.Net.WebUtility.UrlEncode(c.Url) == System.Net.WebUtility.UrlEncode(url)); //} //internal static void ClearContextCache() //{ // ContextCache.Clear(); //} internal void InitializeTelemetry(ClientContext context, InitializationType initializationType) { var enableTelemetry = false; var userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); var telemetryFile = System.IO.Path.Combine(userProfile, ".pnppowershelltelemetry"); if (!System.IO.File.Exists(telemetryFile)) { enableTelemetry = true; } else { if (System.IO.File.ReadAllText(telemetryFile).ToLower() == "allow") { enableTelemetry = true; } } if (enableTelemetry) { var serverLibraryVersion = ""; var serverVersion = ""; if (context != null) { try { if (context.ServerLibraryVersion != null) { serverLibraryVersion = context.ServerLibraryVersion.ToString(); } if (context.ServerVersion != null) { serverVersion = context.ServerVersion.ToString(); } } catch { } } ApplicationInsights = new ApplicationInsights(); var coreAssembly = Assembly.GetExecutingAssembly(); ApplicationInsights.Initialize(serverLibraryVersion, serverVersion, initializationType.ToString(), ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString()); ApplicationInsights.TrackEvent("Connect-PnPOnline"); //TelemetryConfiguration config = TelemetryConfiguration.CreateDefault(); //TelemetryClient = new TelemetryClient(config); //config.InstrumentationKey = "a301024a-9e21-4273-aca5-18d0ef5d80fb"; ////config..Context.Session.Id = Guid.NewGuid().ToString(); //TelemetryClient.Context.Cloud.RoleInstance = "PnPPowerShell"; //TelemetryClient.Context.Device.OperatingSystem = Environment.OSVersion.ToString(); //TelemetryProperties = new Dictionary<string, string>(10); //TelemetryProperties.Add("ServerLibraryVersion", serverLibraryVersion); //TelemetryProperties.Add("ServerVersion", serverVersion); //TelemetryProperties.Add("ConnectionMethod", initializationType.ToString()); //var coreAssembly = Assembly.GetExecutingAssembly(); //TelemetryProperties.Add("Version", ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString()); //TelemetryProperties.Add("Platform", "SPO"); //TelemetryClient.TrackEvent("Connect-PnPOnline", TelemetryProperties); //TelemetryClient = new TelemetryClient(; //TelemetryClient.InstrumentationKey = "a301024a-9e21-4273-aca5-18d0ef5d80fb"; //TelemetryClient.Context.Session.Id = Guid.NewGuid().ToString(); //TelemetryClient.Context.Cloud.RoleInstance = "PnPPowerShell"; //TelemetryClient.Context.Device.OperatingSystem = Environment.OSVersion.ToString(); //TelemetryClient.Context.Properties.Add("ServerLibraryVersion", serverLibraryVersion); //TelemetryClient.Context.Properties.Add("ServerVersion", serverVersion); //TelemetryClient.Context.Properties.Add("ConnectionMethod", initializationType.ToString()); //var coreAssembly = Assembly.GetExecutingAssembly(); //TelemetryClient.Context.Properties.Add("Version", ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString()); //TelemetryClient.Context.Properties.Add("Platform", "SPO"); //TelemetryClient.TrackEvent("Connect-PnPOnline"); } }
public void InitializeRegistersTelemetryContextInitializer() { var applicationInsightsConfiguration = ApplicationInsights.Initialize(InstrumentationKey); Assert.Contains(applicationInsightsConfiguration.TelemetryConfiguration.TelemetryInitializers, ti => ti is TelemetryContextInitializer); }
/// <summary> /// This is a static method to run a job whose args are passed in /// By default, /// a) The job will be run continuously in a while loop. Could be overridden using 'once' argument /// b) The sleep duration between each run when running continuously is 5000 milliSeconds. Could be overridden using '-Sleep' argument /// </summary> /// <param name="job">Job to run</param> /// <param name="commandLineArgs">Args contains args to the job runner like (dbg, once and so on) and for the job itself</param> /// <returns></returns> public static async Task Run(JobBase job, string[] commandLineArgs) { if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase)) { commandLineArgs = commandLineArgs.Skip(1).ToArray(); Debugger.Launch(); } // Configure logging before Application Insights is enabled. // This is done so, in case Application Insights fails to initialize, we still see output. var loggerFactory = ConfigureLogging(job); try { _logger.LogInformation("Started..."); // Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue> var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(loggerFactory.CreateLogger(typeof(JobConfigurationManager)), commandLineArgs, job.JobName, (ISecretReaderFactory)ServiceContainer.GetService(typeof(ISecretReaderFactory))); // Setup logging if (!ApplicationInsights.Initialized) { string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); if (!string.IsNullOrWhiteSpace(instrumentationKey)) { ApplicationInsights.Initialize(instrumentationKey); } } // Configure our logging again with Application Insights initialized. loggerFactory = ConfigureLogging(job); var runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once); var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds if (!sleepDuration.HasValue) { sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval); if (sleepDuration.HasValue) { sleepDuration = sleepDuration.Value * 1000; // interval is in seconds } } if (sleepDuration == null) { _logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms..."); sleepDuration = 5000; } // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled. ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; // Run the job loop await JobLoop(job, runContinuously, sleepDuration.Value, jobArgsDictionary); } catch (Exception ex) { _logger.LogError("Job runner threw an exception: {Exception}", ex); } }
private static IServiceProvider InitializeAndGetServiceProvider() { ServicePointManager.DefaultConnectionLimit = 64; ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; var configurationRoot = GetConfigurationRoot(); var instrumentationKey = configurationRoot .GetSection(ConfigurationSectionName) .GetValue <string>(nameof(V3PerPackageConfiguration.InstrumentationKey)); ApplicationInsights.Initialize(instrumentationKey); var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true); var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Information); var serviceCollection = new ServiceCollection(); serviceCollection.Configure <V3PerPackageConfiguration>(configurationRoot.GetSection(ConfigurationSectionName)); serviceCollection.AddOptions(); serviceCollection.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot <>), typeof(NonCachingOptionsSnapshot <>))); serviceCollection.AddLogging(); serviceCollection.AddSingleton(loggerFactory); serviceCollection.AddSingleton(new ControlledDisposeHttpClientHandler()); serviceCollection.AddTransient <HttpMessageHandler>(x => x.GetRequiredService <ControlledDisposeHttpClientHandler>()); serviceCollection.AddSingleton(x => new HttpClient(x.GetRequiredService <HttpMessageHandler>())); serviceCollection.AddTransient <Func <HttpMessageHandler> >(x => () => x.GetRequiredService <HttpMessageHandler>()); serviceCollection.AddTransient <PerBatchProcessor>(); serviceCollection.AddTransient <ITelemetryService, TelemetryService>(); serviceCollection.AddTransient <PerWorkerProcessor>(); serviceCollection.AddTransient <PerProcessProcessor>(); serviceCollection.AddSingleton <StringLocker>(); serviceCollection.AddTransient <EnqueueCollector>(); serviceCollection.AddTransient <EnqueueCommand>(); serviceCollection.AddTransient <CleanUpCommand>(); serviceCollection.AddSingleton(x => { var globalContext = x.GetRequiredService <GlobalContext>(); var perProcessContext = new PerProcessContext( globalContext, UniqueName.New("process"), WorkerCount, MessageCount, BatchSize); var blobClient = BlobStorageUtilities.GetBlobClient(globalContext); var flatContainerUrl = $"{blobClient.BaseUri.AbsoluteUri}/{globalContext.FlatContainerContainerName}/{perProcessContext.Name}"; RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerUrl); return(perProcessContext); }); serviceCollection.AddTransient(x => { var settings = x.GetRequiredService <IOptionsSnapshot <V3PerPackageConfiguration> >(); return(new GlobalContext( settings.Value.StorageBaseAddress, settings.Value.StorageAccountName, settings.Value.StorageKeyValue, settings.Value.ContentBaseAddress)); }); serviceCollection.AddSingleton(new TelemetryClient()); serviceCollection.AddTransient <IStorageQueue <PackageMessage> >(x => { var globalContext = x.GetRequiredService <GlobalContext>(); var storageCredentials = new StorageCredentials(globalContext.StorageAccountName, globalContext.StorageKeyValue); var storageAccount = new CloudStorageAccount(storageCredentials, useHttps: true); return(new StorageQueue <PackageMessage>( new AzureStorageQueue(storageAccount, "v3perpackage"), new JsonMessageSerializer <PackageMessage>(JsonSerializerUtility.SerializerSettings), PackageMessage.Version)); }); return(serviceCollection.BuildServiceProvider()); }
public override bool Init(IDictionary <string, string> jobArgsDictionary) { try { if (!ApplicationInsights.Initialized) { string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey); if (!string.IsNullOrWhiteSpace(instrumentationKey)) { ApplicationInsights.Initialize(instrumentationKey); } } _loggerFactory = LoggingSetup.CreateLoggerFactory(); _logger = _loggerFactory.CreateLogger <Job>(); // Configure job _galleryBaseAddress = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryBaseAddress); var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount); _cloudStorageAccount = CreateCloudStorageAccount(JobArgumentNames.DataStorageAccount, storageConnectionString); _containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName); _runValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RunValidationTasks).Split(';'); _requestValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RequestValidationTasks).Split(';'); // Add validators if (_runValidationTasks.Contains(UnzipValidator.ValidatorName)) { _validators.Add(new UnzipValidator(_loggerFactory)); } if (_runValidationTasks.Contains(VcsValidator.ValidatorName)) { // if contact alias set, use it, if not, use submitter alias. string submitterAlias = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorSubmitterAlias); string contactAlias = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.VcsContactAlias) ?? submitterAlias; _validators.Add(new VcsValidator( JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorServiceUrl), JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorCallbackUrl), contactAlias, submitterAlias, JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsPackageUrlTemplate), _loggerFactory)); } return(true); } catch (Exception ex) { if (_logger != null) { _logger.LogError(TraceEvent.CommandLineProcessingFailed, ex, "Exception occurred while processing command line arguments"); } else { Trace.TraceError("Exception occurred while processing command line arguments: {0}", ex); } } return(false); }
void SetupApplicationInsights() { ApplicationInsights.Setup(InstrumentationKey); ApplicationInsights.SetDebugLogEnabled(true); ApplicationInsights.Start(); }
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile) { var logStream = await OpenCompressedBlobAsync(logFile); var blobUri = logFile.Uri; var blobName = logFile.Blob.Name; var packageStatistics = new List <PackageStatistics>(); var toolStatistics = new List <ToolStatistics>(); var dnxStatistics = new List <DnxStatistics>(); var stopwatch = Stopwatch.StartNew(); try { // parse the log into table entities _jobEventSource.BeginningParseLog(blobUri); using (var logStreamReader = new StreamReader(logStream)) { do { var rawLogLine = logStreamReader.ReadLine(); if (rawLogLine != null) { var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (logEntry != null) { var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry); if (statistic != null) { packageStatistics.Add(statistic); } else { // check if this is a dist.nuget.org download if (logEntry.RequestUrl.Contains("dist.nuget.org/")) { var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry); if (toolInfo != null) { toolStatistics.Add(toolInfo); } } if (logEntry.RequestUrl.Contains("dist.asp.net")) { var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry); if (dnxInfo != null) { dnxStatistics.Add(dnxInfo); } } } } } } while (!logStreamReader.EndOfStream); } _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count); stopwatch.Stop(); ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName); } catch (Exception exception) { if (stopwatch.IsRunning) { stopwatch.Stop(); } _jobEventSource.FailedParseLog(blobUri); ApplicationInsights.TrackException(exception, blobName); throw; } finally { logStream.Dispose(); } var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics); return(cdnStatistics); }
private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime) { var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_statisticsDatabase, reportGenerationTime); if (!dirtyPackageIds.Any()) { return; } // first process the top 100 packages var top100 = dirtyPackageIds.Take(100); var reportDataCollector = new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase); var top100Task = Parallel.ForEach(top100, new ParallelOptions { MaxDegreeOfParallelism = 4 }, dirtyPackageId => { var packageId = dirtyPackageId.PackageId.ToLowerInvariant(); var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId); ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait(); ApplicationInsights.TrackReportProcessed(reportBuilder.ReportName + " report", packageId); }); // once top 100 is processed, continue with the rest if (top100Task.IsCompleted) { var excludingTop100 = dirtyPackageIds.Skip(100); top100Task = Parallel.ForEach(excludingTop100, new ParallelOptions { MaxDegreeOfParallelism = 8 }, dirtyPackageId => { // generate all reports var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector> { { new RecentPopularityDetailByPackageReportBuilder( ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + dirtyPackageId.PackageId.ToLowerInvariant()), new ReportDataCollector( _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase) } }; foreach (var reportGenerator in reportGenerators) { ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait(); ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report", dirtyPackageId.PackageId.ToLowerInvariant()); } }); if (top100Task.IsCompleted) { var runToCursor = dirtyPackageIds.First().RunToCuror; await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDatabase, runToCursor); } } }
public static LoggerConfiguration ConfigureTraceLogger(LoggerConfiguration loggerConfig, ApplicationInsights appInsightsConfig, LoggingLevelSwitch levelSwitch = null) { ConfigureLoggerConfiguration(loggerConfig, levelSwitch ?? LoggingLevelSwitch); if (appInsightsConfig != null && appInsightsConfig.Enabled && !string.IsNullOrEmpty(appInsightsConfig.InstrumentationKey)) { loggerConfig.WriteTo.ApplicationInsights(TelemetryConverter.Traces); } return(loggerConfig); }
// This method is auto-detected by the OWIN pipeline. DO NOT RENAME IT! public static void Configuration(IAppBuilder app) { // Tune ServicePointManager // (based on http://social.technet.microsoft.com/Forums/en-US/windowsazuredata/thread/d84ba34b-b0e0-4961-a167-bbe7618beb83 and https://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.aspx) ServicePointManager.DefaultConnectionLimit = 500; ServicePointManager.UseNagleAlgorithm = false; ServicePointManager.Expect100Continue = false; // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled. ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; // Setting time out for all RegEx objects. Noted in remarks at https://msdn.microsoft.com/en-us/library/system.text.regularexpressions.regex.matchtimeout%28v=vs.110%29.aspx AppDomain.CurrentDomain.SetData("REGEX_DEFAULT_MATCH_TIMEOUT", TimeSpan.FromSeconds(10)); // Register IoC app.UseAutofacInjection(GlobalConfiguration.Configuration); var dependencyResolver = DependencyResolver.Current; // Register Elmah var elmahServiceCenter = new DependencyResolverServiceProviderAdapter(dependencyResolver); ServiceCenter.Current = _ => elmahServiceCenter; // Get config var config = dependencyResolver.GetService <IGalleryConfigurationService>(); var auth = dependencyResolver.GetService <AuthenticationService>(); // Configure machine key for session persistence across slots SessionPersistence.Setup(config); // Refresh the content for the ContentObjectService to guarantee it has loaded the latest configuration on startup. var contentObjectService = dependencyResolver.GetService <IContentObjectService>(); HostingEnvironment.QueueBackgroundWorkItem(async token => { while (!token.IsCancellationRequested) { await contentObjectService.Refresh(); await Task.Delay(ContentObjectService.RefreshInterval, token); } }); // Setup telemetry var instrumentationKey = config.Current.AppInsightsInstrumentationKey; if (!string.IsNullOrEmpty(instrumentationKey)) { var heartbeatIntervalSeconds = config.Current.AppInsightsHeartbeatIntervalSeconds; if (heartbeatIntervalSeconds > 0) { // Configure instrumentation key, heartbeat interval, // and register NuGet.Services.Logging.TelemetryContextInitializer. ApplicationInsights.Initialize(instrumentationKey, TimeSpan.FromSeconds(heartbeatIntervalSeconds)); } else { // Configure instrumentation key, // and register NuGet.Services.Logging.TelemetryContextInitializer. ApplicationInsights.Initialize(instrumentationKey); } // Add enrichers TelemetryConfiguration.Active.TelemetryInitializers.Add(new DeploymentIdTelemetryEnricher()); TelemetryConfiguration.Active.TelemetryInitializers.Add(new ClientInformationTelemetryEnricher()); var telemetryProcessorChainBuilder = TelemetryConfiguration.Active.TelemetryProcessorChainBuilder; // Add processors telemetryProcessorChainBuilder.Use(next => { var processor = new RequestTelemetryProcessor(next); processor.SuccessfulResponseCodes.Add(400); processor.SuccessfulResponseCodes.Add(404); processor.SuccessfulResponseCodes.Add(405); return(processor); }); telemetryProcessorChainBuilder.Use(next => new ClientTelemetryPIIProcessor(next)); var telemetry = dependencyResolver.GetService <TelemetryClientWrapper>(); telemetryProcessorChainBuilder.Use( next => new ExceptionTelemetryProcessor(next, telemetry.UnderlyingClient)); // Note: sampling rate must be a factor 100/N where N is a whole number // e.g.: 50 (= 100/2), 33.33 (= 100/3), 25 (= 100/4), ... // https://azure.microsoft.com/en-us/documentation/articles/app-insights-sampling/ var instrumentationSamplingPercentage = config.Current.AppInsightsSamplingPercentage; if (instrumentationSamplingPercentage > 0 && instrumentationSamplingPercentage < 100) { telemetryProcessorChainBuilder.UseSampling(instrumentationSamplingPercentage); } telemetryProcessorChainBuilder.Build(); } // Configure logging app.SetLoggerFactory(new DiagnosticsLoggerFactory()); // Remove X-AspNetMvc-Version header MvcHandler.DisableMvcResponseHeader = true; if (config.Current.RequireSSL) { // Put a middleware at the top of the stack to force the user over to SSL if (config.Current.ForceSslExclusion == null) { app.UseForceSsl(config.Current.SSLPort); } else { app.UseForceSsl(config.Current.SSLPort, config.Current.ForceSslExclusion); } } // Get the local user auth provider, if present and attach it first Authenticator localUserAuthenticator; if (auth.Authenticators.TryGetValue(Authenticator.GetName(typeof(LocalUserAuthenticator)), out localUserAuthenticator)) { // Configure cookie auth now localUserAuthenticator.Startup(config, app).Wait(); } // Attach external sign-in cookie middleware app.SetDefaultSignInAsAuthenticationType(AuthenticationTypes.External); app.UseCookieAuthentication(new CookieAuthenticationOptions { AuthenticationType = AuthenticationTypes.External, AuthenticationMode = AuthenticationMode.Passive, CookieName = ".AspNet." + AuthenticationTypes.External, ExpireTimeSpan = TimeSpan.FromMinutes(5) }); // Attach non-cookie auth providers var nonCookieAuthers = auth .Authenticators .Where(p => !String.Equals( p.Key, Authenticator.GetName(typeof(LocalUserAuthenticator)), StringComparison.OrdinalIgnoreCase)) .Select(p => p.Value); foreach (var auther in nonCookieAuthers) { auther.Startup(config, app).Wait(); } var featureFlags = DependencyResolver.Current.GetService <IFeatureFlagCacheService>(); if (featureFlags != null) { StartFeatureFlags(featureFlags); } // Catch unobserved exceptions from threads before they cause IIS to crash: TaskScheduler.UnobservedTaskException += (sender, exArgs) => { // Send to AppInsights try { var telemetryClient = new TelemetryClient(); telemetryClient.TrackException(exArgs.Exception, new Dictionary <string, string>() { { "ExceptionOrigin", "UnobservedTaskException" } }); } catch (Exception) { // this is a tragic moment... swallow Exception to prevent crashing IIS } // Send to ELMAH try { HttpContext current = HttpContext.Current; if (current != null) { var errorSignal = ErrorSignal.FromContext(current); if (errorSignal != null) { errorSignal.Raise(exArgs.Exception, current); } } } catch (Exception) { // more tragedy... swallow Exception to prevent crashing IIS } exArgs.SetObserved(); }; HasRun = true; }