Exemplo n.º 1
0
        public static async Task MainAsync(string[] args)
        {
            if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase))
            {
                args = args.Skip(1).ToArray();
                Debugger.Launch();
            }

            NgJob job = null;

            try
            {
                // Get arguments
                var arguments = CommandHelpers.GetArguments(args, 1);

                // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
                ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
                ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;

                // Determine the job name
                if (args.Length == 0)
                {
                    throw new ArgumentException("Missing job name argument.");
                }

                var jobName = args[0];
                TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName));

                // Configure ApplicationInsights
                ApplicationInsights.Initialize(arguments.GetOrDefault <string>(Arguments.InstrumentationKey));

                // Create an ILoggerFactory
                var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
                loggerConfiguration.WriteTo.File("Log.txt", retainedFileCountLimit: 3, fileSizeLimitBytes: 1000000, rollOnFileSizeLimit: true);

                var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Debug);

                // Create a logger that is scoped to this class (only)
                _logger = loggerFactory.CreateLogger <Program>();

                var cancellationTokenSource = new CancellationTokenSource();

                // Create an ITelemetryService
                var telemetryService = new TelemetryService(new TelemetryClient());

                // Allow jobs to set global custom dimensions
                TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobPropertiesTelemetryInitializer(telemetryService));

                job = NgJobFactory.GetJob(jobName, telemetryService, loggerFactory);
                await job.RunAsync(arguments, cancellationTokenSource.Token);
            }
            catch (ArgumentException ae)
            {
                _logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae);

                Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
            }
            catch (Exception e)
            {
                _logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e);
            }

            Trace.Close();
            TelemetryConfiguration.Active.TelemetryChannel.Flush();
        }
Exemplo n.º 2
0
        private static async Task Run(JobBase job, string[] commandLineArgs, bool?runContinuously)
        {
            if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase))
            {
                commandLineArgs = commandLineArgs.Skip(1).ToArray();
                Debugger.Launch();
            }

            // Configure logging before Application Insights is enabled.
            // This is done so, in case Application Insights fails to initialize, we still see output.
            var loggerFactory = ConfigureLogging(job);

            try
            {
                _logger.LogInformation("Started...");

                // Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
                var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(
                    ServiceContainer,
                    loggerFactory.CreateLogger(typeof(JobConfigurationManager)),
                    commandLineArgs);

                // Determine job and instance name, for logging.
                var jobName      = job.GetType().Assembly.GetName().Name;
                var instanceName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstanceName) ?? jobName;
                TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName, instanceName));

                // Setup logging
                if (!ApplicationInsights.Initialized)
                {
                    string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
                    if (!string.IsNullOrWhiteSpace(instrumentationKey))
                    {
                        ApplicationInsights.Initialize(instrumentationKey);
                    }
                }

                // Configure our logging again with Application Insights initialized.
                loggerFactory = ConfigureLogging(job);

                var hasOnceArgument = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);

                if (runContinuously.HasValue && hasOnceArgument)
                {
                    _logger.LogWarning(
                        $"This job is designed to {(runContinuously.Value ? "run continuously" : "run once")} so " +
                        $"the -{JobArgumentNames.Once} argument is {(runContinuously.Value ? "ignored" : "redundant")}.");
                }

                runContinuously = runContinuously ?? !hasOnceArgument;
                var reinitializeAfterSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.ReinitializeAfterSeconds);
                var sleepDuration            = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds

                if (!sleepDuration.HasValue)
                {
                    sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
                    if (sleepDuration.HasValue)
                    {
                        sleepDuration = sleepDuration.Value * 1000; // interval is in seconds
                    }
                }

                if (!sleepDuration.HasValue)
                {
                    if (runContinuously.Value)
                    {
                        _logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
                    }

                    sleepDuration = 5000;
                }
                else if (!runContinuously.Value)
                {
                    _logger.LogWarning(
                        $"The job is designed to run once so the -{JobArgumentNames.Sleep} and " +
                        $"-{JobArgumentNames.Interval} arguments are ignored.");
                }

                if (!reinitializeAfterSeconds.HasValue)
                {
                    _logger.LogInformation(
                        $"{JobArgumentNames.ReinitializeAfterSeconds} command line argument is not provided or is not a valid integer. " +
                        "The job will reinitialize on every iteration");
                }
                else if (!runContinuously.Value)
                {
                    _logger.LogWarning(
                        $"The job is designed to run once so the -{JobArgumentNames.ReinitializeAfterSeconds} " +
                        $"argument is ignored.");
                }

                // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
                ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
                ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;

                // Run the job loop
                await JobLoop(job, runContinuously.Value, sleepDuration.Value, reinitializeAfterSeconds, jobArgsDictionary);
            }
            catch (Exception ex)
            {
                _logger.LogError("Job runner threw an exception: {Exception}", ex);
            }

            Trace.Close();
            TelemetryConfiguration.Active.TelemetryChannel.Flush();
        }
		public static void Init(){
			var forceLoad = new ApplicationInsights ();
		}
Exemplo n.º 4
0
        private async Task <IReadOnlyCollection <T> > GetDimension <T>(string dimension, string logFileName, Func <SqlConnection, Task <IReadOnlyCollection <T> > > retrieve)
        {
            var stopwatch = Stopwatch.StartNew();
            var count     = _maxRetryCount;

            while (count > 0)
            {
                try
                {
                    _jobEventSource.BeginningRetrieveDimension(dimension);

                    IReadOnlyCollection <T> dimensions;
                    using (var connection = await _targetDatabase.ConnectTo())
                    {
                        dimensions = await retrieve(connection);
                    }

                    stopwatch.Stop();
                    _jobEventSource.FinishedRetrieveDimension(dimension, stopwatch.ElapsedMilliseconds);
                    ApplicationInsights.TrackRetrieveDimensionDuration(dimension, stopwatch.ElapsedMilliseconds, logFileName);

                    return(dimensions);
                }
                catch (SqlException e)
                {
                    --count;
                    if (count <= 0)
                    {
                        throw;
                    }

                    if (e.Number == 1205)
                    {
                        Trace.TraceWarning("Deadlock, retrying...");
                        ApplicationInsights.TrackSqlException("SQL Deadlock", e, logFileName, dimension);
                    }
                    else if (e.Number == -2)
                    {
                        Trace.TraceWarning("Timeout, retrying...");
                        ApplicationInsights.TrackSqlException("SQL Timeout", e, logFileName, dimension);
                    }
                    else if (e.Number == 2601)
                    {
                        Trace.TraceWarning("Duplicate key, retrying...");
                        ApplicationInsights.TrackSqlException("SQL Duplicate Key", e, logFileName, dimension);
                    }
                    else
                    {
                        throw;
                    }

                    Task.Delay(_retryDelay).Wait();
                }
                catch (Exception exception)
                {
                    _jobEventSource.FailedRetrieveDimension(dimension);
                    ApplicationInsights.TrackException(exception, logFileName);

                    if (stopwatch.IsRunning)
                    {
                        stopwatch.Stop();
                    }

                    throw;
                }
            }
            return(Enumerable.Empty <T>().ToList());
        }
Exemplo n.º 5
0
        public async Task <DataTable> CreateAsync(IReadOnlyCollection <PackageStatistics> sourceData, string logFileName)
        {
            var stopwatch = Stopwatch.StartNew();

            // insert any new dimension data first
            if (_times == null)
            {
                // this call is only needed once in the lifetime of the service
                _times = await GetDimension("time", logFileName, connection => RetrieveTimeDimensions(connection));
            }

            var packagesTask            = GetDimension("package", logFileName, connection => RetrievePackageDimensions(sourceData, connection));
            var operationsTask          = GetDimension("operation", logFileName, connection => RetrieveOperationDimensions(sourceData, connection));
            var projectTypesTask        = GetDimension("project type", logFileName, connection => RetrieveProjectTypeDimensions(sourceData, connection));
            var clientsTask             = GetDimension("client", logFileName, connection => RetrieveClientDimensions(sourceData, connection));
            var platformsTask           = GetDimension("platform", logFileName, connection => RetrievePlatformDimensions(sourceData, connection));
            var datesTask               = GetDimension("date", logFileName, connection => RetrieveDateDimensions(connection, sourceData.Min(e => e.EdgeServerTimeDelivered), sourceData.Max(e => e.EdgeServerTimeDelivered)));
            var packageTranslationsTask = GetDimension("package translations", logFileName, connection => RetrievePackageTranslations(sourceData, connection));

            await Task.WhenAll(operationsTask, projectTypesTask, clientsTask, platformsTask, datesTask, packagesTask, packageTranslationsTask);

            var operations   = operationsTask.Result;
            var projectTypes = projectTypesTask.Result;
            var clients      = clientsTask.Result;
            var platforms    = platformsTask.Result;

            var dates               = datesTask.Result;
            var packages            = packagesTask.Result;
            var packageTranslations = packageTranslationsTask.Result;

            // create facts data rows by linking source data with dimensions
            var dataImporter = new DataImporter(_targetDatabase);
            var dataTable    = await dataImporter.GetDataTableAsync("Fact_Download");

            // ensure all dimension IDs are set to the Unknown equivalent if no dimension data is available
            int?operationId   = !operations.Any() ? DimensionId.Unknown : (int?)null;
            int?projectTypeId = !projectTypes.Any() ? DimensionId.Unknown : (int?)null;
            int?clientId      = !clients.Any() ? DimensionId.Unknown : (int?)null;
            int?platformId    = !platforms.Any() ? DimensionId.Unknown : (int?)null;

            Trace.WriteLine("Creating facts...");
            foreach (var groupedByPackageId in sourceData.GroupBy(e => e.PackageId, StringComparer.OrdinalIgnoreCase))
            {
                var packagesForId = packages.Where(e => string.Equals(e.PackageId, groupedByPackageId.Key, StringComparison.OrdinalIgnoreCase)).ToList();

                foreach (var groupedByPackageIdAndVersion in groupedByPackageId.GroupBy(e => e.PackageVersion, StringComparer.OrdinalIgnoreCase))
                {
                    int packageId;
                    var package = packagesForId.FirstOrDefault(e => string.Equals(e.PackageVersion, groupedByPackageIdAndVersion.Key, StringComparison.OrdinalIgnoreCase));
                    if (package == null)
                    {
                        // This package id and version could not be 100% accurately parsed from the CDN Request URL,
                        // likely due to weird package ID which could be interpreted as a version string.
                        // Look for a mapping in the support table in an attempt to auto-correct this entry.
                        var packageTranslation = packageTranslations.FirstOrDefault(t => t.IncorrectPackageId == groupedByPackageId.Key && t.IncorrectPackageVersion == groupedByPackageIdAndVersion.Key);
                        if (packageTranslation != null)
                        {
                            // there seems to be a mapping
                            packageId = packageTranslation.CorrectedPackageId;
                        }
                        else
                        {
                            // Track it in Application Insights.
                            ApplicationInsights.TrackPackageNotFound(groupedByPackageId.Key, groupedByPackageIdAndVersion.Key, logFileName);

                            continue;
                        }
                    }
                    else
                    {
                        packageId = package.Id;
                    }

                    foreach (var element in groupedByPackageIdAndVersion)
                    {
                        // required dimensions
                        var dateId = dates.First(e => e.Date.Equals(element.EdgeServerTimeDelivered.Date)).Id;
                        var timeId = _times.First(e => e.HourOfDay == element.EdgeServerTimeDelivered.Hour).Id;

                        // dimensions that could be "(unknown)"
                        if (!operationId.HasValue)
                        {
                            if (!operations.ContainsKey(element.Operation))
                            {
                                operationId = DimensionId.Unknown;
                            }
                            else
                            {
                                operationId = operations[element.Operation];
                            }
                        }
                        if (!platformId.HasValue)
                        {
                            if (!platforms.ContainsKey(element.UserAgent))
                            {
                                platformId = DimensionId.Unknown;
                            }
                            else
                            {
                                platformId = platforms[element.UserAgent];
                            }
                        }
                        if (!clientId.HasValue)
                        {
                            if (!clients.ContainsKey(element.UserAgent))
                            {
                                clientId = DimensionId.Unknown;
                            }
                            else
                            {
                                clientId = clients[element.UserAgent];
                            }
                        }

                        if (!projectTypeId.HasValue)
                        {
                            // foreach project type
                            foreach (var projectGuid in element.ProjectGuids.Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries))
                            {
                                projectTypeId = projectTypes[projectGuid];

                                var dataRow = dataTable.NewRow();
                                FillDataRow(dataRow, dateId, timeId, packageId, operationId.Value, platformId.Value, projectTypeId.Value, clientId.Value, logFileName, element.UserAgent);
                                dataTable.Rows.Add(dataRow);
                            }
                        }
                        else
                        {
                            var dataRow = dataTable.NewRow();
                            FillDataRow(dataRow, dateId, timeId, packageId, operationId.Value, platformId.Value, projectTypeId.Value, clientId.Value, logFileName, element.UserAgent);
                            dataTable.Rows.Add(dataRow);
                        }
                    }
                }
            }
            stopwatch.Stop();
            Trace.Write("  DONE (" + dataTable.Rows.Count + " records, " + stopwatch.ElapsedMilliseconds + "ms)");
            ApplicationInsights.TrackMetric("Blob record count", dataTable.Rows.Count, logFileName);

            return(dataTable);
        }
 public UserProfile()
 {
     Roles = new List <string>();
     ApplicationInsights = new ApplicationInsights();
 }
Exemplo n.º 7
0
        //internal static ClientContext GetCachedContext(string url)
        //{
        //    return ContextCache.FirstOrDefault(c => System.Net.WebUtility.UrlEncode(c.Url) == System.Net.WebUtility.UrlEncode(url));
        //}

        //internal static void ClearContextCache()
        //{
        //    ContextCache.Clear();
        //}

        internal void InitializeTelemetry(ClientContext context, InitializationType initializationType)
        {
            var enableTelemetry = false;
            var userProfile     = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
            var telemetryFile   = System.IO.Path.Combine(userProfile, ".pnppowershelltelemetry");

            if (!System.IO.File.Exists(telemetryFile))
            {
                enableTelemetry = true;
            }
            else
            {
                if (System.IO.File.ReadAllText(telemetryFile).ToLower() == "allow")
                {
                    enableTelemetry = true;
                }
            }
            if (enableTelemetry)
            {
                var serverLibraryVersion = "";
                var serverVersion        = "";
                if (context != null)
                {
                    try
                    {
                        if (context.ServerLibraryVersion != null)
                        {
                            serverLibraryVersion = context.ServerLibraryVersion.ToString();
                        }
                        if (context.ServerVersion != null)
                        {
                            serverVersion = context.ServerVersion.ToString();
                        }
                    }
                    catch { }
                }

                ApplicationInsights = new ApplicationInsights();
                var coreAssembly = Assembly.GetExecutingAssembly();
                ApplicationInsights.Initialize(serverLibraryVersion, serverVersion, initializationType.ToString(), ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString());
                ApplicationInsights.TrackEvent("Connect-PnPOnline");

                //TelemetryConfiguration config = TelemetryConfiguration.CreateDefault();
                //TelemetryClient = new TelemetryClient(config);
                //config.InstrumentationKey = "a301024a-9e21-4273-aca5-18d0ef5d80fb";
                ////config..Context.Session.Id = Guid.NewGuid().ToString();
                //TelemetryClient.Context.Cloud.RoleInstance = "PnPPowerShell";
                //TelemetryClient.Context.Device.OperatingSystem = Environment.OSVersion.ToString();

                //TelemetryProperties = new Dictionary<string, string>(10);
                //TelemetryProperties.Add("ServerLibraryVersion", serverLibraryVersion);
                //TelemetryProperties.Add("ServerVersion", serverVersion);
                //TelemetryProperties.Add("ConnectionMethod", initializationType.ToString());
                //var coreAssembly = Assembly.GetExecutingAssembly();
                //TelemetryProperties.Add("Version", ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString());
                //TelemetryProperties.Add("Platform", "SPO");
                //TelemetryClient.TrackEvent("Connect-PnPOnline", TelemetryProperties);


                //TelemetryClient = new TelemetryClient(;
                //TelemetryClient.InstrumentationKey = "a301024a-9e21-4273-aca5-18d0ef5d80fb";
                //TelemetryClient.Context.Session.Id = Guid.NewGuid().ToString();
                //TelemetryClient.Context.Cloud.RoleInstance = "PnPPowerShell";
                //TelemetryClient.Context.Device.OperatingSystem = Environment.OSVersion.ToString();
                //TelemetryClient.Context.Properties.Add("ServerLibraryVersion", serverLibraryVersion);
                //TelemetryClient.Context.Properties.Add("ServerVersion", serverVersion);
                //TelemetryClient.Context.Properties.Add("ConnectionMethod", initializationType.ToString());
                //var coreAssembly = Assembly.GetExecutingAssembly();
                //TelemetryClient.Context.Properties.Add("Version", ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString());
                //TelemetryClient.Context.Properties.Add("Platform", "SPO");
                //TelemetryClient.TrackEvent("Connect-PnPOnline");
            }
        }
        public void InitializeRegistersTelemetryContextInitializer()
        {
            var applicationInsightsConfiguration = ApplicationInsights.Initialize(InstrumentationKey);

            Assert.Contains(applicationInsightsConfiguration.TelemetryConfiguration.TelemetryInitializers, ti => ti is TelemetryContextInitializer);
        }
Exemplo n.º 9
0
        /// <summary>
        /// This is a static method to run a job whose args are passed in
        /// By default,
        ///     a) The job will be run continuously in a while loop. Could be overridden using 'once' argument
        ///     b) The sleep duration between each run when running continuously is 5000 milliSeconds. Could be overridden using '-Sleep' argument
        /// </summary>
        /// <param name="job">Job to run</param>
        /// <param name="commandLineArgs">Args contains args to the job runner like (dbg, once and so on) and for the job itself</param>
        /// <returns></returns>
        public static async Task Run(JobBase job, string[] commandLineArgs)
        {
            if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase))
            {
                commandLineArgs = commandLineArgs.Skip(1).ToArray();
                Debugger.Launch();
            }

            // Configure logging before Application Insights is enabled.
            // This is done so, in case Application Insights fails to initialize, we still see output.
            var loggerFactory = ConfigureLogging(job);

            try
            {
                _logger.LogInformation("Started...");

                // Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
                var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(loggerFactory.CreateLogger(typeof(JobConfigurationManager)), commandLineArgs, job.JobName, (ISecretReaderFactory)ServiceContainer.GetService(typeof(ISecretReaderFactory)));

                // Setup logging
                if (!ApplicationInsights.Initialized)
                {
                    string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
                    if (!string.IsNullOrWhiteSpace(instrumentationKey))
                    {
                        ApplicationInsights.Initialize(instrumentationKey);
                    }
                }

                // Configure our logging again with Application Insights initialized.
                loggerFactory = ConfigureLogging(job);

                var runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
                var sleepDuration   = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds
                if (!sleepDuration.HasValue)
                {
                    sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
                    if (sleepDuration.HasValue)
                    {
                        sleepDuration = sleepDuration.Value * 1000; // interval is in seconds
                    }
                }

                if (sleepDuration == null)
                {
                    _logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
                    sleepDuration = 5000;
                }

                // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
                ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
                ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;

                // Run the job loop
                await JobLoop(job, runContinuously, sleepDuration.Value, jobArgsDictionary);
            }
            catch (Exception ex)
            {
                _logger.LogError("Job runner threw an exception: {Exception}", ex);
            }
        }
Exemplo n.º 10
0
        private static IServiceProvider InitializeAndGetServiceProvider()
        {
            ServicePointManager.DefaultConnectionLimit = 64;
            ServicePointManager.SecurityProtocol      &= ~SecurityProtocolType.Ssl3;
            ServicePointManager.SecurityProtocol      |= SecurityProtocolType.Tls12;

            var configurationRoot  = GetConfigurationRoot();
            var instrumentationKey = configurationRoot
                                     .GetSection(ConfigurationSectionName)
                                     .GetValue <string>(nameof(V3PerPackageConfiguration.InstrumentationKey));

            ApplicationInsights.Initialize(instrumentationKey);

            var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
            var loggerFactory       = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Information);

            var serviceCollection = new ServiceCollection();

            serviceCollection.Configure <V3PerPackageConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
            serviceCollection.AddOptions();
            serviceCollection.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot <>), typeof(NonCachingOptionsSnapshot <>)));

            serviceCollection.AddLogging();
            serviceCollection.AddSingleton(loggerFactory);

            serviceCollection.AddSingleton(new ControlledDisposeHttpClientHandler());
            serviceCollection.AddTransient <HttpMessageHandler>(x => x.GetRequiredService <ControlledDisposeHttpClientHandler>());
            serviceCollection.AddSingleton(x => new HttpClient(x.GetRequiredService <HttpMessageHandler>()));
            serviceCollection.AddTransient <Func <HttpMessageHandler> >(x => () => x.GetRequiredService <HttpMessageHandler>());
            serviceCollection.AddTransient <PerBatchProcessor>();
            serviceCollection.AddTransient <ITelemetryService, TelemetryService>();
            serviceCollection.AddTransient <PerWorkerProcessor>();
            serviceCollection.AddTransient <PerProcessProcessor>();
            serviceCollection.AddSingleton <StringLocker>();
            serviceCollection.AddTransient <EnqueueCollector>();
            serviceCollection.AddTransient <EnqueueCommand>();
            serviceCollection.AddTransient <CleanUpCommand>();

            serviceCollection.AddSingleton(x =>
            {
                var globalContext = x.GetRequiredService <GlobalContext>();

                var perProcessContext = new PerProcessContext(
                    globalContext,
                    UniqueName.New("process"),
                    WorkerCount,
                    MessageCount,
                    BatchSize);

                var blobClient       = BlobStorageUtilities.GetBlobClient(globalContext);
                var flatContainerUrl = $"{blobClient.BaseUri.AbsoluteUri}/{globalContext.FlatContainerContainerName}/{perProcessContext.Name}";
                RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerUrl);

                return(perProcessContext);
            });

            serviceCollection.AddTransient(x =>
            {
                var settings = x.GetRequiredService <IOptionsSnapshot <V3PerPackageConfiguration> >();
                return(new GlobalContext(
                           settings.Value.StorageBaseAddress,
                           settings.Value.StorageAccountName,
                           settings.Value.StorageKeyValue,
                           settings.Value.ContentBaseAddress));
            });

            serviceCollection.AddSingleton(new TelemetryClient());

            serviceCollection.AddTransient <IStorageQueue <PackageMessage> >(x =>
            {
                var globalContext      = x.GetRequiredService <GlobalContext>();
                var storageCredentials = new StorageCredentials(globalContext.StorageAccountName, globalContext.StorageKeyValue);
                var storageAccount     = new CloudStorageAccount(storageCredentials, useHttps: true);

                return(new StorageQueue <PackageMessage>(
                           new AzureStorageQueue(storageAccount, "v3perpackage"),
                           new JsonMessageSerializer <PackageMessage>(JsonSerializerUtility.SerializerSettings),
                           PackageMessage.Version));
            });

            return(serviceCollection.BuildServiceProvider());
        }
Exemplo n.º 11
0
        public override bool Init(IDictionary <string, string> jobArgsDictionary)
        {
            try
            {
                if (!ApplicationInsights.Initialized)
                {
                    string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
                    if (!string.IsNullOrWhiteSpace(instrumentationKey))
                    {
                        ApplicationInsights.Initialize(instrumentationKey);
                    }
                }

                _loggerFactory = LoggingSetup.CreateLoggerFactory();
                _logger        = _loggerFactory.CreateLogger <Job>();

                // Configure job
                _galleryBaseAddress = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryBaseAddress);

                var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount);
                _cloudStorageAccount = CreateCloudStorageAccount(JobArgumentNames.DataStorageAccount, storageConnectionString);

                _containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName);

                _runValidationTasks     = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RunValidationTasks).Split(';');
                _requestValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RequestValidationTasks).Split(';');

                // Add validators
                if (_runValidationTasks.Contains(UnzipValidator.ValidatorName))
                {
                    _validators.Add(new UnzipValidator(_loggerFactory));
                }
                if (_runValidationTasks.Contains(VcsValidator.ValidatorName))
                {
                    // if contact alias set, use it, if not, use submitter alias.
                    string submitterAlias = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorSubmitterAlias);
                    string contactAlias   = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.VcsContactAlias)
                                            ?? submitterAlias;

                    _validators.Add(new VcsValidator(
                                        JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorServiceUrl),
                                        JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorCallbackUrl),
                                        contactAlias,
                                        submitterAlias,
                                        JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsPackageUrlTemplate),
                                        _loggerFactory));
                }

                return(true);
            }
            catch (Exception ex)
            {
                if (_logger != null)
                {
                    _logger.LogError(TraceEvent.CommandLineProcessingFailed, ex, "Exception occurred while processing command line arguments");
                }
                else
                {
                    Trace.TraceError("Exception occurred while processing command line arguments: {0}", ex);
                }
            }

            return(false);
        }
Exemplo n.º 12
0
 void SetupApplicationInsights()
 {
     ApplicationInsights.Setup(InstrumentationKey);
     ApplicationInsights.SetDebugLogEnabled(true);
     ApplicationInsights.Start();
 }
Exemplo n.º 13
0
        private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile)
        {
            var logStream = await OpenCompressedBlobAsync(logFile);

            var blobUri  = logFile.Uri;
            var blobName = logFile.Blob.Name;

            var packageStatistics = new List <PackageStatistics>();
            var toolStatistics    = new List <ToolStatistics>();
            var dnxStatistics     = new List <DnxStatistics>();

            var stopwatch = Stopwatch.StartNew();

            try
            {
                // parse the log into table entities
                _jobEventSource.BeginningParseLog(blobUri);

                using (var logStreamReader = new StreamReader(logStream))
                {
                    do
                    {
                        var rawLogLine = logStreamReader.ReadLine();
                        if (rawLogLine != null)
                        {
                            var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
                            if (logEntry != null)
                            {
                                var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
                                if (statistic != null)
                                {
                                    packageStatistics.Add(statistic);
                                }
                                else
                                {
                                    // check if this is a dist.nuget.org download
                                    if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
                                    {
                                        var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
                                        if (toolInfo != null)
                                        {
                                            toolStatistics.Add(toolInfo);
                                        }
                                    }
                                    if (logEntry.RequestUrl.Contains("dist.asp.net"))
                                    {
                                        var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry);
                                        if (dnxInfo != null)
                                        {
                                            dnxStatistics.Add(dnxInfo);
                                        }
                                    }
                                }
                            }
                        }
                    } while (!logStreamReader.EndOfStream);
                }

                _jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);

                stopwatch.Stop();
                ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
            }
            catch (Exception exception)
            {
                if (stopwatch.IsRunning)
                {
                    stopwatch.Stop();
                }

                _jobEventSource.FailedParseLog(blobUri);
                ApplicationInsights.TrackException(exception, blobName);
                throw;
            }
            finally
            {
                logStream.Dispose();
            }

            var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics);

            return(cdnStatistics);
        }
Exemplo n.º 14
0
        private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
        {
            var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_statisticsDatabase, reportGenerationTime);

            if (!dirtyPackageIds.Any())
            {
                return;
            }

            // first process the top 100 packages
            var top100 = dirtyPackageIds.Take(100);
            var reportDataCollector = new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase);
            var top100Task          = Parallel.ForEach(top100, new ParallelOptions {
                MaxDegreeOfParallelism = 4
            }, dirtyPackageId =>
            {
                var packageId     = dirtyPackageId.PackageId.ToLowerInvariant();
                var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId);

                ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
                ApplicationInsights.TrackReportProcessed(reportBuilder.ReportName + " report", packageId);
            });

            // once top 100 is processed, continue with the rest
            if (top100Task.IsCompleted)
            {
                var excludingTop100 = dirtyPackageIds.Skip(100);

                top100Task = Parallel.ForEach(excludingTop100, new ParallelOptions {
                    MaxDegreeOfParallelism = 8
                },
                                              dirtyPackageId =>
                {
                    // generate all reports
                    var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
                    {
                        {
                            new RecentPopularityDetailByPackageReportBuilder(
                                ReportNames.RecentPopularityDetailByPackageId,
                                "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName +
                                dirtyPackageId.PackageId.ToLowerInvariant()),
                            new ReportDataCollector(
                                _storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId],
                                _statisticsDatabase)
                        }
                    };

                    foreach (var reportGenerator in reportGenerators)
                    {
                        ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value,
                                      reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
                        ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report",
                                                                 dirtyPackageId.PackageId.ToLowerInvariant());
                    }
                });

                if (top100Task.IsCompleted)
                {
                    var runToCursor = dirtyPackageIds.First().RunToCuror;
                    await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDatabase, runToCursor);
                }
            }
        }
Exemplo n.º 15
0
        public static LoggerConfiguration ConfigureTraceLogger(LoggerConfiguration loggerConfig, ApplicationInsights appInsightsConfig,
                                                               LoggingLevelSwitch levelSwitch = null)
        {
            ConfigureLoggerConfiguration(loggerConfig, levelSwitch ?? LoggingLevelSwitch);

            if (appInsightsConfig != null && appInsightsConfig.Enabled &&
                !string.IsNullOrEmpty(appInsightsConfig.InstrumentationKey))
            {
                loggerConfig.WriteTo.ApplicationInsights(TelemetryConverter.Traces);
            }

            return(loggerConfig);
        }
Exemplo n.º 16
0
        // This method is auto-detected by the OWIN pipeline. DO NOT RENAME IT!
        public static void Configuration(IAppBuilder app)
        {
            // Tune ServicePointManager
            // (based on http://social.technet.microsoft.com/Forums/en-US/windowsazuredata/thread/d84ba34b-b0e0-4961-a167-bbe7618beb83 and https://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.aspx)
            ServicePointManager.DefaultConnectionLimit = 500;
            ServicePointManager.UseNagleAlgorithm      = false;
            ServicePointManager.Expect100Continue      = false;

            // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
            ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
            ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;

            // Setting time out for all RegEx objects. Noted in remarks at https://msdn.microsoft.com/en-us/library/system.text.regularexpressions.regex.matchtimeout%28v=vs.110%29.aspx
            AppDomain.CurrentDomain.SetData("REGEX_DEFAULT_MATCH_TIMEOUT", TimeSpan.FromSeconds(10));

            // Register IoC
            app.UseAutofacInjection(GlobalConfiguration.Configuration);
            var dependencyResolver = DependencyResolver.Current;

            // Register Elmah
            var elmahServiceCenter = new DependencyResolverServiceProviderAdapter(dependencyResolver);

            ServiceCenter.Current = _ => elmahServiceCenter;

            // Get config
            var config = dependencyResolver.GetService <IGalleryConfigurationService>();
            var auth   = dependencyResolver.GetService <AuthenticationService>();

            // Configure machine key for session persistence across slots
            SessionPersistence.Setup(config);
            // Refresh the content for the ContentObjectService to guarantee it has loaded the latest configuration on startup.
            var contentObjectService = dependencyResolver.GetService <IContentObjectService>();

            HostingEnvironment.QueueBackgroundWorkItem(async token =>
            {
                while (!token.IsCancellationRequested)
                {
                    await contentObjectService.Refresh();
                    await Task.Delay(ContentObjectService.RefreshInterval, token);
                }
            });

            // Setup telemetry
            var instrumentationKey = config.Current.AppInsightsInstrumentationKey;

            if (!string.IsNullOrEmpty(instrumentationKey))
            {
                var heartbeatIntervalSeconds = config.Current.AppInsightsHeartbeatIntervalSeconds;

                if (heartbeatIntervalSeconds > 0)
                {
                    // Configure instrumentation key, heartbeat interval,
                    // and register NuGet.Services.Logging.TelemetryContextInitializer.
                    ApplicationInsights.Initialize(instrumentationKey, TimeSpan.FromSeconds(heartbeatIntervalSeconds));
                }
                else
                {
                    // Configure instrumentation key,
                    // and register NuGet.Services.Logging.TelemetryContextInitializer.
                    ApplicationInsights.Initialize(instrumentationKey);
                }

                // Add enrichers
                TelemetryConfiguration.Active.TelemetryInitializers.Add(new DeploymentIdTelemetryEnricher());
                TelemetryConfiguration.Active.TelemetryInitializers.Add(new ClientInformationTelemetryEnricher());

                var telemetryProcessorChainBuilder = TelemetryConfiguration.Active.TelemetryProcessorChainBuilder;

                // Add processors
                telemetryProcessorChainBuilder.Use(next =>
                {
                    var processor = new RequestTelemetryProcessor(next);

                    processor.SuccessfulResponseCodes.Add(400);
                    processor.SuccessfulResponseCodes.Add(404);
                    processor.SuccessfulResponseCodes.Add(405);

                    return(processor);
                });

                telemetryProcessorChainBuilder.Use(next => new ClientTelemetryPIIProcessor(next));

                var telemetry = dependencyResolver.GetService <TelemetryClientWrapper>();
                telemetryProcessorChainBuilder.Use(
                    next => new ExceptionTelemetryProcessor(next, telemetry.UnderlyingClient));

                // Note: sampling rate must be a factor 100/N where N is a whole number
                // e.g.: 50 (= 100/2), 33.33 (= 100/3), 25 (= 100/4), ...
                // https://azure.microsoft.com/en-us/documentation/articles/app-insights-sampling/
                var instrumentationSamplingPercentage = config.Current.AppInsightsSamplingPercentage;
                if (instrumentationSamplingPercentage > 0 && instrumentationSamplingPercentage < 100)
                {
                    telemetryProcessorChainBuilder.UseSampling(instrumentationSamplingPercentage);
                }

                telemetryProcessorChainBuilder.Build();
            }

            // Configure logging
            app.SetLoggerFactory(new DiagnosticsLoggerFactory());

            // Remove X-AspNetMvc-Version header
            MvcHandler.DisableMvcResponseHeader = true;

            if (config.Current.RequireSSL)
            {
                // Put a middleware at the top of the stack to force the user over to SSL
                if (config.Current.ForceSslExclusion == null)
                {
                    app.UseForceSsl(config.Current.SSLPort);
                }
                else
                {
                    app.UseForceSsl(config.Current.SSLPort, config.Current.ForceSslExclusion);
                }
            }

            // Get the local user auth provider, if present and attach it first
            Authenticator localUserAuthenticator;

            if (auth.Authenticators.TryGetValue(Authenticator.GetName(typeof(LocalUserAuthenticator)), out localUserAuthenticator))
            {
                // Configure cookie auth now
                localUserAuthenticator.Startup(config, app).Wait();
            }

            // Attach external sign-in cookie middleware
            app.SetDefaultSignInAsAuthenticationType(AuthenticationTypes.External);
            app.UseCookieAuthentication(new CookieAuthenticationOptions
            {
                AuthenticationType = AuthenticationTypes.External,
                AuthenticationMode = AuthenticationMode.Passive,
                CookieName         = ".AspNet." + AuthenticationTypes.External,
                ExpireTimeSpan     = TimeSpan.FromMinutes(5)
            });

            // Attach non-cookie auth providers
            var nonCookieAuthers = auth
                                   .Authenticators
                                   .Where(p => !String.Equals(
                                              p.Key,
                                              Authenticator.GetName(typeof(LocalUserAuthenticator)),
                                              StringComparison.OrdinalIgnoreCase))
                                   .Select(p => p.Value);

            foreach (var auther in nonCookieAuthers)
            {
                auther.Startup(config, app).Wait();
            }

            var featureFlags = DependencyResolver.Current.GetService <IFeatureFlagCacheService>();

            if (featureFlags != null)
            {
                StartFeatureFlags(featureFlags);
            }

            // Catch unobserved exceptions from threads before they cause IIS to crash:
            TaskScheduler.UnobservedTaskException += (sender, exArgs) =>
            {
                // Send to AppInsights
                try
                {
                    var telemetryClient = new TelemetryClient();
                    telemetryClient.TrackException(exArgs.Exception, new Dictionary <string, string>()
                    {
                        { "ExceptionOrigin", "UnobservedTaskException" }
                    });
                }
                catch (Exception)
                {
                    // this is a tragic moment... swallow Exception to prevent crashing IIS
                }

                // Send to ELMAH
                try
                {
                    HttpContext current = HttpContext.Current;
                    if (current != null)
                    {
                        var errorSignal = ErrorSignal.FromContext(current);
                        if (errorSignal != null)
                        {
                            errorSignal.Raise(exArgs.Exception, current);
                        }
                    }
                }
                catch (Exception)
                {
                    // more tragedy... swallow Exception to prevent crashing IIS
                }

                exArgs.SetObserved();
            };

            HasRun = true;
        }