public static async Task MainAsync(string[] args)
{
if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase))
{
args = args.Skip(1).ToArray();
Debugger.Launch();
}
NgJob job = null;
try
{
// Get arguments
var arguments = CommandHelpers.GetArguments(args, 1);
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Determine the job name
if (args.Length == 0)
{
throw new ArgumentException("Missing job name argument.");
}
var jobName = args[0];
TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName));
// Configure ApplicationInsights
ApplicationInsights.Initialize(arguments.GetOrDefault <string>(Arguments.InstrumentationKey));
// Create an ILoggerFactory
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
loggerConfiguration.WriteTo.File("Log.txt", retainedFileCountLimit: 3, fileSizeLimitBytes: 1000000, rollOnFileSizeLimit: true);
var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Debug);
// Create a logger that is scoped to this class (only)
_logger = loggerFactory.CreateLogger <Program>();
var cancellationTokenSource = new CancellationTokenSource();
// Create an ITelemetryService
var telemetryService = new TelemetryService(new TelemetryClient());
// Allow jobs to set global custom dimensions
TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobPropertiesTelemetryInitializer(telemetryService));
job = NgJobFactory.GetJob(jobName, telemetryService, loggerFactory);
await job.RunAsync(arguments, cancellationTokenSource.Token);
}
catch (ArgumentException ae)
{
_logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae);
Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
}
catch (Exception e)
{
_logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e);
}
Trace.Close();
TelemetryConfiguration.Active.TelemetryChannel.Flush();
}
private static async Task Run(JobBase job, string[] commandLineArgs, bool?runContinuously)
{
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
Debugger.Launch();
}
// Configure logging before Application Insights is enabled.
// This is done so, in case Application Insights fails to initialize, we still see output.
var loggerFactory = ConfigureLogging(job);
try
{
_logger.LogInformation("Started...");
// Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(
ServiceContainer,
loggerFactory.CreateLogger(typeof(JobConfigurationManager)),
commandLineArgs);
// Determine job and instance name, for logging.
var jobName = job.GetType().Assembly.GetName().Name;
var instanceName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstanceName) ?? jobName;
TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName, instanceName));
// Setup logging
if (!ApplicationInsights.Initialized)
{
string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
if (!string.IsNullOrWhiteSpace(instrumentationKey))
{
ApplicationInsights.Initialize(instrumentationKey);
}
}
// Configure our logging again with Application Insights initialized.
loggerFactory = ConfigureLogging(job);
var hasOnceArgument = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
if (runContinuously.HasValue && hasOnceArgument)
{
_logger.LogWarning(
$"This job is designed to {(runContinuously.Value ? "run continuously" : "run once")} so " +
$"the -{JobArgumentNames.Once} argument is {(runContinuously.Value ? "ignored" : "redundant")}.");
}
runContinuously = runContinuously ?? !hasOnceArgument;
var reinitializeAfterSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.ReinitializeAfterSeconds);
var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds
if (!sleepDuration.HasValue)
{
sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
if (sleepDuration.HasValue)
{
sleepDuration = sleepDuration.Value * 1000; // interval is in seconds
}
}
if (!sleepDuration.HasValue)
{
if (runContinuously.Value)
{
_logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
}
sleepDuration = 5000;
}
else if (!runContinuously.Value)
{
_logger.LogWarning(
$"The job is designed to run once so the -{JobArgumentNames.Sleep} and " +
$"-{JobArgumentNames.Interval} arguments are ignored.");
}
if (!reinitializeAfterSeconds.HasValue)
{
_logger.LogInformation(
$"{JobArgumentNames.ReinitializeAfterSeconds} command line argument is not provided or is not a valid integer. " +
"The job will reinitialize on every iteration");
}
else if (!runContinuously.Value)
{
_logger.LogWarning(
$"The job is designed to run once so the -{JobArgumentNames.ReinitializeAfterSeconds} " +
$"argument is ignored.");
}
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Run the job loop
await JobLoop(job, runContinuously.Value, sleepDuration.Value, reinitializeAfterSeconds, jobArgsDictionary);
}
catch (Exception ex)
{
_logger.LogError("Job runner threw an exception: {Exception}", ex);
}
Trace.Close();
TelemetryConfiguration.Active.TelemetryChannel.Flush();
}
public static void Init(){
var forceLoad = new ApplicationInsights ();
}
private async Task <IReadOnlyCollection <T> > GetDimension <T>(string dimension, string logFileName, Func <SqlConnection, Task <IReadOnlyCollection <T> > > retrieve)
{
var stopwatch = Stopwatch.StartNew();
var count = _maxRetryCount;
while (count > 0)
{
try
{
_jobEventSource.BeginningRetrieveDimension(dimension);
IReadOnlyCollection <T> dimensions;
using (var connection = await _targetDatabase.ConnectTo())
{
dimensions = await retrieve(connection);
}
stopwatch.Stop();
_jobEventSource.FinishedRetrieveDimension(dimension, stopwatch.ElapsedMilliseconds);
ApplicationInsights.TrackRetrieveDimensionDuration(dimension, stopwatch.ElapsedMilliseconds, logFileName);
return(dimensions);
}
catch (SqlException e)
{
--count;
if (count <= 0)
{
throw;
}
if (e.Number == 1205)
{
Trace.TraceWarning("Deadlock, retrying...");
ApplicationInsights.TrackSqlException("SQL Deadlock", e, logFileName, dimension);
}
else if (e.Number == -2)
{
Trace.TraceWarning("Timeout, retrying...");
ApplicationInsights.TrackSqlException("SQL Timeout", e, logFileName, dimension);
}
else if (e.Number == 2601)
{
Trace.TraceWarning("Duplicate key, retrying...");
ApplicationInsights.TrackSqlException("SQL Duplicate Key", e, logFileName, dimension);
}
else
{
throw;
}
Task.Delay(_retryDelay).Wait();
}
catch (Exception exception)
{
_jobEventSource.FailedRetrieveDimension(dimension);
ApplicationInsights.TrackException(exception, logFileName);
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
throw;
}
}
return(Enumerable.Empty <T>().ToList());
}
public async Task <DataTable> CreateAsync(IReadOnlyCollection <PackageStatistics> sourceData, string logFileName)
{
var stopwatch = Stopwatch.StartNew();
// insert any new dimension data first
if (_times == null)
{
// this call is only needed once in the lifetime of the service
_times = await GetDimension("time", logFileName, connection => RetrieveTimeDimensions(connection));
}
var packagesTask = GetDimension("package", logFileName, connection => RetrievePackageDimensions(sourceData, connection));
var operationsTask = GetDimension("operation", logFileName, connection => RetrieveOperationDimensions(sourceData, connection));
var projectTypesTask = GetDimension("project type", logFileName, connection => RetrieveProjectTypeDimensions(sourceData, connection));
var clientsTask = GetDimension("client", logFileName, connection => RetrieveClientDimensions(sourceData, connection));
var platformsTask = GetDimension("platform", logFileName, connection => RetrievePlatformDimensions(sourceData, connection));
var datesTask = GetDimension("date", logFileName, connection => RetrieveDateDimensions(connection, sourceData.Min(e => e.EdgeServerTimeDelivered), sourceData.Max(e => e.EdgeServerTimeDelivered)));
var packageTranslationsTask = GetDimension("package translations", logFileName, connection => RetrievePackageTranslations(sourceData, connection));
await Task.WhenAll(operationsTask, projectTypesTask, clientsTask, platformsTask, datesTask, packagesTask, packageTranslationsTask);
var operations = operationsTask.Result;
var projectTypes = projectTypesTask.Result;
var clients = clientsTask.Result;
var platforms = platformsTask.Result;
var dates = datesTask.Result;
var packages = packagesTask.Result;
var packageTranslations = packageTranslationsTask.Result;
// create facts data rows by linking source data with dimensions
var dataImporter = new DataImporter(_targetDatabase);
var dataTable = await dataImporter.GetDataTableAsync("Fact_Download");
// ensure all dimension IDs are set to the Unknown equivalent if no dimension data is available
int?operationId = !operations.Any() ? DimensionId.Unknown : (int?)null;
int?projectTypeId = !projectTypes.Any() ? DimensionId.Unknown : (int?)null;
int?clientId = !clients.Any() ? DimensionId.Unknown : (int?)null;
int?platformId = !platforms.Any() ? DimensionId.Unknown : (int?)null;
Trace.WriteLine("Creating facts...");
foreach (var groupedByPackageId in sourceData.GroupBy(e => e.PackageId, StringComparer.OrdinalIgnoreCase))
{
var packagesForId = packages.Where(e => string.Equals(e.PackageId, groupedByPackageId.Key, StringComparison.OrdinalIgnoreCase)).ToList();
foreach (var groupedByPackageIdAndVersion in groupedByPackageId.GroupBy(e => e.PackageVersion, StringComparer.OrdinalIgnoreCase))
{
int packageId;
var package = packagesForId.FirstOrDefault(e => string.Equals(e.PackageVersion, groupedByPackageIdAndVersion.Key, StringComparison.OrdinalIgnoreCase));
if (package == null)
{
// This package id and version could not be 100% accurately parsed from the CDN Request URL,
// likely due to weird package ID which could be interpreted as a version string.
// Look for a mapping in the support table in an attempt to auto-correct this entry.
var packageTranslation = packageTranslations.FirstOrDefault(t => t.IncorrectPackageId == groupedByPackageId.Key && t.IncorrectPackageVersion == groupedByPackageIdAndVersion.Key);
if (packageTranslation != null)
{
// there seems to be a mapping
packageId = packageTranslation.CorrectedPackageId;
}
else
{
// Track it in Application Insights.
ApplicationInsights.TrackPackageNotFound(groupedByPackageId.Key, groupedByPackageIdAndVersion.Key, logFileName);
continue;
}
}
else
{
packageId = package.Id;
}
foreach (var element in groupedByPackageIdAndVersion)
{
// required dimensions
var dateId = dates.First(e => e.Date.Equals(element.EdgeServerTimeDelivered.Date)).Id;
var timeId = _times.First(e => e.HourOfDay == element.EdgeServerTimeDelivered.Hour).Id;
// dimensions that could be "(unknown)"
if (!operationId.HasValue)
{
if (!operations.ContainsKey(element.Operation))
{
operationId = DimensionId.Unknown;
}
else
{
operationId = operations[element.Operation];
}
}
if (!platformId.HasValue)
{
if (!platforms.ContainsKey(element.UserAgent))
{
platformId = DimensionId.Unknown;
}
else
{
platformId = platforms[element.UserAgent];
}
}
if (!clientId.HasValue)
{
if (!clients.ContainsKey(element.UserAgent))
{
clientId = DimensionId.Unknown;
}
else
{
clientId = clients[element.UserAgent];
}
}
if (!projectTypeId.HasValue)
{
// foreach project type
foreach (var projectGuid in element.ProjectGuids.Split(new[] { ";" }, StringSplitOptions.RemoveEmptyEntries))
{
projectTypeId = projectTypes[projectGuid];
var dataRow = dataTable.NewRow();
FillDataRow(dataRow, dateId, timeId, packageId, operationId.Value, platformId.Value, projectTypeId.Value, clientId.Value, logFileName, element.UserAgent);
dataTable.Rows.Add(dataRow);
}
}
else
{
var dataRow = dataTable.NewRow();
FillDataRow(dataRow, dateId, timeId, packageId, operationId.Value, platformId.Value, projectTypeId.Value, clientId.Value, logFileName, element.UserAgent);
dataTable.Rows.Add(dataRow);
}
}
}
}
stopwatch.Stop();
Trace.Write(" DONE (" + dataTable.Rows.Count + " records, " + stopwatch.ElapsedMilliseconds + "ms)");
ApplicationInsights.TrackMetric("Blob record count", dataTable.Rows.Count, logFileName);
return(dataTable);
}
public UserProfile()
{
Roles = new List <string>();
ApplicationInsights = new ApplicationInsights();
}
//internal static ClientContext GetCachedContext(string url)
//{
// return ContextCache.FirstOrDefault(c => System.Net.WebUtility.UrlEncode(c.Url) == System.Net.WebUtility.UrlEncode(url));
//}
//internal static void ClearContextCache()
//{
// ContextCache.Clear();
//}
internal void InitializeTelemetry(ClientContext context, InitializationType initializationType)
{
var enableTelemetry = false;
var userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
var telemetryFile = System.IO.Path.Combine(userProfile, ".pnppowershelltelemetry");
if (!System.IO.File.Exists(telemetryFile))
{
enableTelemetry = true;
}
else
{
if (System.IO.File.ReadAllText(telemetryFile).ToLower() == "allow")
{
enableTelemetry = true;
}
}
if (enableTelemetry)
{
var serverLibraryVersion = "";
var serverVersion = "";
if (context != null)
{
try
{
if (context.ServerLibraryVersion != null)
{
serverLibraryVersion = context.ServerLibraryVersion.ToString();
}
if (context.ServerVersion != null)
{
serverVersion = context.ServerVersion.ToString();
}
}
catch { }
}
ApplicationInsights = new ApplicationInsights();
var coreAssembly = Assembly.GetExecutingAssembly();
ApplicationInsights.Initialize(serverLibraryVersion, serverVersion, initializationType.ToString(), ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString());
ApplicationInsights.TrackEvent("Connect-PnPOnline");
//TelemetryConfiguration config = TelemetryConfiguration.CreateDefault();
//TelemetryClient = new TelemetryClient(config);
//config.InstrumentationKey = "a301024a-9e21-4273-aca5-18d0ef5d80fb";
////config..Context.Session.Id = Guid.NewGuid().ToString();
//TelemetryClient.Context.Cloud.RoleInstance = "PnPPowerShell";
//TelemetryClient.Context.Device.OperatingSystem = Environment.OSVersion.ToString();
//TelemetryProperties = new Dictionary<string, string>(10);
//TelemetryProperties.Add("ServerLibraryVersion", serverLibraryVersion);
//TelemetryProperties.Add("ServerVersion", serverVersion);
//TelemetryProperties.Add("ConnectionMethod", initializationType.ToString());
//var coreAssembly = Assembly.GetExecutingAssembly();
//TelemetryProperties.Add("Version", ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString());
//TelemetryProperties.Add("Platform", "SPO");
//TelemetryClient.TrackEvent("Connect-PnPOnline", TelemetryProperties);
//TelemetryClient = new TelemetryClient(;
//TelemetryClient.InstrumentationKey = "a301024a-9e21-4273-aca5-18d0ef5d80fb";
//TelemetryClient.Context.Session.Id = Guid.NewGuid().ToString();
//TelemetryClient.Context.Cloud.RoleInstance = "PnPPowerShell";
//TelemetryClient.Context.Device.OperatingSystem = Environment.OSVersion.ToString();
//TelemetryClient.Context.Properties.Add("ServerLibraryVersion", serverLibraryVersion);
//TelemetryClient.Context.Properties.Add("ServerVersion", serverVersion);
//TelemetryClient.Context.Properties.Add("ConnectionMethod", initializationType.ToString());
//var coreAssembly = Assembly.GetExecutingAssembly();
//TelemetryClient.Context.Properties.Add("Version", ((AssemblyFileVersionAttribute)coreAssembly.GetCustomAttribute(typeof(AssemblyFileVersionAttribute))).Version.ToString());
//TelemetryClient.Context.Properties.Add("Platform", "SPO");
//TelemetryClient.TrackEvent("Connect-PnPOnline");
}
}
public void InitializeRegistersTelemetryContextInitializer()
{
var applicationInsightsConfiguration = ApplicationInsights.Initialize(InstrumentationKey);
Assert.Contains(applicationInsightsConfiguration.TelemetryConfiguration.TelemetryInitializers, ti => ti is TelemetryContextInitializer);
}
/// <summary>
/// This is a static method to run a job whose args are passed in
/// By default,
/// a) The job will be run continuously in a while loop. Could be overridden using 'once' argument
/// b) The sleep duration between each run when running continuously is 5000 milliSeconds. Could be overridden using '-Sleep' argument
/// </summary>
/// <param name="job">Job to run</param>
/// <param name="commandLineArgs">Args contains args to the job runner like (dbg, once and so on) and for the job itself</param>
/// <returns></returns>
public static async Task Run(JobBase job, string[] commandLineArgs)
{
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
Debugger.Launch();
}
// Configure logging before Application Insights is enabled.
// This is done so, in case Application Insights fails to initialize, we still see output.
var loggerFactory = ConfigureLogging(job);
try
{
_logger.LogInformation("Started...");
// Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(loggerFactory.CreateLogger(typeof(JobConfigurationManager)), commandLineArgs, job.JobName, (ISecretReaderFactory)ServiceContainer.GetService(typeof(ISecretReaderFactory)));
// Setup logging
if (!ApplicationInsights.Initialized)
{
string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
if (!string.IsNullOrWhiteSpace(instrumentationKey))
{
ApplicationInsights.Initialize(instrumentationKey);
}
}
// Configure our logging again with Application Insights initialized.
loggerFactory = ConfigureLogging(job);
var runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds
if (!sleepDuration.HasValue)
{
sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
if (sleepDuration.HasValue)
{
sleepDuration = sleepDuration.Value * 1000; // interval is in seconds
}
}
if (sleepDuration == null)
{
_logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
sleepDuration = 5000;
}
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Run the job loop
await JobLoop(job, runContinuously, sleepDuration.Value, jobArgsDictionary);
}
catch (Exception ex)
{
_logger.LogError("Job runner threw an exception: {Exception}", ex);
}
}
private static IServiceProvider InitializeAndGetServiceProvider()
{
ServicePointManager.DefaultConnectionLimit = 64;
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
var configurationRoot = GetConfigurationRoot();
var instrumentationKey = configurationRoot
.GetSection(ConfigurationSectionName)
.GetValue <string>(nameof(V3PerPackageConfiguration.InstrumentationKey));
ApplicationInsights.Initialize(instrumentationKey);
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Information);
var serviceCollection = new ServiceCollection();
serviceCollection.Configure <V3PerPackageConfiguration>(configurationRoot.GetSection(ConfigurationSectionName));
serviceCollection.AddOptions();
serviceCollection.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot <>), typeof(NonCachingOptionsSnapshot <>)));
serviceCollection.AddLogging();
serviceCollection.AddSingleton(loggerFactory);
serviceCollection.AddSingleton(new ControlledDisposeHttpClientHandler());
serviceCollection.AddTransient <HttpMessageHandler>(x => x.GetRequiredService <ControlledDisposeHttpClientHandler>());
serviceCollection.AddSingleton(x => new HttpClient(x.GetRequiredService <HttpMessageHandler>()));
serviceCollection.AddTransient <Func <HttpMessageHandler> >(x => () => x.GetRequiredService <HttpMessageHandler>());
serviceCollection.AddTransient <PerBatchProcessor>();
serviceCollection.AddTransient <ITelemetryService, TelemetryService>();
serviceCollection.AddTransient <PerWorkerProcessor>();
serviceCollection.AddTransient <PerProcessProcessor>();
serviceCollection.AddSingleton <StringLocker>();
serviceCollection.AddTransient <EnqueueCollector>();
serviceCollection.AddTransient <EnqueueCommand>();
serviceCollection.AddTransient <CleanUpCommand>();
serviceCollection.AddSingleton(x =>
{
var globalContext = x.GetRequiredService <GlobalContext>();
var perProcessContext = new PerProcessContext(
globalContext,
UniqueName.New("process"),
WorkerCount,
MessageCount,
BatchSize);
var blobClient = BlobStorageUtilities.GetBlobClient(globalContext);
var flatContainerUrl = $"{blobClient.BaseUri.AbsoluteUri}/{globalContext.FlatContainerContainerName}/{perProcessContext.Name}";
RegistrationMakerCatalogItem.PackagePathProvider = new FlatContainerPackagePathProvider(flatContainerUrl);
return(perProcessContext);
});
serviceCollection.AddTransient(x =>
{
var settings = x.GetRequiredService <IOptionsSnapshot <V3PerPackageConfiguration> >();
return(new GlobalContext(
settings.Value.StorageBaseAddress,
settings.Value.StorageAccountName,
settings.Value.StorageKeyValue,
settings.Value.ContentBaseAddress));
});
serviceCollection.AddSingleton(new TelemetryClient());
serviceCollection.AddTransient <IStorageQueue <PackageMessage> >(x =>
{
var globalContext = x.GetRequiredService <GlobalContext>();
var storageCredentials = new StorageCredentials(globalContext.StorageAccountName, globalContext.StorageKeyValue);
var storageAccount = new CloudStorageAccount(storageCredentials, useHttps: true);
return(new StorageQueue <PackageMessage>(
new AzureStorageQueue(storageAccount, "v3perpackage"),
new JsonMessageSerializer <PackageMessage>(JsonSerializerUtility.SerializerSettings),
PackageMessage.Version));
});
return(serviceCollection.BuildServiceProvider());
}
public override bool Init(IDictionary <string, string> jobArgsDictionary)
{
try
{
if (!ApplicationInsights.Initialized)
{
string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
if (!string.IsNullOrWhiteSpace(instrumentationKey))
{
ApplicationInsights.Initialize(instrumentationKey);
}
}
_loggerFactory = LoggingSetup.CreateLoggerFactory();
_logger = _loggerFactory.CreateLogger <Job>();
// Configure job
_galleryBaseAddress = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.GalleryBaseAddress);
var storageConnectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.DataStorageAccount);
_cloudStorageAccount = CreateCloudStorageAccount(JobArgumentNames.DataStorageAccount, storageConnectionString);
_containerName = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.ContainerName);
_runValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RunValidationTasks).Split(';');
_requestValidationTasks = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.RequestValidationTasks).Split(';');
// Add validators
if (_runValidationTasks.Contains(UnzipValidator.ValidatorName))
{
_validators.Add(new UnzipValidator(_loggerFactory));
}
if (_runValidationTasks.Contains(VcsValidator.ValidatorName))
{
// if contact alias set, use it, if not, use submitter alias.
string submitterAlias = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorSubmitterAlias);
string contactAlias = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.VcsContactAlias)
?? submitterAlias;
_validators.Add(new VcsValidator(
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorServiceUrl),
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsValidatorCallbackUrl),
contactAlias,
submitterAlias,
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.VcsPackageUrlTemplate),
_loggerFactory));
}
return(true);
}
catch (Exception ex)
{
if (_logger != null)
{
_logger.LogError(TraceEvent.CommandLineProcessingFailed, ex, "Exception occurred while processing command line arguments");
}
else
{
Trace.TraceError("Exception occurred while processing command line arguments: {0}", ex);
}
}
return(false);
}
void SetupApplicationInsights()
{
ApplicationInsights.Setup(InstrumentationKey);
ApplicationInsights.SetDebugLogEnabled(true);
ApplicationInsights.Start();
}
private async Task <CdnStatistics> ParseLogEntries(ILeasedLogFile logFile)
{
var logStream = await OpenCompressedBlobAsync(logFile);
var blobUri = logFile.Uri;
var blobName = logFile.Blob.Name;
var packageStatistics = new List <PackageStatistics>();
var toolStatistics = new List <ToolStatistics>();
var dnxStatistics = new List <DnxStatistics>();
var stopwatch = Stopwatch.StartNew();
try
{
// parse the log into table entities
_jobEventSource.BeginningParseLog(blobUri);
using (var logStreamReader = new StreamReader(logStream))
{
do
{
var rawLogLine = logStreamReader.ReadLine();
if (rawLogLine != null)
{
var logEntry = CdnLogEntryParser.ParseLogEntryFromLine(rawLogLine);
if (logEntry != null)
{
var statistic = PackageStatisticsParser.FromCdnLogEntry(logEntry);
if (statistic != null)
{
packageStatistics.Add(statistic);
}
else
{
// check if this is a dist.nuget.org download
if (logEntry.RequestUrl.Contains("dist.nuget.org/"))
{
var toolInfo = ToolStatisticsParser.FromCdnLogEntry(logEntry);
if (toolInfo != null)
{
toolStatistics.Add(toolInfo);
}
}
if (logEntry.RequestUrl.Contains("dist.asp.net"))
{
var dnxInfo = DnxStatisticsParser.FromCdnLogEntry(logEntry);
if (dnxInfo != null)
{
dnxStatistics.Add(dnxInfo);
}
}
}
}
}
} while (!logStreamReader.EndOfStream);
}
_jobEventSource.FinishingParseLog(blobUri, packageStatistics.Count);
stopwatch.Stop();
ApplicationInsights.TrackMetric("Blob parsing duration (ms)", stopwatch.ElapsedMilliseconds, blobName);
}
catch (Exception exception)
{
if (stopwatch.IsRunning)
{
stopwatch.Stop();
}
_jobEventSource.FailedParseLog(blobUri);
ApplicationInsights.TrackException(exception, blobName);
throw;
}
finally
{
logStream.Dispose();
}
var cdnStatistics = new CdnStatistics(packageStatistics, toolStatistics, dnxStatistics);
return(cdnStatistics);
}
private async Task RebuildPackageReports(CloudBlobContainer destinationContainer, DateTime reportGenerationTime)
{
var dirtyPackageIds = await ReportDataCollector.GetDirtyPackageIds(_statisticsDatabase, reportGenerationTime);
if (!dirtyPackageIds.Any())
{
return;
}
// first process the top 100 packages
var top100 = dirtyPackageIds.Take(100);
var reportDataCollector = new ReportDataCollector(_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId], _statisticsDatabase);
var top100Task = Parallel.ForEach(top100, new ParallelOptions {
MaxDegreeOfParallelism = 4
}, dirtyPackageId =>
{
var packageId = dirtyPackageId.PackageId.ToLowerInvariant();
var reportBuilder = new RecentPopularityDetailByPackageReportBuilder(ReportNames.RecentPopularityDetailByPackageId, "recentpopularity/" + _recentPopularityDetailByPackageReportBaseName + packageId);
ProcessReport(destinationContainer, reportBuilder, reportDataCollector, reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
ApplicationInsights.TrackReportProcessed(reportBuilder.ReportName + " report", packageId);
});
// once top 100 is processed, continue with the rest
if (top100Task.IsCompleted)
{
var excludingTop100 = dirtyPackageIds.Skip(100);
top100Task = Parallel.ForEach(excludingTop100, new ParallelOptions {
MaxDegreeOfParallelism = 8
},
dirtyPackageId =>
{
// generate all reports
var reportGenerators = new Dictionary <ReportBuilder, ReportDataCollector>
{
{
new RecentPopularityDetailByPackageReportBuilder(
ReportNames.RecentPopularityDetailByPackageId,
"recentpopularity/" + _recentPopularityDetailByPackageReportBaseName +
dirtyPackageId.PackageId.ToLowerInvariant()),
new ReportDataCollector(
_storedProceduresPerPackageId[ReportNames.RecentPopularityDetailByPackageId],
_statisticsDatabase)
}
};
foreach (var reportGenerator in reportGenerators)
{
ProcessReport(destinationContainer, reportGenerator.Key, reportGenerator.Value,
reportGenerationTime, Tuple.Create("@PackageId", 128, dirtyPackageId.PackageId)).Wait();
ApplicationInsights.TrackReportProcessed(reportGenerator.Key.ReportName + " report",
dirtyPackageId.PackageId.ToLowerInvariant());
}
});
if (top100Task.IsCompleted)
{
var runToCursor = dirtyPackageIds.First().RunToCuror;
await ReportDataCollector.UpdateDirtyPackageIdCursor(_statisticsDatabase, runToCursor);
}
}
}
public static LoggerConfiguration ConfigureTraceLogger(LoggerConfiguration loggerConfig, ApplicationInsights appInsightsConfig,
LoggingLevelSwitch levelSwitch = null)
{
ConfigureLoggerConfiguration(loggerConfig, levelSwitch ?? LoggingLevelSwitch);
if (appInsightsConfig != null && appInsightsConfig.Enabled &&
!string.IsNullOrEmpty(appInsightsConfig.InstrumentationKey))
{
loggerConfig.WriteTo.ApplicationInsights(TelemetryConverter.Traces);
}
return(loggerConfig);
}
// This method is auto-detected by the OWIN pipeline. DO NOT RENAME IT!
public static void Configuration(IAppBuilder app)
{
// Tune ServicePointManager
// (based on http://social.technet.microsoft.com/Forums/en-US/windowsazuredata/thread/d84ba34b-b0e0-4961-a167-bbe7618beb83 and https://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.aspx)
ServicePointManager.DefaultConnectionLimit = 500;
ServicePointManager.UseNagleAlgorithm = false;
ServicePointManager.Expect100Continue = false;
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Setting time out for all RegEx objects. Noted in remarks at https://msdn.microsoft.com/en-us/library/system.text.regularexpressions.regex.matchtimeout%28v=vs.110%29.aspx
AppDomain.CurrentDomain.SetData("REGEX_DEFAULT_MATCH_TIMEOUT", TimeSpan.FromSeconds(10));
// Register IoC
app.UseAutofacInjection(GlobalConfiguration.Configuration);
var dependencyResolver = DependencyResolver.Current;
// Register Elmah
var elmahServiceCenter = new DependencyResolverServiceProviderAdapter(dependencyResolver);
ServiceCenter.Current = _ => elmahServiceCenter;
// Get config
var config = dependencyResolver.GetService <IGalleryConfigurationService>();
var auth = dependencyResolver.GetService <AuthenticationService>();
// Configure machine key for session persistence across slots
SessionPersistence.Setup(config);
// Refresh the content for the ContentObjectService to guarantee it has loaded the latest configuration on startup.
var contentObjectService = dependencyResolver.GetService <IContentObjectService>();
HostingEnvironment.QueueBackgroundWorkItem(async token =>
{
while (!token.IsCancellationRequested)
{
await contentObjectService.Refresh();
await Task.Delay(ContentObjectService.RefreshInterval, token);
}
});
// Setup telemetry
var instrumentationKey = config.Current.AppInsightsInstrumentationKey;
if (!string.IsNullOrEmpty(instrumentationKey))
{
var heartbeatIntervalSeconds = config.Current.AppInsightsHeartbeatIntervalSeconds;
if (heartbeatIntervalSeconds > 0)
{
// Configure instrumentation key, heartbeat interval,
// and register NuGet.Services.Logging.TelemetryContextInitializer.
ApplicationInsights.Initialize(instrumentationKey, TimeSpan.FromSeconds(heartbeatIntervalSeconds));
}
else
{
// Configure instrumentation key,
// and register NuGet.Services.Logging.TelemetryContextInitializer.
ApplicationInsights.Initialize(instrumentationKey);
}
// Add enrichers
TelemetryConfiguration.Active.TelemetryInitializers.Add(new DeploymentIdTelemetryEnricher());
TelemetryConfiguration.Active.TelemetryInitializers.Add(new ClientInformationTelemetryEnricher());
var telemetryProcessorChainBuilder = TelemetryConfiguration.Active.TelemetryProcessorChainBuilder;
// Add processors
telemetryProcessorChainBuilder.Use(next =>
{
var processor = new RequestTelemetryProcessor(next);
processor.SuccessfulResponseCodes.Add(400);
processor.SuccessfulResponseCodes.Add(404);
processor.SuccessfulResponseCodes.Add(405);
return(processor);
});
telemetryProcessorChainBuilder.Use(next => new ClientTelemetryPIIProcessor(next));
var telemetry = dependencyResolver.GetService <TelemetryClientWrapper>();
telemetryProcessorChainBuilder.Use(
next => new ExceptionTelemetryProcessor(next, telemetry.UnderlyingClient));
// Note: sampling rate must be a factor 100/N where N is a whole number
// e.g.: 50 (= 100/2), 33.33 (= 100/3), 25 (= 100/4), ...
// https://azure.microsoft.com/en-us/documentation/articles/app-insights-sampling/
var instrumentationSamplingPercentage = config.Current.AppInsightsSamplingPercentage;
if (instrumentationSamplingPercentage > 0 && instrumentationSamplingPercentage < 100)
{
telemetryProcessorChainBuilder.UseSampling(instrumentationSamplingPercentage);
}
telemetryProcessorChainBuilder.Build();
}
// Configure logging
app.SetLoggerFactory(new DiagnosticsLoggerFactory());
// Remove X-AspNetMvc-Version header
MvcHandler.DisableMvcResponseHeader = true;
if (config.Current.RequireSSL)
{
// Put a middleware at the top of the stack to force the user over to SSL
if (config.Current.ForceSslExclusion == null)
{
app.UseForceSsl(config.Current.SSLPort);
}
else
{
app.UseForceSsl(config.Current.SSLPort, config.Current.ForceSslExclusion);
}
}
// Get the local user auth provider, if present and attach it first
Authenticator localUserAuthenticator;
if (auth.Authenticators.TryGetValue(Authenticator.GetName(typeof(LocalUserAuthenticator)), out localUserAuthenticator))
{
// Configure cookie auth now
localUserAuthenticator.Startup(config, app).Wait();
}
// Attach external sign-in cookie middleware
app.SetDefaultSignInAsAuthenticationType(AuthenticationTypes.External);
app.UseCookieAuthentication(new CookieAuthenticationOptions
{
AuthenticationType = AuthenticationTypes.External,
AuthenticationMode = AuthenticationMode.Passive,
CookieName = ".AspNet." + AuthenticationTypes.External,
ExpireTimeSpan = TimeSpan.FromMinutes(5)
});
// Attach non-cookie auth providers
var nonCookieAuthers = auth
.Authenticators
.Where(p => !String.Equals(
p.Key,
Authenticator.GetName(typeof(LocalUserAuthenticator)),
StringComparison.OrdinalIgnoreCase))
.Select(p => p.Value);
foreach (var auther in nonCookieAuthers)
{
auther.Startup(config, app).Wait();
}
var featureFlags = DependencyResolver.Current.GetService <IFeatureFlagCacheService>();
if (featureFlags != null)
{
StartFeatureFlags(featureFlags);
}
// Catch unobserved exceptions from threads before they cause IIS to crash:
TaskScheduler.UnobservedTaskException += (sender, exArgs) =>
{
// Send to AppInsights
try
{
var telemetryClient = new TelemetryClient();
telemetryClient.TrackException(exArgs.Exception, new Dictionary <string, string>()
{
{ "ExceptionOrigin", "UnobservedTaskException" }
});
}
catch (Exception)
{
// this is a tragic moment... swallow Exception to prevent crashing IIS
}
// Send to ELMAH
try
{
HttpContext current = HttpContext.Current;
if (current != null)
{
var errorSignal = ErrorSignal.FromContext(current);
if (errorSignal != null)
{
errorSignal.Raise(exArgs.Exception, current);
}
}
}
catch (Exception)
{
// more tragedy... swallow Exception to prevent crashing IIS
}
exArgs.SetObserved();
};
HasRun = true;
}