public static void Run(string[] args)
{
IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);
if (arguments == null || arguments.Count == 0)
{
PrintUsage();
return;
}
string gallery = CommandHelpers.GetGallery(arguments);
if (gallery == null)
{
PrintUsage();
return;
}
bool verbose = CommandHelpers.GetVerbose(arguments);
int interval = CommandHelpers.GetInterval(arguments);
DateTime startDate = CommandHelpers.GetStartDate(arguments);
StorageFactory storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
if (storageFactory == null)
{
PrintUsage();
return;
}
if (verbose)
{
Trace.Listeners.Add(new ConsoleTraceListener());
Trace.AutoFlush = true;
}
Trace.TraceInformation("CONFIG source: \"{0}\" storage: \"{1}\" interval: {2}", gallery, storageFactory, interval);
DateTime?nullableStartDate = null;
if (startDate != DateTime.MinValue)
{
nullableStartDate = startDate;
}
Loop(gallery, storageFactory, verbose, interval, nullableStartDate).Wait();
}
public static void Package(string[] args)
{
IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);
if (arguments == null || arguments.Count == 0)
{
PackagePrintUsage();
return;
}
string gallery = CommandHelpers.GetGallery(arguments);
if (gallery == null)
{
PackagePrintUsage();
return;
}
bool verbose = CommandHelpers.GetVerbose(arguments);
string id = CommandHelpers.GetId(arguments);
if (id == null)
{
PackagePrintUsage();
return;
}
string version = CommandHelpers.GetVersion(arguments);
StorageFactory storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
if (storageFactory == null)
{
PrintUsage();
return;
}
if (verbose)
{
Trace.Listeners.Add(new ConsoleTraceListener());
Trace.AutoFlush = true;
}
ProcessPackages(gallery, storageFactory, id, version, verbose).Wait();
}
public static void Run(string[] args)
{
IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);
if (arguments == null || arguments.Count == 0)
{
PrintUsage();
return;
}
string source = CommandHelpers.GetSource(arguments);
if (source == null)
{
PrintUsage();
return;
}
bool verbose = CommandHelpers.GetVerbose(arguments);
int interval = CommandHelpers.GetInterval(arguments);
string contentBaseAddress = CommandHelpers.GetContentBaseAddress(arguments);
StorageFactory storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);
if (storageFactory == null)
{
PrintUsage();
return;
}
if (verbose)
{
Trace.Listeners.Add(new ConsoleTraceListener());
Trace.AutoFlush = true;
}
Trace.TraceInformation("CONFIG source: \"{0}\" storage: \"{1}\" interval: {2} seconds", source, storageFactory, interval);
Loop(source, storageFactory, contentBaseAddress, verbose, interval).Wait();
}
static async Task Loop(string source, string registration, Lucene.Net.Store.Directory directory, string catalogBaseAddress, string storageBaseAddress, bool verbose, int interval)
{
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose, catalogBaseAddress, storageBaseAddress);
CommitCollector collector = new SearchIndexFromCatalogCollector(new Uri(source), directory, catalogBaseAddress, handlerFunc);
ReadWriteCursor front = new LuceneCursor(directory, MemoryCursor.Min.Value);
ReadCursor back = (registration == null) ? (ReadCursor)MemoryCursor.Max : new HttpReadCursor(new Uri(registration), handlerFunc);
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back);
}while (run);
Thread.Sleep(interval * 1000);
}
}
public static void Run(string[] args)
{
IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);
if (arguments == null || arguments.Count == 0)
{
PrintUsage();
return;
}
Lucene.Net.Store.Directory directory = CommandHelpers.GetLuceneDirectory(arguments);
if (directory == null)
{
PrintUsage();
return;
}
using (IndexReader reader = IndexReader.Open(directory, true))
{
Console.WriteLine("Lucene index contains: {0} documents", reader.NumDocs());
IDictionary <string, string> commitUserData = reader.CommitUserData;
if (commitUserData == null)
{
Console.WriteLine("commitUserData is null");
}
else
{
Console.WriteLine("commitUserData:");
foreach (var entry in commitUserData)
{
Console.WriteLine(" {0} = {1}", entry.Key, entry.Value);
}
}
}
}
static async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose)
{
int timeout = 300;
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
AllowPipelining = true
};
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// if teh version is specified a single package is processed otherwise all the packages corresponding to that id are processed
Uri uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > packages = await GetPackages(client, uri, "Created");
Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());
Storage storage = storageFactory.Create();
// the idea here is to leave the lastCreated and lastEdited values exactly as they were
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? DateTime.MinValue.ToUniversalTime();
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? DateTime.MinValue.ToUniversalTime();
DateTime d = await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited);
}
}
static async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool verbose, int interval)
{
CommitCollector collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
{
ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress)
};
Storage storage = storageFactory.Create();
ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
ReadCursor back = MemoryCursor.Max;
while (true)
{
bool run = false;
do
{
run = await collector.Run(front, back);
}while (run);
Thread.Sleep(interval * 1000);
}
}
public static async Task MainAsync(string[] args)
{
if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase))
{
args = args.Skip(1).ToArray();
Debugger.Launch();
}
NgJob job = null;
try
{
// Get arguments
var arguments = CommandHelpers.GetArguments(args, 1);
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Determine the job name
if (args.Length == 0)
{
throw new ArgumentException("Missing job name argument.");
}
var jobName = args[0];
TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName));
// Configure ApplicationInsights
ApplicationInsights.Initialize(arguments.GetOrDefault <string>(Arguments.InstrumentationKey));
// Create an ILoggerFactory
var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
loggerConfiguration.WriteTo.File("Log.txt", retainedFileCountLimit: 3, fileSizeLimitBytes: 1000000, rollOnFileSizeLimit: true);
var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Debug);
// Create a logger that is scoped to this class (only)
_logger = loggerFactory.CreateLogger <Program>();
var cancellationTokenSource = new CancellationTokenSource();
// Create an ITelemetryService
var telemetryService = new TelemetryService(new TelemetryClient());
// Allow jobs to set global custom dimensions
TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobPropertiesTelemetryInitializer(telemetryService));
job = NgJobFactory.GetJob(jobName, telemetryService, loggerFactory);
await job.RunAsync(arguments, cancellationTokenSource.Token);
}
catch (ArgumentException ae)
{
_logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae);
Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
}
catch (Exception e)
{
_logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e);
}
Trace.Close();
TelemetryConfiguration.Active.TelemetryChannel.Flush();
}
public static async Task MainAsync(string[] args)
{
if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase))
{
args = args.Skip(1).ToArray();
Debugger.Launch();
}
NgJob job = null;
ApplicationInsightsConfiguration applicationInsightsConfiguration = null;
int exitCode = 0;
try
{
// Get arguments
var arguments = CommandHelpers.GetArguments(args, 1, out var secretInjector);
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Determine the job name
if (args.Length == 0)
{
throw new ArgumentException("Missing job name argument.");
}
var jobName = args[0];
var instanceName = arguments.GetOrDefault(Arguments.InstanceName, jobName);
var instrumentationKey = arguments.GetOrDefault <string>(Arguments.InstrumentationKey);
var heartbeatIntervalSeconds = arguments.GetOrDefault <int>(Arguments.HeartbeatIntervalSeconds);
applicationInsightsConfiguration = ConfigureApplicationInsights(
instrumentationKey,
heartbeatIntervalSeconds,
jobName,
instanceName,
out var telemetryClient,
out var telemetryGlobalDimensions);
var loggerFactory = ConfigureLoggerFactory(applicationInsightsConfiguration);
InitializeServiceProvider(
arguments,
secretInjector,
applicationInsightsConfiguration,
telemetryClient,
loggerFactory);
job = NgJobFactory.GetJob(jobName, loggerFactory, telemetryClient, telemetryGlobalDimensions);
job.SetSecretInjector(secretInjector);
// This tells Application Insights that, even though a heartbeat is reported,
// the state of the application is unhealthy when the exitcode is different from zero.
// The heartbeat metadata is enriched with the job loop exit code.
applicationInsightsConfiguration.DiagnosticsTelemetryModule?.AddOrSetHeartbeatProperty(
HeartbeatProperty_JobLoopExitCode,
exitCode.ToString(),
isHealthy: exitCode == 0);
var cancellationTokenSource = new CancellationTokenSource();
await job.RunAsync(arguments, cancellationTokenSource.Token);
exitCode = 0;
}
catch (ArgumentException ae)
{
exitCode = 1;
_logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae);
Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
}
catch (KeyNotFoundException knfe)
{
exitCode = 1;
_logger?.LogError("An expected key was not found. One possible cause of this is required argument has not been provided: {Exception}", knfe);
Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
}
catch (Exception e)
{
exitCode = 1;
_logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e);
}
applicationInsightsConfiguration.DiagnosticsTelemetryModule?.SetHeartbeatProperty(
HeartbeatProperty_JobLoopExitCode,
exitCode.ToString(),
isHealthy: exitCode == 0);
Trace.Close();
applicationInsightsConfiguration?.TelemetryConfiguration.TelemetryChannel.Flush();
}
static async Task Loop(string gallery, StorageFactory storageFactory, bool verbose, int interval, DateTime?startDate)
{
Storage storage = storageFactory.Create();
const string LastCreated = "nuget:lastCreated";
const string LastEdited = "nuget:lastEdited";
int top = 20;
int timeout = 300;
while (true)
{
Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);
HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
AllowPipelining = true
};
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromSeconds(timeout);
// fetch and add all newly CREATED packages - in order
DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? (startDate ?? DateTime.MinValue.ToUniversalTime());
DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? lastCreated;
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > createdPackages;
DateTime previousLastCreated = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastCreated: {0}", lastCreated.ToString("O"));
createdPackages = await GetCreatedPackages(client, gallery, lastCreated, top);
Trace.TraceInformation("FEED CreatedPackages: {0}", createdPackages.Count);
lastCreated = await DownloadMetadata2Catalog(client, createdPackages, storage, lastCreated, lastEdited, createdPackages : true);
if (previousLastCreated == lastCreated)
{
break;
}
previousLastCreated = lastCreated;
}while (createdPackages.Count > 0);
// THEN fetch and add all EDITED packages - in order
SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > editedPackages;
DateTime previousLastEdited = DateTime.MinValue;
do
{
Trace.TraceInformation("CATALOG LastEdited: {0}", lastEdited.ToString("O"));
editedPackages = await GetEditedPackages(client, gallery, lastEdited, top);
Trace.TraceInformation("FEED EditedPackages: {0}", editedPackages.Count);
lastEdited = await DownloadMetadata2Catalog(client, editedPackages, storage, lastCreated, lastEdited, createdPackages : false);
if (previousLastEdited == lastEdited)
{
break;
}
previousLastEdited = lastEdited;
}while (editedPackages.Count > 0);
}
Thread.Sleep(interval * 1000);
}
}