public static void Run(string[] args)
        {
            IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);

            if (arguments == null || arguments.Count == 0)
            {
                PrintUsage();
                return;
            }

            string gallery = CommandHelpers.GetGallery(arguments);

            if (gallery == null)
            {
                PrintUsage();
                return;
            }

            bool verbose = CommandHelpers.GetVerbose(arguments);

            int interval = CommandHelpers.GetInterval(arguments);

            DateTime startDate = CommandHelpers.GetStartDate(arguments);

            StorageFactory storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);

            if (storageFactory == null)
            {
                PrintUsage();
                return;
            }

            if (verbose)
            {
                Trace.Listeners.Add(new ConsoleTraceListener());
                Trace.AutoFlush = true;
            }

            Trace.TraceInformation("CONFIG source: \"{0}\" storage: \"{1}\" interval: {2}", gallery, storageFactory, interval);
            DateTime?nullableStartDate = null;

            if (startDate != DateTime.MinValue)
            {
                nullableStartDate = startDate;
            }
            Loop(gallery, storageFactory, verbose, interval, nullableStartDate).Wait();
        }
        public static void Package(string[] args)
        {
            IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);

            if (arguments == null || arguments.Count == 0)
            {
                PackagePrintUsage();
                return;
            }

            string gallery = CommandHelpers.GetGallery(arguments);

            if (gallery == null)
            {
                PackagePrintUsage();
                return;
            }

            bool verbose = CommandHelpers.GetVerbose(arguments);

            string id = CommandHelpers.GetId(arguments);

            if (id == null)
            {
                PackagePrintUsage();
                return;
            }

            string version = CommandHelpers.GetVersion(arguments);

            StorageFactory storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);

            if (storageFactory == null)
            {
                PrintUsage();
                return;
            }

            if (verbose)
            {
                Trace.Listeners.Add(new ConsoleTraceListener());
                Trace.AutoFlush = true;
            }

            ProcessPackages(gallery, storageFactory, id, version, verbose).Wait();
        }
Пример #3
0
        public static void Run(string[] args)
        {
            IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);

            if (arguments == null || arguments.Count == 0)
            {
                PrintUsage();
                return;
            }

            string source = CommandHelpers.GetSource(arguments);

            if (source == null)
            {
                PrintUsage();
                return;
            }

            bool verbose = CommandHelpers.GetVerbose(arguments);

            int interval = CommandHelpers.GetInterval(arguments);

            string contentBaseAddress = CommandHelpers.GetContentBaseAddress(arguments);

            StorageFactory storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose);

            if (storageFactory == null)
            {
                PrintUsage();
                return;
            }

            if (verbose)
            {
                Trace.Listeners.Add(new ConsoleTraceListener());
                Trace.AutoFlush = true;
            }

            Trace.TraceInformation("CONFIG source: \"{0}\" storage: \"{1}\" interval: {2} seconds", source, storageFactory, interval);

            Loop(source, storageFactory, contentBaseAddress, verbose, interval).Wait();
        }
        static async Task Loop(string source, string registration, Lucene.Net.Store.Directory directory, string catalogBaseAddress, string storageBaseAddress, bool verbose, int interval)
        {
            Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose, catalogBaseAddress, storageBaseAddress);

            CommitCollector collector = new SearchIndexFromCatalogCollector(new Uri(source), directory, catalogBaseAddress, handlerFunc);

            ReadWriteCursor front = new LuceneCursor(directory, MemoryCursor.Min.Value);

            ReadCursor back = (registration == null) ? (ReadCursor)MemoryCursor.Max : new HttpReadCursor(new Uri(registration), handlerFunc);

            while (true)
            {
                bool run = false;
                do
                {
                    run = await collector.Run(front, back);
                }while (run);

                Thread.Sleep(interval * 1000);
            }
        }
        public static void Run(string[] args)
        {
            IDictionary <string, string> arguments = CommandHelpers.GetArguments(args, 1);

            if (arguments == null || arguments.Count == 0)
            {
                PrintUsage();
                return;
            }

            Lucene.Net.Store.Directory directory = CommandHelpers.GetLuceneDirectory(arguments);
            if (directory == null)
            {
                PrintUsage();
                return;
            }

            using (IndexReader reader = IndexReader.Open(directory, true))
            {
                Console.WriteLine("Lucene index contains: {0} documents", reader.NumDocs());

                IDictionary <string, string> commitUserData = reader.CommitUserData;

                if (commitUserData == null)
                {
                    Console.WriteLine("commitUserData is null");
                }
                else
                {
                    Console.WriteLine("commitUserData:");
                    foreach (var entry in commitUserData)
                    {
                        Console.WriteLine("  {0} = {1}", entry.Key, entry.Value);
                    }
                }
            }
        }
        static async Task ProcessPackages(string gallery, StorageFactory storageFactory, string id, string version, bool verbose)
        {
            int timeout = 300;

            Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);

            HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
                AllowPipelining = true
            };

            using (HttpClient client = new HttpClient(handler))
            {
                client.Timeout = TimeSpan.FromSeconds(timeout);

                //  if teh version is specified a single package is processed otherwise all the packages corresponding to that id are processed

                Uri uri = (version == null) ? MakePackageUri(gallery, id) : MakePackageUri(gallery, id, version);

                SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > packages = await GetPackages(client, uri, "Created");

                Trace.TraceInformation("downloading {0} packages", packages.Select(t => t.Value.Count).Sum());

                Storage storage = storageFactory.Create();

                //  the idea here is to leave the lastCreated and lastEdited values exactly as they were

                const string LastCreated = "nuget:lastCreated";
                const string LastEdited  = "nuget:lastEdited";

                DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? DateTime.MinValue.ToUniversalTime();

                DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? DateTime.MinValue.ToUniversalTime();

                DateTime d = await DownloadMetadata2Catalog(client, packages, storage, lastCreated, lastEdited);
            }
        }
Пример #7
0
        static async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool verbose, int interval)
        {
            CommitCollector collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose))
            {
                ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress)
            };

            Storage         storage = storageFactory.Create();
            ReadWriteCursor front   = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value);
            ReadCursor      back    = MemoryCursor.Max;

            while (true)
            {
                bool run = false;
                do
                {
                    run = await collector.Run(front, back);
                }while (run);

                Thread.Sleep(interval * 1000);
            }
        }
Пример #8
0
        public static async Task MainAsync(string[] args)
        {
            if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase))
            {
                args = args.Skip(1).ToArray();
                Debugger.Launch();
            }

            NgJob job = null;

            try
            {
                // Get arguments
                var arguments = CommandHelpers.GetArguments(args, 1);

                // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
                ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
                ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;

                // Determine the job name
                if (args.Length == 0)
                {
                    throw new ArgumentException("Missing job name argument.");
                }

                var jobName = args[0];
                TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobNameTelemetryInitializer(jobName));

                // Configure ApplicationInsights
                ApplicationInsights.Initialize(arguments.GetOrDefault <string>(Arguments.InstrumentationKey));

                // Create an ILoggerFactory
                var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true);
                loggerConfiguration.WriteTo.File("Log.txt", retainedFileCountLimit: 3, fileSizeLimitBytes: 1000000, rollOnFileSizeLimit: true);

                var loggerFactory = LoggingSetup.CreateLoggerFactory(loggerConfiguration, LogEventLevel.Debug);

                // Create a logger that is scoped to this class (only)
                _logger = loggerFactory.CreateLogger <Program>();

                var cancellationTokenSource = new CancellationTokenSource();

                // Create an ITelemetryService
                var telemetryService = new TelemetryService(new TelemetryClient());

                // Allow jobs to set global custom dimensions
                TelemetryConfiguration.Active.TelemetryInitializers.Add(new JobPropertiesTelemetryInitializer(telemetryService));

                job = NgJobFactory.GetJob(jobName, telemetryService, loggerFactory);
                await job.RunAsync(arguments, cancellationTokenSource.Token);
            }
            catch (ArgumentException ae)
            {
                _logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae);

                Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
            }
            catch (Exception e)
            {
                _logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e);
            }

            Trace.Close();
            TelemetryConfiguration.Active.TelemetryChannel.Flush();
        }
Пример #9
0
        public static async Task MainAsync(string[] args)
        {
            if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase))
            {
                args = args.Skip(1).ToArray();
                Debugger.Launch();
            }

            NgJob job = null;
            ApplicationInsightsConfiguration applicationInsightsConfiguration = null;
            int exitCode = 0;

            try
            {
                // Get arguments
                var arguments = CommandHelpers.GetArguments(args, 1, out var secretInjector);

                // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
                ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
                ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;

                // Determine the job name
                if (args.Length == 0)
                {
                    throw new ArgumentException("Missing job name argument.");
                }

                var jobName                  = args[0];
                var instanceName             = arguments.GetOrDefault(Arguments.InstanceName, jobName);
                var instrumentationKey       = arguments.GetOrDefault <string>(Arguments.InstrumentationKey);
                var heartbeatIntervalSeconds = arguments.GetOrDefault <int>(Arguments.HeartbeatIntervalSeconds);

                applicationInsightsConfiguration = ConfigureApplicationInsights(
                    instrumentationKey,
                    heartbeatIntervalSeconds,
                    jobName,
                    instanceName,
                    out var telemetryClient,
                    out var telemetryGlobalDimensions);

                var loggerFactory = ConfigureLoggerFactory(applicationInsightsConfiguration);

                InitializeServiceProvider(
                    arguments,
                    secretInjector,
                    applicationInsightsConfiguration,
                    telemetryClient,
                    loggerFactory);

                job = NgJobFactory.GetJob(jobName, loggerFactory, telemetryClient, telemetryGlobalDimensions);
                job.SetSecretInjector(secretInjector);

                // This tells Application Insights that, even though a heartbeat is reported,
                // the state of the application is unhealthy when the exitcode is different from zero.
                // The heartbeat metadata is enriched with the job loop exit code.
                applicationInsightsConfiguration.DiagnosticsTelemetryModule?.AddOrSetHeartbeatProperty(
                    HeartbeatProperty_JobLoopExitCode,
                    exitCode.ToString(),
                    isHealthy: exitCode == 0);

                var cancellationTokenSource = new CancellationTokenSource();
                await job.RunAsync(arguments, cancellationTokenSource.Token);

                exitCode = 0;
            }
            catch (ArgumentException ae)
            {
                exitCode = 1;
                _logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae);

                Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
            }
            catch (KeyNotFoundException knfe)
            {
                exitCode = 1;
                _logger?.LogError("An expected key was not found. One possible cause of this is required argument has not been provided: {Exception}", knfe);

                Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase());
            }
            catch (Exception e)
            {
                exitCode = 1;
                _logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e);
            }

            applicationInsightsConfiguration.DiagnosticsTelemetryModule?.SetHeartbeatProperty(
                HeartbeatProperty_JobLoopExitCode,
                exitCode.ToString(),
                isHealthy: exitCode == 0);

            Trace.Close();
            applicationInsightsConfiguration?.TelemetryConfiguration.TelemetryChannel.Flush();
        }
        static async Task Loop(string gallery, StorageFactory storageFactory, bool verbose, int interval, DateTime?startDate)
        {
            Storage storage = storageFactory.Create();

            const string LastCreated = "nuget:lastCreated";
            const string LastEdited  = "nuget:lastEdited";

            int top     = 20;
            int timeout = 300;

            while (true)
            {
                Func <HttpMessageHandler> handlerFunc = CommandHelpers.GetHttpMessageHandlerFactory(verbose);

                HttpMessageHandler handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler {
                    AllowPipelining = true
                };

                using (HttpClient client = new HttpClient(handler))
                {
                    client.Timeout = TimeSpan.FromSeconds(timeout);

                    //  fetch and add all newly CREATED packages - in order
                    DateTime lastCreated = await GetCatalogProperty(storage, LastCreated) ?? (startDate ?? DateTime.MinValue.ToUniversalTime());

                    DateTime lastEdited = await GetCatalogProperty(storage, LastEdited) ?? lastCreated;

                    SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > createdPackages;
                    DateTime previousLastCreated = DateTime.MinValue;
                    do
                    {
                        Trace.TraceInformation("CATALOG LastCreated: {0}", lastCreated.ToString("O"));

                        createdPackages = await GetCreatedPackages(client, gallery, lastCreated, top);

                        Trace.TraceInformation("FEED CreatedPackages: {0}", createdPackages.Count);

                        lastCreated = await DownloadMetadata2Catalog(client, createdPackages, storage, lastCreated, lastEdited, createdPackages : true);

                        if (previousLastCreated == lastCreated)
                        {
                            break;
                        }
                        previousLastCreated = lastCreated;
                    }while (createdPackages.Count > 0);

                    //  THEN fetch and add all EDITED packages - in order

                    SortedList <DateTime, IList <Tuple <Uri, PackageDates> > > editedPackages;
                    DateTime previousLastEdited = DateTime.MinValue;
                    do
                    {
                        Trace.TraceInformation("CATALOG LastEdited: {0}", lastEdited.ToString("O"));

                        editedPackages = await GetEditedPackages(client, gallery, lastEdited, top);

                        Trace.TraceInformation("FEED EditedPackages: {0}", editedPackages.Count);

                        lastEdited = await DownloadMetadata2Catalog(client, editedPackages, storage, lastCreated, lastEdited, createdPackages : false);

                        if (previousLastEdited == lastEdited)
                        {
                            break;
                        }
                        previousLastEdited = lastEdited;
                    }while (editedPackages.Count > 0);
                }

                Thread.Sleep(interval * 1000);
            }
        }