public override async Task RunCore() { if (!_complete) { _complete = true; var queueClient = Account.CreateCloudQueueClient(); await CreateQueue(queueClient, CantonConstants.GalleryPagesQueue); await CreateQueue(queueClient, CantonConstants.CatalogCommitQueue); await CreateQueue(queueClient, CantonConstants.CatalogPageQueue); var blobClient = Account.CreateCloudBlobClient(); await CreateContainer(blobClient, Config.GetProperty("CatalogContainer")); await CreateContainer(blobClient, Config.GetProperty("RegistrationContainer")); await CreateContainer(blobClient, Config.GetProperty("GalleryPageContainer")); await CreateContainer(blobClient, Config.GetProperty("tmp")); var tableClient = Account.CreateCloudTableClient(); var cursorTable = tableClient.GetTableReference(CantonConstants.CursorTable); await cursorTable.CreateIfNotExistsAsync(); string localTmp = Config.GetProperty("localtmp"); DirectoryInfo tmpDir = new DirectoryInfo(localTmp); if (!tmpDir.Exists) { tmpDir.Create(); } } }
/// <summary> /// Canton jobs that can run many instances. /// </summary> static void Main(string[] args) { if (args.Length != 2) { Console.WriteLine(".exe <config path> <thread count>"); Environment.Exit(1); } Console.CancelKeyPress += Console_CancelKeyPress; CantonUtilities.Init(); Config config = new Config(args[0]); int threadCount = 1; Int32.TryParse(args[1], out threadCount); CloudStorageAccount account = CloudStorageAccount.Parse(config.GetProperty("StorageConnectionString")); Queue <Func <CantonJob> > jobs = new Queue <Func <CantonJob> >(); // process gallery pages and nupkgs into catalog pages jobs.Enqueue(() => new CatalogPageJob(config, new AzureStorage(account, config.GetProperty("tmp")), CantonConstants.GalleryPagesQueue)); Stopwatch timer = new Stopwatch(); // avoid flooding storage TimeSpan minWait = TimeSpan.FromMinutes(2); while (_run) { timer.Restart(); CantonUtilities.RunManyJobs(jobs, threadCount); TimeSpan waitTime = minWait.Subtract(timer.Elapsed); Console.WriteLine("Completed jobs in: " + timer.Elapsed); if (waitTime.TotalMilliseconds > 0 && _run) { Console.WriteLine("Sleeping: " + waitTime.TotalSeconds + "s"); Thread.Sleep(waitTime); } } }
/// <summary> /// Canton jobs that can run many instances. /// </summary> static void Main(string[] args) { if (args.Length != 2) { Console.WriteLine(".exe <config path> <thread count>"); Environment.Exit(1); } Console.CancelKeyPress += Console_CancelKeyPress; CantonUtilities.Init(); Config config = new Config(args[0]); int threadCount = 1; Int32.TryParse(args[1], out threadCount); CloudStorageAccount account = CloudStorageAccount.Parse(config.GetProperty("StorageConnectionString")); Queue<Func<CantonJob>> jobs = new Queue<Func<CantonJob>>(); // process gallery pages and nupkgs into catalog pages jobs.Enqueue(() => new CatalogPageJob(config, new AzureStorage(account, config.GetProperty("tmp")), CantonConstants.GalleryPagesQueue)); Stopwatch timer = new Stopwatch(); // avoid flooding storage TimeSpan minWait = TimeSpan.FromMinutes(2); while (_run) { timer.Restart(); CantonUtilities.RunManyJobs(jobs, threadCount); TimeSpan waitTime = minWait.Subtract(timer.Elapsed); Console.WriteLine("Completed jobs in: " + timer.Elapsed); if (waitTime.TotalMilliseconds > 0 && _run) { Console.WriteLine("Sleeping: " + waitTime.TotalSeconds + "s"); Thread.Sleep(waitTime); } } }
public CatalogPageJob(Config config, Storage storage, string queueName) : base(config, storage, queueName) { _packagesStorageAccount = CloudStorageAccount.Parse(config.GetProperty("PackagesStorageConnectionString")); _queueTasks = new ConcurrentQueue <Task>(); _workQueueStatus = new ConcurrentDictionary <int, bool>(); _tmpDir = new DirectoryInfo(Config.GetProperty("localtmp")); if (!_tmpDir.Exists) { _tmpDir.Create(); } }
public CatalogPageJob(Config config, Storage storage, string queueName) : base(config, storage, queueName) { _packagesStorageAccount = CloudStorageAccount.Parse(config.GetProperty("PackagesStorageConnectionString")); _queueTasks = new ConcurrentQueue<Task>(); _workQueueStatus = new ConcurrentDictionary<int, bool>(); _tmpDir = new DirectoryInfo(Config.GetProperty("localtmp")); if (!_tmpDir.Exists) { _tmpDir.Create(); } }
private async Task <FileInfo> GetNupkg(string id, string version) { // TODO: Use the MD5 hash on the prod packgae to see if we have the current one //NuGetVersion nugetVersion = new NuGetVersion(version); var tmpBlobClient = Account.CreateCloudBlobClient(); string packageName = String.Format(CultureInfo.InvariantCulture, "{0}.{1}.nupkg", id, version).ToLowerInvariant(); FileInfo file = new FileInfo(Path.Combine(_tmpDir.FullName, Guid.NewGuid().ToString() + ".nupkg")); var tmpContainer = tmpBlobClient.GetContainerReference(Config.GetProperty("tmp")); string tmpFile = String.Format(CultureInfo.InvariantCulture, "packages/{0}", packageName); var tmpBlob = tmpContainer.GetBlockBlobReference(tmpFile); if (await tmpBlob.ExistsAsync()) { Log("Downloading from tmp: " + packageName); await tmpBlob.DownloadToFileAsync(file.FullName, FileMode.CreateNew); } else { var blobClient = _packagesStorageAccount.CreateCloudBlobClient(); var prodPackages = blobClient.GetContainerReference("packages"); var prodBlob = prodPackages.GetBlockBlobReference(packageName); if (prodBlob.Exists()) { Log("Downloading from prod: " + packageName); await prodBlob.DownloadToFileAsync(file.FullName, FileMode.CreateNew); // store this in tmp also await tmpBlob.UploadFromFileAsync(file.FullName, FileMode.Open); } else { throw new FileNotFoundException(prodBlob.Uri.AbsoluteUri); } } return(file); }
/// <summary> /// Canton jobs that can only run as single instances. /// </summary> static void Main(string[] args) { if (args.Length != 1) { Console.WriteLine(".exe <config path>"); Environment.Exit(1); } Console.CancelKeyPress += Console_CancelKeyPress; CantonUtilities.Init(); Config config = new Config(args[0]); CloudStorageAccount account = CloudStorageAccount.Parse(config.GetProperty("StorageConnectionString")); CloudStorageAccount outputAccount = CloudStorageAccount.Parse(config.GetProperty("OutputStorageConnectionString")); Uri baseAddress = new Uri(config.GetProperty("BaseAddress")); Queue<CantonJob> jobs = new Queue<CantonJob>(); // set up the storage account jobs.Enqueue(new InitStorageJob(config)); // read the gallery to find new packages // jobs.Enqueue(new QueueNewPackagesFromGallery(config, new AzureStorage(account, config.GetProperty("GalleryPageContainer")))); // this does the work of Many, just so this can all be in one .exe jobs.Enqueue(new CatalogPageJob(config, new AzureStorage(account, config.GetProperty("tmp")), CantonConstants.GalleryPagesQueue)); // commit pages to the catalog jobs.Enqueue(new CatalogPageCommitJob(config, new AzureStorage(outputAccount, config.GetProperty("CatalogContainer"), string.Empty, new Uri(baseAddress.AbsoluteUri + config.GetProperty("CatalogContainer") + "/")))); // registrations Uri regBase = new Uri(baseAddress, config.GetProperty("RegistrationContainer") + "/"); TransHttpClient httpClient = new TransHttpClient(outputAccount, config.GetProperty("BaseAddress")); jobs.Enqueue(new PartitionedRegJob(config, new AzureStorage(outputAccount, config.GetProperty("RegistrationContainer"), string.Empty, regBase), new AzureStorageFactory(outputAccount, config.GetProperty("RegistrationContainer"), null, regBase), httpClient)); Stopwatch timer = new Stopwatch(); // avoid flooding the gallery TimeSpan minWait = TimeSpan.FromMinutes(10); while (_run) { timer.Restart(); CantonUtilities.RunJobs(jobs); TimeSpan waitTime = minWait.Subtract(timer.Elapsed); Console.WriteLine("Completed jobs in: " + timer.Elapsed); // min sleep Thread.Sleep(60 * 1000); if (waitTime.TotalSeconds > 0 && _run) { Console.WriteLine("Sleeping: " + waitTime.TotalSeconds + "s"); Thread.Sleep(waitTime); } } }
public override async Task RunCore() { int nextMasterRegId = 0; DateTime position = Cursor.Position; JToken nextMasterRegIdToken = null; if (Cursor.Metadata.TryGetValue("nextMasterRegId", out nextMasterRegIdToken)) { nextMasterRegId = nextMasterRegIdToken.ToObject <int>(); } // Get the catalog index Uri catalogIndexUri = new Uri(Config.GetProperty("CatalogIndex")); Log("Reading index entries"); var indexReader = new CatalogIndexReader(catalogIndexUri); var indexEntries = await indexReader.GetRolledUpEntries(); var context = indexReader.GetContext(); Log("Finding new or editted entries"); var changedEntries = new HashSet <string>(indexEntries.Where(e => e.CommitTimeStamp.CompareTo(position) > 0) .Select(e => e.Id.ToLowerInvariant()), StringComparer.OrdinalIgnoreCase); DateTime newPosition = indexEntries.Select(e => e.CommitTimeStamp).OrderByDescending(e => e).FirstOrDefault(); ConcurrentDictionary <string, ConcurrentBag <Uri> > batches = new ConcurrentDictionary <string, ConcurrentBag <Uri> >(StringComparer.OrdinalIgnoreCase); ParallelOptions options = new ParallelOptions(); options.MaxDegreeOfParallelism = 8; Parallel.ForEach(indexEntries, options, entry => { if (changedEntries.Contains(entry.Id)) { batches.AddOrUpdate(entry.Id, new ConcurrentBag <Uri>() { entry.Uri }, (id, uris) => { uris.Add(entry.Uri); return(uris); }); } }); Uri contentBaseAddress = new Uri(Config.GetProperty("ContentBaseAddress")); if (batches.Count > 0) { Log("Building registrations from: " + position.ToString("O")); options.MaxDegreeOfParallelism = 4; for (int i = 0; i < 3 && batches.Count > 0; i++) { if (i != 0) { options.MaxDegreeOfParallelism = 1; Console.WriteLine("Single batch run."); } var ids = batches.Keys.OrderBy(s => s).ToArray(); Stopwatch buildTimer = new Stopwatch(); buildTimer.Start(); int startingCount = ids.Length; Parallel.ForEach(ids, options, id => { try { BatchRegistrationCollector regCollector = new BatchRegistrationCollector(null, _factory); regCollector.ContentBaseAddress = contentBaseAddress; Stopwatch timer = new Stopwatch(); timer.Start(); var uriGroup = batches[id].ToArray(); regCollector.ProcessGraphs(_client, id, uriGroup, context).Wait(); int rem = batches.Count; timer.Stop(); string log = String.Format("Completed: {0} Duration: {1} Uris: {2} Remaining Ids: {3} Loop: {4}", id, timer.Elapsed, uriGroup.Length, rem, i); Console.WriteLine(log); // stats double perPackage = buildTimer.Elapsed.TotalSeconds / (double)(startingCount - rem + 1); DateTime finish = DateTime.Now.AddSeconds(Math.Ceiling(perPackage * rem)); Console.WriteLine("Estimated Finish: " + finish.ToString("O")); ConcurrentBag <Uri> vals; if (!batches.TryRemove(id, out vals)) { Console.WriteLine("Unable to remove!"); } } catch (Exception ex) { LogError("Registration failed: " + id + " " + ex.ToString()); } }); } // mark this with the last commit we included Cursor.Position = newPosition; await Cursor.Save(); Log("Finished registrations: " + newPosition.ToString("O")); } }