public async Task MonitorAsync(ServiceProvider provider) { DateTime? lastCheck = null; this.Logger.LogInformation("Monitoring Bucket {0}", this.BucketName); Console.Out.Flush(); for (; ; ) { try { var now = DateTime.Now; if (!lastCheck.HasValue || DateTime.Now.Subtract(lastCheck.Value) > this.PollingInterval) { using (var scoped = provider.CreateScope()) { var dbContext = scoped.ServiceProvider.GetService<BucketMonitorContext>(); this.Logger.LogDebug("Scanning Bucket {0}", this.BucketName); lastCheck = DateTime.Now; Console.WriteLine("\n=> Scanning Bucket {0} ({1})", this.BucketName, DateTime.Now); var pending = await this.ListPendingAsync(provider); var count = pending.Count(); var tracker = new DownloadTracker(count, pending.Select(x => x.TotalBytes).Sum()); var dbTracker = new DatabaseTracker(dbContext: dbContext); var entryMap = await this.GetImageMapAsync(dbContext); if (count > 0) { this.Logger.LogDebug("Downloading {0} Pending Images", pending.Count()); var throttler = new SemaphoreSlim(initialCount: this.MaxDownloads); var tasks = new List<Task<FileInfo>>(); foreach (var image in pending) { await throttler.WaitAsync(); tasks.Add( Task.Run(async () => { try { return await this.DownloadAsync( image, entryMap[image.Key], tracker, dbTracker); } finally { throttler.Release(); } })); } await Task.WhenAll(tasks); Console.WriteLine(); Console.WriteLine("Downloads Complete"); this.Logger.LogDebug("Downloads Complete"); } else { Console.WriteLine("No Pending Images"); this.Logger.LogDebug("No Pending Images"); } } } else { await Task.Delay(1000); } } catch (Exception e) { this.Logger.LogError("Exception: {0}", e.ToString()); } } }
public async Task<FileInfo> DownloadAsync( SourceImage image, ImageEntry entry, DownloadTracker tracker, DatabaseTracker dbTracker) { var file = image.File; var started = false; long downloaded = 0; try { var response = await this.Client.GetObjectAsync(new GetObjectRequest() { BucketName = this.BucketName, Key = image.Key }); var tmp = Path.GetTempFileName(); long transferred = 0; response.WriteObjectProgressEvent += (sender, p) => { if (!started) { tracker.Start(); } started = true; downloaded = p.TransferredBytes; tracker.Downloaded(downloaded - transferred); transferred = downloaded; }; await response.WriteResponseStreamToFileAsync(tmp, false, CancellationToken.None); if (this.DebugMode) { File.Delete(tmp); } else { if (!this.SafeMove(tmp, file.FullName)) { image.MarkFailed(); dbTracker.Update(entry, ImageStatus.Failed); tracker.Fail(); return null; } } this.Logger.LogDebug("Download Complete: {0} -> {1}", image.Key, file?.FullName ?? "-"); image.MarkCompleted(); dbTracker.Update(entry, ImageStatus.Completed); if (started) { tracker.Complete(); } return file; } catch (Exception ec) { this.Logger.LogDebug("Download Failed: {0} -> {1}", image.Key, file?.FullName ?? "-"); this.Logger.LogDebug("Exception: {0}", ec.ToString()); image.MarkFailed(); dbTracker.Update(entry, ImageStatus.Failed); if (started) { tracker.Fail(); tracker.Downloaded(image.TotalBytes - downloaded); } return null; } }