Exemplo n.º 1
0
        public async Task MonitorAsync(ServiceProvider provider)
        {
            DateTime? lastCheck = null;

            this.Logger.LogInformation("Monitoring Bucket {0}", this.BucketName);
            Console.Out.Flush();
            for (; ; )
            {
                try
                {
                    var now = DateTime.Now;
                    if (!lastCheck.HasValue || DateTime.Now.Subtract(lastCheck.Value) > this.PollingInterval)
                    {
                        using (var scoped = provider.CreateScope())
                        {
                            var dbContext = scoped.ServiceProvider.GetService<BucketMonitorContext>();
                            this.Logger.LogDebug("Scanning Bucket {0}", this.BucketName);
                            lastCheck = DateTime.Now;

                            Console.WriteLine("\n=> Scanning Bucket {0} ({1})", this.BucketName, DateTime.Now);

                            var pending = await this.ListPendingAsync(provider);
                            var count = pending.Count();
                            var tracker = new DownloadTracker(count, pending.Select(x => x.TotalBytes).Sum());
                            var dbTracker = new DatabaseTracker(dbContext: dbContext);

                            var entryMap = await this.GetImageMapAsync(dbContext);
                            if (count > 0)
                            {
                                this.Logger.LogDebug("Downloading {0} Pending Images", pending.Count());
                                var throttler = new SemaphoreSlim(initialCount: this.MaxDownloads);

                                var tasks = new List<Task<FileInfo>>();
                                foreach (var image in pending)
                                {
                                    await throttler.WaitAsync();
                                    tasks.Add(
                                        Task.Run(async () =>
                                        {
                                            try
                                            {
                                                return await this.DownloadAsync(
                                                    image,
                                                    entryMap[image.Key],
                                                    tracker,
                                                    dbTracker);
                                            }
                                            finally
                                            {
                                                throttler.Release();
                                            }
                                        }));
                                }
                                await Task.WhenAll(tasks);

                                Console.WriteLine();
                                Console.WriteLine("Downloads Complete");
                                this.Logger.LogDebug("Downloads Complete");

                            }
                            else
                            {
                                Console.WriteLine("No Pending Images");
                                this.Logger.LogDebug("No Pending Images");
                            }
                        }
                    }
                    else
                    {
                        await Task.Delay(1000);
                    }
                }
                catch (Exception e)
                {
                    this.Logger.LogError("Exception: {0}", e.ToString());
                }
            }
        }
Exemplo n.º 2
0
        public async Task<FileInfo> DownloadAsync(
            SourceImage image,
            ImageEntry entry,
            DownloadTracker tracker,
            DatabaseTracker dbTracker)
        {
            var file = image.File;
            var started = false;
            long downloaded = 0;

            try
            {
                var response = await this.Client.GetObjectAsync(new GetObjectRequest()
                {
                    BucketName = this.BucketName,
                    Key = image.Key
                });

                var tmp = Path.GetTempFileName();

                long transferred = 0;
                response.WriteObjectProgressEvent += (sender, p) =>
                {
                    if (!started)
                    {
                        tracker.Start();
                    }
                    started = true;

                    downloaded = p.TransferredBytes;
                    tracker.Downloaded(downloaded - transferred);
                    transferred = downloaded;
                };

                await response.WriteResponseStreamToFileAsync(tmp, false, CancellationToken.None);

                if (this.DebugMode)
                {
                    File.Delete(tmp);
                }
                else
                {
                    if (!this.SafeMove(tmp, file.FullName))
                    {
                        image.MarkFailed();
                        dbTracker.Update(entry, ImageStatus.Failed);
                        tracker.Fail();
                        return null;
                    }
                }

                this.Logger.LogDebug("Download Complete: {0} -> {1}", image.Key, file?.FullName ?? "-");

                image.MarkCompleted();
                dbTracker.Update(entry, ImageStatus.Completed);

                if (started)
                {
                    tracker.Complete();
                }

                return file;
            }
            catch (Exception ec)
            {
                this.Logger.LogDebug("Download Failed:  {0} -> {1}", image.Key, file?.FullName ?? "-");
                this.Logger.LogDebug("Exception: {0}", ec.ToString());

                image.MarkFailed();
                dbTracker.Update(entry, ImageStatus.Failed);

                if (started)
                {
                    tracker.Fail();
                    tracker.Downloaded(image.TotalBytes - downloaded);
                }

                return null;
            }
        }