public async Task MonitorAsync(ServiceProvider provider)
{
DateTime? lastCheck = null;
this.Logger.LogInformation("Monitoring Bucket {0}", this.BucketName);
Console.Out.Flush();
for (; ; )
{
try
{
var now = DateTime.Now;
if (!lastCheck.HasValue || DateTime.Now.Subtract(lastCheck.Value) > this.PollingInterval)
{
using (var scoped = provider.CreateScope())
{
var dbContext = scoped.ServiceProvider.GetService<BucketMonitorContext>();
this.Logger.LogDebug("Scanning Bucket {0}", this.BucketName);
lastCheck = DateTime.Now;
Console.WriteLine("\n=> Scanning Bucket {0} ({1})", this.BucketName, DateTime.Now);
var pending = await this.ListPendingAsync(provider);
var count = pending.Count();
var tracker = new DownloadTracker(count, pending.Select(x => x.TotalBytes).Sum());
var dbTracker = new DatabaseTracker(dbContext: dbContext);
var entryMap = await this.GetImageMapAsync(dbContext);
if (count > 0)
{
this.Logger.LogDebug("Downloading {0} Pending Images", pending.Count());
var throttler = new SemaphoreSlim(initialCount: this.MaxDownloads);
var tasks = new List<Task<FileInfo>>();
foreach (var image in pending)
{
await throttler.WaitAsync();
tasks.Add(
Task.Run(async () =>
{
try
{
return await this.DownloadAsync(
image,
entryMap[image.Key],
tracker,
dbTracker);
}
finally
{
throttler.Release();
}
}));
}
await Task.WhenAll(tasks);
Console.WriteLine();
Console.WriteLine("Downloads Complete");
this.Logger.LogDebug("Downloads Complete");
}
else
{
Console.WriteLine("No Pending Images");
this.Logger.LogDebug("No Pending Images");
}
}
}
else
{
await Task.Delay(1000);
}
}
catch (Exception e)
{
this.Logger.LogError("Exception: {0}", e.ToString());
}
}
}
public async Task<FileInfo> DownloadAsync(
SourceImage image,
ImageEntry entry,
DownloadTracker tracker,
DatabaseTracker dbTracker)
{
var file = image.File;
var started = false;
long downloaded = 0;
try
{
var response = await this.Client.GetObjectAsync(new GetObjectRequest()
{
BucketName = this.BucketName,
Key = image.Key
});
var tmp = Path.GetTempFileName();
long transferred = 0;
response.WriteObjectProgressEvent += (sender, p) =>
{
if (!started)
{
tracker.Start();
}
started = true;
downloaded = p.TransferredBytes;
tracker.Downloaded(downloaded - transferred);
transferred = downloaded;
};
await response.WriteResponseStreamToFileAsync(tmp, false, CancellationToken.None);
if (this.DebugMode)
{
File.Delete(tmp);
}
else
{
if (!this.SafeMove(tmp, file.FullName))
{
image.MarkFailed();
dbTracker.Update(entry, ImageStatus.Failed);
tracker.Fail();
return null;
}
}
this.Logger.LogDebug("Download Complete: {0} -> {1}", image.Key, file?.FullName ?? "-");
image.MarkCompleted();
dbTracker.Update(entry, ImageStatus.Completed);
if (started)
{
tracker.Complete();
}
return file;
}
catch (Exception ec)
{
this.Logger.LogDebug("Download Failed: {0} -> {1}", image.Key, file?.FullName ?? "-");
this.Logger.LogDebug("Exception: {0}", ec.ToString());
image.MarkFailed();
dbTracker.Update(entry, ImageStatus.Failed);
if (started)
{
tracker.Fail();
tracker.Downloaded(image.TotalBytes - downloaded);
}
return null;
}
}