private SourceImage CreateSourceImage(string key, DateTime lastModified, long totalBytes)
{
SourceImage image;
if (this.TryConvertPath(key, out var file))
{
image = new SourceImage(
key: key,
file: file,
lastModified: lastModified,
totalBytes: totalBytes,
status: ImageStatus.Pending);
}
else
{
image = new SourceImage(
key: key,
file: null,
lastModified: lastModified,
totalBytes: totalBytes,
status: ImageStatus.Skipped);
}
return image;
}
public bool TryGet(string key, DateTime lastModified, out SourceImage image)
{
if (this.Cache.TryGetValue(key, out image))
{
if (image.LastModified == lastModified)
{
return(true);
}
else
{
Console.WriteLine("Key is outdated: {key}", key);
return(false);
}
}
else
{
return(false);
}
}
public async Task <FileInfo> DownloadAsync(
SourceImage image)
{
var file = image.File;
if (!file.Directory.Exists)
{
file.Directory.Create();
}
try
{
var response = await this.Client.GetObjectAsync(new GetObjectRequest()
{
BucketName = this.BucketName,
Key = image.Key
});
var tmp = Path.GetTempFileName();
await response.WriteResponseStreamToFileAsync(tmp, false, CancellationToken.None);
File.Move(tmp, file.FullName, overwrite: true);
await this.UpdateImageStatusAsync(image.Key, true);
Console.WriteLine("\rDownload Complete: {0} -> {1}", image.Key, file.FullName);
image.MarkCompleted();
this.Cache.Put(image);
return(file);
}
catch
{
Console.WriteLine("\rDownload Failed: {0} -> {1}", image.Key, file.FullName);
image.MarkFailed();
this.Cache.Put(image);
return(null);
}
}
private async Task <SourceImage> LoadSourceImageAsync(string key, DateTime lastModified)
{
SourceImage image;
if (this.TryConvertPath(key, out var file))
{
var tags = await this.ListTagsAsync(key);
ImageStatus status;
if (tags.TryGetValue(TAG_NAME_PROCESSED, out var value) && value == true.ToString())
{
status = ImageStatus.Completed;
}
else
{
status = ImageStatus.Pending;
}
image = new SourceImage(
key: key,
file: file,
lastModified: lastModified,
status: status);
}
else
{
image = new SourceImage(
key: key,
file: null,
lastModified: lastModified,
status: ImageStatus.Skipped);
}
this.Cache.Put(image);
return(image);
}
public void Put(SourceImage image)
{
this.Cache[image.Key] = image;
}
public async Task<FileInfo> DownloadAsync(
SourceImage image,
ImageEntry entry,
DownloadTracker tracker,
DatabaseTracker dbTracker)
{
var file = image.File;
var started = false;
long downloaded = 0;
try
{
var response = await this.Client.GetObjectAsync(new GetObjectRequest()
{
BucketName = this.BucketName,
Key = image.Key
});
var tmp = Path.GetTempFileName();
long transferred = 0;
response.WriteObjectProgressEvent += (sender, p) =>
{
if (!started)
{
tracker.Start();
}
started = true;
downloaded = p.TransferredBytes;
tracker.Downloaded(downloaded - transferred);
transferred = downloaded;
};
await response.WriteResponseStreamToFileAsync(tmp, false, CancellationToken.None);
if (this.DebugMode)
{
File.Delete(tmp);
}
else
{
if (!this.SafeMove(tmp, file.FullName))
{
image.MarkFailed();
dbTracker.Update(entry, ImageStatus.Failed);
tracker.Fail();
return null;
}
}
this.Logger.LogDebug("Download Complete: {0} -> {1}", image.Key, file?.FullName ?? "-");
image.MarkCompleted();
dbTracker.Update(entry, ImageStatus.Completed);
if (started)
{
tracker.Complete();
}
return file;
}
catch (Exception ec)
{
this.Logger.LogDebug("Download Failed: {0} -> {1}", image.Key, file?.FullName ?? "-");
this.Logger.LogDebug("Exception: {0}", ec.ToString());
image.MarkFailed();
dbTracker.Update(entry, ImageStatus.Failed);
if (started)
{
tracker.Fail();
tracker.Downloaded(image.TotalBytes - downloaded);
}
return null;
}
}
public bool TryGet(string key, DateTime lastModified, out SourceImage image)
{
return(this.Cache.TryGetValue(key, out image));
}