private void ReportStart()
{
StartedEventArgs args = new StartedEventArgs();
args.File = File;
OnDownloadStarted?.Invoke(this, args);
}
/// <summary>
/// Download photos for a page.
/// </summary>
/// <param name="pageNumber"></param>
public bool GetRecentPhotos(int pageNumber)
{
if (pageNumber > pageCount)
{
return(false);
}
OnDownloadStarted?.Invoke();
client.GetRecentPhotos(PerPage, pageNumber);
return(true);
}
/// <summary>
/// </summary>
/// <param name="zipPath"></param>
/// <param name="zipName"></param>
/// <param name="urlName"></param>
private async Task DownloadExecutable(string zipPath, string zipName, string urlName)
{
var url = $"{Endpoint}/distr/{urlName}/{Path.GetFileName(ZipDirectory)}/{zipName}.zip";
using (var client = new WebClient())
{
OnDownloadStarted?.Invoke();
await client.DownloadFileTaskAsync(url, zipPath).ConfigureAwait(false);
OnDownloadEnded?.Invoke();
}
}
private async Task Download(FSItem item, Stream result, Downloader downloader)
{
try
{
Log.Trace($"Started download: {item.Name} - {item.Id}");
var start = Stopwatch.StartNew();
var buf = new byte[64 << 10];
var uncommitedSize = 0;
const int commitSize = 512 << 10;
using (result)
using (var writer = new BufferedStream(result))
{
OnDownloadStarted?.Invoke(item);
while (writer.Length < item.Length)
{
var stream = await cloud.Files.Download(item.Id);
stream.Position = writer.Length;
int red;
do
{
red = await stream.ReadAsync(buf, 0, buf.Length);
if (writer.Length == 0)
{
Log.Trace($"Got first part: {item.Name} - {item.Id} in {start.ElapsedMilliseconds}");
}
await writer.WriteAsync(buf, 0, red);
uncommitedSize += red;
if (uncommitedSize <= commitSize)
{
continue;
}
uncommitedSize = 0;
await writer.FlushAsync();
downloader.Downloaded = writer.Length;
}while (red > 0);
}
await writer.FlushAsync();
downloader.Downloaded = writer.Length;
}
Log.Trace($"Finished download: {item.Name} - {item.Id}");
OnDownloaded?.Invoke(item);
if (access.TryAdd(
item.Id,
new CacheEntry {
Id = item.Id, AccessTime = DateTime.UtcNow, Length = item.Length
}))
{
TotalSizeIncrease(item.Length);
}
if (TotalSize > CacheSize)
{
var task = cleanSizeWorker.Run(TotalSize - CacheSize);
}
if (start.ElapsedMilliseconds > 29000)
{
Log.Warn($"Downloading {item.Path} took: {start.ElapsedMilliseconds}");
}
}
catch (Exception ex)
{
Log.ErrorTrace($"Download failed: {item.Name} - {item.Id}\r\n{ex}");
await downloader.Failed();
OnDownloadFailed?.Invoke(item);
}
finally
{
lock (DownloadersLock)
{
Downloaders.Remove(item.Path);
}
}
}
public void StartDownload()
{
_downloadClient.DownloadFileAsync(new Uri(DownloadUrl), SavePath);
OnDownloadStarted?.Invoke();
}
/// <summary>
/// </summary>
private void UpdateDownloads(bool bHasConnection)
{
List <Guid> ActiveManifestIds = new List <Guid>();
foreach (DownloadState State in StateCollection.States)
{
bool UpdateManifestId = true;
// If not set to auto update and we already have a completed download, do nothing else.
if (!State.UpdateAutomatically && State.ActiveManifestId != Guid.Empty)
{
ManifestDownloadState OldVersionDownloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (OldVersionDownloader != null && OldVersionDownloader.State == ManifestDownloadProgressState.Complete)
{
UpdateManifestId = false;
}
}
if (bHasConnection && UpdateManifestId)
{
Guid NewManifestId = GetTargetManifestForState(State);
if (NewManifestId != Guid.Empty)
{
if (State.ActiveManifestId != NewManifestId && !State.Paused)
{
OnDownloadStarted?.Invoke(State);
}
State.ActiveManifestId = NewManifestId;
}
}
if (State.ActiveManifestId == Guid.Empty)
{
continue;
}
ActiveManifestIds.Add(State.ActiveManifestId);
ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (Downloader != null)
{
// If state is downloading but we've paused it, pause it and ignore.
if (State.Paused)
{
if (!Downloader.Paused)
{
ManifestDownloader.PauseDownload(State.ActiveManifestId);
continue;
}
}
else
{
if (Downloader.Paused)
{
ManifestDownloader.ResumeDownload(State.ActiveManifestId);
}
}
// Hackily change the priority - should change this to a SetDownloadPriority method.
Downloader.Priority = State.Priority;
Downloader.InstallOnComplete = State.InstallAutomatically;
Downloader.InstallDeviceName = State.InstallDeviceName;
Downloader.InstallLocation = State.InstallLocation;
// Have we finished this download?
if (Downloader.State == ManifestDownloadProgressState.Complete &&
State.PreviousDownloaderState != Downloader.State &&
State.PreviousDownloaderState != ManifestDownloadProgressState.Unknown)
{
OnDownloadFinished?.Invoke(State);
}
State.PreviousDownloaderState = Downloader.State;
// Store the amount of time the download is in each state to make
// time estimates a bit better.
if (Downloader.State != ManifestDownloadProgressState.Complete && !State.Paused && Downloader.Manifest != null && (bHasConnection || Downloader.State != ManifestDownloadProgressState.Downloading))
{
if (State.PendingDurationHistory == null)
{
State.PendingDurationHistory = new DownloadStateDuration();
State.PendingDurationHistory.TotalSize = Downloader.Manifest.GetTotalSize();
}
ulong CurrentTime = TimeUtils.Ticks;
if (State.PendingDurationTimer > 0)
{
ulong Elapsed = TimeUtils.Ticks - State.PendingDurationTimer;
ulong StartTime = State.PendingDurationHistory.GetDuration(Downloader.State);
ulong NewTime = StartTime + Elapsed;
State.PendingDurationHistory.SetDuration(Downloader.State, NewTime);
//Console.WriteLine("State:{0} Time:{1}", Downloader.State.ToString(), StringUtils.FormatAsDuration((long)State.PendingDurationHistory.StateDurations[Downloader.State] / 1000));
}
State.PendingDurationTimer = CurrentTime;
}
else if (Downloader.State == ManifestDownloadProgressState.Complete)
{
if (State.PendingDurationHistory != null)
{
State.DurationHistory.Add(State.PendingDurationHistory);
State.PendingDurationHistory = null;
// Only keep a few history entries.
while (State.DurationHistory.Count > 15)
{
State.DurationHistory.RemoveAt(0);
}
}
}
}
else
{
// Start downloading this manifest.
ManifestDownloader.StartDownload(State.ActiveManifestId, State.Priority);
}
}
// Go through each manifest download and pause any that are no longer relevant.
foreach (ManifestDownloadState Downloader in ManifestDownloader.States.States)
{
if (!ActiveManifestIds.Contains(Downloader.ManifestId))
{
if (!Downloader.Paused)
{
ManifestDownloader.PauseDownload(Downloader.ManifestId);
}
Downloader.Active = false;
}
else
{
Downloader.Active = true;
Downloader.LastActive = DateTime.Now;
}
}
// Only download a single auto-replication download at a time, more efficient than dozens downloading at the same time.
Guid UnpausedAutoDownload = Guid.Empty;
// Remove auto replication downloads as required.
for (int i = 0; i < StateCollection.States.Count; i++)
{
DownloadState State = StateCollection.States[i];
if (State.ActiveManifestId == Guid.Empty)
{
continue;
}
if (!State.IsAutomaticReplication)
{
continue;
}
ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (Downloader != null)
{
bool ShouldRemove = false;
// Always remove downloads that have completed.
if (Downloader.State == ManifestDownloadProgressState.Complete)
{
ShouldRemove = true;
}
// Download has not recieved any blocks in a long time, remove.
TimeSpan Elapsed = DateTime.UtcNow - Downloader.LastRecievedData;
if (Elapsed.TotalHours > ReplicationMaxIdleHours)
{
ShouldRemove = true;
}
// Automatically resume replication downloads if they are in an error state.
if (UnpausedAutoDownload == Guid.Empty)
{
UnpausedAutoDownload = State.ActiveManifestId;
}
State.Paused = (UnpausedAutoDownload != State.ActiveManifestId);
if (ShouldRemove)
{
StateCollection.States.RemoveAt(i);
AreStatesDirty = false;
i--;
}
}
}
}
private void SinglePartUnknownDownloader_OnDownloadStarted(object sender, StartedEventArgs e)
{
DownloadStatus = Status.NotDownloading;
OnDownloadStarted?.Invoke(this, e);
}
private async Task Download(FSItem item, Stream writer, Downloader downloader)
{
Log.Trace("Started download: " + item.Id);
var start = Stopwatch.StartNew();
var buf = new byte[4096];
using (writer)
try
{
OnDownloadStarted?.Invoke(item.Id);
while (writer.Length < item.Length)
{
await amazon.Files.Download(item.Id, fileOffset : writer.Length, streammer : async(response) =>
{
var partial = response.StatusCode == System.Net.HttpStatusCode.PartialContent;
ContentRangeHeaderValue contentRange = null;
if (partial)
{
contentRange = response.Headers.GetContentRange();
if (contentRange.From != writer.Length)
{
throw new InvalidOperationException("Content range does not match request");
}
}
using (var stream = response.GetResponseStream())
{
int red = 0;
do
{
red = await stream.ReadAsync(buf, 0, buf.Length);
if (writer.Length == 0)
{
Log.Trace("Got first part: " + item.Id + " in " + start.ElapsedMilliseconds);
}
writer.Write(buf, 0, red);
downloader.Downloaded = writer.Length;
}while (red > 0);
}
});
if (writer.Length < item.Length)
{
await Task.Delay(500);
}
}
Log.Trace("Finished download: " + item.Id);
OnDownloaded?.Invoke(item.Id);
access.TryAdd(item.Id, new CacheEntry {
Id = item.Id, AccessTime = DateTime.UtcNow, Length = item.Length
});
TotalSize += item.Length;
if (TotalSize > CacheSize)
{
StartClear(TotalSize - CacheSize);
}
}
catch (Exception ex)
{
OnDownloadFailed?.Invoke(item.Id);
Log.Error($"Download failed: {item.Id}\r\n{ex}");
}
finally
{
Downloader remove;
downloaders.TryRemove(item.Path, out remove);
}
}
internal void TriggerDownloadStartedEvent(DownloadDetails details, long fileSize)
{
details.UpdateStatus(DownloadDetails.Status.Downloading);
OnDownloadStarted?.Invoke(this, details, fileSize);
}
