/// <summary>
/// </summary>
/// <param name="Name"></param>
/// <param name="VirtualPath"></param>
/// <param name="Priority"></param>
/// <param name="KeepUpToDate"></param>
public DownloadState AddDownload(string Name, string VirtualPath, int Priority, BuildSelectionRule Rule, BuildSelectionFilter Filter, string SelectionFilterFilePath, string ScmWorkspaceLocation, bool AutomaticallyUpdate, bool AutomaticallyInstall, string InstallDeviceName, string InstallLocation, List <Guid> IncludeTags, List <Guid> ExcludeTags, bool DeleteOnComplete = false)
{
DownloadState State = new DownloadState();
State.Id = Guid.NewGuid();
State.Name = Name;
State.VirtualPath = VirtualPath;
State.Priority = Priority;
State.UpdateAutomatically = AutomaticallyUpdate;
State.InstallAutomatically = AutomaticallyInstall;
State.InstallDeviceName = InstallDeviceName;
State.InstallLocation = InstallLocation;
State.SelectionRule = Rule;
State.SelectionFilter = Filter;
State.SelectionFilterFilePath = SelectionFilterFilePath;
State.ScmWorkspaceLocation = ScmWorkspaceLocation;
State.IncludeTags = IncludeTags;
State.ExcludeTags = ExcludeTags;
State.IsAutomaticReplication = DeleteOnComplete;
StateCollection.States.Add(State);
AreStatesDirty = true;
return(State);
}
/// <summary>
///
/// </summary>
/// <param name="State"></param>
/// <param name="TotalDownloadSize"></param>
/// <param name="CurrentProgress"></param>
public long GetEstimatedTimeRemaining(DownloadState State)
{
ulong SumCount = 0;
ulong SumValue = 0;
long TotalSize = 0;
ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (Downloader != null && Downloader.Manifest != null)
{
TotalSize = Downloader.Manifest.GetTotalSize();
}
foreach (DownloadStateDuration Duration in State.DurationHistory)
{
if (Duration.Values.Count > 0)
{
ulong Unadjusted = 0;
foreach (DownloadStateDurationValue Value in Duration.Values)
{
Unadjusted += Value.Elapsed;
}
float Adjustment = TotalSize == 0 ? 1.0f : (float)Duration.TotalSize / (float)TotalSize;
ulong AdjustedDuration = (ulong)(Unadjusted * Adjustment);
SumValue += AdjustedDuration;
SumCount++;
}
}
ulong AverageDuration = SumCount == 0 ? 0 : (SumValue / SumCount);
ulong CurrentElapsed = 0;
if (State.PendingDurationHistory != null)
{
foreach (DownloadStateDurationValue Value in State.PendingDurationHistory.Values)
{
CurrentElapsed += Value.Elapsed;
}
}
return(Math.Max(0, (long)AverageDuration - (long)CurrentElapsed) / 1000);
}
/// <summary>
///
/// </summary>
/// <param name="State"></param>
/// <param name="TotalDownloadSize"></param>
/// <param name="CurrentProgress"></param>
public long GetEstimatedTimeRemainingForState(DownloadState State, ManifestDownloadProgressState ProgressState)
{
double DurationSum = 0;
int SumCount = 0;
long TotalSize = 0;
double RawEstimateSeconds = 0.0f;
double RawProgress = 0.0f;
double CurrentStateDuration = 0.0f;
if (State.PendingDurationHistory != null)
{
CurrentStateDuration = State.PendingDurationHistory.GetDuration(ProgressState);
}
ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (Downloader != null && Downloader.Manifest != null)
{
TotalSize = Downloader.Manifest.GetTotalSize();
// Calculate raw estimate based on speeds and data remaining.
switch (Downloader.State)
{
case ManifestDownloadProgressState.Initializing:
{
/*RawEstimateSeconds = Downloader.InitializeBytesRemaining / (double)Downloader.InitializeRateStats.RateIn;
* RawProgress = Downloader.InitializeProgress;
* if (Downloader.InitializeRateStats.RateIn == 0)
* {
* RawEstimateSeconds = 0;
* }
* break;*/
Downloader.InitializeRateEstimater.SetProgress(Downloader.InitializeProgress);
Downloader.InitializeRateEstimater.Poll();
RawEstimateSeconds = Downloader.InitializeRateEstimater.EstimatedSeconds;
RawProgress = Downloader.InitializeRateEstimater.EstimatedProgress;
break;
}
case ManifestDownloadProgressState.DeltaCopying:
{
/*RawEstimateSeconds = Downloader.DeltaCopyBytesRemaining / (double)Downloader.DeltaCopyRateStats.RateIn;
* RawProgress = Downloader.DeltaCopyProgress;
* if (Downloader.DeltaCopyRateStats.RateIn == 0)
* {
* RawEstimateSeconds = 0;
* }
* break;*/
Downloader.DeltaCopyRateEstimater.SetProgress(Downloader.DeltaCopyProgress);
Downloader.DeltaCopyRateEstimater.Poll();
RawEstimateSeconds = Downloader.DeltaCopyRateEstimater.EstimatedSeconds;
RawProgress = Downloader.DeltaCopyRateEstimater.EstimatedProgress;
break;
}
case ManifestDownloadProgressState.Validating:
{
/*RawEstimateSeconds = Downloader.ValidateBytesRemaining / (double)Downloader.ValidateRateStats.RateOut;
* RawProgress = Downloader.ValidateProgress;
* if (Downloader.ValidateRateStats.RateOut == 0)
* {
* RawEstimateSeconds = 0;
* }
* break;*/
Downloader.ValidateRateEstimater.SetProgress(Downloader.ValidateProgress);
Downloader.ValidateRateEstimater.Poll();
RawEstimateSeconds = Downloader.ValidateRateEstimater.EstimatedSeconds;
RawProgress = Downloader.ValidateRateEstimater.EstimatedProgress;
break;
}
case ManifestDownloadProgressState.Downloading:
{
/*RawEstimateSeconds = Downloader.BytesRemaining / (double)Downloader.BandwidthStats.RateIn;
* RawProgress = Downloader.Progress;
* if (Downloader.BandwidthStats.RateIn == 0)
* {
* RawEstimateSeconds = 0;
* }
* break;*/
Downloader.DownloadRateEstimater.SetProgress(Downloader.Progress);
Downloader.DownloadRateEstimater.Poll();
RawEstimateSeconds = Downloader.DownloadRateEstimater.EstimatedSeconds;
RawProgress = Downloader.DownloadRateEstimater.EstimatedProgress;
break;
}
case ManifestDownloadProgressState.Installing:
{
Downloader.InstallRateEstimater.SetProgress(Downloader.InstallProgress);
Downloader.InstallRateEstimater.Poll();
RawEstimateSeconds = Downloader.InstallRateEstimater.EstimatedSeconds;
RawProgress = Downloader.InstallRateEstimater.EstimatedProgress;
break;
}
}
}
// Calculate average historic duration adjusted to the difference in download size.
foreach (DownloadStateDuration Duration in State.DurationHistory)
{
if (Duration.HasDuration(ProgressState))
{
ulong Unadjusted = Duration.GetDuration(ProgressState);
float Adjustment = TotalSize == 0 ? 1.0f : (float)Duration.TotalSize / (float)TotalSize;
ulong AdjustedDuration = (ulong)(Unadjusted * Adjustment);
DurationSum += AdjustedDuration;
SumCount++;
}
}
if (SumCount != 0)
{
double Historic = Math.Max(0, (DurationSum / SumCount) - CurrentStateDuration) / 1000.0f;// ((DurationSum / SumCount) * (1.0f - RawProgress)) / 1000.0f;
if (Historic == 0.0f)
{
return((long)RawEstimateSeconds);
}
else
{
double Combined = (Historic * 0.75f) + (RawEstimateSeconds * 0.25f);
return((long)Combined);
}
}
else
{
return((long)RawEstimateSeconds);
}
}
/// <summary>
/// </summary>
private void UpdateDownloads(bool bHasConnection)
{
List <Guid> ActiveManifestIds = new List <Guid>();
foreach (DownloadState State in StateCollection.States)
{
bool UpdateManifestId = true;
// If not set to auto update and we already have a completed download, do nothing else.
if (!State.UpdateAutomatically && State.ActiveManifestId != Guid.Empty)
{
ManifestDownloadState OldVersionDownloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (OldVersionDownloader != null && OldVersionDownloader.State == ManifestDownloadProgressState.Complete)
{
UpdateManifestId = false;
}
}
if (bHasConnection && UpdateManifestId)
{
Guid NewManifestId = GetTargetManifestForState(State);
if (NewManifestId != Guid.Empty)
{
if (State.ActiveManifestId != NewManifestId && !State.Paused)
{
OnDownloadStarted?.Invoke(State);
}
State.ActiveManifestId = NewManifestId;
}
}
if (State.ActiveManifestId == Guid.Empty)
{
continue;
}
ActiveManifestIds.Add(State.ActiveManifestId);
ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (Downloader != null)
{
// If state is downloading but we've paused it, pause it and ignore.
if (State.Paused)
{
if (!Downloader.Paused)
{
ManifestDownloader.PauseDownload(State.ActiveManifestId);
continue;
}
}
else
{
if (Downloader.Paused)
{
ManifestDownloader.ResumeDownload(State.ActiveManifestId);
}
}
// Hackily change the priority - should change this to a SetDownloadPriority method.
Downloader.Priority = State.Priority;
Downloader.InstallOnComplete = State.InstallAutomatically;
Downloader.InstallDeviceName = State.InstallDeviceName;
Downloader.InstallLocation = State.InstallLocation;
// Have we finished this download?
if (Downloader.State == ManifestDownloadProgressState.Complete &&
State.PreviousDownloaderState != Downloader.State &&
State.PreviousDownloaderState != ManifestDownloadProgressState.Unknown)
{
OnDownloadFinished?.Invoke(State);
}
State.PreviousDownloaderState = Downloader.State;
// Store the amount of time the download is in each state to make
// time estimates a bit better.
if (Downloader.State != ManifestDownloadProgressState.Complete && !State.Paused && Downloader.Manifest != null && (bHasConnection || Downloader.State != ManifestDownloadProgressState.Downloading))
{
if (State.PendingDurationHistory == null)
{
State.PendingDurationHistory = new DownloadStateDuration();
State.PendingDurationHistory.TotalSize = Downloader.Manifest.GetTotalSize();
}
ulong CurrentTime = TimeUtils.Ticks;
if (State.PendingDurationTimer > 0)
{
ulong Elapsed = TimeUtils.Ticks - State.PendingDurationTimer;
ulong StartTime = State.PendingDurationHistory.GetDuration(Downloader.State);
ulong NewTime = StartTime + Elapsed;
State.PendingDurationHistory.SetDuration(Downloader.State, NewTime);
//Console.WriteLine("State:{0} Time:{1}", Downloader.State.ToString(), StringUtils.FormatAsDuration((long)State.PendingDurationHistory.StateDurations[Downloader.State] / 1000));
}
State.PendingDurationTimer = CurrentTime;
}
else if (Downloader.State == ManifestDownloadProgressState.Complete)
{
if (State.PendingDurationHistory != null)
{
State.DurationHistory.Add(State.PendingDurationHistory);
State.PendingDurationHistory = null;
// Only keep a few history entries.
while (State.DurationHistory.Count > 15)
{
State.DurationHistory.RemoveAt(0);
}
}
}
}
else
{
// Start downloading this manifest.
ManifestDownloader.StartDownload(State.ActiveManifestId, State.Priority);
}
}
// Go through each manifest download and pause any that are no longer relevant.
foreach (ManifestDownloadState Downloader in ManifestDownloader.States.States)
{
if (!ActiveManifestIds.Contains(Downloader.ManifestId))
{
if (!Downloader.Paused)
{
ManifestDownloader.PauseDownload(Downloader.ManifestId);
}
Downloader.Active = false;
}
else
{
Downloader.Active = true;
Downloader.LastActive = DateTime.Now;
}
}
// Only download a single auto-replication download at a time, more efficient than dozens downloading at the same time.
Guid UnpausedAutoDownload = Guid.Empty;
// Remove auto replication downloads as required.
for (int i = 0; i < StateCollection.States.Count; i++)
{
DownloadState State = StateCollection.States[i];
if (State.ActiveManifestId == Guid.Empty)
{
continue;
}
if (!State.IsAutomaticReplication)
{
continue;
}
ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId);
if (Downloader != null)
{
bool ShouldRemove = false;
// Always remove downloads that have completed.
if (Downloader.State == ManifestDownloadProgressState.Complete)
{
ShouldRemove = true;
}
// Download has not recieved any blocks in a long time, remove.
TimeSpan Elapsed = DateTime.UtcNow - Downloader.LastRecievedData;
if (Elapsed.TotalHours > ReplicationMaxIdleHours)
{
ShouldRemove = true;
}
// Automatically resume replication downloads if they are in an error state.
if (UnpausedAutoDownload == Guid.Empty)
{
UnpausedAutoDownload = State.ActiveManifestId;
}
State.Paused = (UnpausedAutoDownload != State.ActiveManifestId);
if (ShouldRemove)
{
StateCollection.States.RemoveAt(i);
AreStatesDirty = false;
i--;
}
}
}
}
/// <summary>
/// </summary>
/// <param name="State"></param>
/// <returns></returns>
public Guid GetTargetManifestForState(DownloadState State)
{
string VirtualPath = State.VirtualPath;
VirtualFileSystemNode Node = BuildFileSystem.GetNodeByPath(VirtualPath);
if (Node == null)
{
return(Guid.Empty);
}
// Node is a build in and of itself, use its id.
if (Node.Metadata != null)
{
NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Node.Metadata;
Guid ManifestId = (Guid)BuildInfo.Guid;
if (ManifestId != Guid.Empty)
{
return(ManifestId);
}
}
List <VirtualFileSystemNode> Children = BuildFileSystem.GetChildren(VirtualPath);
VirtualFileSystemNode SelectedChild = null;
List <VirtualFileSystemNode> BuildChildren = new List <VirtualFileSystemNode>();
foreach (VirtualFileSystemNode Child in Children)
{
if (Child.Metadata != null)
{
NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Child.Metadata;
Guid ManifestId = (Guid)BuildInfo.Guid;
if (ManifestId != Guid.Empty)
{
BuildChildren.Add(Child);
}
}
}
// Remove all children without included tags.
for (int i = 0; i < BuildChildren.Count; i++)
{
VirtualFileSystemNode Child = BuildChildren[i];
if (Child.Metadata != null)
{
NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Child.Metadata;
if (!BuildHasAllTags(BuildInfo, State.IncludeTags))
{
BuildChildren.RemoveAt(i);
i--;
}
}
}
// Remove all children with any tags..
for (int i = 0; i < BuildChildren.Count; i++)
{
VirtualFileSystemNode Child = BuildChildren[i];
if (Child.Metadata != null)
{
NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Child.Metadata;
if (BuildHasAnyTags(BuildInfo, State.ExcludeTags))
{
BuildChildren.RemoveAt(i);
i--;
}
}
}
List <VirtualFileSystemNode> FilteredChildren = new List <VirtualFileSystemNode>();
switch (State.SelectionFilter)
{
case BuildSelectionFilter.None:
{
FilteredChildren = BuildChildren;
break;
}
case BuildSelectionFilter.BuildTimeBeforeScmSyncTime:
{
IScmProvider Workspace = ScmManager.GetProvider(State.ScmWorkspaceLocation);
if (Workspace != null)
{
DateTime ScmSyncTime = Workspace.GetSyncTime();
if (ScmSyncTime != DateTime.MinValue)
{
foreach (VirtualFileSystemNode Child in BuildChildren)
{
if (Child.CreateTime <= ScmSyncTime)
{
FilteredChildren.Add(Child);
}
}
}
}
break;
}
case BuildSelectionFilter.BuildTimeAfterScmSyncTime:
{
IScmProvider Workspace = ScmManager.GetProvider(State.ScmWorkspaceLocation);
if (Workspace != null)
{
DateTime ScmSyncTime = Workspace.GetSyncTime();
if (ScmSyncTime != DateTime.MinValue)
{
foreach (VirtualFileSystemNode Child in BuildChildren)
{
if (Child.CreateTime >= ScmSyncTime)
{
FilteredChildren.Add(Child);
}
}
}
}
break;
}
case BuildSelectionFilter.BuildNameBelowFileContents:
{
string FilePath = Path.Combine(State.ScmWorkspaceLocation, State.SelectionFilterFilePath);
string FileContents = FileContentsCache.Get(FilePath);
int Value = 0;
if (int.TryParse(FileContents, out Value))
{
foreach (VirtualFileSystemNode Child in BuildChildren)
{
int ChildValue = 0;
if (int.TryParse(Child.Name, out ChildValue))
{
if (Value <= ChildValue)
{
FilteredChildren.Add(Child);
}
}
}
}
break;
}
case BuildSelectionFilter.BuildNameAboveFileContents:
{
string FilePath = Path.Combine(State.ScmWorkspaceLocation, State.SelectionFilterFilePath);
string FileContents = FileContentsCache.Get(FilePath);
int Value = 0;
if (int.TryParse(FileContents, out Value))
{
foreach (VirtualFileSystemNode Child in BuildChildren)
{
int ChildValue = 0;
if (int.TryParse(Child.Name, out ChildValue))
{
if (Value >= ChildValue)
{
FilteredChildren.Add(Child);
}
}
}
}
break;
}
case BuildSelectionFilter.BuildNameEqualsFileContents:
{
string FilePath = Path.Combine(State.ScmWorkspaceLocation, State.SelectionFilterFilePath);
string FileContents = FileContentsCache.Get(FilePath);
foreach (VirtualFileSystemNode Child in BuildChildren)
{
if (FileContents == Child.Name)
{
FilteredChildren.Add(Child);
}
}
break;
}
default:
{
Debug.Assert(false);
break;
}
}
switch (State.SelectionRule)
{
case BuildSelectionRule.Newest:
{
foreach (VirtualFileSystemNode Child in FilteredChildren)
{
if (SelectedChild == null || SelectedChild.CreateTime < Child.CreateTime)
{
SelectedChild = Child;
}
}
break;
}
case BuildSelectionRule.Oldest:
{
foreach (VirtualFileSystemNode Child in FilteredChildren)
{
if (SelectedChild == null || SelectedChild.CreateTime > Child.CreateTime)
{
SelectedChild = Child;
}
}
break;
}
}
if (SelectedChild != null)
{
NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)SelectedChild.Metadata;
return(BuildInfo.Guid);
}
return(Guid.Empty);
}
/// <summary>
/// </summary>
public void RemoveDownload(DownloadState State)
{
StateCollection.States.Remove(State);
AreStatesDirty = false;
}
