/// <summary> /// </summary> /// <param name="Name"></param> /// <param name="VirtualPath"></param> /// <param name="Priority"></param> /// <param name="KeepUpToDate"></param> public DownloadState AddDownload(string Name, string VirtualPath, int Priority, BuildSelectionRule Rule, BuildSelectionFilter Filter, string SelectionFilterFilePath, string ScmWorkspaceLocation, bool AutomaticallyUpdate, bool AutomaticallyInstall, string InstallDeviceName, string InstallLocation, List <Guid> IncludeTags, List <Guid> ExcludeTags, bool DeleteOnComplete = false) { DownloadState State = new DownloadState(); State.Id = Guid.NewGuid(); State.Name = Name; State.VirtualPath = VirtualPath; State.Priority = Priority; State.UpdateAutomatically = AutomaticallyUpdate; State.InstallAutomatically = AutomaticallyInstall; State.InstallDeviceName = InstallDeviceName; State.InstallLocation = InstallLocation; State.SelectionRule = Rule; State.SelectionFilter = Filter; State.SelectionFilterFilePath = SelectionFilterFilePath; State.ScmWorkspaceLocation = ScmWorkspaceLocation; State.IncludeTags = IncludeTags; State.ExcludeTags = ExcludeTags; State.IsAutomaticReplication = DeleteOnComplete; StateCollection.States.Add(State); AreStatesDirty = true; return(State); }
/// <summary> /// /// </summary> /// <param name="State"></param> /// <param name="TotalDownloadSize"></param> /// <param name="CurrentProgress"></param> public long GetEstimatedTimeRemaining(DownloadState State) { ulong SumCount = 0; ulong SumValue = 0; long TotalSize = 0; ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId); if (Downloader != null && Downloader.Manifest != null) { TotalSize = Downloader.Manifest.GetTotalSize(); } foreach (DownloadStateDuration Duration in State.DurationHistory) { if (Duration.Values.Count > 0) { ulong Unadjusted = 0; foreach (DownloadStateDurationValue Value in Duration.Values) { Unadjusted += Value.Elapsed; } float Adjustment = TotalSize == 0 ? 1.0f : (float)Duration.TotalSize / (float)TotalSize; ulong AdjustedDuration = (ulong)(Unadjusted * Adjustment); SumValue += AdjustedDuration; SumCount++; } } ulong AverageDuration = SumCount == 0 ? 0 : (SumValue / SumCount); ulong CurrentElapsed = 0; if (State.PendingDurationHistory != null) { foreach (DownloadStateDurationValue Value in State.PendingDurationHistory.Values) { CurrentElapsed += Value.Elapsed; } } return(Math.Max(0, (long)AverageDuration - (long)CurrentElapsed) / 1000); }
/// <summary> /// /// </summary> /// <param name="State"></param> /// <param name="TotalDownloadSize"></param> /// <param name="CurrentProgress"></param> public long GetEstimatedTimeRemainingForState(DownloadState State, ManifestDownloadProgressState ProgressState) { double DurationSum = 0; int SumCount = 0; long TotalSize = 0; double RawEstimateSeconds = 0.0f; double RawProgress = 0.0f; double CurrentStateDuration = 0.0f; if (State.PendingDurationHistory != null) { CurrentStateDuration = State.PendingDurationHistory.GetDuration(ProgressState); } ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId); if (Downloader != null && Downloader.Manifest != null) { TotalSize = Downloader.Manifest.GetTotalSize(); // Calculate raw estimate based on speeds and data remaining. switch (Downloader.State) { case ManifestDownloadProgressState.Initializing: { /*RawEstimateSeconds = Downloader.InitializeBytesRemaining / (double)Downloader.InitializeRateStats.RateIn; * RawProgress = Downloader.InitializeProgress; * if (Downloader.InitializeRateStats.RateIn == 0) * { * RawEstimateSeconds = 0; * } * break;*/ Downloader.InitializeRateEstimater.SetProgress(Downloader.InitializeProgress); Downloader.InitializeRateEstimater.Poll(); RawEstimateSeconds = Downloader.InitializeRateEstimater.EstimatedSeconds; RawProgress = Downloader.InitializeRateEstimater.EstimatedProgress; break; } case ManifestDownloadProgressState.DeltaCopying: { /*RawEstimateSeconds = Downloader.DeltaCopyBytesRemaining / (double)Downloader.DeltaCopyRateStats.RateIn; * RawProgress = Downloader.DeltaCopyProgress; * if (Downloader.DeltaCopyRateStats.RateIn == 0) * { * RawEstimateSeconds = 0; * } * break;*/ Downloader.DeltaCopyRateEstimater.SetProgress(Downloader.DeltaCopyProgress); Downloader.DeltaCopyRateEstimater.Poll(); RawEstimateSeconds = Downloader.DeltaCopyRateEstimater.EstimatedSeconds; RawProgress = Downloader.DeltaCopyRateEstimater.EstimatedProgress; break; } case ManifestDownloadProgressState.Validating: { /*RawEstimateSeconds = Downloader.ValidateBytesRemaining / (double)Downloader.ValidateRateStats.RateOut; * RawProgress = Downloader.ValidateProgress; * if (Downloader.ValidateRateStats.RateOut == 0) * { * RawEstimateSeconds = 0; * } * break;*/ Downloader.ValidateRateEstimater.SetProgress(Downloader.ValidateProgress); Downloader.ValidateRateEstimater.Poll(); RawEstimateSeconds = Downloader.ValidateRateEstimater.EstimatedSeconds; RawProgress = Downloader.ValidateRateEstimater.EstimatedProgress; break; } case ManifestDownloadProgressState.Downloading: { /*RawEstimateSeconds = Downloader.BytesRemaining / (double)Downloader.BandwidthStats.RateIn; * RawProgress = Downloader.Progress; * if (Downloader.BandwidthStats.RateIn == 0) * { * RawEstimateSeconds = 0; * } * break;*/ Downloader.DownloadRateEstimater.SetProgress(Downloader.Progress); Downloader.DownloadRateEstimater.Poll(); RawEstimateSeconds = Downloader.DownloadRateEstimater.EstimatedSeconds; RawProgress = Downloader.DownloadRateEstimater.EstimatedProgress; break; } case ManifestDownloadProgressState.Installing: { Downloader.InstallRateEstimater.SetProgress(Downloader.InstallProgress); Downloader.InstallRateEstimater.Poll(); RawEstimateSeconds = Downloader.InstallRateEstimater.EstimatedSeconds; RawProgress = Downloader.InstallRateEstimater.EstimatedProgress; break; } } } // Calculate average historic duration adjusted to the difference in download size. foreach (DownloadStateDuration Duration in State.DurationHistory) { if (Duration.HasDuration(ProgressState)) { ulong Unadjusted = Duration.GetDuration(ProgressState); float Adjustment = TotalSize == 0 ? 1.0f : (float)Duration.TotalSize / (float)TotalSize; ulong AdjustedDuration = (ulong)(Unadjusted * Adjustment); DurationSum += AdjustedDuration; SumCount++; } } if (SumCount != 0) { double Historic = Math.Max(0, (DurationSum / SumCount) - CurrentStateDuration) / 1000.0f;// ((DurationSum / SumCount) * (1.0f - RawProgress)) / 1000.0f; if (Historic == 0.0f) { return((long)RawEstimateSeconds); } else { double Combined = (Historic * 0.75f) + (RawEstimateSeconds * 0.25f); return((long)Combined); } } else { return((long)RawEstimateSeconds); } }
/// <summary> /// </summary> private void UpdateDownloads(bool bHasConnection) { List <Guid> ActiveManifestIds = new List <Guid>(); foreach (DownloadState State in StateCollection.States) { bool UpdateManifestId = true; // If not set to auto update and we already have a completed download, do nothing else. if (!State.UpdateAutomatically && State.ActiveManifestId != Guid.Empty) { ManifestDownloadState OldVersionDownloader = ManifestDownloader.GetDownload(State.ActiveManifestId); if (OldVersionDownloader != null && OldVersionDownloader.State == ManifestDownloadProgressState.Complete) { UpdateManifestId = false; } } if (bHasConnection && UpdateManifestId) { Guid NewManifestId = GetTargetManifestForState(State); if (NewManifestId != Guid.Empty) { if (State.ActiveManifestId != NewManifestId && !State.Paused) { OnDownloadStarted?.Invoke(State); } State.ActiveManifestId = NewManifestId; } } if (State.ActiveManifestId == Guid.Empty) { continue; } ActiveManifestIds.Add(State.ActiveManifestId); ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId); if (Downloader != null) { // If state is downloading but we've paused it, pause it and ignore. if (State.Paused) { if (!Downloader.Paused) { ManifestDownloader.PauseDownload(State.ActiveManifestId); continue; } } else { if (Downloader.Paused) { ManifestDownloader.ResumeDownload(State.ActiveManifestId); } } // Hackily change the priority - should change this to a SetDownloadPriority method. Downloader.Priority = State.Priority; Downloader.InstallOnComplete = State.InstallAutomatically; Downloader.InstallDeviceName = State.InstallDeviceName; Downloader.InstallLocation = State.InstallLocation; // Have we finished this download? if (Downloader.State == ManifestDownloadProgressState.Complete && State.PreviousDownloaderState != Downloader.State && State.PreviousDownloaderState != ManifestDownloadProgressState.Unknown) { OnDownloadFinished?.Invoke(State); } State.PreviousDownloaderState = Downloader.State; // Store the amount of time the download is in each state to make // time estimates a bit better. if (Downloader.State != ManifestDownloadProgressState.Complete && !State.Paused && Downloader.Manifest != null && (bHasConnection || Downloader.State != ManifestDownloadProgressState.Downloading)) { if (State.PendingDurationHistory == null) { State.PendingDurationHistory = new DownloadStateDuration(); State.PendingDurationHistory.TotalSize = Downloader.Manifest.GetTotalSize(); } ulong CurrentTime = TimeUtils.Ticks; if (State.PendingDurationTimer > 0) { ulong Elapsed = TimeUtils.Ticks - State.PendingDurationTimer; ulong StartTime = State.PendingDurationHistory.GetDuration(Downloader.State); ulong NewTime = StartTime + Elapsed; State.PendingDurationHistory.SetDuration(Downloader.State, NewTime); //Console.WriteLine("State:{0} Time:{1}", Downloader.State.ToString(), StringUtils.FormatAsDuration((long)State.PendingDurationHistory.StateDurations[Downloader.State] / 1000)); } State.PendingDurationTimer = CurrentTime; } else if (Downloader.State == ManifestDownloadProgressState.Complete) { if (State.PendingDurationHistory != null) { State.DurationHistory.Add(State.PendingDurationHistory); State.PendingDurationHistory = null; // Only keep a few history entries. while (State.DurationHistory.Count > 15) { State.DurationHistory.RemoveAt(0); } } } } else { // Start downloading this manifest. ManifestDownloader.StartDownload(State.ActiveManifestId, State.Priority); } } // Go through each manifest download and pause any that are no longer relevant. foreach (ManifestDownloadState Downloader in ManifestDownloader.States.States) { if (!ActiveManifestIds.Contains(Downloader.ManifestId)) { if (!Downloader.Paused) { ManifestDownloader.PauseDownload(Downloader.ManifestId); } Downloader.Active = false; } else { Downloader.Active = true; Downloader.LastActive = DateTime.Now; } } // Only download a single auto-replication download at a time, more efficient than dozens downloading at the same time. Guid UnpausedAutoDownload = Guid.Empty; // Remove auto replication downloads as required. for (int i = 0; i < StateCollection.States.Count; i++) { DownloadState State = StateCollection.States[i]; if (State.ActiveManifestId == Guid.Empty) { continue; } if (!State.IsAutomaticReplication) { continue; } ManifestDownloadState Downloader = ManifestDownloader.GetDownload(State.ActiveManifestId); if (Downloader != null) { bool ShouldRemove = false; // Always remove downloads that have completed. if (Downloader.State == ManifestDownloadProgressState.Complete) { ShouldRemove = true; } // Download has not recieved any blocks in a long time, remove. TimeSpan Elapsed = DateTime.UtcNow - Downloader.LastRecievedData; if (Elapsed.TotalHours > ReplicationMaxIdleHours) { ShouldRemove = true; } // Automatically resume replication downloads if they are in an error state. if (UnpausedAutoDownload == Guid.Empty) { UnpausedAutoDownload = State.ActiveManifestId; } State.Paused = (UnpausedAutoDownload != State.ActiveManifestId); if (ShouldRemove) { StateCollection.States.RemoveAt(i); AreStatesDirty = false; i--; } } } }
/// <summary> /// </summary> /// <param name="State"></param> /// <returns></returns> public Guid GetTargetManifestForState(DownloadState State) { string VirtualPath = State.VirtualPath; VirtualFileSystemNode Node = BuildFileSystem.GetNodeByPath(VirtualPath); if (Node == null) { return(Guid.Empty); } // Node is a build in and of itself, use its id. if (Node.Metadata != null) { NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Node.Metadata; Guid ManifestId = (Guid)BuildInfo.Guid; if (ManifestId != Guid.Empty) { return(ManifestId); } } List <VirtualFileSystemNode> Children = BuildFileSystem.GetChildren(VirtualPath); VirtualFileSystemNode SelectedChild = null; List <VirtualFileSystemNode> BuildChildren = new List <VirtualFileSystemNode>(); foreach (VirtualFileSystemNode Child in Children) { if (Child.Metadata != null) { NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Child.Metadata; Guid ManifestId = (Guid)BuildInfo.Guid; if (ManifestId != Guid.Empty) { BuildChildren.Add(Child); } } } // Remove all children without included tags. for (int i = 0; i < BuildChildren.Count; i++) { VirtualFileSystemNode Child = BuildChildren[i]; if (Child.Metadata != null) { NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Child.Metadata; if (!BuildHasAllTags(BuildInfo, State.IncludeTags)) { BuildChildren.RemoveAt(i); i--; } } } // Remove all children with any tags.. for (int i = 0; i < BuildChildren.Count; i++) { VirtualFileSystemNode Child = BuildChildren[i]; if (Child.Metadata != null) { NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)Child.Metadata; if (BuildHasAnyTags(BuildInfo, State.ExcludeTags)) { BuildChildren.RemoveAt(i); i--; } } } List <VirtualFileSystemNode> FilteredChildren = new List <VirtualFileSystemNode>(); switch (State.SelectionFilter) { case BuildSelectionFilter.None: { FilteredChildren = BuildChildren; break; } case BuildSelectionFilter.BuildTimeBeforeScmSyncTime: { IScmProvider Workspace = ScmManager.GetProvider(State.ScmWorkspaceLocation); if (Workspace != null) { DateTime ScmSyncTime = Workspace.GetSyncTime(); if (ScmSyncTime != DateTime.MinValue) { foreach (VirtualFileSystemNode Child in BuildChildren) { if (Child.CreateTime <= ScmSyncTime) { FilteredChildren.Add(Child); } } } } break; } case BuildSelectionFilter.BuildTimeAfterScmSyncTime: { IScmProvider Workspace = ScmManager.GetProvider(State.ScmWorkspaceLocation); if (Workspace != null) { DateTime ScmSyncTime = Workspace.GetSyncTime(); if (ScmSyncTime != DateTime.MinValue) { foreach (VirtualFileSystemNode Child in BuildChildren) { if (Child.CreateTime >= ScmSyncTime) { FilteredChildren.Add(Child); } } } } break; } case BuildSelectionFilter.BuildNameBelowFileContents: { string FilePath = Path.Combine(State.ScmWorkspaceLocation, State.SelectionFilterFilePath); string FileContents = FileContentsCache.Get(FilePath); int Value = 0; if (int.TryParse(FileContents, out Value)) { foreach (VirtualFileSystemNode Child in BuildChildren) { int ChildValue = 0; if (int.TryParse(Child.Name, out ChildValue)) { if (Value <= ChildValue) { FilteredChildren.Add(Child); } } } } break; } case BuildSelectionFilter.BuildNameAboveFileContents: { string FilePath = Path.Combine(State.ScmWorkspaceLocation, State.SelectionFilterFilePath); string FileContents = FileContentsCache.Get(FilePath); int Value = 0; if (int.TryParse(FileContents, out Value)) { foreach (VirtualFileSystemNode Child in BuildChildren) { int ChildValue = 0; if (int.TryParse(Child.Name, out ChildValue)) { if (Value >= ChildValue) { FilteredChildren.Add(Child); } } } } break; } case BuildSelectionFilter.BuildNameEqualsFileContents: { string FilePath = Path.Combine(State.ScmWorkspaceLocation, State.SelectionFilterFilePath); string FileContents = FileContentsCache.Get(FilePath); foreach (VirtualFileSystemNode Child in BuildChildren) { if (FileContents == Child.Name) { FilteredChildren.Add(Child); } } break; } default: { Debug.Assert(false); break; } } switch (State.SelectionRule) { case BuildSelectionRule.Newest: { foreach (VirtualFileSystemNode Child in FilteredChildren) { if (SelectedChild == null || SelectedChild.CreateTime < Child.CreateTime) { SelectedChild = Child; } } break; } case BuildSelectionRule.Oldest: { foreach (VirtualFileSystemNode Child in FilteredChildren) { if (SelectedChild == null || SelectedChild.CreateTime > Child.CreateTime) { SelectedChild = Child; } } break; } } if (SelectedChild != null) { NetMessage_GetBuildsResponse.BuildInfo BuildInfo = (NetMessage_GetBuildsResponse.BuildInfo)SelectedChild.Metadata; return(BuildInfo.Guid); } return(Guid.Empty); }
/// <summary> /// </summary> public void RemoveDownload(DownloadState State) { StateCollection.States.Remove(State); AreStatesDirty = false; }