public async Task<string[]> Download( string targetFolder, string[] urls ) { Directory.CreateDirectory( targetFolder ); List<string> files = new List<string>( urls.Length ); const int MaxTries = 5; int triesLeft = MaxTries; while ( files.Count < urls.Length ) { if ( triesLeft <= 0 ) { throw new Exception( "Failed to download individual parts after " + MaxTries + " tries, aborting." ); } files.Clear(); for ( int i = 0; i < urls.Length; ++i ) { string url = urls[i]; string outpath = Path.Combine( targetFolder, "part" + i.ToString( "D8" ) + ".ts" ); string outpath_temp = outpath + ".tmp"; if ( await Util.FileExists( outpath_temp ) ) { await Util.DeleteFile( outpath_temp ); } if ( await Util.FileExists( outpath ) ) { if ( i % 100 == 99 ) { Status = "Already have part " + ( i + 1 ) + "/" + urls.Length + "..."; } files.Add( outpath ); continue; } bool success = false; using ( var client = new KeepAliveWebClient() ) { try { Status = "Downloading files... (" + ( files.Count + 1 ) + "/" + urls.Length + ")"; byte[] data = await client.DownloadDataTaskAsync( url ); using ( FileStream fs = File.Create( outpath_temp ) ) { await fs.WriteAsync( data, 0, data.Length ); } success = true; } catch ( System.Net.WebException ex ) { Console.WriteLine( ex.ToString() ); continue; } } if ( success ) { File.Move( outpath_temp, outpath ); files.Add( outpath ); } } --triesLeft; } return files.ToArray(); }
public static async Task <(ResultType result, string[] files)> Download( IVideoJob job, CancellationToken cancellationToken, string targetFolder, List <DownloadInfo> downloadInfos, int delayPerDownload = 0 ) { Directory.CreateDirectory(targetFolder); List <string> files = new List <string>(downloadInfos.Count); const int MaxTries = 5; int triesLeft = MaxTries; while (files.Count < downloadInfos.Count) { if (triesLeft <= 0) { job.Status = "Failed to download individual parts after " + MaxTries + " tries, aborting."; return(ResultType.Failure, null); } files.Clear(); for (int i = 0; i < downloadInfos.Count; ++i) { //for (int i = downloadInfos.Count - 1; i >= 0; --i) { if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled, null); } DownloadInfo downloadInfo = downloadInfos[i]; string outpath = Path.Combine(targetFolder, downloadInfo.FilesystemId + ".ts"); string outpath_temp = outpath + ".tmp"; if (await Util.FileExists(outpath_temp)) { await Util.DeleteFile(outpath_temp); } if (outpath.EndsWith("--2d--muted--2e--ts__.ts")) { string alt_outpath = outpath.Substring(0, outpath.Length - "--2d--muted--2e--ts__.ts".Length) + "--2e--ts__.ts"; if (await Util.FileExists(alt_outpath)) { if (i % 100 == 99) { job.Status = "Already have part " + (i + 1) + "/" + downloadInfos.Count + "..."; } files.Add(alt_outpath); continue; } } if (await Util.FileExists(outpath)) { if (i % 100 == 99) { job.Status = "Already have part " + (i + 1) + "/" + downloadInfos.Count + "..."; } files.Add(outpath); continue; } bool success = false; { System.Net.WebClient client = null; try { if (downloadInfo.Length != null && downloadInfo.Offset != null) { client = new KeepAliveWebClientWithRange(downloadInfo.Offset.Value, downloadInfo.Offset.Value + downloadInfo.Length.Value - 1); } else { client = new KeepAliveWebClient(); } job.Status = "Downloading files... (" + (files.Count + 1) + "/" + downloadInfos.Count + ")"; byte[] data = await client.DownloadDataTaskAsync(downloadInfo.Url); await job.StallWrite(outpath_temp, data.LongLength, cancellationToken); if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled, null); } using (FileStream fs = File.Create(outpath_temp)) { await fs.WriteAsync(data, 0, data.Length); } success = true; } catch (System.Net.WebException ex) { System.Net.HttpWebResponse httpWebResponse = ex.Response as System.Net.HttpWebResponse; if (httpWebResponse != null) { switch (httpWebResponse.StatusCode) { case System.Net.HttpStatusCode.NotFound: Newtonsoft.Json.Linq.JObject reply = Newtonsoft.Json.Linq.JObject.Parse(new StreamReader(httpWebResponse.GetResponseStream()).ReadToEnd()); string detail = reply["errors"][0]["detail"].ToObject <string>(); if (detail == "No chats for this Video") { return(ResultType.Dead, null); } break; default: Console.WriteLine("Server returned unhandled error code: " + httpWebResponse.StatusCode); break; } } else { Console.WriteLine(ex.ToString()); } continue; } finally { if (client != null) { client.Dispose(); } } } if (success) { await job.StallWrite(outpath, new FileInfo( outpath_temp ).Length, cancellationToken); if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled, null); } File.Move(outpath_temp, outpath); files.Add(outpath); } if (delayPerDownload > 0) { try { await Task.Delay(delayPerDownload, cancellationToken); } catch (TaskCanceledException) { return(ResultType.Cancelled, null); } } } if (files.Count < downloadInfos.Count) { try { await Task.Delay(60000, cancellationToken); } catch (TaskCanceledException) { return(ResultType.Cancelled, null); } --triesLeft; } } return(ResultType.Success, files.ToArray()); }
public override async Task <ResultType> Run(CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } JobStatus = VideoJobStatus.Running; Status = "Retrieving video info..."; var video_json = await TwitchYTDL.GetVideoJson(long.Parse(VideoInfo.VideoId)); VideoInfo = new TwitchVideoInfo(TwitchYTDL.VideoFromJson(video_json), StreamService.TwitchChatReplay); if (!AssumeFinished && VideoInfo.VideoRecordingState == RecordingState.Live) { _UserInputRequest = new UserInputRequestStreamLive(this); return(ResultType.TemporarilyUnavailable); } string tempname = Path.Combine(Util.TempFolderPath, GetTempFilenameWithoutExtension() + ".json.tmp"); string finalintmpname = Path.Combine(Util.TempFolderPath, GetTempFilenameWithoutExtension() + ".json"); string filename = Path.Combine(Util.TargetFolderPath, GetTargetFilenameWithoutExtension() + ".json"); Random rng = new Random(int.Parse(VideoInfo.VideoId)); if (!await Util.FileExists(filename)) { if (!await Util.FileExists(finalintmpname)) { Status = "Downloading chat (Initial)..."; StringBuilder concatJson = new StringBuilder(); string url = GetStartUrl(VideoInfo); int attemptsLeft = 5; TimeSpan? lastTimeSpan = new TimeSpan(0); int nextDelayMilliseconds = 0; while (true) { using (var client = new KeepAliveWebClient()) using (var cancellationCallback = cancellationToken.Register(client.CancelAsync)) { try { try { if (nextDelayMilliseconds != 0) { await Task.Delay(nextDelayMilliseconds > 0?nextDelayMilliseconds : rng.Next(90000, 270000), cancellationToken); } } catch (TaskCanceledException) { return(ResultType.Cancelled); } string commentJson = await TwitchAPI.GetLegacy(url, Util.TwitchClientId); JObject responseObject = JObject.Parse(commentJson); if (responseObject["comments"] == null) { throw new Exception("Nonsense JSON returned, no comments."); } string offset = "Unknown"; try { JToken c = ((JArray)responseObject["comments"]).Last; double val = (double)c["content_offset_seconds"]; TimeSpan ts = TimeSpan.FromSeconds(val); if (lastTimeSpan != null) { TimeSpan diff = ts - lastTimeSpan.Value; double delay = diff.TotalMilliseconds; if (delay < 0.0) { nextDelayMilliseconds = -1; } else if (delay < 90000.0) { nextDelayMilliseconds = (int)delay; } else if (delay < 270000.0) { nextDelayMilliseconds = rng.Next(90000, (int)delay); } else { nextDelayMilliseconds = -1; } } else { nextDelayMilliseconds = -1; } lastTimeSpan = ts; offset = ts.ToString(); } catch (Exception) { lastTimeSpan = null; nextDelayMilliseconds = -1; } concatJson.Append(commentJson); if (responseObject["_next"] != null) { string next = (string)responseObject["_next"]; attemptsLeft = 5; Status = "Downloading chat (Last offset: " + offset + "; next file: " + next + ")..."; url = GetNextUrl(VideoInfo, next); } else { // presumably done? break; } } catch (System.Net.WebException ex) { Console.WriteLine(ex.ToString()); --attemptsLeft; Status = "Downloading chat (Error; " + attemptsLeft + " attempts left)..."; if (attemptsLeft <= 0) { throw; } continue; } } } await StallWrite(tempname, concatJson.Length * 4, cancellationToken); // size not accurate because encoding but whatever if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } File.WriteAllText(tempname, concatJson.ToString()); File.Move(tempname, finalintmpname); } if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } Status = "Moving to final location..."; await StallWrite(filename, new FileInfo( finalintmpname ).Length, cancellationToken); if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } Util.MoveFileOverwrite(finalintmpname, filename); } Status = "Done!"; JobStatus = VideoJobStatus.Finished; return(ResultType.Success); }
public override async Task <ResultType> Run(CancellationToken cancellationToken) { JobStatus = VideoJobStatus.Running; Status = "Downloading..."; string tempFilename = Path.Combine(Util.TempFolderPath, GetTempFilename()); string movedFilename = Path.Combine(Util.TempFolderPath, GetTargetFilename()); string targetFilename = Path.Combine(Util.TargetFolderPath, GetTargetFilename()); if (!await Util.FileExists(targetFilename)) { if (!await Util.FileExists(movedFilename)) { if (await Util.FileExists(tempFilename)) { await Util.DeleteFile(tempFilename); } if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } bool success = false; using (var client = new KeepAliveWebClient()) using (var cancellationCallback = cancellationToken.Register(client.CancelAsync)) { client.DownloadProgressChanged += new DownloadProgressChangedEventHandler(ReportDownloadProgress); try { byte[] data = await client.DownloadDataTaskAsync(VideoInfo.VideoId); await StallWrite(tempFilename, data.LongLength, cancellationToken); if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } using (FileStream fs = File.Create(tempFilename)) { await fs.WriteAsync(data, 0, data.Length); success = true; } } catch (WebException) { if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } throw; } } if (success) { await StallWrite(movedFilename, new FileInfo( tempFilename ).Length, cancellationToken); if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } File.Move(tempFilename, movedFilename); } } if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } await StallWrite(targetFilename, new FileInfo( movedFilename ).Length, cancellationToken); if (cancellationToken.IsCancellationRequested) { return(ResultType.Cancelled); } File.Move(movedFilename, targetFilename); } Status = "Done!"; JobStatus = VideoJobStatus.Finished; return(ResultType.Success); }