public async Task<string[]> Download( string targetFolder, string[] urls ) {
Directory.CreateDirectory( targetFolder );
List<string> files = new List<string>( urls.Length );
const int MaxTries = 5;
int triesLeft = MaxTries;
while ( files.Count < urls.Length ) {
if ( triesLeft <= 0 ) {
throw new Exception( "Failed to download individual parts after " + MaxTries + " tries, aborting." );
}
files.Clear();
for ( int i = 0; i < urls.Length; ++i ) {
string url = urls[i];
string outpath = Path.Combine( targetFolder, "part" + i.ToString( "D8" ) + ".ts" );
string outpath_temp = outpath + ".tmp";
if ( await Util.FileExists( outpath_temp ) ) {
await Util.DeleteFile( outpath_temp );
}
if ( await Util.FileExists( outpath ) ) {
if ( i % 100 == 99 ) {
Status = "Already have part " + ( i + 1 ) + "/" + urls.Length + "...";
}
files.Add( outpath );
continue;
}
bool success = false;
using ( var client = new KeepAliveWebClient() ) {
try {
Status = "Downloading files... (" + ( files.Count + 1 ) + "/" + urls.Length + ")";
byte[] data = await client.DownloadDataTaskAsync( url );
using ( FileStream fs = File.Create( outpath_temp ) ) {
await fs.WriteAsync( data, 0, data.Length );
}
success = true;
} catch ( System.Net.WebException ex ) {
Console.WriteLine( ex.ToString() );
continue;
}
}
if ( success ) {
File.Move( outpath_temp, outpath );
files.Add( outpath );
}
}
--triesLeft;
}
return files.ToArray();
}
public static async Task <(ResultType result, string[] files)> Download(
IVideoJob job, CancellationToken cancellationToken, string targetFolder, List <DownloadInfo> downloadInfos, int delayPerDownload = 0
)
{
Directory.CreateDirectory(targetFolder);
List <string> files = new List <string>(downloadInfos.Count);
const int MaxTries = 5;
int triesLeft = MaxTries;
while (files.Count < downloadInfos.Count)
{
if (triesLeft <= 0)
{
job.Status = "Failed to download individual parts after " + MaxTries + " tries, aborting.";
return(ResultType.Failure, null);
}
files.Clear();
for (int i = 0; i < downloadInfos.Count; ++i)
{
//for (int i = downloadInfos.Count - 1; i >= 0; --i) {
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled, null);
}
DownloadInfo downloadInfo = downloadInfos[i];
string outpath = Path.Combine(targetFolder, downloadInfo.FilesystemId + ".ts");
string outpath_temp = outpath + ".tmp";
if (await Util.FileExists(outpath_temp))
{
await Util.DeleteFile(outpath_temp);
}
if (outpath.EndsWith("--2d--muted--2e--ts__.ts"))
{
string alt_outpath = outpath.Substring(0, outpath.Length - "--2d--muted--2e--ts__.ts".Length) + "--2e--ts__.ts";
if (await Util.FileExists(alt_outpath))
{
if (i % 100 == 99)
{
job.Status = "Already have part " + (i + 1) + "/" + downloadInfos.Count + "...";
}
files.Add(alt_outpath);
continue;
}
}
if (await Util.FileExists(outpath))
{
if (i % 100 == 99)
{
job.Status = "Already have part " + (i + 1) + "/" + downloadInfos.Count + "...";
}
files.Add(outpath);
continue;
}
bool success = false;
{
System.Net.WebClient client = null;
try {
if (downloadInfo.Length != null && downloadInfo.Offset != null)
{
client = new KeepAliveWebClientWithRange(downloadInfo.Offset.Value, downloadInfo.Offset.Value + downloadInfo.Length.Value - 1);
}
else
{
client = new KeepAliveWebClient();
}
job.Status = "Downloading files... (" + (files.Count + 1) + "/" + downloadInfos.Count + ")";
byte[] data = await client.DownloadDataTaskAsync(downloadInfo.Url);
await job.StallWrite(outpath_temp, data.LongLength, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled, null);
}
using (FileStream fs = File.Create(outpath_temp)) {
await fs.WriteAsync(data, 0, data.Length);
}
success = true;
} catch (System.Net.WebException ex) {
System.Net.HttpWebResponse httpWebResponse = ex.Response as System.Net.HttpWebResponse;
if (httpWebResponse != null)
{
switch (httpWebResponse.StatusCode)
{
case System.Net.HttpStatusCode.NotFound:
Newtonsoft.Json.Linq.JObject reply = Newtonsoft.Json.Linq.JObject.Parse(new StreamReader(httpWebResponse.GetResponseStream()).ReadToEnd());
string detail = reply["errors"][0]["detail"].ToObject <string>();
if (detail == "No chats for this Video")
{
return(ResultType.Dead, null);
}
break;
default:
Console.WriteLine("Server returned unhandled error code: " + httpWebResponse.StatusCode);
break;
}
}
else
{
Console.WriteLine(ex.ToString());
}
continue;
} finally {
if (client != null)
{
client.Dispose();
}
}
}
if (success)
{
await job.StallWrite(outpath, new FileInfo( outpath_temp ).Length, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled, null);
}
File.Move(outpath_temp, outpath);
files.Add(outpath);
}
if (delayPerDownload > 0)
{
try {
await Task.Delay(delayPerDownload, cancellationToken);
} catch (TaskCanceledException) {
return(ResultType.Cancelled, null);
}
}
}
if (files.Count < downloadInfos.Count)
{
try {
await Task.Delay(60000, cancellationToken);
} catch (TaskCanceledException) {
return(ResultType.Cancelled, null);
}
--triesLeft;
}
}
return(ResultType.Success, files.ToArray());
}
public override async Task <ResultType> Run(CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
JobStatus = VideoJobStatus.Running;
Status = "Retrieving video info...";
var video_json = await TwitchYTDL.GetVideoJson(long.Parse(VideoInfo.VideoId));
VideoInfo = new TwitchVideoInfo(TwitchYTDL.VideoFromJson(video_json), StreamService.TwitchChatReplay);
if (!AssumeFinished && VideoInfo.VideoRecordingState == RecordingState.Live)
{
_UserInputRequest = new UserInputRequestStreamLive(this);
return(ResultType.TemporarilyUnavailable);
}
string tempname = Path.Combine(Util.TempFolderPath, GetTempFilenameWithoutExtension() + ".json.tmp");
string finalintmpname = Path.Combine(Util.TempFolderPath, GetTempFilenameWithoutExtension() + ".json");
string filename = Path.Combine(Util.TargetFolderPath, GetTargetFilenameWithoutExtension() + ".json");
Random rng = new Random(int.Parse(VideoInfo.VideoId));
if (!await Util.FileExists(filename))
{
if (!await Util.FileExists(finalintmpname))
{
Status = "Downloading chat (Initial)...";
StringBuilder concatJson = new StringBuilder();
string url = GetStartUrl(VideoInfo);
int attemptsLeft = 5;
TimeSpan? lastTimeSpan = new TimeSpan(0);
int nextDelayMilliseconds = 0;
while (true)
{
using (var client = new KeepAliveWebClient())
using (var cancellationCallback = cancellationToken.Register(client.CancelAsync)) {
try {
try {
if (nextDelayMilliseconds != 0)
{
await Task.Delay(nextDelayMilliseconds > 0?nextDelayMilliseconds : rng.Next(90000, 270000), cancellationToken);
}
} catch (TaskCanceledException) {
return(ResultType.Cancelled);
}
string commentJson = await TwitchAPI.GetLegacy(url, Util.TwitchClientId);
JObject responseObject = JObject.Parse(commentJson);
if (responseObject["comments"] == null)
{
throw new Exception("Nonsense JSON returned, no comments.");
}
string offset = "Unknown";
try {
JToken c = ((JArray)responseObject["comments"]).Last;
double val = (double)c["content_offset_seconds"];
TimeSpan ts = TimeSpan.FromSeconds(val);
if (lastTimeSpan != null)
{
TimeSpan diff = ts - lastTimeSpan.Value;
double delay = diff.TotalMilliseconds;
if (delay < 0.0)
{
nextDelayMilliseconds = -1;
}
else if (delay < 90000.0)
{
nextDelayMilliseconds = (int)delay;
}
else if (delay < 270000.0)
{
nextDelayMilliseconds = rng.Next(90000, (int)delay);
}
else
{
nextDelayMilliseconds = -1;
}
}
else
{
nextDelayMilliseconds = -1;
}
lastTimeSpan = ts;
offset = ts.ToString();
} catch (Exception) {
lastTimeSpan = null;
nextDelayMilliseconds = -1;
}
concatJson.Append(commentJson);
if (responseObject["_next"] != null)
{
string next = (string)responseObject["_next"];
attemptsLeft = 5;
Status = "Downloading chat (Last offset: " + offset + "; next file: " + next + ")...";
url = GetNextUrl(VideoInfo, next);
}
else
{
// presumably done?
break;
}
} catch (System.Net.WebException ex) {
Console.WriteLine(ex.ToString());
--attemptsLeft;
Status = "Downloading chat (Error; " + attemptsLeft + " attempts left)...";
if (attemptsLeft <= 0)
{
throw;
}
continue;
}
}
}
await StallWrite(tempname, concatJson.Length * 4, cancellationToken); // size not accurate because encoding but whatever
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
File.WriteAllText(tempname, concatJson.ToString());
File.Move(tempname, finalintmpname);
}
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
Status = "Moving to final location...";
await StallWrite(filename, new FileInfo( finalintmpname ).Length, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
Util.MoveFileOverwrite(finalintmpname, filename);
}
Status = "Done!";
JobStatus = VideoJobStatus.Finished;
return(ResultType.Success);
}
public override async Task <ResultType> Run(CancellationToken cancellationToken)
{
JobStatus = VideoJobStatus.Running;
Status = "Downloading...";
string tempFilename = Path.Combine(Util.TempFolderPath, GetTempFilename());
string movedFilename = Path.Combine(Util.TempFolderPath, GetTargetFilename());
string targetFilename = Path.Combine(Util.TargetFolderPath, GetTargetFilename());
if (!await Util.FileExists(targetFilename))
{
if (!await Util.FileExists(movedFilename))
{
if (await Util.FileExists(tempFilename))
{
await Util.DeleteFile(tempFilename);
}
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
bool success = false;
using (var client = new KeepAliveWebClient())
using (var cancellationCallback = cancellationToken.Register(client.CancelAsync)) {
client.DownloadProgressChanged += new DownloadProgressChangedEventHandler(ReportDownloadProgress);
try {
byte[] data = await client.DownloadDataTaskAsync(VideoInfo.VideoId);
await StallWrite(tempFilename, data.LongLength, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
using (FileStream fs = File.Create(tempFilename)) {
await fs.WriteAsync(data, 0, data.Length);
success = true;
}
} catch (WebException) {
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
throw;
}
}
if (success)
{
await StallWrite(movedFilename, new FileInfo( tempFilename ).Length, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
File.Move(tempFilename, movedFilename);
}
}
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
await StallWrite(targetFilename, new FileInfo( movedFilename ).Length, cancellationToken);
if (cancellationToken.IsCancellationRequested)
{
return(ResultType.Cancelled);
}
File.Move(movedFilename, targetFilename);
}
Status = "Done!";
JobStatus = VideoJobStatus.Finished;
return(ResultType.Success);
}
