public async void Fetch(string ClientID, string TwitchName, string downloadpath, string jsonpath = null) { string url = $"https://api.twitch.tv/kraken/clips/top?channel={TwitchName}&period=all&trending=false&limit=100"; //change path = downloadpath; try { string cursor = null; string startingcursor = null; var ps = new ProjectSpecific(); if (jsonpath != null) { List <ClipModel> tempj = JsonConvert.DeserializeObject <List <ClipModel> >(File.ReadAllText(jsonpath)); if (tempj != null) { var rcc = await ps.clipCount(tempj); if (tempj.Last()._cursor == "") { clipjson = rcc.Item2; Download(downloadpath, rcc.Item1); return; } clipjson = rcc.Item2; startingcursor = clipjson.First()._cursor; cursor = clipjson.Last()._cursor; } } File.Create($@".\{TwitchName}.json").Close(); Loop: //garbage check, let this slide, brain no work. if (cursor != null) { if (cursor != "") { url = $"{url + "&cursor=" + cursor}"; } } HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); request.Method = "GET"; request.Accept = "application/vnd.twitchtv.v5+json"; request.Headers.Add($"Client-ID: {ClientID}"); var response = await request.GetResponseAsync(); string resp = await new StreamReader(response.GetResponseStream()).ReadToEndAsync(); var json = JsonConvert.DeserializeObject <ClipModel>(resp); if (json.clips.Length == 0 && clipjson.Count == 0) { ps.ConsoleRedX($"Could not locate any clips for {TwitchName}", false); return; } if (json._cursor == startingcursor & cursor != null) { var tresp = await ps.clipCount(clipjson); int clips = tresp.Item1; clipjson = tresp.Item2; Console.Clear(); Console.WriteLine($"Found {clips} clips!... Starting download!"); File.WriteAllText($@".\{clipjson.First().clips.First().broadcaster.name}.json", JsonConvert.SerializeObject(clipjson)); totalclips = clips; Download(downloadpath, clips); return; } //if for some reason the json.clips is empty, don't add it that way there's no chance of empty arrays causing things to break. if (json.clips.Length != 0) { cursor = json._cursor; clipjson.Add(json); startingcursor = json._cursor; foreach (clips clip in json.clips) { ps.ConsoleGreenCheck($"{clip.url}"); } } goto Loop; } catch (WebException ex) { var ps = new ProjectSpecific(); var dt = DateTime.Now; File.WriteAllText($@".\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt", ex.ToString()); if (ex.Status == WebExceptionStatus.ProtocolError) { ps.ConsoleRedX("Invalid client ID or issue with twitch servers!", true); } else { ps.ConsoleRedX($"Oh no! An error occured with twitch! Please screenshot this and provide the crash log located at ({Directory.GetCurrentDirectory() })\\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt)" + $"to masamesa via submitting an issue on github!\r\n" + ex, true); } } catch (Exception ex) { var dt = DateTime.Now; File.WriteAllText($@".\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt", ex.ToString()); new ProjectSpecific().ConsoleRedX($"Oh no! An error occured! Please screenshot this and provide the crash log located at ({Directory.GetCurrentDirectory() })\\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt)" + $"to masamesa via submitting an issue on github!\r\n" + ex, true); } }
public void Download(string path, int lcount, int currentpos = 0) { try { var ps = new ProjectSpecific(); int i = currentpos; foreach (ClipModel model in clipjson) { foreach (clips clip in model.clips) { int retry = 0; i++; current++; string[] url = Regex.Split(clip.thumbnails.medium, "-preview"); url[0] = url[0] + ".mp4"; if (!Regex.IsMatch(url[0], "offset")) { Retry: var htmldoc = new HtmlDocument(); FirefoxOptions op = new FirefoxOptions(); op.AddArgument("--headless"); op.SetLoggingPreference("driver", LogLevel.Off); IWebDriver driver = new FirefoxDriver(op); driver.Navigate().GoToUrl("https://clips.twitch.tv/embed?clip=" + clip.slug); driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(60); htmldoc.LoadHtml(driver.PageSource); driver.Quit(); retry++; var node = htmldoc.DocumentNode.SelectSingleNode("//div[@class='clips-embed-page tw-c-background-base tw-full-height']/div[@class='video-player']/div[@class='tw-absolute tw-bottom-0 tw-left-0 tw-overflow-hidden tw-right-0 tw-top-0 video-player__container']/video/@src"); if (retry == 3) { ps.ConsoleRedX($"Failed to download clip {current}/{totalclips} - {clipjson.First().clips.First().title}", false); clips[] carray = { clipjson.First().clips.First() }; if (File.Exists($@".\{clipjson.First().clips.First().broadcaster.name}-faileddump.json")) { Failedclipsjson = JsonConvert.DeserializeObject <List <ClipModel> >(File.ReadAllText($@".\{clipjson.First().clips.First().broadcaster.name}-faileddump.json")); } Failedclipsjson.Add(new ClipModel { clips = carray, _cursor = "" }); File.WriteAllText($@".\{clipjson.First().clips.First().broadcaster.name}-faileddump.json", JsonConvert.SerializeObject(Failedclipsjson)); clipjson.First().clips = clipjson.First().clips.Where((source, index) => index != 0).ToArray(); goto End; } if (node == null) { goto Retry; } url[0] = node.Attributes[2].Value; } WebClient wc = new WebClient(); string clean = null; //temp fix to remove illegal characters foreach (var c in (new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars()))) { if ($"{clip.broadcaster.name}_{clip.title}_{clip.created_at}".Contains(c.ToString())) { clean = $"{clip.broadcaster.name}_{clip.title}_{clip.created_at}".Replace(c, '-'); } } clean = clean.Replace('<', '-').Replace('>', '-').Replace('?', '-').Replace('|', '-').Replace('*', '-').Replace(':', '-').Replace('"', '_').Replace('\'', '-').Replace('\\', '_').Replace('/', '_').Trim(); wc.DownloadFile(new Uri(url[0]), path + '\\' + clean + ".mp4"); ps.ConsoleGreenCheck($"Downloaded clip {i}/{lcount} - {clip.title} - {clip.url.Split('?')[0]}"); End: if (model.clips.Length != 0) { model.clips = model.clips.Where((source, index) => index != 0).ToArray(); } else { break; } } } if (clipjson.Count != 0) { clipjson.RemoveAt(0); } else { ps.ConsoleGreenCheck("Finished downloading!"); return; } } catch (Exception ex) { try { var ps = new ProjectSpecific(); var dt = DateTime.Now; ps.ConsoleRedX($"Oh no! An error occured! Please screenshot this and provide the crash log located at ({Directory.GetCurrentDirectory() })\\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt)" + $"to masamesa via submitting an issue on github!\r\n" + ex, true); File.WriteAllText($@".\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt", ex.ToString()); //error handling if for whatever reason the clipjson.clips were to be empty. start: if (clipjson.Count != 0 && clipjson.First().clips.Length != 0) { ps.ConsoleRedX($"Failed to download clip {current}/{totalclips} - {clipjson.First().clips.First().title}", false); clips[] carray = { clipjson.First().clips.First() }; if (File.Exists($@".\{clipjson.First().clips.First().broadcaster.name}-faileddump.json")) { Failedclipsjson = JsonConvert.DeserializeObject <List <ClipModel> >(File.ReadAllText($@".\{clipjson.First().clips.First().broadcaster.name}-faileddump.json")); } Failedclipsjson.Add(new ClipModel { clips = carray, _cursor = "" }); File.WriteAllText($@".\{clipjson.First().clips.First().broadcaster.name}-faileddump.json", JsonConvert.SerializeObject(Failedclipsjson)); clipjson.First().clips = clipjson.First().clips.Where((source, index) => index != 0).ToArray(); ps.ConsoleGreenCheck($"Recovered! Using backup {Directory.GetCurrentDirectory() + '\\' + clipjson.First().clips.First().broadcaster.name}-backup.json..."); if (!File.Exists($@".\{clipjson.First().clips.First().broadcaster.name}-backup.json")) { File.Create($@".\{clipjson.First().clips.First().broadcaster.name}-backup.json").Close(); } File.WriteAllText($@".\{clipjson.First().clips.First().broadcaster.name}-backup.json", JsonConvert.SerializeObject(clipjson)); Download(path, totalclips, current); } else if (clipjson.Count != 0 && clipjson.First().clips.Length == 0) { clipjson.RemoveAt(0); goto start; } else { ps.ConsoleRedX("Failed to save and restart! Something went very wrong...", false); } } catch (Exception err) { var ps = new ProjectSpecific(); var dt = DateTime.Now; ps.ConsoleRedX($"EMERGENCY BACKUP MADE.\r\nPLEASE TRY AGAIN AND SUBMIT AN ISSUE ON GITHUB WITH A SCREENSHOT AND THE CRASH REPORT LOCATED AT ({Directory.GetCurrentDirectory() })\\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt), THIS IS VERY ABNORMAL.\r\n{err}", true); File.WriteAllText($@".\CrashReport_{dt.ToString().Replace(':', '-').Replace('/', '-')}.txt", err.ToString() + "\r\nEx:\r\n" + ex.ToString()); ps.ConsoleGreenCheck($"Recovered! Using backup {Directory.GetCurrentDirectory() + '\\' + clipjson.First().clips.First().broadcaster.name}-backup.json..."); if (!File.Exists($@".\{clipjson.First().clips.First().broadcaster.name}-backup.json")) { File.Create($@".\{clipjson.First().clips.First().broadcaster.name}-backup.json").Close(); } File.WriteAllText($@".\{clipjson.First().clips.First().broadcaster.name}-backup.json", JsonConvert.SerializeObject(clipjson)); } } }