private async Task <ICollection <AnimeStreamInfo> > GetAnimeSeriesAsync(string apiToken, CancellationToken cancellationToken)
{
// https://public-api.viewster.com/series?genreId=58&pageSize=25
int pageSize = 100;
int pageIndex = 1;
HashSet <AnimeStreamInfo> streams = new HashSet <AnimeStreamInfo>(new ProjectionEqualityComparer <AnimeStreamInfo, string>(stream => stream.Url));
string baseUrl = "https://public-api.viewster.com/series?genreId=58";
ICollection <AnimeStreamInfo> streamsFromThisRequest;
do
{
string url = baseUrl + string.Format("&pageSize={0}&pageIndex={1}", pageSize, pageIndex);
Console.WriteLine("Getting Viewster anime page {0}.", pageIndex);
WebClientRequest request = new WebClientRequest(url);
request.Headers.Add(new KeyValuePair <string, string>("Auth-token", apiToken));
request.Accept = "application/json";
string json = await _webClient.GetStringAsync(request, cancellationToken).ConfigureAwait(continueOnCapturedContext: false);
streamsFromThisRequest = ParseAnimeJson(json);
streams.UnionWith(streamsFromThisRequest);
pageIndex++;
} while (streamsFromThisRequest.Count == pageSize);
if (streams.Count == 0)
{
throw new Exception("No Viewster streams found, something must be wrong.");
}
return(streams.ToList());
}
public async Task <ICollection <AnimeStreamInfo> > GetAnimeStreamInfoAsync(CancellationToken cancellationToken)
{
Console.WriteLine("Getting HTML for {0}", Url);
WebClientRequest request = new WebClientRequest(Url);
ModifyRequestBeforeSending(request);
string responseBody = await WebClient.GetStringAsync(request, cancellationToken).ConfigureAwait(continueOnCapturedContext: false);
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(responseBody);
HtmlNodeCollection matchingNodes = htmlDoc.DocumentNode.SelectNodes(XPath);
if (matchingNodes == null || matchingNodes.Count == 0)
{
throw new NoMatchingHtmlException(string.Format("Could not extract information from {0}. The site's HTML format probably changed.", Url));
}
List <AnimeStreamInfo> streams = new List <AnimeStreamInfo>();
foreach (HtmlNode matchingNode in matchingNodes)
{
AnimeStreamInfo stream = GetStreamInfoFromMatch(matchingNode);
// Convert possibly relative url to an absolute url
stream = new AnimeStreamInfo(stream.AnimeName, Utils.PossiblyRelativeUrlToAbsoluteUrl(stream.Url, Url), stream.Service);
streams.Add(stream);
}
OnStreamsExtracted(htmlDoc, streams);
return(streams);
}
private async Task <string> GetAPITokenAsync(CancellationToken cancellationToken)
{
WebClientRequest request = new WebClientRequest(InitialPageUrl);
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.UserAgent = "animerecs.com stream update tool";
Console.WriteLine("Getting Viewster API token.");
using (IWebClientResult result = await _webClient.GetAsync(request, cancellationToken).ConfigureAwait(continueOnCapturedContext: false))
{
CookieCollection viewsterCookies = _webClient.Cookies.GetCookies(new Uri("https://www.viewster.com/"));
if (viewsterCookies == null)
{
throw new Exception("No cookie collection for Viewster somehow.");
}
Cookie apiTokenCookie = viewsterCookies["api_token"];
if (apiTokenCookie == null)
{
throw new Exception("Viewster api_token cookie not set on initial page request. Maybe Viewster changed their site.");
}
return(WebUtility.UrlDecode(apiTokenCookie.Value));
}
}
public static async Task <string> GetStringAsync(this IWebClient webClient, WebClientRequest request, CancellationToken cancellationToken)
{
using (IWebClientResult result = await webClient.GetAsync(request, cancellationToken).ConfigureAwait(continueOnCapturedContext: false))
{
return(await result.ReadResponseAsStringAsync(cancellationToken).ConfigureAwait(continueOnCapturedContext: false));
}
}
private async Task <IWebClientResult> DoRequestAsync(WebClientRequest request, HttpMethod method, CancellationToken cancellationToken)
{
using (HttpRequestMessage translatedRequest = TranslateRequest(request, method))
{
HttpResponseMessage response = await _client.SendAsync(translatedRequest, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(continueOnCapturedContext: false);
try
{
if (!response.IsSuccessStatusCode)
{
throw new Exception(string.Format("Request to {0} failed with status code {1}.", request.URL, response.StatusCode));
}
}
catch (Exception)
{
response.Dispose();
throw;
}
return(new WebClientResult(response));
}
}
private HttpRequestMessage TranslateRequest(WebClientRequest request, HttpMethod method)
{
HttpRequestMessage translatedRequest = new HttpRequestMessage(method, request.URL);
if (request.UserAgent != null)
{
if (!translatedRequest.Headers.TryAddWithoutValidation("User-Agent", request.UserAgent))
{
throw new Exception("Failed adding user agent string to request.");
}
}
if (request.Accept != null)
{
if (!translatedRequest.Headers.TryAddWithoutValidation("Accept", request.Accept))
{
throw new Exception("Failed adding Accept header to request.");
}
}
if (request.Headers != null)
{
foreach (KeyValuePair <string, string> header in request.Headers)
{
if (!translatedRequest.Headers.TryAddWithoutValidation(header.Key, header.Value))
{
throw new Exception(string.Format("Failed adding {0} header with value {1} to request.", header.Key, header.Value));
}
}
}
if (method == HttpMethod.Post && request.PostParameters != null && request.PostParameters.Count > 0)
{
FormUrlEncodedContent encodedContent = new FormUrlEncodedContent(request.PostParameters);
translatedRequest.Content = encodedContent;
}
return(translatedRequest);
}
/// <summary>
/// Subclasses can add headers here.
/// </summary>
protected virtual void ModifyRequestBeforeSending(WebClientRequest request)
{
}
/// <summary>
/// Not currently supported.
/// </summary>
/// <param name="request"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public Task <IWebClientResult> PostAsync(WebClientRequest request, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
private IWebClientResult GetSync(WebClientRequest request)
{
_driver.Navigate().GoToUrl(request.URL);
return(new StringWebClientResult(_driver.PageSource));
}
/// <summary>
/// Does not currently honor any of the properties of <paramref name="request"/> other than the URL.
/// Does not currently honor the cancellation token.
/// </summary>
/// <param name="request"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public Task <IWebClientResult> GetAsync(WebClientRequest request, CancellationToken cancellationToken)
{
// Selenium's Chrome Driver support has no async support. So do the request on a separate thread.
return(Task.Factory.StartNew(() => GetSync(request), cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Default));
}
protected override void ModifyRequestBeforeSending(WebClientRequest request)
{
// Hidive returns a 403 forbidden if we don't use a browser's user agent.
request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0";
}
public Task <IWebClientResult> GetAsync(WebClientRequest request, CancellationToken cancellationToken)
{
return(DoRequestAsync(request, HttpMethod.Get, cancellationToken));
}
internal WebPageStreamInfoSource(string url, IWebClient webClient)
{
Request = new WebClientRequest(url);
WebClient = webClient;
}