/// <summary>
/// Logs into Pawoo, generating an access token.
/// </summary>
/// <param name="client"></param>
/// <param name="username"></param>
/// <param name="password"></param>
/// <returns></returns>
private static async Task <ApiKey> GetApiKeyAsync(IDownloaderClient client, string username, string password)
{
if (client.ApiKeys.TryGetValue(_Type, out var key))
{
return(key);
}
var authTokenQuery = new Uri("https://pawoo.net/auth/sign_in");
var authTokenResult = await client.GetHtmlAsync(() => client.GenerateReq(authTokenQuery)).CAF();
if (!authTokenResult.IsSuccess)
{
throw new HttpRequestException("Unable to gather the authenticity token for login.");
}
var input = authTokenResult.Value.DocumentNode.Descendants("input");
var authTokenElement = input.Single(x => x.GetAttributeValue("name", null) == "authenticity_token");
var authToken = authTokenElement.GetAttributeValue("value", null);
if (string.IsNullOrWhiteSpace(authToken))
{
throw new HttpRequestException("Unable to find the authenticity token for login.");
}
using var data = new FormUrlEncodedContent(new Dictionary <string, string>
{
{ "utf8", "✓" },
{ "authenticity_token", authToken },
{ "user[email]", username },
{ "user[password]", password },
});
var loginQuery = new Uri("https://pawoo.net/auth/sign_in");
var loginResult = await client.GetTextAsync(() =>
{
var req = client.GenerateReq(loginQuery, HttpMethod.Post);
req.Content = data;
return(req);
}).CAF();
if (!loginResult.IsSuccess)
{
throw new InvalidOperationException("Unable to login to Pawoo.");
}
//Load a random user's page so we can scrape the API key from it
var query = new Uri("https://pawoo.net/web/timelines/home");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the access token.");
}
const string search = "\"access_token\":\"";
var cut = result.Value.Substring(result.Value.IndexOf(search) + search.Length);
return(client.ApiKeys[_Type] = new ApiKey(cut.Substring(0, cut.IndexOf('"'))));
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetAnimePicturesPostAsync(IDownloaderClient client, string id)
{
var query = new Uri($"https://anime-pictures.net/pictures/view_post/{id}?type=json&lang=en");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
return(result.IsSuccess ? JsonConvert.DeserializeObject <Model>(result.Value) : null);
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetVscoPostAsync(IDownloaderClient client, string id)
{
var query = new Uri($"https://vsco.co/ajxp/{await GetApiKeyAsync(client).CAF()}/2.0/medias/{id}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
return(result.IsSuccess ? JObject.Parse(result.Value)["media"].ToObject <Model>() : null);
}
/// <summary>
/// Logs into pixiv, generating an access token.
/// </summary>
/// <param name="client"></param>
/// <param name="username"></param>
/// <param name="password"></param>
/// <returns></returns>
private static async Task <ApiKey> GetApiKeyAsync(IDownloaderClient client, string username, string password)
{
if (client.ApiKeys.TryGetValue(_Type, out var key) && (key.CreatedAt + key.ValidFor) > DateTime.UtcNow)
{
return(key);
}
var query = new Uri("https://oauth.secure.pixiv.net/auth/token");
var result = await client.GetTextAsync(() =>
{
var req = client.GenerateReq(query, HttpMethod.Post);
req.Content = new FormUrlEncodedContent(new Dictionary <string, string>
{
{ "get_secure_url", "1" },
{ "client_id", _ClientId },
{ "client_secret", _ClientSecret },
{ "grant_type", "password" },
{ "username", username },
{ "password", password },
});
return(req);
}).CAF();
if (!result.IsSuccess)
{
throw new InvalidOperationException("Unable to login to Pixiv.");
}
var jObj = JObject.Parse(result.Value);
var accessToken = jObj["response"]["access_token"].ToObject <string>();
var expiresIn = TimeSpan.FromSeconds(jObj["response"]["expires_in"].ToObject <int>());
return(client.ApiKeys[_Type] = new ApiKey(accessToken, expiresIn));
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetArtstationPostAsync(IDownloaderClient client, string id)
{
var query = new Uri($"https://www.artstation.com/projects/{id}.json");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
return(result.IsSuccess ? JsonConvert.DeserializeObject <Model>(result.Value) : null);
}
/// <inheritdoc />
public async Task <ImageResponse> GetImagesAsync(IDownloaderClient client)
{
//Only one image, so return the one image's url
if (PageCount == 1)
{
return(ImageResponse.FromUrl(ImageUrls["large"]));
}
//Metadata is null so we need to get it again
else if (Metadata == null)
{
var query = new Uri($"https://public-api.secure.pixiv.net/v1/works/{Id}.json" +
$"?access_token={client.ApiKeys[typeof(PixivPostDownloader)]}" +
"&include_stats=1" +
"&include_sanity_level=1" +
"&image_sizes=large" +
"&inclue_metadata=1" +
"&include_content_type=1");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new InvalidOperationException("Unable to use the Pixiv api.");
}
//First b/c returns a list
Metadata = JObject.Parse(result.Value)["response"].First["metadata"].ToObject <PixivPostMetadata>();
}
return(ImageResponse.FromImages(Metadata.Pages.Select(x => x.ImageUrls["large"])));
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetYanderePostAsync(IDownloaderClient client, string id)
{
var query = GenerateYandereQuery($"id:{id}", 0);
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
return(result.IsSuccess ? ParseYanderePosts(result.Value)[0] : null);
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetWeiboPostAsync(IDownloaderClient client, string id)
{
var query = new Uri($"https://m.weibo.cn/api/statuses/show?id={id}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
return(result.IsSuccess ? JsonConvert.DeserializeObject <Model>(result.Value) : null);
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetInstagramPostAsync(IDownloaderClient client, string id)
{
var query = new Uri($"https://www.instagram.com/p/{id}/?__a=1");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
return(result.IsSuccess ? JsonConvert.DeserializeObject <InstagramGraphqlResult>(result.Value).Graphql.ShortcodeMedia : null);
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetPawooPostAsync(IDownloaderClient client, string id)
{
//This API call does not require authentication.
var query = new Uri($"https://www.pawoo.net/api/v1/statuses/{id}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
return(result.IsSuccess ? JsonConvert.DeserializeObject <Model>(result.Value) : null);
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var parsed = new PixivPage();
//Iterate because it's easy and has less strain
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Count >= 100); ++i)
{
token.ThrowIfCancellationRequested();
var query = new Uri($"https://public-api.secure.pixiv.net/v1/users/{UserId}/works.json" +
$"?access_token={await GetApiKeyAsync(client, LoginUsername, LoginPassword).CAF()}" +
$"&page={i + 1}" +
"&per_page=100" +
"&include_stats=1" +
"&include_sanity_level=1" +
"&image_sizes=large"); //This is an array of sizes separated by commas, but we only care about large
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
//If there's an error with the access token, try to get another one
if (result.Value.Contains("access token"))
{
//Means the access token cannot be gotten
if (!client.ApiKeys.ContainsKey(_Type))
{
return;
}
client.ApiKeys.Remove(_Type);
--i; //Decrement since this iteration is useless
continue;
}
return;
}
parsed = JsonConvert.DeserializeObject <PixivPage>(result.Value);
foreach (var post in parsed.Posts)
{
token.ThrowIfCancellationRequested();
if (post.CreatedAt < OldestAllowed)
{
return;
}
if (post.Score < MinScore)
{
continue;
}
//Don't think the API has an endpoint that holds the sizes of every image?
//if (!HasValidSize(post, out _))
//{
// continue;
//}
if (!Add(list, post))
{
return;
}
}
}
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var parsed = new List <Model>();
var validToken = false;
//Iterate to get the next page of results
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Count >= 60); ++i)
{
token.ThrowIfCancellationRequested();
var query = new Uri($"https://api.imgur.com/3/gallery/search/time/all/{i}/" +
$"?client_id={await GetApiKeyAsync(client, validToken).CAF()}" +
$"&q={WebUtility.UrlEncode(Tags)}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
//If there's an error with the api key, try to get another one
if (result.Value.Contains("client_id"))
{
validToken = false;
continue;
}
throw new HttpRequestException("Unable to gather more Imgur posts.\n\n" + result.Value);
}
validToken = true;
parsed = JObject.Parse(result.Value)["data"].ToObject <List <Model> >();
foreach (var post in parsed)
{
token.ThrowIfCancellationRequested();
if (post.CreatedAt < OldestAllowed)
{
return;
}
if (post.Score < MinScore)
{
continue;
}
//Get all images then make sure they're valid
await post.SetAllImages(client).CAF();
foreach (var image in post.Images.Where(x => !HasValidSize(x, out _)).ToList())
{
post.Images.Remove(image);
}
if (post.Images.Count == 0)
{
continue;
}
if (!Add(list, post))
{
return;
}
}
}
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var parsed = new TumblrPage();
//Iterate because the results are in pages
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Posts?.Count > 0); i += parsed.Posts?.Count ?? 0)
{
token.ThrowIfCancellationRequested();
var query = new Uri($"http://{Username}.tumblr.com/api/read/json" +
"?debug=1" +
"&type=photo" +
"&filter=text" +
"&num=50" +
$"&start={i}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
return;
}
parsed = JsonConvert.DeserializeObject <TumblrPage>(result.Value.Split(new[] { '=' }, 2)[1].Trim().TrimEnd(';'));
foreach (var post in parsed.Posts)
{
token.ThrowIfCancellationRequested();
if (post.CreatedAt < OldestAllowed)
{
return;
}
if (post.Score < MinScore)
{
continue;
}
if (post.Photos.Count > 0) //Going into here means there is more than one photo
{
foreach (var photo in post.Photos.Where(x => !HasValidSize(x, out _)).ToList())
{
post.Photos.Remove(photo);
}
if (post.Photos.Count == 0)
{
continue;
}
}
else if (!HasValidSize(post, out _)) //Going into here means there is one photo
{
continue;
}
if (!Add(list, post))
{
return;
}
}
}
}
/// <summary>
/// Gets the id of the Vsco user.
/// </summary>
/// <param name="client"></param>
/// <param name="username"></param>
/// <returns></returns>
public static async Task <ulong> GetUserIdAsync(IDownloaderClient client, string username)
{
var query = new Uri($"https://vsco.co/ajxp/{await GetApiKeyAsync(client).CAF()}/2.0/sites?subdomain={username}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the user's id.");
}
return(JsonConvert.DeserializeObject <VscoUserResults>(result.Value).Users.Single().Id);
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <TwitterOAuthPost> GetTwitterPostAsync(IDownloaderClient client, string id)
{
var query = new Uri($"https://api.twitter.com/1.1/statuses/show.json?id={id}");
var result = await client.GetTextAsync(() =>
{
var req = client.GenerateReq(query);
req.Headers.Add("Authorization", $"Bearer {_Token}");
return(req);
}, TimeSpan.FromMinutes(15)).CAF();
return(result.IsSuccess ? JsonConvert.DeserializeObject <TwitterOAuthPost>(result.Value) : null);
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="url">The url to gather. Can be in fav.me/id or post link format.</param>
/// <returns></returns>
public static async Task <DeviantArtOEmbedPost> GetDeviantArtPostAsync(IDownloaderClient client, Uri url)
{
var query = new Uri($"https://backend.deviantart.com/oembed?url={url}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (result.IsSuccess)
{
var jObj = JObject.Parse(result.Value);
jObj.Add(nameof(DeviantArtOEmbedPost.PostUrl), url);
return(jObj.ToObject <DeviantArtOEmbedPost>());
}
return(null);
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var userId = await GetUserIdAsync(client, Username).CAF();
var parsed = new List <Model>();
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Count >= 10); ++i)
{
token.ThrowIfCancellationRequested();
var query = new Uri("https://m.weibo.cn/api/container/getIndex" +
$"?containerid=230413{userId}_-_longbloglist" +
$"&page={i + 1}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
return;
}
parsed = JObject.Parse(result.Value)["data"]["cards"].Select(x => x["mblog"].ToObject <Model>()).ToList();
foreach (var post in parsed)
{
token.ThrowIfCancellationRequested();
if (post.CreatedAt < OldestAllowed)
{
return;
}
if (post.Score < MinScore)
{
continue;
}
if (post.Pictures == null)
{
continue;
}
//Remove all images that are too small to be downloaded
foreach (var picture in post.Pictures.Where(x => !HasValidSize(x.Large.Geo, out _)).ToList())
{
post.Pictures.Remove(picture);
}
if (post.Pictures.Count == 0)
{
continue;
}
if (!Add(list, post))
{
return;
}
}
}
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var userId = 0UL;
var parsed = new VscoPage();
//Iterate because the results are in pages
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Posts.Count > 0); ++i)
{
token.ThrowIfCancellationRequested();
if (userId == 0UL)
{
userId = await GetUserIdAsync(client, Username).CAF();
}
var query = new Uri($"https://vsco.co/ajxp/{await GetApiKeyAsync(client).CAF()}/2.0/medias" +
$"?site_id={userId}" +
$"&page={i}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
//If there's an error with authorization, try to get a new key
if (result.StatusCode == HttpStatusCode.Unauthorized)
{
client.ApiKeys.Remove(_Type);
--i;
continue;
}
return;
}
parsed = JsonConvert.DeserializeObject <VscoPage>(result.Value);
foreach (var post in parsed.Posts)
{
token.ThrowIfCancellationRequested();
if (post.CreatedAt < OldestAllowed)
{
return;
}
if (!HasValidSize(post, out _))
{
continue;
}
if (!Add(list, post))
{
return;
}
}
}
}
/// <summary>
/// Returns the id of the user.
/// </summary>
/// <param name="client"></param>
/// <param name="username"></param>
/// <returns></returns>
private static async Task <ulong> GetUserIdAsync(IDownloaderClient client, string username)
{
var query = new Uri($"https://pawoo.net/{username}/");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the user id.");
}
const string search = "/api/salmon/";
var cut = result.Value.Substring(result.Value.IndexOf(search) + search.Length);
return(Convert.ToUInt64(cut.Substring(0, cut.IndexOf('\''))));
}
/// <summary>
/// Gets the id of a board.
/// </summary>
/// <returns></returns>
private static async Task <string> GetBoardId(IDownloaderClient client, Uri url)
{
var result = await client.GetTextAsync(() => client.GenerateReq(url)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the board id.");
}
const string search = "\"board_id\": \"";
var cut = result.Value.Substring(result.Value.IndexOf(search) + search.Length);
return(cut.Substring(0, cut.IndexOf('"')));
}
/// <summary>
/// Gets the id of the user.
/// </summary>
/// <param name="client"></param>
/// <param name="username"></param>
/// <returns></returns>
private static async Task <string> GetUserIdAsync(IDownloaderClient client, string username)
{
var query = new Uri($"https://www.flickr.com/photos/{username}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the Flickr user's id.");
}
const string search = "\"ownerNsid\":\"";
var cut = result.Value.Substring(result.Value.IndexOf(search) + search.Length);
return(cut.Substring(0, cut.IndexOf('"')));
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="username"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetTumblrPostAsync(IDownloaderClient client, string username, string id)
{
var query = new Uri($"http://{username}.tumblr.com/api/read/json?debug=1&id={id}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (result.IsSuccess)
{
var post = JObject.Parse(result.Value.Split(new[] { '=' }, 2)[1].Trim().TrimEnd(';'))["posts"].First;
//If the id doesn't match, then that means it just got random values and the id is invalid
if (post["id"].ToString() == id)
{
return(post.ToObject <Model>());
}
}
return(null);
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var parsed = new ArtstationPage();
//Iterate to get the next page of results
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Posts.Count >= 50); ++i)
{
token.ThrowIfCancellationRequested();
var query = new Uri($"https://www.artstation.com/users/{Username}/projects.json?page={i}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
return;
}
parsed = JsonConvert.DeserializeObject <ArtstationPage>(result.Value);
foreach (var post in parsed.Posts)
{
token.ThrowIfCancellationRequested();
var fullPost = await GetArtstationPostAsync(client, post.Id).CAF();
if (fullPost.CreatedAt < OldestAllowed)
{
return;
}
if (fullPost.Score < MinScore)
{
continue;
}
//Remove all images that don't meet the size requirements
foreach (var image in fullPost.Assets.Where(x => x.AssetType != "image" || !HasValidSize(x, out _)).ToList())
{
fullPost.Assets.Remove(image);
}
if (fullPost.Assets.Count == 0)
{
continue;
}
if (!Add(list, fullPost))
{
return;
}
}
}
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetPinterestPostAsync(IDownloaderClient client, string id)
{
var options = new Dictionary <string, object>
{
{ "id", id },
{ "field_set_key", "detailed" },
};
const string endpoint = "/PinResource/get/";
var result = await client.GetTextAsync(() =>
{
var req = client.GenerateReq(GenerateQuery(endpoint, options));
req.Headers.Add("X-Requested-With", "XMLHttpRequest");
return(req);
}).CAF();
return(result.IsSuccess ? JObject.Parse(result.Value)["resource_response"]["data"].ToObject <Model>() : null);
}
/// <summary>
/// Gets the post with the specified id.
/// </summary>
/// <param name="client"></param>
/// <param name="id"></param>
/// <returns></returns>
public static async Task <Model> GetFlickrPostAsync(IDownloaderClient client, string id)
{
var query = new Uri($"{await GenerateApiQueryAsync(client, 0).CAF()}&method=flickr.photos.getInfo&photo_id={id}");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
return(null);
}
return(JObject.Parse(result.Value)["photo"].ToObject <Model>(JsonSerializer.Create(new JsonSerializerSettings
{
Error = (obj, e) =>
{
//Eat any exceptions for unexpected start objects, since json doesn't like unmapped nested objects
e.ErrorContext.Handled = e.ErrorContext.Error.ToString().Contains("StartObject. Path 'photo.");
},
})));
}
/// <summary>
/// Gets an api key for Vsco.
/// </summary>
/// <param name="client"></param>
/// <returns></returns>
public static async Task <ApiKey> GetApiKeyAsync(IDownloaderClient client)
{
if (client.ApiKeys.TryGetValue(_Type, out var key))
{
return(key);
}
var time = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var query = new Uri($"https://vsco.co/content/Static/userinfo?callback=jsonp_{time}_0");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the api key.");
}
var cookie = client.Cookies.GetCookies(query)["vs"].Value;
return(client.ApiKeys[_Type] = new ApiKey(cookie));
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var userId = await GetUserIdAsync(client, Username).CAF();
var parsed = new List <Model>();
//Iterate becasue there's a limit of 20 per page
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Count >= 20); ++i)
{
token.ThrowIfCancellationRequested();
var query = new Uri("https://bcy.net/home/timeline/loaduserposts" +
$"?since={(parsed.Count > 0 ? parsed.Last().Id : "0")}" +
$"&uid={userId}" +
"&limit=20" +
"&source=all" +
"&filter=origin");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
return;
}
parsed = JObject.Parse(result.Value)["data"].Select(x => x["item_detail"].ToObject <Model>()).ToList();
foreach (var post in parsed)
{
token.ThrowIfCancellationRequested();
if (post.CreatedAt < OldestAllowed)
{
return;
}
//First check indicates the post doesn't have any images
//Due to the way this site's api is set up can't check image sizes
if (post.PicNum < 1 || post.Score < MinScore)
{
continue;
}
if (!Add(list, post))
{
return;
}
}
}
}
/// <summary>
/// Gets an api key for Flickr.
/// </summary>
/// <param name="client"></param>
/// <returns></returns>
private static async Task <ApiKey> GetApiKeyAsync(IDownloaderClient client)
{
if (client.ApiKeys.TryGetValue(_Type, out var key))
{
return(key);
}
var query = new Uri("https://www.flickr.com");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the Flickr api key.");
}
const string search = "api.site_key = \"";
var cut = result.Value.Substring(result.Value.IndexOf(search) + search.Length);
return(client.ApiKeys[_Type] = new ApiKey(cut.Substring(0, cut.IndexOf('"'))));
}
/// <summary>
/// Gets an api key for Instagram.
/// </summary>
/// <param name="client"></param>
/// <returns></returns>
private static async Task <ApiKey> GetApiKeyAsync(IDownloaderClient client)
{
if (client.ApiKeys.TryGetValue(_Type, out var key))
{
return(key);
}
//Load the page regularly first so we can get some data from it
var query = new Uri("https://www.instagram.com/instagram/?hl=en");
var result = await client.GetHtmlAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
throw new HttpRequestException("Unable to get the first request to the user's account.");
}
//Find the direct link to ProfilePageContainer.js
var jsLink = result.Value.DocumentNode.Descendants("link")
.Select(x => x.GetAttributeValue("href", ""))
.First(x => x.Contains("ProfilePageContainer.js"));
var jsQuery = new Uri($"https://www.instagram.com{jsLink}");
var jsResult = await client.GetTextAsync(() => client.GenerateReq(jsQuery)).CAF();
if (!jsResult.IsSuccess)
{
throw new HttpRequestException("Unable to get the request to the Javascript holding the query hash.");
}
//Read ProfilePageContainer.js and find the query id
//There are multiple query ids in this file, and Instagram likes changing the location of each valid one,
//so we have to do some searching with long strings and regex
//(o=e.profilePosts.byUserId.get(t))||void 0===o?void 0:o.pagination},queryId:\"
//(n=e.profilePosts.byUserId.get(t))||void 0===n?void 0:n.pagination},queryId:\"
const string alpha = "[a-zA-Z]";
var qSearch = $@"\({alpha}={alpha}\.profilePosts\.byUserId\.get\(t\)\)\|\|void 0==={alpha}\?void 0:{alpha}\.pagination}},queryId:""";
var qMatch = Regex.Matches(jsResult.Value, qSearch).Cast <Match>().Single();
var qCut = jsResult.Value.Substring(qMatch.Index + qMatch.Length);
var q = qCut.Substring(0, qCut.IndexOf('"'));
return(client.ApiKeys[_Type] = new ApiKey(q));
}
/// <inheritdoc />
protected override async Task GatherAsync(IDownloaderClient client, List <IPost> list, CancellationToken token)
{
var parsed = new List <Model>();
for (var i = 0; list.Count < AmountOfPostsToGather && (i == 0 || parsed.Count >= 24); ++i, parsed.Clear())
{
token.ThrowIfCancellationRequested();
var query = new Uri($"https://www.zerochan.net/{WebUtility.UrlEncode(Tags)}" +
"?s=id" +
$"&p={i + 1}" +
"&json");
var result = await client.GetTextAsync(() => client.GenerateReq(query)).CAF();
if (!result.IsSuccess)
{
return;
}
foreach (var item in JObject.Parse(result.Value)["items"])
{
token.ThrowIfCancellationRequested();
var post = await GenerateModel(client, item).CAF();
parsed.Add(post);
if (post.CreatedAt < OldestAllowed)
{
return;
}
if (!HasValidSize(post, out _) || post.Score < MinScore)
{
continue;
}
if (!Add(list, post))
{
return;
}
}
}
}
