public async Task <Int32> TotalCountAsync()
{
// Under the condition of using a "per_page" count of 1 and then taking
// the number of received last page is indeed a trick (or better a hack)
// but it solves the problem of getting the "total number of something".
// But be aware this trick should only be used together with method HEAD.
// Otherwise at least one record is returned by the GitHub API!
RepositoryQueryArguments arguments = new RepositoryQueryArguments
{
Paging = new Paging(null, 1)
};
using (HttpClient client = new HttpClient())
using (HttpRequestMessage request = this.GetRequestMessage(HttpMethod.Head, arguments))
using (HttpResponseMessage response = await client.SendAsync(request))
{
// TODO: Handle limitation results
if (response.IsSuccessStatusCode)
{
if (PaginationParser.TryParse(response, out Pagination pagination))
{
return(pagination.Last.ToIndex());
}
return(0);
}
throw new AccessorRequestException(response);
}
}
public PaginationQueryStringParameterReader(IJsonApiRequest request, IResourceContextProvider resourceContextProvider, IJsonApiOptions options)
: base(request, resourceContextProvider)
{
ArgumentGuard.NotNull(options, nameof(options));
_options = options;
_paginationParser = new PaginationParser(resourceContextProvider);
}
private List <PaginationElementQueryStringValueExpression> ParsePageSizeExpression(string pageSizeParameterValue)
{
if (pageSizeParameterValue == null)
{
return(new List <PaginationElementQueryStringValueExpression>());
}
var requestResource = _request.SecondaryResource ?? _request.PrimaryResource;
var parser = new PaginationParser(_provider);
var paginationExpression = parser.Parse(pageSizeParameterValue, requestResource);
return(paginationExpression.Elements.ToList());
}
protected void ApplyCommonResults(HttpResponseMessage response, QueryArguments arguments, RequestResult result)
{
if (LimitationParser.TryParse(response, out Limitation limitation))
{
result.Limitation = limitation;
}
if (PaginationParser.TryParse(response, out Pagination pagination))
{
// Pagination workaround. Because of the fact that working with pagination
// is a bit more complicated as actually expected, it becomes necessary to
// give back the provided "current" page to the caller. For that reason, take
// requested page and put it into the returned pagination settings.
pagination.Current = new Paging(arguments.Paging?.Index ?? 1, arguments.Paging.Count);
result.Pagination = pagination;
}
}
public PaginationQueryStringParameterReader(ICurrentRequest currentRequest, IResourceContextProvider resourceContextProvider, IJsonApiOptions options)
: base(currentRequest, resourceContextProvider)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_paginationParser = new PaginationParser(resourceContextProvider);
}
public async Task Parse(CancellationToken cancellationToken)
{
// return;
var response = "";
var parser = new MangaParser();
var authors = context.Authors.ToList();
// var translators = context.Translators.ToList();
var tags = context.Tags.ToList();
var types = context.Types.ToList();
var mangas = context.Mangas.ToList();
#region parse Tags, Translators, Types, Authors
response = await client
.GetAsync("https://mangachan.me/tags/", cancellationToken).Result.Content
.ReadAsStringAsync();
parser.cq = CQ.Create(response);
var allTags = parser.ParseAllTags();
var changesTags = allTags.Where(x => tags.All(y => x.Name != y.Name)).ToList();
if (changesTags.Count != 0)
{
await context.Tags.AddRangeAsync(changesTags, cancellationToken);
await context.SaveChangesAsync(cancellationToken);
tags = context.Tags.ToList();
}
response = await client
.GetAsync("https://mangachan.me/type/", cancellationToken).Result.Content
.ReadAsStringAsync();
parser.cq = CQ.Create(response);
var allTypes = parser.ParseAllTypes();
var changesTypes = allTypes.Where(x => types.All(y => x.Name != y.Name)).ToList();
if (changesTypes.Count != 0)
{
await context.Types.AddRangeAsync(changesTypes, cancellationToken);
await context.SaveChangesAsync(cancellationToken);
types = context.Types.ToList();
}
//
// response = await client
// .GetAsync("https://mangachan.me/mangaka/", cancellationToken).Result.Content
// .ReadAsStringAsync();
// var allAuthors = await new MangaParser {cq = CQ.Create(response)}.ParseAllAuthors();
// await context.Authors.AddRangeAsync(allAuthors, cancellationToken);
// await context.SaveChangesAsync(cancellationToken);
// response = await client
// .GetAsync("https://mangachan.me/translation/", cancellationToken).Result.Content
// .ReadAsStringAsync();
// var allTranslators = await new MangaParser {cq = CQ.Create(response)}.ParseAllTranslators();
// var changesTranslators = translators.Where(x => allTranslators.All(y => x.Name != y.Name)).ToList();
// if (changesTypes.Count != 0)
// {
// await context.Translators.AddRangeAsync(changesTranslators, cancellationToken);
// await context.SaveChangesAsync(cancellationToken);
// translators = context.Translators.ToList();
// }
#endregion
response = await client
.GetAsync("https://mangachan.me/catalog", cancellationToken).Result.Content
.ReadAsStringAsync();
var pagination = PaginationParser
.Parse(response);
foreach (var page in pagination)
{
response = await client
.GetAsync($"https://mangachan.me/catalog?offset={page}", cancellationToken).Result.Content
.ReadAsStringAsync();
var mangaListUrLs = parser.ParseMangaList(response);
foreach (var mangaListUrl in mangaListUrLs)
{
response = await client
.GetAsync(mangaListUrl, cancellationToken).Result.Content
.ReadAsStringAsync();
parser = new MangaParser
{
cq = CQ.Create(response)
};
var title = parser.ParseTitle();
if (mangas.Any(x => x.Title == title))
{
continue;
}
var description = parser.ParseDescription();
var image = parser.ParseImage();
var type = parser.ParseType();
var parsedAuthors = parser.ParseAuthors();
var parsedTranslators = parser.ParseTranslators();
var parsedTags = parser.ParseTags();
var parsedChapters = parser.ParseChapters();
var mangaAuthors = new List <Author>();
foreach (var author in parsedAuthors)
{
var containsAuthors = authors.Where(x => x.Name == author.Name).ToList();
if (containsAuthors.Count != 0)
{
mangaAuthors.AddRange(containsAuthors);
}
else
{
mangaAuthors.Add(author);
}
}
var manga = new Manga
{
Title = title,
Description = description,
Image = image,
Type = types.FirstOrDefault(x => x.Name == type.Name),
Tags = parsedTags
.Select(tag =>
new MangaTag
{
Tag = tags.FirstOrDefault(x => x.Name == tag.Name)
}).ToList(),
Authors = mangaAuthors
.Select(author =>
new MangaAuthor
{
Author = context.Authors.FirstOrDefault(x => x.Name == author.Name) ?? author
}).ToList(),
Translators = parsedTranslators
.Select(translator =>
new MangaTranslator
{
Translator = context.Translators.FirstOrDefault(x => x.Name == translator.Name) ??
translator
}).ToList(),
Chapters = parsedChapters
.Select(chapter =>
new Chapter
{
Title = chapter.Result.Title,
UploadDate = chapter.Result.UploadDate,
Order = chapter.Result.Order,
Pages = chapter.Result.Pages
}).ToList()
};
context.Mangas.Add(manga);
await context.SaveChangesAsync(cancellationToken);
Console.WriteLine($"{title} done");
}
}
}
