private EverythingRequest GenerateEverythingRequest()
{
/*
* Creates Everything Request between selected dates
*/
EverythingRequest request = new EverythingRequest();
request.Q = "spotify";
request.Language = Languages.EN;
request.SortBy = SortBys.PublishedAt;
if (Check_ToDate.Enabled && Check_FromDate.Enabled)
{
if (ToDate.Value.Date < FromDate.Value.Date)
{
ToDate.Value = ToDate.MaxDate;
}
}
if (Check_FromDate.Checked)
{
request.From = FromDate.Value.Date;
}
if (Check_ToDate.Checked)
{
request.To = ToDate.Value.Date;
}
return(request);
}
public ArticlesResult GetEverything()
{
var everythingRequest = new EverythingRequest();
var articlesResult = _client.GetEverything(everythingRequest);
return(articlesResult);
}
public void EverythingRequestWithDomainsWorks()
{
var everythingRequest = new EverythingRequest
{
Domains = new List <String>(new[] { "wsj.com", "nytimes.com" })
};
var result = NewsApiClient.GetEverything(everythingRequest);
Assert.Equal(Statuses.Ok, result.Status);
Assert.True(result.TotalResults > 0);
Assert.True(result.Articles.Count > 0);
Assert.Null(result.Error);
}
public async Task Execute(IJobExecutionContext context)
{
using (var scope = _serviceProvider.CreateScope())
{
var dbContext = scope.ServiceProvider.GetService <ApplicationDbContext>();
var client = scope.ServiceProvider.GetService <NewsApiClient>();
var websites = dbContext.Websites.Include(item => item.Articles);
foreach (var website in websites)
{
var request = new EverythingRequest
{
Domains = new List <string> {
website.Url
},
Q = "*",
SortBy = SortBys.Popularity,
From = website.LastMonitoringDate,
Language = Languages.UK,
};
var result = await client.GetEverythingAsync(request);
if (result.Status == Statuses.Ok)
{
website.LastMonitoringDate = DateTime.UtcNow;
website.LastPublishedArticleCount = result.TotalResults;
website.Articles.AddRange(result.Articles.Select(item => new Data.Article
{
Author = item.Author,
Description = item.Description,
ImageUrl = item.UrlToImage,
PublishedAt = item.PublishedAt,
Source = item.Source.Name,
Title = item.Title,
Url = item.Url
}));
await dbContext.SaveChangesAsync();
//todo: send request on Vlad API
//var articles = result.Articles.Select(item => new ArticleSchema
//{
// Title = item.Title,
// Text = item.Description,
// Source = item.Source.Name
//}).ToList();
}
}
}
}
public void BasicEverythingRequestWorks()
{
var everythingRequest = new EverythingRequest
{
Q = "bitcoin"
};
var result = NewsApiClient.GetEverything(everythingRequest);
Assert.Equal(Statuses.Ok, result.Status);
Assert.True(result.TotalResults > 0);
Assert.True(result.Articles.Count > 0);
Assert.Null(result.Error);
}
public ArticlesResult GetEverything(EverythingRequest requestParams)
{
ValidateParams(requestParams);
var result = ApiConnector.GetEverything(requestParams);
//Limits the description of each article to 250 characters
foreach (var article in result.Articles)
{
if (!(article.Description is null) && article.Description.Length > 250)
{
article.Description = article.Description.Substring(0, 250) + "...";
}
}
return(result);
}
public void ComplexEverythingRequestWorks()
{
var everythingRequest = new EverythingRequest
{
Q = "apple",
SortBy = SortBys.PublishedAt,
Language = Languages.EN
};
var result = NewsApiClient.GetEverything(everythingRequest);
Assert.Equal(Statuses.Ok, result.Status);
Assert.True(result.TotalResults > 0);
Assert.True(result.Articles.Count > 0);
Assert.Null(result.Error);
}
public void BadEverythingRequestReturnsError()
{
var everythingRequest = new EverythingRequest
{
Q = "bitcoin"
};
var brokenClient = new NewsApiClient("nokey");
var result = brokenClient.GetEverything(everythingRequest);
Assert.Equal(Statuses.Error, result.Status);
Assert.Null(result.Articles);
Assert.NotNull(result.Error);
Assert.Equal(ErrorCodes.ApiKeyInvalid, result.Error.Code);
}
private void ValidateParams(EverythingRequest requestParams)
{
if (requestParams.Domains.Count == 0 &&
requestParams.Q is null &&
requestParams.Sources.Count == 0)
{
throw new EverythingRequestInputException(EVERYTHING_REQUEST_MISSING_INFO_MESSAGE);
}
if (requestParams.PageSize <= 0 || requestParams.PageSize > 100)
{
throw new EverythingRequestInputException(PAGE_SIZE_VIOLATION);
}
if (requestParams.Page <= 0)
{
throw new EverythingRequestInputException(PAGE_NUMBER_VIOLATION);
}
}
public async Task <IActionResult> GetNewsEverything([FromQuery] EverythingRequest requestEverything)
{
var invokingUser = int.Parse(User.FindFirst(claim => claim.Type == ClaimTypes.NameIdentifier).Value);
var newsApiClient = new NewsApiClient(_configuration.GetSection("ApiKey").Value);
var articlesResponse = await newsApiClient.GetEverythingAsync(requestEverything);
if (articlesResponse.Status == Statuses.Ok)
{
var queryObject = _mapper.Map <EverythingRequest, QueryObjectEverythingRequest>(requestEverything);
await _repository.PostQueryObjectEverythingRequestAsync(invokingUser, queryObject);
var articlesResponseDto = _mapper.Map <ArticlesResult, ArticlesResultViewModel>(articlesResponse);
return(Ok(articlesResponseDto));
}
return(BadRequest(articlesResponse.Error.Message));
}
///Returns list of Articles
public static ArticlesResult GetEverything(EverythingRequest reqParams)
{
try {
var newsApiClient = new NewsApiClient(apiKey);
var articleresult = newsApiClient.GetEverything(reqParams);
if (articleresult.Status == Statuses.Ok)
{
return(articleresult);
}
else
{
throw new ApiResponseException(articleresult.Error.Message);
}
} catch (AggregateException e)
{
throw new ApiResponseException(e.Message);
}
}
public static EverythingRequest DTOToEverything(EverythingParamsDTO requestParamsDTO)
{
EverythingRequest requestParam = new EverythingRequest();
requestParam.Q = requestParamsDTO.Keywords;
requestParam.Page = requestParamsDTO.Page;
requestParam.PageSize = requestParamsDTO.PageSize;
requestParam.From = requestParamsDTO.From;
requestParam.To = requestParamsDTO.To;
if (!(requestParamsDTO.Sources is null))
{
requestParam.Sources = requestParamsDTO.Sources.Split(",").ToList();
}
if (!(requestParamsDTO.Domains is null))
{
requestParam.Domains = requestParamsDTO.Domains.Split(",").ToList();
}
if (!(requestParamsDTO.Language is null))
{
if (Enum.TryParse(requestParamsDTO.Language.ToUpper(), out Languages language))
{
requestParam.Language = language;
}
else
{
throw new EverythingRequestInputException(string.Format(
NO_SUCH_LANGUAGE_CODE, requestParamsDTO.Language));
}
}
if (!(requestParamsDTO.SortBy is null))
{
if (Enum.TryParse(requestParamsDTO.SortBy, out SortBys sortBy))
{
requestParam.SortBy = sortBy;
}
else
{
throw new EverythingRequestInputException(string.Format(
NO_SUCH_SORT_BY, requestParamsDTO.SortBy));
}
}
return(requestParam);
}
public async Task <List <Article> > SearchAsync(string query, DateTime?from, DateTime?to, string lang = "EN", params string[] sources)
{
int count = 100; //limited for for free API
EverythingRequest req = new EverythingRequest()
{
Q = query,
From = from,
To = to,
Language = (Languages)Enum.Parse(typeof(Languages), lang),
PageSize = 100,
Sources = sources.ToList()
};
var result = await Client.GetEverythingAsync(req);
if (result.Status != Statuses.Ok)
{
throw new Exception();
}
List <Article> articles = new List <Article>();
articles.AddRange(GetArticlesFromResult(result));
if (count > 100 && result.TotalResults > 100)
{
int pages = result.TotalResults / 100 + 1;
while (++req.Page <= pages)
{
var r = await Client.GetEverythingAsync(req);
if (r.Status != Statuses.Ok)
{
Error("NewsAPI returned status {0} for page {1} of {2}.", r.Status, req.Page, pages);
continue;
}
articles.AddRange(GetArticlesFromResult(r));
}
}
return(articles);
}
public async Task <ArticlesResult> GetNewsByTopicAsync(string topic, List <string> domains, List <string> sources, DateTime?from, DateTime?to, Languages?language)
{
var request = new EverythingRequest();
if (!string.IsNullOrEmpty(topic) && !string.IsNullOrWhiteSpace(topic))
{
request.Q = topic;
}
if (domains != null)
{
request.Domains = domains;
}
if (sources != null)
{
request.Sources = sources;
}
request.From = from;
request.To = to;
request.Language = language;
request.SortBy = SortBys.Relevancy;
return(await newsApiClient.GetEverythingAsync(request));
}
/// <summary>
/// Query the /v2/everything endpoint for recent articles all over the web.
/// </summary>
/// <param name="request">The params and filters for the request.</param>
/// <returns></returns>
public ArticlesResult GetEverything(EverythingRequest request)
{
return(GetEverythingAsync(request).Result);
}
/// <summary>
/// Query the /v2/everything endpoint for recent articles all over the web.
/// </summary>
/// <param name="request">The params and filters for the request.</param>
/// <returns></returns>
public async Task <ArticlesResult> GetEverythingAsync(EverythingRequest request)
{
// build the querystring
var queryParams = new List <string>();
// q
if (!string.IsNullOrWhiteSpace(request.Q))
{
queryParams.Add("q=" + request.Q);
}
// qInTitle
if (!string.IsNullOrWhiteSpace(request.QInTitle))
{
queryParams.Add("qInTitle=" + request.QInTitle);
}
// sources
if (request.Sources.Count > 0)
{
queryParams.Add("sources=" + string.Join(",", request.Sources));
}
// domains
if (request.Domains.Count > 0)
{
queryParams.Add("domains=" + string.Join(",", request.Domains));
}
// from
if (request.From.HasValue)
{
queryParams.Add("from=" + string.Format("{0:s}", request.From.Value));
}
// to
if (request.To.HasValue)
{
queryParams.Add("to=" + string.Format("{0:s}", request.To.Value));
}
// language
if (request.Language.HasValue)
{
queryParams.Add("language=" + request.Language.Value.ToString().ToLowerInvariant());
}
// sortBy
if (request.SortBy.HasValue)
{
queryParams.Add("sortBy=" + request.SortBy.Value.ToString());
}
// page
if (request.Page > 1)
{
queryParams.Add("page=" + request.Page);
}
// page size
if (request.PageSize > 0)
{
queryParams.Add("pageSize=" + request.PageSize);
}
// join them together
var querystring = string.Join("&", queryParams.ToArray());
return(await MakeRequest("everything", querystring));
}
public static async Task <IEnumerable <Article> > GetArticlesAsync(EverythingRequest request)
{
var articles = await client.GetEverythingAsync(request);
return(articles.Articles.AsEnumerable());
}
