public async Task PublishPageInTelegraph(DownloadedPost post)
{
try
{
var result = new List <NodeElement>();
foreach (var paragraph in post.PostContents)
{
result.Add(paragraph);
result.Add(new NodeElement("br", null));
}
var operationResult = await _secureClient.CreatePageAsync(post.Title, result.ToArray(), returnContent : true);
if (!string.IsNullOrWhiteSpace(operationResult.Path))
{
_logger.LogInformation("Post was successfully saved on the API {0}", operationResult.Url);
}
else
{
_logger.LogInformation("Post was not saved on the API {0}", post.Title);
}
}
catch (Exception e)
{
if (e.Message.Contains("FLOOD_WAIT_"))
{
_logger.LogError("Error while storing in the API {0}, waiting 30 seconds.", e.Message);
Thread.Sleep(TimeSpan.FromSeconds(30));
}
}
}
public async Task <List <DownloadedPost> > ProcessPostTask(IEnumerable <string> urls)
{
var angleSharpConfig = Configuration.Default
.WithCulture("es-es")
.WithDefaultLoader()
.WithCss()
.WithJs()
.WithXPath();
var angleSharpContext = BrowsingContext.New(angleSharpConfig);
var result = new List <DownloadedPost>();
try
{
foreach (var url in urls)
{
_logger.LogInformation("Starts processing posts for URL {0}", url);
var mainPageDocument = await angleSharpContext.OpenAsync(url);
var links = await _strategyLoaderService.ParseLinksByStrategy(mainPageDocument);
var groupedResult = links.GroupBy(x => x.Href);
_logger.LogInformation("Got a total of {0} links for URL {1}", groupedResult.Count(), url);
foreach (var link in groupedResult)
{
var downloadedPost = new DownloadedPost();
var postDocument = await angleSharpContext.OpenAsync(link.First().Href);
var title = await _strategyLoaderService.ParseTitleByStrategy(postDocument);
var content = await _strategyLoaderService.ParseContentByStrategy(postDocument);
if (string.IsNullOrWhiteSpace(title))
{
var extractedTitle = string.Empty;
foreach (var words in content.ToList().Select(paragraph => paragraph.SplitSpaces()))
{
extractedTitle = string.Join("", words.Take(7));
if (words.Length == 7)
{
break;
}
}
title = extractedTitle;
}
downloadedPost.Title = title;
downloadedPost.PostContents = content;
result.Add(downloadedPost);
}
}
}
catch (Exception e)
{
_logger.LogError(e, "Rest in pepperoni, innerEx: {0}", e.InnerException);
}
result.RemoveAll(x => !x.PostContents.Any() || x.PostContents.All(string.IsNullOrWhiteSpace));
return(result);
}
