private static async Task <bool> TryUpdateArticleData(ArticlePost articlePost, Reddit r, CloudTable articleTable, ILogger log)
{
bool result = false;
Comment comment = r.GetComment(new Uri("https://www.reddit.com" + articlePost.CommentUri));
Post post = (Post)comment.Parent;
string articleUrl = post.Url.GetComponents(UriComponents.Host | UriComponents.Path | UriComponents.Scheme, UriFormat.SafeUnescaped);
using (HttpResponseMessage articleResponse = await client.GetAsync(articleUrl))
{
SeattleTimesArticle seattleTimesArticle = new SeattleTimesArticle(articleResponse);
if (seattleTimesArticle.Headline != articlePost.ArticleHeadline)
{
log.LogInformation("new headline: " + seattleTimesArticle.Headline);
articlePost.ArticleHeadline = seattleTimesArticle.Headline;
result = true;
}
if (seattleTimesArticle.ByLineAuthors.FirstOrDefault() != articlePost.ArticleAuthor)
{
string author = seattleTimesArticle.ByLineAuthors.FirstOrDefault();
log.LogInformation("new author: " + author);
articlePost.ArticleAuthor = author;
result = true;
}
if (result)
{
await articleTable.ExecuteAsync(TableOperation.InsertOrReplace(articlePost));
}
}
return(result);
}
public ArticlePost(SeattleTimesArticle seattleTimesArticle, Comment comment)
{
ArticleAuthor = seattleTimesArticle.ByLineAuthors.First();
ArticleHeadline = seattleTimesArticle.Headline;
ArticleDate = seattleTimesArticle.PublishDate.Date;
CommentUri = comment.Permalink;
PartitionKey = comment.Subreddit;
RowKey = comment.Id;
}
public void TestMethod2()
{
SeattleTimesArticle article = null;
using (HttpClient client = new HttpClient())
using (HttpResponseMessage message = client.GetAsync(new Uri("https://www.seattletimes.com/life/outdoors/starter-kit-what-you-need-to-survive-seattles-rain/")).Result)
{
article = new SeattleTimesArticle(message);
}
Assert.IsNotNull(article);
Assert.IsNotNull(article.Headline);
Assert.IsNotNull(article.ByLineAuthors);
}
public void TestMethod1()
{
SeattleTimesArticle article = null;
using (HttpClient client = new HttpClient())
using (HttpResponseMessage message = client.GetAsync(new Uri("https://www.seattletimes.com/business/retail/dying-fad-fewer-stores-are-staying-open-on-thanksgiving-day-as-online-shopping-takes-off/")).Result)
{
article = new SeattleTimesArticle(message);
}
Assert.IsNotNull(article);
Assert.IsNotNull(article.Headline);
Assert.IsNotNull(article.ByLineAuthors);
}
private static async Task <bool> ProcessPost(Post p, WaybackClient waybackClient, ILogger log, CloudTable articleTable, NewsBankClient newsBankClient)
{
bool successProcessPost = true;
try
{
if (!Debug)
{
p.Hide();
}
Uri archivedUrl = null;
log.LogInformation(p.Url.ToString());
Uri target = new Uri(p.Url.GetComponents(UriComponents.Host | UriComponents.Path | UriComponents.Scheme, UriFormat.SafeUnescaped));
using (Task <AvailableResponse> response = waybackClient.AvailableAsync(target))
using (Task <HttpResponseMessage> targetGetResponse = client.GetAsync(target))
{
Task <Comment> commentTask = response.ContinueWith(async x =>
{
AvailableResponse availableResponse = x.Result;
short attempts = 2;
bool success = false;
do
{
attempts--;
if (availableResponse?.archived_snapshots?.closest?.available == true)
{
archivedUrl = availableResponse.archived_snapshots.closest.url;
log.LogInformation("using available snapshot.");
success = true;
}
else
{
log.LogInformation("creating snapshot.");
archivedUrl = await waybackClient.SaveAsync(target);
short validationAttempts = 2;
do
{
validationAttempts--;
using (HttpResponseMessage responseCheck = await client.GetAsync(archivedUrl))
{
if (!responseCheck.IsSuccessStatusCode || responseCheck.StatusCode == HttpStatusCode.NotFound)
{
log.LogWarning($"404 returned from archive.org using provided response url. \nstatuscode:{responseCheck.StatusCode} \narchiveURL:{archivedUrl}");
Thread.Sleep(100);
}
else
{
log.LogInformation("check returned success.");
success = true;
}
}
} while (validationAttempts > 0 && !success);
}
} while (attempts > 0 && !success);
if (!success)
{
successProcessPost = false;
throw new ApplicationException("Wayback machine wouldn't cache content.");
}
string msg =
$@"[Archive.org version.]({archivedUrl.ToString()})
:0:
----
^^You ^^can ^^support ^^Archive.org ^^via [^^(Amazon) ^^(Smile)](https://smile.amazon.com/ch/94-3242767)
^^You ^^can ^^support ^^Seattle ^^Public ^^Library ^^via [^^(Amazon) ^^(Smile)](https://smile.amazon.com/ch/91-1140642)
^^I'm ^^a ^^bot, ^^beep ^^boop [ ^^((fork) ^^(me) ^^(on) ^^(github))](https://github.com/czf/ArchiveBot)";
log.LogInformation(msg);
Comment comment = null;
if (!Debug)
{
comment = p.Comment(msg);
}
return(comment);
}, TaskContinuationOptions.OnlyOnRanToCompletion).Unwrap();
Comment c = await commentTask;
await Task.WhenAll(targetGetResponse, commentTask).ContinueWith(
x =>
{
log.LogInformation("start newsbank");
Comment comment = commentTask.Result;
using (HttpResponseMessage articleResponse = targetGetResponse.Result)
{
if (articleResponse == null)
{
log.LogInformation("articleResponse is null");
}
SeattleTimesArticle seattleTimesArticle = new SeattleTimesArticle(articleResponse);
if (seattleTimesArticle.PublishDate.Date < DateTime.Now.Date)
{
log.LogInformation("article post is at least a day old, will make newsbank edit.");
EditForNewsbank.GetCommentLine(new ArticlePost(seattleTimesArticle, comment), log, newsBankClient
).ContinueWith(y =>
{
if (!String.IsNullOrEmpty(y.Result))
{
EditForNewsbank.EditComment(y.Result, comment);
log.LogInformation("article post has been edited.");
}
else
{
log.LogInformation("commentline null or empty will store article post");
articleTable.Execute(TableOperation.InsertOrReplace(new ArticlePost(seattleTimesArticle, comment)));
}
});
}
else
{
log.LogInformation("will store article post");
articleTable.Execute(TableOperation.InsertOrReplace(new ArticlePost(seattleTimesArticle, comment)));
}
}
}, TaskContinuationOptions.OnlyOnRanToCompletion);
//TODO Dispose AvailableResponse;
}
}
catch (Exception e)
{
log.LogError("", e);
successProcessPost = false;
}
return(successProcessPost);
}
private static async Task ProcessPost(Post p, WaybackClient waybackClient, TraceWriter log, CloudTable articleTable, NewsBankClient newsBankClient)
{
try
{
if (!Debug)
{
p.Hide();
}
Uri archivedUrl = null;
log.Info(p.Url.ToString());
Uri target = new Uri(p.Url.GetComponents(UriComponents.Host | UriComponents.Path | UriComponents.Scheme, UriFormat.SafeUnescaped));
using (Task <AvailableResponse> response = waybackClient.AvailableAsync(target))
using (Task <HttpResponseMessage> targetGetResponse = client.GetAsync(target))
{
Comment commentResult = null;
Task <Comment> commentTask = response.ContinueWith(async x =>
{
AvailableResponse availableResponse = x.Result;
int attempts = 2;
bool success = false;
do
{
attempts--;
if (availableResponse?.archived_snapshots?.closest?.available == true)
{
archivedUrl = availableResponse.archived_snapshots.closest.url;
log.Info("using available snapshot.");
success = true;
}
else
{
log.Info("creating snapshot.");
archivedUrl = await waybackClient.SaveAsync(target);
using (HttpResponseMessage responseCheck = await client.GetAsync(archivedUrl))
{
if (!responseCheck.IsSuccessStatusCode || responseCheck.StatusCode == HttpStatusCode.NotFound)
{
log.Warning($"404 returned from archive.org using provided response url. \nstatuscode:{responseCheck.StatusCode} \narchiveURL:{archivedUrl}");
}
else
{
log.Info("check returned success.");
success = true;
}
}
}
} while (attempts > 0 && !success);
if (!success)
{
throw new ApplicationException("Wayback machine wouldn't cache content.");
}
string msg =
$@"[Archive.org version.]({archivedUrl.ToString()})
:0:
----
^^I'm ^^a ^^bot, ^^beep ^^boop";
log.Info(msg);
Comment comment = null;
if (!Debug)
{
comment = p.Comment(msg);
}
return(comment);
}, TaskContinuationOptions.OnlyOnRanToCompletion).Unwrap();
await Task.WhenAll(targetGetResponse, commentTask).ContinueWith(
x =>
{
Comment comment = commentTask.Result;
using (HttpResponseMessage articleResponse = targetGetResponse.Result)
{
SeattleTimesArticle seattleTimesArticle = new SeattleTimesArticle(articleResponse);
if (seattleTimesArticle.PublishDate.Date < DateTime.Now.Date)
{
log.Info("article post is at least a day old, will make newsbank edit.");
EditForNewsbank.GetCommentLine(new ArticlePost(seattleTimesArticle, comment), log, newsBankClient
).ContinueWith(y => {
if (!String.IsNullOrEmpty(y.Result))
{
EditForNewsbank.EditComment(y.Result, comment);
log.Info("article post has been edited.");
}
else
{
log.Info("commentline null or empty will store article post");
articleTable.Execute(TableOperation.InsertOrReplace(new ArticlePost(seattleTimesArticle, comment)));
}
}
);
}
else
{
log.Info("will store article post");
articleTable.Execute(TableOperation.InsertOrReplace(new ArticlePost(seattleTimesArticle, comment)));
}
}
}, TaskContinuationOptions.OnlyOnRanToCompletion);
//TODO Dispose AvailableResponse;
}
}
catch (Exception e)
{
log.Error("", e);
throw;
}
}
