public async Task <HNArticle> Extract_HN_article(WallabagClient client, WallabagItem item)
{
// Not a HN article
if (!item.Url.Contains("news.ycombinator"))
{
return(null);
}
try
{
string content_url = await Extract_story_link(item.Url);
// Ignore no match posts
if (content_url == null)
{
return(null);
}
Console.Write($"Adding {content_url}");
WallabagItem newItem = await client.AddAsync(
new Uri(content_url),
tags : new string[] { "hacker-news" }
);
if (newItem == null)
{
Console.WriteLine("\nFailed to extract HN article: article add call failed");
Console.WriteLine($"\nURI: {content_url}");
return(null);
}
Console.WriteLine(" ✓");
if (newItem.Url != content_url)
{
Console.WriteLine($"URLs don't match: {content_url} {newItem.Url}");
}
return(new HNArticle
{
SourceWBId = item.Id,
SourceUrl = item.Url,
ContentWBId = newItem.Id,
ContentUrl = content_url
});
}
catch (HttpRequestException e)
{
Console.WriteLine("\nException Caught!");
Console.WriteLine("Message :{0} ", e.Message);
}
return(null);
}
public async Task Process(WallabagClient client, WallabagItem item)
{
foreach (var mapping in mappings)
{
// Not a matching entry
if (!item.Url.Contains(mapping.domain))
{
continue;
}
// Already tagged
if (item.Tags.Any(t => t.Label == mapping.tag))
{
continue;
}
Console.Write(item.Title.Replace("\n", " "));
await client.AddTagsAsync(item, new[] { mapping.tag });
Console.WriteLine($" {mapping.tag} ✓");
}
}
public async Task Process(WallabagClient client, WallabagItem item)
{
using (var db = new WallabagContext())
{
// Filter out HN articles that have already been processed
var exists = db.ProcessedHNArticles.Any(a => a.SourceWBId == item.Id || a.ContentWBId == item.Id);
if (!exists)
{
var hn = await Extract_HN_article(client, item);
if (hn != null)
{
db.ProcessedHNArticles.Add(hn);
var count = await db.SaveChangesAsync();
Console.WriteLine("{0} HN records saved to database", count);
}
}
}
}
public async Task Process(WallabagClient client, WallabagItem item)
{
bool filter_match = false;
String url = null;
foreach (var filter in config.url_filter)
{
// Prefer matches on OriginalUrl
if (!String.IsNullOrWhiteSpace(item.OriginalUrl) && item.OriginalUrl.Contains(filter))
{
filter_match = true;
url = item.OriginalUrl;
break;
}
if (item.Url.Contains(filter))
{
filter_match = true;
url = item.Url;
break;
}
}
// Not from a matching domain
if (!filter_match)
{
return;
}
foreach (var bl in blacklist)
{
if (url.Contains(bl))
{
var oldurl = url;
url = extract_yt_lastditch(url);
if (url == null)
{
Console.WriteLine($"Warning: YoutubeProcessor detected blacklisted pattern; skipping {oldurl}");
}
else
{
Console.WriteLine($"Info: YoutubeProcessor detected blacklisted pattern; extracted {url} from {oldurl}");
}
return;
}
}
// Already tagged
if (item.Tags.Any(t => t.Label == config.tag_name))
{
return;
}
Console.Write(item.Title.Replace("\n", " "));
// Send DL request to Youtube-DL-Server
var content = new FormUrlEncodedContent(new[]
{
new KeyValuePair <string, string>("url", url),
new KeyValuePair <string, string>("format", config.format),
});
try {
var dl_request = await fetcher.PostAsync(config.youtube_dl_server, content);
if (dl_request.StatusCode != System.Net.HttpStatusCode.OK)
{
Console.WriteLine($" ✗ yt-dl-server returned error response");
return;
}
var response = await dl_request.Content.ReadAsStringAsync();
await client.AddTagsAsync(item, new[] { config.tag_name });
if (config.mark_read)
{
await client.ArchiveAsync(item);
}
Console.WriteLine($" ✓");
}
catch (Exception e) {
Console.WriteLine($"Error submitting {url}\n{e.ToString()}\n{e.StackTrace.ToString()}");
}
}
static async Task runOne(WallabagClient client, IEnumerable <IProcessor> processors, WallabagItem item)
{
foreach (var processor in processors)
{
await processor.Process(client, item);
}
}
