static async Task run(WallabagClient client, IEnumerable <IProcessor> processors, int poll_duration)
{
var valid_processors = processors.Where(proc => proc.should_run);
// Run all existing items
var list = await GetAllEntries(client);
list.ForEach(async item =>
{
await runOne(client, valid_processors, item);
});
// Get max Wallabag ID from list
var maxWBId = list.MaxBy(i => i.Id).First().Id + 1;
Console.WriteLine($"Next Wallabag ID: {maxWBId}");
// Poll for new items & run them
while (true)
{
var item = await client.GetItemAsync(maxWBId);
if (item == null)
{
await Task.Delay(poll_duration);
continue;
}
await runOne(client, valid_processors, item);
maxWBId++;
Console.WriteLine($"Next Wallabag ID: {maxWBId}");
}
}
static async Task runOne(WallabagClient client, IEnumerable <IProcessor> processors, WallabagItem item)
{
foreach (var processor in processors)
{
await processor.Process(client, item);
}
}
public async Task <HNArticle> Extract_HN_article(WallabagClient client, WallabagItem item)
{
// Not a HN article
if (!item.Url.Contains("news.ycombinator"))
{
return(null);
}
try
{
string content_url = await Extract_story_link(item.Url);
// Ignore no match posts
if (content_url == null)
{
return(null);
}
Console.Write($"Adding {content_url}");
WallabagItem newItem = await client.AddAsync(
new Uri(content_url),
tags : new string[] { "hacker-news" }
);
if (newItem == null)
{
Console.WriteLine("\nFailed to extract HN article: article add call failed");
Console.WriteLine($"\nURI: {content_url}");
return(null);
}
Console.WriteLine(" ✓");
if (newItem.Url != content_url)
{
Console.WriteLine($"URLs don't match: {content_url} {newItem.Url}");
}
return(new HNArticle
{
SourceWBId = item.Id,
SourceUrl = item.Url,
ContentWBId = newItem.Id,
ContentUrl = content_url
});
}
catch (HttpRequestException e)
{
Console.WriteLine("\nException Caught!");
Console.WriteLine("Message :{0} ", e.Message);
}
return(null);
}
public async Task Process(WallabagClient client, WallabagItem item)
{
foreach (var mapping in mappings)
{
// Not a matching entry
if (!item.Url.Contains(mapping.domain))
{
continue;
}
// Already tagged
if (item.Tags.Any(t => t.Label == mapping.tag))
{
continue;
}
Console.Write(item.Title.Replace("\n", " "));
await client.AddTagsAsync(item, new[] { mapping.tag });
Console.WriteLine($" {mapping.tag} ✓");
}
}
static async Task <List <WallabagItem> > GetAllEntries(WallabagClient client)
{
List <WallabagItem> list = new List <WallabagItem>();
var page = 1;
Console.Write("Fetching all entries ");
while (true)
{
Console.Write(".");
var items = await client.GetItemsAsync(itemsPerPage : 100, pageNumber : page++);
if (items == null)
{
break;
}
list.AddRange(items);
}
Console.WriteLine();
return(list);
}
public async Task Process(WallabagClient client, WallabagItem item)
{
using (var db = new WallabagContext())
{
// Filter out HN articles that have already been processed
var exists = db.ProcessedHNArticles.Any(a => a.SourceWBId == item.Id || a.ContentWBId == item.Id);
if (!exists)
{
var hn = await Extract_HN_article(client, item);
if (hn != null)
{
db.ProcessedHNArticles.Add(hn);
var count = await db.SaveChangesAsync();
Console.WriteLine("{0} HN records saved to database", count);
}
}
}
}
public async Task Process(WallabagClient client, WallabagItem item)
{
bool filter_match = false;
String url = null;
foreach (var filter in config.url_filter)
{
// Prefer matches on OriginalUrl
if (!String.IsNullOrWhiteSpace(item.OriginalUrl) && item.OriginalUrl.Contains(filter))
{
filter_match = true;
url = item.OriginalUrl;
break;
}
if (item.Url.Contains(filter))
{
filter_match = true;
url = item.Url;
break;
}
}
// Not from a matching domain
if (!filter_match)
{
return;
}
foreach (var bl in blacklist)
{
if (url.Contains(bl))
{
var oldurl = url;
url = extract_yt_lastditch(url);
if (url == null)
{
Console.WriteLine($"Warning: YoutubeProcessor detected blacklisted pattern; skipping {oldurl}");
}
else
{
Console.WriteLine($"Info: YoutubeProcessor detected blacklisted pattern; extracted {url} from {oldurl}");
}
return;
}
}
// Already tagged
if (item.Tags.Any(t => t.Label == config.tag_name))
{
return;
}
Console.Write(item.Title.Replace("\n", " "));
// Send DL request to Youtube-DL-Server
var content = new FormUrlEncodedContent(new[]
{
new KeyValuePair <string, string>("url", url),
new KeyValuePair <string, string>("format", config.format),
});
try {
var dl_request = await fetcher.PostAsync(config.youtube_dl_server, content);
if (dl_request.StatusCode != System.Net.HttpStatusCode.OK)
{
Console.WriteLine($" ✗ yt-dl-server returned error response");
return;
}
var response = await dl_request.Content.ReadAsStringAsync();
await client.AddTagsAsync(item, new[] { config.tag_name });
if (config.mark_read)
{
await client.ArchiveAsync(item);
}
Console.WriteLine($" ✓");
}
catch (Exception e) {
Console.WriteLine($"Error submitting {url}\n{e.ToString()}\n{e.StackTrace.ToString()}");
}
}
public ViewModelLocator()
{
ServiceLocator.SetLocatorProvider(() => SimpleIoc.Default);
if (GalaSoft.MvvmLight.ViewModelBase.IsInDesignModeStatic)
{
return;
}
if (!SimpleIoc.Default.IsRegistered <IWallabagClient>())
{
SimpleIoc.Default.Register <IWallabagClient>(() =>
{
var client = new WallabagClient(Settings.Authentication.WallabagUri, Settings.Authentication.ClientId, Settings.Authentication.ClientSecret);
if (!string.IsNullOrEmpty(Settings.Authentication.AccessToken) &&
!string.IsNullOrEmpty(Settings.Authentication.RefreshToken))
{
client.AccessToken = Settings.Authentication.AccessToken;
client.RefreshToken = Settings.Authentication.RefreshToken;
client.LastTokenRefreshDateTime = Settings.Authentication.LastTokenRefreshDateTime;
}
client.CredentialsRefreshed += (s, e) =>
{
Settings.Authentication.ClientId = client.ClientId;
Settings.Authentication.ClientSecret = client.ClientSecret;
Settings.Authentication.AccessToken = client.AccessToken;
Settings.Authentication.RefreshToken = client.RefreshToken;
Settings.Authentication.LastTokenRefreshDateTime = client.LastTokenRefreshDateTime;
};
client.AfterRequestExecution += (s, e) =>
{
var logging = SimpleIoc.Default.GetInstance <ILoggingService>();
logging.WriteLine($"AfterRequestExecution: {e.RequestUriSubString}");
if (e.RequestUriSubString.Contains("oauth") &&
e.Parameters?["grant_type"].ToString() == "refresh_token" &&
e.Response?.StatusCode == System.Net.HttpStatusCode.BadRequest)
{
logging.WriteLine("App tried to refresh the tokens but failed. Informing the user.");
Messenger.Default.Send(new ShowLoginMessage());
}
else if (e.Response?.IsSuccessStatusCode == false)
{
logging.WriteLine("HTTP request to server failed.");
logging.WriteLine($"{e.RequestMethod.ToString().ToUpper()} {e.RequestUriSubString} with {e.Parameters?.Count} parameters.");
if (e.Parameters?.Count > 0)
{
foreach (var param in e.Parameters)
{
logging.WriteLine($"param '{param.Key}': {param.Value}");
}
}
}
};
return(client);
});
SimpleIoc.Default.Register <SQLiteConnection>(() =>
{
var device = SimpleIoc.Default.GetInstance <Interfaces.IPlatformSpecific>();
var db = new SQLiteConnection(device.GetSQLitePlatform(), device.GetDatabasePath(), serializer: new CustomBlobSerializer());
db.CreateTable <Item>();
db.CreateTable <Tag>();
db.CreateTable <OfflineTask>();
return(db);
});
SimpleIoc.Default.Register(() => new Dictionary <string, object>(), "SessionState");
// TODO: Re-enable the generic ApiClientCreationService if cookie issue is done.
//SimpleIoc.Default.Register<IApiClientCreationService, ApiClientCreationService>();
SimpleIoc.Default.Register <IOfflineTaskService, OfflineTaskService>();
SimpleIoc.Default.Register <AddItemViewModel>();
SimpleIoc.Default.Register <EditTagsViewModel>();
SimpleIoc.Default.Register <ItemPageViewModel>();
SimpleIoc.Default.Register <LoginPageViewModel>();
SimpleIoc.Default.Register <MainViewModel>();
SimpleIoc.Default.Register <SettingsPageViewModel>();
SimpleIoc.Default.Register <MultipleSelectionViewModel>();
SimpleIoc.Default.Register <QRScanPageViewModel>();
}
}
static int Main(string[] args)
{
var appConfig = new AppConfig
{
url = Environment.GetEnvironmentVariable("WALLABAG_URL"),
client_id = Environment.GetEnvironmentVariable("WALLABAG_CLIENT_ID"),
client_secret = Environment.GetEnvironmentVariable("WALLABAG_CLIENT_SECRET"),
username = Environment.GetEnvironmentVariable("WALLABAG_USERNAME"),
password = Environment.GetEnvironmentVariable("WALLABAG_PASSWORD"),
poll_duration = int.Parse(Environment.GetEnvironmentVariable("WALLABAG_POLL_DURATION_SECONDS") ?? "60") * 1000,
data_dir = Environment.GetEnvironmentVariable("DATA_DIR") ?? "/config",
database_file = "wallabag_reducer.sqlite3",
config_file = "config.json"
};
WallabagContext.database_file = Path.Combine(appConfig.data_dir, appConfig.database_file);
var config_path = Path.Combine(appConfig.data_dir, appConfig.config_file);
if (!File.Exists(config_path))
{
Console.WriteLine($"No config file, exiting (expected at {config_path}");
return(-1);
}
try
{
appConfig.Validate();
}
catch (Exception e)
{
Console.WriteLine($"Invalid environment configuration: {e.Message}");
return(-1);
}
// Load processor config file
var processorConfig = JObject.Parse(File.ReadAllText(config_path));
// Create DB & run any necessary migrations
using (var db = new WallabagContext())
{
db.Database.Migrate();
}
// Create & auth with Wallabag instance
WallabagClient client = new WallabagClient(new System.Uri(appConfig.url), appConfig.client_id, appConfig.client_secret);
var tok = client.RequestTokenAsync(appConfig.username, appConfig.password);
tok.Wait();
// Prep processors
var processors = new IProcessor[] {
new HNProcessor(),
new GenericTagProcessor(processorConfig),
new YoutubeDownloader(processorConfig),
};
var ex = run(client, processors, appConfig.poll_duration);
ex.Wait();
return(0);
}