public void FetchTest()
{
var s = RedditSubmissionFetcher.GetSubmissions("science").GetAwaiter().GetResult();
var l = s.Count(x => !string.IsNullOrWhiteSpace(x.LinkFlairText));
Assert.IsFalse(l > 2700, l.ToString());
}
public override async Task Execute(string[] args)
{
if (args.Length < 1)
{
Controlo.ReportError("Usage: {0}", ShortHelp);
return;
}
string subcommand = args[0];
if (subcommand == "status")
{
if (Current == null)
{
Controlo.ReportError("Fetching not in progress");
return;
}
Controlo.ReportInfo("Total: {0}, Complete: {1}", Current.Items.Count, Current.Complete);
}
else if (subcommand == "reddit")
{
if (Current != null)
{
Controlo.ReportError("Fetching in progress");
return;
}
if (args.Length < 3)
{
Controlo.ReportError("Usage: {0}", ShortHelp);
return;
}
if (!Guid.TryParse(args[2], out Guid destinationWorkspaceId))
{
Controlo.ReportError("No destination workspace id specified or invalid GUID: {0}", args[2]);
return;
}
string subreddit = args[1];
Controlo.ReportInfo("Getting the URLs from r/{0}", subreddit);
var submissions = await RedditSubmissionFetcher.GetSubmissions(subreddit);
Controlo.ReportInfo("{0} submissions extracted from r/{1}", submissions.Count(), subreddit);
Current = new FetchingTask();
await SetupWorkspaceDataWriter();
await SendItemsForQueuing(submissions.Select(x => MakeQueueItem(x, destinationWorkspaceId)));
Controlo.ReportInfo("Task created");
}
else if (subcommand == "hackernews")
{
if (Current != null)
{
Controlo.ReportError("Fetching in progress");
return;
}
if (args.Length < 2)
{
Controlo.ReportError("Usage: {0}", ShortHelp);
return;
}
if (!Guid.TryParse(args[1], out Guid destinationWorkspaceId))
{
Controlo.ReportError("No destination workspace id specified or invalid GUID: {0}", args[2]);
return;
}
Controlo.ReportInfo("Getting the URLs from hackernews");
Current = new FetchingTask();
await SetupWorkspaceDataWriter();
int numberOfDaysToFetch = 100;
DateTime start = DateTime.Now.AddDays(-numberOfDaysToFetch - 1);
for (int i = 0; i < numberOfDaysToFetch; i++)
{
var hnis = await HackerNewsFrontPageFetcher.GetLinks(start);
start = start.AddDays(1);
Controlo.ReportInfo("{0} submissions extracted from hackernews", hnis.Count());
await SendItemsForQueuing(hnis.Select(x => MakeQueueItem(x, destinationWorkspaceId)));
}
Controlo.ReportInfo("Task created");
}
else
{
Controlo.ReportError("Usage: {0}", ShortHelp);
}
}
