public async Task FetchTest() { int numberOfDaysToDownload = 3; DateTime start = DateTime.Now.AddDays(-numberOfDaysToDownload); for (int i = 0; i < numberOfDaysToDownload; i++) { var s = await HackerNewsFrontPageFetcher.GetLinks(start); Assert.IsTrue(s.Any()); foreach (var u in s) { Console.WriteLine(u); } start = start.AddDays(1); } }
public override async Task Execute(string[] args) { if (args.Length < 1) { Controlo.ReportError("Usage: {0}", ShortHelp); return; } string subcommand = args[0]; if (subcommand == "status") { if (Current == null) { Controlo.ReportError("Fetching not in progress"); return; } Controlo.ReportInfo("Total: {0}, Complete: {1}", Current.Items.Count, Current.Complete); } else if (subcommand == "reddit") { if (Current != null) { Controlo.ReportError("Fetching in progress"); return; } if (args.Length < 3) { Controlo.ReportError("Usage: {0}", ShortHelp); return; } if (!Guid.TryParse(args[2], out Guid destinationWorkspaceId)) { Controlo.ReportError("No destination workspace id specified or invalid GUID: {0}", args[2]); return; } string subreddit = args[1]; Controlo.ReportInfo("Getting the URLs from r/{0}", subreddit); var submissions = await RedditSubmissionFetcher.GetSubmissions(subreddit); Controlo.ReportInfo("{0} submissions extracted from r/{1}", submissions.Count(), subreddit); Current = new FetchingTask(); await SetupWorkspaceDataWriter(); await SendItemsForQueuing(submissions.Select(x => MakeQueueItem(x, destinationWorkspaceId))); Controlo.ReportInfo("Task created"); } else if (subcommand == "hackernews") { if (Current != null) { Controlo.ReportError("Fetching in progress"); return; } if (args.Length < 2) { Controlo.ReportError("Usage: {0}", ShortHelp); return; } if (!Guid.TryParse(args[1], out Guid destinationWorkspaceId)) { Controlo.ReportError("No destination workspace id specified or invalid GUID: {0}", args[2]); return; } Controlo.ReportInfo("Getting the URLs from hackernews"); Current = new FetchingTask(); await SetupWorkspaceDataWriter(); int numberOfDaysToFetch = 100; DateTime start = DateTime.Now.AddDays(-numberOfDaysToFetch - 1); for (int i = 0; i < numberOfDaysToFetch; i++) { var hnis = await HackerNewsFrontPageFetcher.GetLinks(start); start = start.AddDays(1); Controlo.ReportInfo("{0} submissions extracted from hackernews", hnis.Count()); await SendItemsForQueuing(hnis.Select(x => MakeQueueItem(x, destinationWorkspaceId))); } Controlo.ReportInfo("Task created"); } else { Controlo.ReportError("Usage: {0}", ShortHelp); } }