public void IsContent_ShouldReturnFalse_IfAtLeastOneRuleIsMatched() { var rule1 = new Mock <IContentExcludeRule>(); rule1.Setup(x => x.ShouldExcludeFromContent(It.IsAny <string>())).Returns(false); var rule2 = new Mock <IContentExcludeRule>(); rule2.Setup(x => x.ShouldExcludeFromContent(It.IsAny <string>())).Returns(true); var rule3 = new Mock <IContentExcludeRule>(); rule3.Setup(x => x.ShouldExcludeFromContent(It.IsAny <string>())).Returns(true); var rules = new List <IContentExcludeRule> { rule1.Object, rule2.Object, rule3.Object }; ContentMatcher sup = new ContentMatcher(rules); var url = "/lol"; var actual = sup.IsContent(url); Assert.False(actual); rule1.Verify(x => x.ShouldExcludeFromContent(url), Times.AtMostOnce); rule2.Verify(x => x.ShouldExcludeFromContent(url), Times.AtMostOnce); rule3.Verify(x => x.ShouldExcludeFromContent(url), Times.Never); }
public MainViewModel(Func <Task <IFlightAcademyClient> > flightAcademyClientFactory, ContentMatcher mediaMatcher, SettingsViewModel settings) { this.mediaMatcher = mediaMatcher; LoadFlights = ReactiveCommand.CreateFromTask(() => LoadFlightsAsync(flightAcademyClientFactory, mediaMatcher)); flights = LoadFlights.ToProperty(this, x => x.Flights); isBusy = LoadFlights.IsExecuting.ToProperty(this, x => x.IsBusy); Settings = settings; }
public FlightViewModel(FlightSummary summary, IFlightAcademyClient flightAcademyClient, ContentMatcher matcher) { this.summary = summary; this.matcher = matcher; LoadItems = ReactiveCommand.CreateFromObservable(() => Load().ToList()); LoadItems.ThrownExceptions.Subscribe(x => { }); items = LoadItems.ToProperty(this, x => x.Items); }
/// <summary> /// Returns all files to download from the feed /// </summary> /// <returns></returns> public List <string> UrlToDownload() { IEnumerable <string> contentMatchingLinks = _feed.Items .Where(item => ContentMatcher.Match(item.Title)) .Select(item => item.Link); var linksToDownload = contentMatchingLinks.Except(DownloadedLinks); return(linksToDownload.ToList()); }
private static async Task <List <FlightViewModel> > LoadFlightsAsync(Func <Task <IFlightAcademyClient> > flightAcademyClientFactory, ContentMatcher mediaMatcher) { var flightAcademyClient = await flightAcademyClientFactory(); var observable = Observable.FromAsync(() => flightAcademyClient.GetFlights(0, 1500)) .Select(x => x.Select(summary => new FlightViewModel(summary, flightAcademyClient, mediaMatcher)).ToList()); return(await observable); }
protected override void OnNavigatedTo(NavigationEventArgs e) { var settingsViewModel = new SettingsViewModel(); Task <IFlightAcademyClient> FactoryClient() => FlightAcademyClient.Create(settingsViewModel.Username, settingsViewModel.Password); var matcher = new ContentMatcher(); DataContext = new MainViewModel(FactoryClient, matcher, settingsViewModel); base.OnNavigatedTo(e); }
public static async Task Main(string[] args) { Console.OutputEncoding = Encoding.UTF8; Stopwatch elapsedParsing = new Stopwatch(); Stopwatch elapsedProcessing = new Stopwatch(); elapsedParsing.Start(); var configuration = new ConfigurationBuilder(); configuration.AddJsonFile(Path.Combine(Directory.GetCurrentDirectory(), "config.json"), false, false); configuration.AddCommandLine(args); var config = configuration.Build().Get <Config>(); var excludeContentListParser = new ContentExcludeListParser(new FileSystem()); _contentMatcher = new ContentMatcher(excludeContentListParser.Parse(config.ExcludeFromContentFilePath)); var parser = new LogParser(config.LogFilesFolderPath, config.CrawlerUserAgentsFilePath); var items = await parser.ParseAsync(); elapsedParsing.Stop(); if (!string.IsNullOrWhiteSpace(config.Url)) { if (config.Date != null) { try { Console.WriteLine($"Data limited to {config.Date.Value.Date:yyyy-MM-dd}"); items = items.Where(x => x.Time.Date == config.Date.Value.Date).ToList(); } catch (Exception e) { Console.WriteLine( $"Error: Please specify date as a second parameter in the yyyy-MM-dd format. Details: {e.Message}"); return; } } items = items.Where(x => x.NormalizedRequestUrl?.Equals(config.Url, StringComparison.OrdinalIgnoreCase) ?? false).ToList(); ShowUrlDetails(items, config.Url); } elapsedProcessing.Start(); //IgnoreStatistics(items); //return; var contentNotCrawlers = items .Where(x => !x.ShouldIgnore && IsContent(x) && (int)x.ResponseCode < 400 && x.ResponseCode != HttpStatusCode.Moved && x.Referrer != "https://druss.co/xmlrpc.php") .ToList(); var notFoundItems = items .Count(x => x.ResponseCode == HttpStatusCode.NotFound); var crawlers = items.Count(x => x.ShouldIgnore); Console.Write($"Total: {items.Count} Crawlers: "); Write(crawlers.ToString(), ConsoleColor.DarkYellow); Console.Write(" 404: "); Write(notFoundItems.ToString(), ConsoleColor.DarkRed); Console.Write(" Content: "); WriteLine(contentNotCrawlers.Count.ToString(), ConsoleColor.DarkGreen); Console.WriteLine(); ShowLastSevenDays(contentNotCrawlers); ShowLastSevenWeeks(contentNotCrawlers); var date = DateTime.UtcNow; if (config.Date.HasValue) { date = config.Date.Value.AddHours(23).AddMinutes(59).AddSeconds(59).AddMilliseconds(999); } DayStats(date, contentNotCrawlers); Console.WriteLine($"-- {date:yyyy-MM-dd}'s Top 15 --"); var result = contentNotCrawlers .Where(x => x.Time.Date == date.Date) .GroupBy(x => x.NormalizedRequestUrl) .Select(x => new { Url = x.Key, Count = x.Count(), LogItems = x }) .OrderByDescending(x => x.Count) .Take(15); foreach (var item in result) { Console.WriteLine($"{item.Count} \t {item.Url}"); } elapsedProcessing.Stop(); Console.WriteLine(); Console.WriteLine("Elapsed parsing: " + elapsedParsing.Elapsed.TotalSeconds + "s"); Console.WriteLine("Elapsed processing: " + elapsedProcessing.Elapsed.TotalSeconds + "s"); Console.WriteLine("Total: " + (elapsedParsing.Elapsed.TotalSeconds + elapsedProcessing.Elapsed.TotalSeconds) + "s"); //Referrers(contentNotCrawlers); }