public void IsContent_ShouldReturnFalse_IfAtLeastOneRuleIsMatched()
{
var rule1 = new Mock <IContentExcludeRule>();
rule1.Setup(x => x.ShouldExcludeFromContent(It.IsAny <string>())).Returns(false);
var rule2 = new Mock <IContentExcludeRule>();
rule2.Setup(x => x.ShouldExcludeFromContent(It.IsAny <string>())).Returns(true);
var rule3 = new Mock <IContentExcludeRule>();
rule3.Setup(x => x.ShouldExcludeFromContent(It.IsAny <string>())).Returns(true);
var rules = new List <IContentExcludeRule>
{
rule1.Object,
rule2.Object,
rule3.Object
};
ContentMatcher sup = new ContentMatcher(rules);
var url = "/lol";
var actual = sup.IsContent(url);
Assert.False(actual);
rule1.Verify(x => x.ShouldExcludeFromContent(url), Times.AtMostOnce);
rule2.Verify(x => x.ShouldExcludeFromContent(url), Times.AtMostOnce);
rule3.Verify(x => x.ShouldExcludeFromContent(url), Times.Never);
}
public MainViewModel(Func <Task <IFlightAcademyClient> > flightAcademyClientFactory, ContentMatcher mediaMatcher, SettingsViewModel settings)
{
this.mediaMatcher = mediaMatcher;
LoadFlights = ReactiveCommand.CreateFromTask(() => LoadFlightsAsync(flightAcademyClientFactory, mediaMatcher));
flights = LoadFlights.ToProperty(this, x => x.Flights);
isBusy = LoadFlights.IsExecuting.ToProperty(this, x => x.IsBusy);
Settings = settings;
}
public FlightViewModel(FlightSummary summary, IFlightAcademyClient flightAcademyClient, ContentMatcher matcher)
{
this.summary = summary;
this.matcher = matcher;
LoadItems = ReactiveCommand.CreateFromObservable(() => Load().ToList());
LoadItems.ThrownExceptions.Subscribe(x => { });
items = LoadItems.ToProperty(this, x => x.Items);
}
/// <summary>
/// Returns all files to download from the feed
/// </summary>
/// <returns></returns>
public List <string> UrlToDownload()
{
IEnumerable <string> contentMatchingLinks = _feed.Items
.Where(item => ContentMatcher.Match(item.Title))
.Select(item => item.Link);
var linksToDownload = contentMatchingLinks.Except(DownloadedLinks);
return(linksToDownload.ToList());
}
private static async Task <List <FlightViewModel> > LoadFlightsAsync(Func <Task <IFlightAcademyClient> > flightAcademyClientFactory,
ContentMatcher mediaMatcher)
{
var flightAcademyClient = await flightAcademyClientFactory();
var observable = Observable.FromAsync(() => flightAcademyClient.GetFlights(0, 1500))
.Select(x => x.Select(summary => new FlightViewModel(summary, flightAcademyClient, mediaMatcher)).ToList());
return(await observable);
}
protected override void OnNavigatedTo(NavigationEventArgs e)
{
var settingsViewModel = new SettingsViewModel();
Task <IFlightAcademyClient> FactoryClient() => FlightAcademyClient.Create(settingsViewModel.Username,
settingsViewModel.Password);
var matcher = new ContentMatcher();
DataContext = new MainViewModel(FactoryClient, matcher, settingsViewModel);
base.OnNavigatedTo(e);
}
public static async Task Main(string[] args)
{
Console.OutputEncoding = Encoding.UTF8;
Stopwatch elapsedParsing = new Stopwatch();
Stopwatch elapsedProcessing = new Stopwatch();
elapsedParsing.Start();
var configuration = new ConfigurationBuilder();
configuration.AddJsonFile(Path.Combine(Directory.GetCurrentDirectory(), "config.json"), false, false);
configuration.AddCommandLine(args);
var config = configuration.Build().Get <Config>();
var excludeContentListParser = new ContentExcludeListParser(new FileSystem());
_contentMatcher = new ContentMatcher(excludeContentListParser.Parse(config.ExcludeFromContentFilePath));
var parser = new LogParser(config.LogFilesFolderPath, config.CrawlerUserAgentsFilePath);
var items = await parser.ParseAsync();
elapsedParsing.Stop();
if (!string.IsNullOrWhiteSpace(config.Url))
{
if (config.Date != null)
{
try
{
Console.WriteLine($"Data limited to {config.Date.Value.Date:yyyy-MM-dd}");
items = items.Where(x => x.Time.Date == config.Date.Value.Date).ToList();
}
catch (Exception e)
{
Console.WriteLine(
$"Error: Please specify date as a second parameter in the yyyy-MM-dd format. Details: {e.Message}");
return;
}
}
items = items.Where(x => x.NormalizedRequestUrl?.Equals(config.Url, StringComparison.OrdinalIgnoreCase) ?? false).ToList();
ShowUrlDetails(items, config.Url);
}
elapsedProcessing.Start();
//IgnoreStatistics(items);
//return;
var contentNotCrawlers = items
.Where(x =>
!x.ShouldIgnore &&
IsContent(x) &&
(int)x.ResponseCode < 400 && x.ResponseCode != HttpStatusCode.Moved &&
x.Referrer != "https://druss.co/xmlrpc.php")
.ToList();
var notFoundItems = items
.Count(x => x.ResponseCode == HttpStatusCode.NotFound);
var crawlers = items.Count(x => x.ShouldIgnore);
Console.Write($"Total: {items.Count} Crawlers: ");
Write(crawlers.ToString(), ConsoleColor.DarkYellow);
Console.Write(" 404: ");
Write(notFoundItems.ToString(), ConsoleColor.DarkRed);
Console.Write(" Content: ");
WriteLine(contentNotCrawlers.Count.ToString(), ConsoleColor.DarkGreen);
Console.WriteLine();
ShowLastSevenDays(contentNotCrawlers);
ShowLastSevenWeeks(contentNotCrawlers);
var date = DateTime.UtcNow;
if (config.Date.HasValue)
{
date = config.Date.Value.AddHours(23).AddMinutes(59).AddSeconds(59).AddMilliseconds(999);
}
DayStats(date, contentNotCrawlers);
Console.WriteLine($"-- {date:yyyy-MM-dd}'s Top 15 --");
var result = contentNotCrawlers
.Where(x => x.Time.Date == date.Date)
.GroupBy(x => x.NormalizedRequestUrl)
.Select(x => new { Url = x.Key, Count = x.Count(), LogItems = x })
.OrderByDescending(x => x.Count)
.Take(15);
foreach (var item in result)
{
Console.WriteLine($"{item.Count} \t {item.Url}");
}
elapsedProcessing.Stop();
Console.WriteLine();
Console.WriteLine("Elapsed parsing: " + elapsedParsing.Elapsed.TotalSeconds + "s");
Console.WriteLine("Elapsed processing: " + elapsedProcessing.Elapsed.TotalSeconds + "s");
Console.WriteLine("Total: " + (elapsedParsing.Elapsed.TotalSeconds + elapsedProcessing.Elapsed.TotalSeconds) + "s");
//Referrers(contentNotCrawlers);
}
