public MarkupAggregator( IHttpAgent httpAgent, VerboseLogger verboseLogger, string hrefRegex, string[] invalidSiteLinkPatterns) { HttpAgent = httpAgent; VerboseLogger = verboseLogger; HrefRegex = hrefRegex; InvalidSiteLinkPatterns = invalidSiteLinkPatterns; }
public ExtractionManager( ExtractionRequest request, IExtractionConfigProvider configProvider, IHttpAgent httpAgent, VerboseLogger verboseLogger) { Request = request; HttpAgent = httpAgent; VerboseLogger = verboseLogger; _csvDelimiter = configProvider.GetCsvDelimiter(); _emailRegex = configProvider.GetEmailRegex(); _hrefRegex = configProvider.GetHrefRegex(); _invalidSiteLinkPatterns = configProvider.GetInvalidSiteLinkPatterns(); Response = new ExtractionResponse(); }
private static void Main(string[] args) { var inputFilePath = GetInputFilePath(); var outputDirectory = GetOutputDirectory(); var verboseEnabled = GetVerboseLoggingEnabled(); var verboseLogger = new VerboseLogger(verboseEnabled, Console.WriteLine); var configProvider = new AppConfigProvider(); var httpWebRequestAgent = new HttpWebRequestAgent(verboseLogger); var extractor = new ExtractionManager(new ExtractionRequest { InputFilePath = inputFilePath, OutputDirectory = outputDirectory, VerboseLogger = verboseLogger }, configProvider, httpWebRequestAgent, verboseLogger); var response = extractor.RunExtraction(); if (!string.IsNullOrEmpty(response.GeneralException)) { Console.WriteLine($"Process failed. Error: {response.GeneralException}."); Console.ReadKey(); return; } if (response.HasExtractionException) { foreach (var extrctException in response.ExtractionExceptions) { Console.WriteLine( $"Extraction did not complete for url: {extrctException.InputUrl}. " + $"Error: {extrctException}"); } Console.ReadKey(); } Console.WriteLine($"Process complete. {response.SuccessfulExtractions} website(s) successfully processed."); Console.ReadKey(); }
public HttpWebRequestAgent( VerboseLogger verboseLogger) { VerboseLogger = verboseLogger; }