public MarkupAggregator(
IHttpAgent httpAgent,
VerboseLogger verboseLogger,
string hrefRegex,
string[] invalidSiteLinkPatterns)
{
HttpAgent = httpAgent;
VerboseLogger = verboseLogger;
HrefRegex = hrefRegex;
InvalidSiteLinkPatterns = invalidSiteLinkPatterns;
}
public ExtractionManager(
ExtractionRequest request,
IExtractionConfigProvider configProvider,
IHttpAgent httpAgent,
VerboseLogger verboseLogger)
{
Request = request;
HttpAgent = httpAgent;
VerboseLogger = verboseLogger;
_csvDelimiter = configProvider.GetCsvDelimiter();
_emailRegex = configProvider.GetEmailRegex();
_hrefRegex = configProvider.GetHrefRegex();
_invalidSiteLinkPatterns = configProvider.GetInvalidSiteLinkPatterns();
Response = new ExtractionResponse();
}
private static void Main(string[] args)
{
var inputFilePath = GetInputFilePath();
var outputDirectory = GetOutputDirectory();
var verboseEnabled = GetVerboseLoggingEnabled();
var verboseLogger = new VerboseLogger(verboseEnabled, Console.WriteLine);
var configProvider = new AppConfigProvider();
var httpWebRequestAgent = new HttpWebRequestAgent(verboseLogger);
var extractor = new ExtractionManager(new ExtractionRequest
{
InputFilePath = inputFilePath,
OutputDirectory = outputDirectory,
VerboseLogger = verboseLogger
}, configProvider, httpWebRequestAgent, verboseLogger);
var response = extractor.RunExtraction();
if (!string.IsNullOrEmpty(response.GeneralException))
{
Console.WriteLine($"Process failed. Error: {response.GeneralException}.");
Console.ReadKey();
return;
}
if (response.HasExtractionException)
{
foreach (var extrctException in response.ExtractionExceptions)
{
Console.WriteLine(
$"Extraction did not complete for url: {extrctException.InputUrl}. " +
$"Error: {extrctException}");
}
Console.ReadKey();
}
Console.WriteLine($"Process complete. {response.SuccessfulExtractions} website(s) successfully processed.");
Console.ReadKey();
}
public HttpWebRequestAgent(
VerboseLogger verboseLogger)
{
VerboseLogger = verboseLogger;
}
