public WebsiteTextExtractor(string storageDirForWebsite, string[] newParams, bool doContinue = false)
{
// Save action requested
DoContinue = doContinue;
// Reload params file
FileInfo paramFileInfo = new FileInfo(Path.Combine(storageDirForWebsite, LogsDirName, ConfigFileName));
if (!paramFileInfo.Exists)
{
throw new Exception("No configuration file found at : " + paramFileInfo.FullName);
}
using (StreamReader sr = new StreamReader(paramFileInfo.FullName))
{
ExtractorParams = WebsiteExtractorParams.ReadFromFile(sr);
}
// Override with new params
if (newParams != null)
{
foreach (string keyValueParam in newParams)
{
ExtractorParams.ParseParam(keyValueParam);
}
}
Init();
}
private void InitLogFiles()
{
Console.CancelKeyPress += (object sender, ConsoleCancelEventArgs e) =>
{
e.Cancel = true;
userCancelEventReceived = true;
};
var logsDirectory = new DirectoryInfo(Path.Combine(ContentDirectory.FullName, LogsDirName));
if (!logsDirectory.Exists)
{
logsDirectory.Create();
}
using (var paramsWriter = new StreamWriter(Path.Combine(logsDirectory.FullName, ConfigFileName), DoContinue))
{
if (DoContinue)
{
paramsWriter.WriteLine();
}
ExtractorParams.WriteToFile(paramsWriter);
}
requestsWriter = new StreamWriter(Path.Combine(logsDirectory.FullName, RequestsLogFileName), DoContinue);
requestsWriter.Write("Clock");
requestsWriter.Write(";");
requestsWriter.Write("Url");
requestsWriter.Write(";");
requestsWriter.Write("Status code");
requestsWriter.Write(";");
requestsWriter.Write("Reponse time (ms)");
requestsWriter.Write(";");
requestsWriter.Write("Download time (ms)");
requestsWriter.Write(";");
requestsWriter.Write("Content size (bytes)");
requestsWriter.Write(";");
requestsWriter.Write("Unique text blocks (%)");
requestsWriter.Write(";");
requestsWriter.Write("Crawl depth");
requestsWriter.Write(";");
requestsWriter.Write("Parent Url");
requestsWriter.Write(";");
requestsWriter.Write("Redirected from");
requestsWriter.Write(";");
requestsWriter.Write("Retry count");
requestsWriter.Write(";");
requestsWriter.Write("Retry after (s)");
requestsWriter.Write(";");
requestsWriter.Write("Error message");
requestsWriter.WriteLine();
messagesWriter = new StreamWriter(Path.Combine(logsDirectory.FullName, MessagesLogFileName), DoContinue);
exceptionsWriter = new StreamWriter(Path.Combine(logsDirectory.FullName, ExceptionsLogFileName), DoContinue);
log4net.LogManager.SetTextWriter(exceptionsWriter);
}
