public void Verify_sanitize_of_single_mhtml_file()
{
var tempFile = Path.ChangeExtension(Path.GetTempFileName(), ".mht");
File.Copy(TestConfig.PathToMht, tempFile);
string mhtml = File.ReadAllText(tempFile);
MHTMLParser parser = new MHTMLParser(mhtml)
{
OutputDirectory = Path.GetDirectoryName(tempFile),
DecodeImageData = true
};
var outFile = Path.ChangeExtension(tempFile, ".html");
File.WriteAllText(outFile, parser.getHTMLText());
_sanitizer = new SafeHtmlConverter(outFile)
{
Logger = NullLogger.Instance
};
var result = _sanitizer.Run("jobtest");
Assert.That(File.Exists(result), "Output pdf file not created");
File.Delete(result);
}
public void SaveDescriptor(FileSystemBlobDescriptor fileSystemBlobDescriptor)
{
if (fileSystemBlobDescriptor == null)
{
throw new ArgumentNullException(nameof(fileSystemBlobDescriptor));
}
var fileName = _directoryManager.GetDescriptorFileNameFromBlobId(fileSystemBlobDescriptor.BlobId);
File.WriteAllText(fileName, JsonConvert.SerializeObject(fileSystemBlobDescriptor));
}
protected async override Task <ProcessResult> OnPolling(PollerJobParameters parameters, string workingFolder)
{
string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder).ConfigureAwait(false);
//String fileName = Path.Combine(Path.GetDirectoryName(pathToFile), parameters.All[JobKeys.FileName]);
//Logger.DebugFormat("Move blob id {0} to real filename {1}", pathToFile, fileName);
//if (File.Exists(fileName)) File.Delete(fileName);
//File.Copy(pathToFile, fileName);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Conversion of HtmlZip to PDF: file {0}", pathToFile);
}
var file = pathToFile;
if (pathToFile.EndsWith(".mht", StringComparison.OrdinalIgnoreCase) || pathToFile.EndsWith(".mhtml", StringComparison.OrdinalIgnoreCase))
{
string mhtml = File.ReadAllText(pathToFile);
MHTMLParser parser = new MHTMLParser(mhtml)
{
OutputDirectory = workingFolder,
DecodeImageData = true
};
var outFile = Path.ChangeExtension(pathToFile, ".html");
File.WriteAllText(outFile, parser.getHTMLText());
file = outFile;
}
var sanitizer = new SafeHtmlConverter(file)
{
Logger = Logger
};
file = sanitizer.Run(parameters.JobId);
var converter = new HtmlToPdfConverterFromDiskFile(file, base.JobsHostConfiguration)
{
Logger = Logger
};
var pdfConvertedFileName = converter.Run(parameters.TenantId, parameters.JobId);
await AddFormatToDocumentFromFile(
parameters.TenantId,
parameters.JobId,
new DocumentFormat(DocumentFormats.Pdf),
pdfConvertedFileName,
new Dictionary <string, object>()).ConfigureAwait(false);
return(ProcessResult.Ok);
}
protected async override Task <ProcessResult> OnPolling(PollerJobParameters parameters, string workingFolder)
{
string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder);
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Conversion of HtmlZip to PDF: file {0}", pathToFile);
}
var file = pathToFile;
if (pathToFile.ToLower().EndsWith(".mht") || pathToFile.ToLower().EndsWith(".mhtml"))
{
string mhtml = File.ReadAllText(pathToFile);
MHTMLParser parser = new MHTMLParser(mhtml);
parser.OutputDirectory = workingFolder;
parser.DecodeImageData = true;
var outFile = Path.ChangeExtension(pathToFile, ".html");
File.WriteAllText(outFile, parser.getHTMLText());
file = outFile;
}
var converter = new HtmlToPdfConverterFromDiskFileOld(file, base.JobsHostConfiguration)
{
Logger = Logger
};
var pdfConvertedFileName = converter.Run(parameters.JobId);
await AddFormatToDocumentFromFile(
parameters.TenantId,
parameters.JobId,
new DocumentFormat(DocumentFormats.Pdf),
pdfConvertedFileName,
new Dictionary <string, object>());
return(ProcessResult.Ok);
}
private string ProcessFile(string pathToFile, string workingFolder)
{
var extension = Path.GetExtension(pathToFile).ToLower();
if (extension == ".htmlzip" || extension == ".htmzip")
{
ZipFile.ExtractToDirectory(pathToFile, workingFolder);
Logger.DebugFormat("Extracted zip to {0}", workingFolder);
var htmlFile = Path.ChangeExtension(pathToFile, "html");
if (File.Exists(htmlFile))
{
Logger.DebugFormat("Html file is {0}", htmlFile);
return(htmlFile);
}
htmlFile = Path.ChangeExtension(pathToFile, "htm");
if (File.Exists(htmlFile))
{
Logger.DebugFormat("Html file is {0}", htmlFile);
return(htmlFile);
}
Logger.ErrorFormat("Invalid HTMLZIP file, name is {0} but corresponding html file not found after decompression", Path.GetFileName(pathToFile));
}
else if (extension == ".mht" || extension == ".mhtml")
{
MHTMLParser parser = new MHTMLParser(File.ReadAllText(pathToFile));
parser.OutputDirectory = workingFolder;
parser.DecodeImageData = false;
var html = parser.getHTMLText();
pathToFile = pathToFile + ".html";
File.WriteAllText(pathToFile, html);
}
return(pathToFile);
}
public string Convert(String jobId, string pathToEml, string workingFolder)
{
Logger.DebugFormat("Coverting {0} in working folder {1}", pathToEml, workingFolder);
var reader = new Reader();
var outFolder = Path.Combine(workingFolder, jobId);
Logger.DebugFormat("Creating message working folder is {0}", outFolder);
Directory.CreateDirectory(outFolder);
Logger.Debug("Extracting files");
var files = reader.ExtractToFolder(pathToEml, outFolder);
if (Logger.IsDebugEnabled)
{
foreach (var file in files)
{
Logger.DebugFormat("\t{0}", Path.GetFileName(file));
}
Logger.DebugFormat("Total files {0}", files.Length);
}
var htmlFileName = files.FirstOrDefault(x => x.EndsWith(".htm", StringComparison.OrdinalIgnoreCase)) ??
files.FirstOrDefault(x => x.EndsWith(".html", StringComparison.OrdinalIgnoreCase));
if (htmlFileName == null)
{
var textFile = files.FirstOrDefault(x => x.EndsWith(".txt", StringComparison.OrdinalIgnoreCase));
if (textFile != null)
{
htmlFileName = textFile + ".html";
var textcontent = File.ReadAllText(textFile);
File.WriteAllText(htmlFileName, String.Format("<html><body><pre>{0}</pre></body></html>", textcontent));
}
else
{
htmlFileName = "contentmissing.html";
File.WriteAllText(htmlFileName, "<html>No content found in mail.</html>");
}
}
var htmlNameWithoutExtension = Path.GetFileNameWithoutExtension(htmlFileName);
var htmlContent = File.ReadAllText(htmlFileName);
var dirInfoFullName = new DirectoryInfo(outFolder).FullName;
htmlContent = Regex.Replace(
htmlContent,
@"src=""(?<src>.+?)""",
new MatchEvaluator((m) => NormalizeImgEvaluator(m, dirInfoFullName)),
RegexOptions.IgnoreCase);
File.WriteAllText(htmlFileName, htmlContent);
var pathToZip = Path.Combine(workingFolder, htmlNameWithoutExtension + ".ezip");
Logger.DebugFormat("New zip file is {0}", pathToZip);
if (File.Exists(pathToZip))
{
Logger.DebugFormat("Deleting previous file: {0}", pathToZip);
File.Delete(pathToZip);
}
Logger.DebugFormat("Creating new file: {0}", pathToZip);
ZipFile.CreateFromDirectory(outFolder, pathToZip);
Logger.DebugFormat("Deleting message working folder", outFolder);
Directory.Delete(outFolder, true);
Logger.DebugFormat(
"Convesion done {0} => {1}",
pathToEml,
pathToZip
);
return(pathToZip);
}
protected async override Task <ProcessResult> OnPolling(
PollerJobParameters parameters,
String workingFolder)
{
Boolean result;
var contentFileName = Path.ChangeExtension(parameters.FileName, ".content");
if (!_formats.Contains(parameters.FileExtension))
{
Logger.DebugFormat("Document for job Id {0} has an extension not supported, setting null content", parameters.JobId);
return(new ProcessResult(await AddNullContentFormat(parameters, contentFileName)));
}
Logger.DebugFormat("Starting tika on job: {0}, file extension {1}", parameters.JobId, parameters.FileExtension);
Logger.DebugFormat("Downloading blob for job: {0}, on local path {1}", parameters.JobId, workingFolder);
string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder);
pathToFile = ProcessFile(pathToFile, workingFolder);
Boolean shouldAnalyze = _filterManager.ShouldAnalyze(parameters.FileName, pathToFile);
if (!shouldAnalyze)
{
Logger.InfoFormat("File {0} for job {1} was discharded!", parameters.FileName, parameters.JobId);
return(new ProcessResult(await AddNullContentFormat(parameters, contentFileName)));
}
Logger.DebugFormat("Search for password JobId:{0}", parameters.JobId);
var passwords = ClientPasswordSet.GetPasswordFor(parameters.FileName).ToArray();
String content = "";
Int32 analyzerOrdinal = 0;
Boolean success = false;
var analyzer = BuildAnalyzer(analyzerOrdinal);
do
{
try
{
if (passwords.Any())
{
//Try with all the password
foreach (var password in passwords)
{
try
{
content = analyzer.GetHtmlContent(pathToFile, password) ?? "";
break; //first password that can decrypt file break the list of password to try
}
catch (Exception)
{
Logger.ErrorFormat("Error opening file {0} with password", parameters.FileName);
}
}
}
else
{
//Simply analyze file without password
Logger.DebugFormat("Analyze content JobId: {0} -> Path: {1}", parameters.JobId, pathToFile);
content = analyzer.GetHtmlContent(pathToFile, "") ?? "";
}
success = true;
}
catch (Exception ex)
{
Logger.ErrorFormat(ex, "Error extracting tika with analyzer {0} on file {1}", analyzer.Describe(), parameters.FileName, parameters.JobId);
analyzer = BuildAnalyzer(++analyzerOrdinal);
if (analyzer != null)
{
Logger.InfoFormat("Retry job {0} with analyzer {1}", parameters.JobId, analyzer.Describe());
}
}
} while (analyzer != null && success == false);
Logger.DebugFormat("Finished tika on job: {0}, charsNum {1}", parameters.JobId, content.Count());
String sanitizedContent = content;
if (!string.IsNullOrWhiteSpace(content))
{
var resultContent = _builder.CreateFromTikaPlain(content);
var documentContent = resultContent.Content;
sanitizedContent = resultContent.SanitizedTikaContent;
var pages = documentContent.Pages.Count();
string lang = null;
if (pages > 1)
{
lang = LanguageDetector.GetLanguage(documentContent.Pages[1].Content);
}
if (lang == null && pages == 1)
{
lang = LanguageDetector.GetLanguage(documentContent.Pages[0].Content);
}
if (lang != null)
{
documentContent.AddMetadata(DocumentContent.MedatataLanguage, lang);
}
result = await AddFormatToDocumentFromObject(
parameters.TenantId,
this.QueueName,
parameters.JobId,
new DocumentFormat(DocumentFormats.Content),
documentContent,
contentFileName,
new Dictionary <string, object>());
Logger.DebugFormat("Added format {0} to jobId {1}, result: {2}", DocumentFormats.Content, parameters.JobId, result);
}
var tikaFileName = Path.Combine(workingFolder, Path.GetFileNameWithoutExtension(parameters.FileName) + ".tika.html");
tikaFileName = SanitizeFileNameForLength(tikaFileName);
File.WriteAllText(tikaFileName, sanitizedContent);
result = await AddFormatToDocumentFromFile(
parameters.TenantId,
parameters.JobId,
new DocumentFormat(DocumentFormats.Tika),
tikaFileName,
new Dictionary <string, object>());
Logger.DebugFormat("Added format {0} to jobId {1}, result: {2}", DocumentFormats.Tika, parameters.JobId, result);
return(ProcessResult.Ok);
}
