public void Verify_sanitize_of_single_mhtml_file() { var tempFile = Path.ChangeExtension(Path.GetTempFileName(), ".mht"); File.Copy(TestConfig.PathToMht, tempFile); string mhtml = File.ReadAllText(tempFile); MHTMLParser parser = new MHTMLParser(mhtml) { OutputDirectory = Path.GetDirectoryName(tempFile), DecodeImageData = true }; var outFile = Path.ChangeExtension(tempFile, ".html"); File.WriteAllText(outFile, parser.getHTMLText()); _sanitizer = new SafeHtmlConverter(outFile) { Logger = NullLogger.Instance }; var result = _sanitizer.Run("jobtest"); Assert.That(File.Exists(result), "Output pdf file not created"); File.Delete(result); }
public void SaveDescriptor(FileSystemBlobDescriptor fileSystemBlobDescriptor) { if (fileSystemBlobDescriptor == null) { throw new ArgumentNullException(nameof(fileSystemBlobDescriptor)); } var fileName = _directoryManager.GetDescriptorFileNameFromBlobId(fileSystemBlobDescriptor.BlobId); File.WriteAllText(fileName, JsonConvert.SerializeObject(fileSystemBlobDescriptor)); }
protected async override Task <ProcessResult> OnPolling(PollerJobParameters parameters, string workingFolder) { string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder).ConfigureAwait(false); //String fileName = Path.Combine(Path.GetDirectoryName(pathToFile), parameters.All[JobKeys.FileName]); //Logger.DebugFormat("Move blob id {0} to real filename {1}", pathToFile, fileName); //if (File.Exists(fileName)) File.Delete(fileName); //File.Copy(pathToFile, fileName); if (Logger.IsDebugEnabled) { Logger.DebugFormat("Conversion of HtmlZip to PDF: file {0}", pathToFile); } var file = pathToFile; if (pathToFile.EndsWith(".mht", StringComparison.OrdinalIgnoreCase) || pathToFile.EndsWith(".mhtml", StringComparison.OrdinalIgnoreCase)) { string mhtml = File.ReadAllText(pathToFile); MHTMLParser parser = new MHTMLParser(mhtml) { OutputDirectory = workingFolder, DecodeImageData = true }; var outFile = Path.ChangeExtension(pathToFile, ".html"); File.WriteAllText(outFile, parser.getHTMLText()); file = outFile; } var sanitizer = new SafeHtmlConverter(file) { Logger = Logger }; file = sanitizer.Run(parameters.JobId); var converter = new HtmlToPdfConverterFromDiskFile(file, base.JobsHostConfiguration) { Logger = Logger }; var pdfConvertedFileName = converter.Run(parameters.TenantId, parameters.JobId); await AddFormatToDocumentFromFile( parameters.TenantId, parameters.JobId, new DocumentFormat(DocumentFormats.Pdf), pdfConvertedFileName, new Dictionary <string, object>()).ConfigureAwait(false); return(ProcessResult.Ok); }
protected async override Task <ProcessResult> OnPolling(PollerJobParameters parameters, string workingFolder) { string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder); if (Logger.IsDebugEnabled) { Logger.DebugFormat("Conversion of HtmlZip to PDF: file {0}", pathToFile); } var file = pathToFile; if (pathToFile.ToLower().EndsWith(".mht") || pathToFile.ToLower().EndsWith(".mhtml")) { string mhtml = File.ReadAllText(pathToFile); MHTMLParser parser = new MHTMLParser(mhtml); parser.OutputDirectory = workingFolder; parser.DecodeImageData = true; var outFile = Path.ChangeExtension(pathToFile, ".html"); File.WriteAllText(outFile, parser.getHTMLText()); file = outFile; } var converter = new HtmlToPdfConverterFromDiskFileOld(file, base.JobsHostConfiguration) { Logger = Logger }; var pdfConvertedFileName = converter.Run(parameters.JobId); await AddFormatToDocumentFromFile( parameters.TenantId, parameters.JobId, new DocumentFormat(DocumentFormats.Pdf), pdfConvertedFileName, new Dictionary <string, object>()); return(ProcessResult.Ok); }
private string ProcessFile(string pathToFile, string workingFolder) { var extension = Path.GetExtension(pathToFile).ToLower(); if (extension == ".htmlzip" || extension == ".htmzip") { ZipFile.ExtractToDirectory(pathToFile, workingFolder); Logger.DebugFormat("Extracted zip to {0}", workingFolder); var htmlFile = Path.ChangeExtension(pathToFile, "html"); if (File.Exists(htmlFile)) { Logger.DebugFormat("Html file is {0}", htmlFile); return(htmlFile); } htmlFile = Path.ChangeExtension(pathToFile, "htm"); if (File.Exists(htmlFile)) { Logger.DebugFormat("Html file is {0}", htmlFile); return(htmlFile); } Logger.ErrorFormat("Invalid HTMLZIP file, name is {0} but corresponding html file not found after decompression", Path.GetFileName(pathToFile)); } else if (extension == ".mht" || extension == ".mhtml") { MHTMLParser parser = new MHTMLParser(File.ReadAllText(pathToFile)); parser.OutputDirectory = workingFolder; parser.DecodeImageData = false; var html = parser.getHTMLText(); pathToFile = pathToFile + ".html"; File.WriteAllText(pathToFile, html); } return(pathToFile); }
public string Convert(String jobId, string pathToEml, string workingFolder) { Logger.DebugFormat("Coverting {0} in working folder {1}", pathToEml, workingFolder); var reader = new Reader(); var outFolder = Path.Combine(workingFolder, jobId); Logger.DebugFormat("Creating message working folder is {0}", outFolder); Directory.CreateDirectory(outFolder); Logger.Debug("Extracting files"); var files = reader.ExtractToFolder(pathToEml, outFolder); if (Logger.IsDebugEnabled) { foreach (var file in files) { Logger.DebugFormat("\t{0}", Path.GetFileName(file)); } Logger.DebugFormat("Total files {0}", files.Length); } var htmlFileName = files.FirstOrDefault(x => x.EndsWith(".htm", StringComparison.OrdinalIgnoreCase)) ?? files.FirstOrDefault(x => x.EndsWith(".html", StringComparison.OrdinalIgnoreCase)); if (htmlFileName == null) { var textFile = files.FirstOrDefault(x => x.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)); if (textFile != null) { htmlFileName = textFile + ".html"; var textcontent = File.ReadAllText(textFile); File.WriteAllText(htmlFileName, String.Format("<html><body><pre>{0}</pre></body></html>", textcontent)); } else { htmlFileName = "contentmissing.html"; File.WriteAllText(htmlFileName, "<html>No content found in mail.</html>"); } } var htmlNameWithoutExtension = Path.GetFileNameWithoutExtension(htmlFileName); var htmlContent = File.ReadAllText(htmlFileName); var dirInfoFullName = new DirectoryInfo(outFolder).FullName; htmlContent = Regex.Replace( htmlContent, @"src=""(?<src>.+?)""", new MatchEvaluator((m) => NormalizeImgEvaluator(m, dirInfoFullName)), RegexOptions.IgnoreCase); File.WriteAllText(htmlFileName, htmlContent); var pathToZip = Path.Combine(workingFolder, htmlNameWithoutExtension + ".ezip"); Logger.DebugFormat("New zip file is {0}", pathToZip); if (File.Exists(pathToZip)) { Logger.DebugFormat("Deleting previous file: {0}", pathToZip); File.Delete(pathToZip); } Logger.DebugFormat("Creating new file: {0}", pathToZip); ZipFile.CreateFromDirectory(outFolder, pathToZip); Logger.DebugFormat("Deleting message working folder", outFolder); Directory.Delete(outFolder, true); Logger.DebugFormat( "Convesion done {0} => {1}", pathToEml, pathToZip ); return(pathToZip); }
protected async override Task <ProcessResult> OnPolling( PollerJobParameters parameters, String workingFolder) { Boolean result; var contentFileName = Path.ChangeExtension(parameters.FileName, ".content"); if (!_formats.Contains(parameters.FileExtension)) { Logger.DebugFormat("Document for job Id {0} has an extension not supported, setting null content", parameters.JobId); return(new ProcessResult(await AddNullContentFormat(parameters, contentFileName))); } Logger.DebugFormat("Starting tika on job: {0}, file extension {1}", parameters.JobId, parameters.FileExtension); Logger.DebugFormat("Downloading blob for job: {0}, on local path {1}", parameters.JobId, workingFolder); string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder); pathToFile = ProcessFile(pathToFile, workingFolder); Boolean shouldAnalyze = _filterManager.ShouldAnalyze(parameters.FileName, pathToFile); if (!shouldAnalyze) { Logger.InfoFormat("File {0} for job {1} was discharded!", parameters.FileName, parameters.JobId); return(new ProcessResult(await AddNullContentFormat(parameters, contentFileName))); } Logger.DebugFormat("Search for password JobId:{0}", parameters.JobId); var passwords = ClientPasswordSet.GetPasswordFor(parameters.FileName).ToArray(); String content = ""; Int32 analyzerOrdinal = 0; Boolean success = false; var analyzer = BuildAnalyzer(analyzerOrdinal); do { try { if (passwords.Any()) { //Try with all the password foreach (var password in passwords) { try { content = analyzer.GetHtmlContent(pathToFile, password) ?? ""; break; //first password that can decrypt file break the list of password to try } catch (Exception) { Logger.ErrorFormat("Error opening file {0} with password", parameters.FileName); } } } else { //Simply analyze file without password Logger.DebugFormat("Analyze content JobId: {0} -> Path: {1}", parameters.JobId, pathToFile); content = analyzer.GetHtmlContent(pathToFile, "") ?? ""; } success = true; } catch (Exception ex) { Logger.ErrorFormat(ex, "Error extracting tika with analyzer {0} on file {1}", analyzer.Describe(), parameters.FileName, parameters.JobId); analyzer = BuildAnalyzer(++analyzerOrdinal); if (analyzer != null) { Logger.InfoFormat("Retry job {0} with analyzer {1}", parameters.JobId, analyzer.Describe()); } } } while (analyzer != null && success == false); Logger.DebugFormat("Finished tika on job: {0}, charsNum {1}", parameters.JobId, content.Count()); String sanitizedContent = content; if (!string.IsNullOrWhiteSpace(content)) { var resultContent = _builder.CreateFromTikaPlain(content); var documentContent = resultContent.Content; sanitizedContent = resultContent.SanitizedTikaContent; var pages = documentContent.Pages.Count(); string lang = null; if (pages > 1) { lang = LanguageDetector.GetLanguage(documentContent.Pages[1].Content); } if (lang == null && pages == 1) { lang = LanguageDetector.GetLanguage(documentContent.Pages[0].Content); } if (lang != null) { documentContent.AddMetadata(DocumentContent.MedatataLanguage, lang); } result = await AddFormatToDocumentFromObject( parameters.TenantId, this.QueueName, parameters.JobId, new DocumentFormat(DocumentFormats.Content), documentContent, contentFileName, new Dictionary <string, object>()); Logger.DebugFormat("Added format {0} to jobId {1}, result: {2}", DocumentFormats.Content, parameters.JobId, result); } var tikaFileName = Path.Combine(workingFolder, Path.GetFileNameWithoutExtension(parameters.FileName) + ".tika.html"); tikaFileName = SanitizeFileNameForLength(tikaFileName); File.WriteAllText(tikaFileName, sanitizedContent); result = await AddFormatToDocumentFromFile( parameters.TenantId, parameters.JobId, new DocumentFormat(DocumentFormats.Tika), tikaFileName, new Dictionary <string, object>()); Logger.DebugFormat("Added format {0} to jobId {1}, result: {2}", DocumentFormats.Tika, parameters.JobId, result); return(ProcessResult.Ok); }