public async Task can_upload_document_with_name_greater_than_250_char() { var handle = DocumentHandle.FromString("Pdf_3"); String longFileName = Path.Combine( Path.GetTempPath(), "_lfn" + new string('X', 240) + ".pdf"); if (!File.Exists(longFileName)) { File.Copy(TestConfig.PathToDocumentPdf, longFileName); } await _documentStoreClient.UploadAsync(longFileName, handle); // wait background projection polling await UpdateAndWaitAsync().ConfigureAwait(false); // check readmodel var tenantAccessor = ContainerAccessor.Instance.Resolve <ITenantAccessor>(); var tenant = tenantAccessor.GetTenant(new TenantId(TestConfig.Tenant)); var docReader = tenant.Container.Resolve <IMongoDbReader <DocumentDescriptorReadModel, DocumentDescriptorId> >(); var allDocuments = docReader.AllUnsorted.Count(); Assert.AreEqual(1, allDocuments); }
private void TestPowerPointFile( List <PollerTestResult> retValue, String fileName, String type, Byte[] fileContent) { try { var tempFile = Path.Combine(Path.GetTempPath(), fileName); if (File.Exists(tempFile)) { File.Delete(tempFile); } File.WriteAllBytes(tempFile, fileContent); var conversionError = PowerPointConverter.ConvertToPdf(tempFile, tempFile + ".pdf"); if (!String.IsNullOrEmpty(conversionError)) { retValue.Add(new PollerTestResult(false, type + "Conversion with powerpoint converter failed:" + conversionError)); } else { retValue.Add(new PollerTestResult(true, type + "Conversion with powerpoint ok.")); } } catch (Exception ex) { retValue.Add(new PollerTestResult(false, type + "Conversion with powerpoint converter failed: " + ex.Message)); } }
private void TestWordFile( List <PollerTestResult> retValue, String fileName, String type, Byte[] fileContent) { try { OfficeUtils.KillOfficeProcess("WINWORD"); var tempFile = Path.Combine(Path.GetTempPath(), fileName); if (File.Exists(tempFile)) { File.Delete(tempFile); } File.WriteAllBytes(tempFile, fileContent); var conversionError = WordConverter.ConvertToPdf(tempFile, tempFile + ".pdf"); if (!String.IsNullOrEmpty(conversionError)) { retValue.Add(new PollerTestResult(false, type + "Conversion with word converter failed: " + conversionError)); } else { retValue.Add(new PollerTestResult(true, type + "Conversion with word ok.")); } } catch (Exception ex) { retValue.Add(new PollerTestResult(false, type + "Conversion with word converter failed: " + ex.Message)); } }
string DownloadLocalCopy(String tenantId, String jobId) { Logger.DebugFormat("Downloaded {0}", _inputFileName); if (IsUnzippedHtmlFile()) { return(_inputFileName); } var workingFolder = Path.GetDirectoryName(_inputFileName); ZipFile.ExtractToDirectory(_inputFileName, workingFolder); Logger.DebugFormat("Extracted zip to {0}", workingFolder); var htmlFile = Path.ChangeExtension(_inputFileName, "html"); if (File.Exists(htmlFile)) { Logger.DebugFormat("Html file is {0}", htmlFile); return(htmlFile); } htmlFile = Path.ChangeExtension(_inputFileName, "htm"); if (File.Exists(htmlFile)) { Logger.DebugFormat("Html file is {0}", htmlFile); return(htmlFile); } var msg = string.Format("Html file not found for {0}!", jobId); Logger.Error(msg); throw new Exception(msg); }
public void Verify_sanitize_of_single_mhtml_file() { var tempFile = Path.ChangeExtension(Path.GetTempFileName(), ".mht"); File.Copy(TestConfig.PathToMht, tempFile); string mhtml = File.ReadAllText(tempFile); MHTMLParser parser = new MHTMLParser(mhtml) { OutputDirectory = Path.GetDirectoryName(tempFile), DecodeImageData = true }; var outFile = Path.ChangeExtension(tempFile, ".html"); File.WriteAllText(outFile, parser.getHTMLText()); _sanitizer = new SafeHtmlConverter(outFile) { Logger = NullLogger.Instance }; var result = _sanitizer.Run("jobtest"); Assert.That(File.Exists(result), "Output pdf file not created"); File.Delete(result); }
private static void TestFile( List <PollerTestResult> retValue, ITikaAnalyzer analyzer, String fileName, String type, String expected, Byte[] fileContent) { var tempFile = Path.Combine(Path.GetTempPath(), fileName); if (File.Exists(tempFile)) { File.Delete(tempFile); } File.WriteAllBytes(tempFile, fileContent); try { string content = analyzer.GetHtmlContent(tempFile, ""); if (content.Contains(expected)) { retValue.Add(new PollerTestResult(true, type + " conversion")); } else { retValue.Add(new PollerTestResult(false, type + " conversion: wrong content")); } } catch (Exception ex) { retValue.Add(new PollerTestResult(false, type + " conversion: " + ex.Message)); } }
private void TestFile( List <PollerTestResult> retValue, String fileName, String type, Byte[] fileContent) { String converter = Conversion.GetType().Name; try { var tempFile = Path.Combine(Path.GetTempPath(), fileName); if (File.Exists(tempFile)) { File.Delete(tempFile); } File.WriteAllBytes(tempFile, fileContent); string content = Conversion.Run(tempFile, "pdf"); if (!String.IsNullOrEmpty(content)) { retValue.Add(new PollerTestResult(true, type + " conversion with converter: " + converter)); } else { retValue.Add(new PollerTestResult(false, type + " conversion: wrong content with converter: " + converter)); } } catch (Exception ex) { retValue.Add(new PollerTestResult(false, type + " conversion with converter " + converter + ex.Message)); } }
protected async override Task <ProcessResult> OnPolling(PollerJobParameters parameters, string workingFolder) { string pathToFile = await DownloadBlob(parameters.TenantId, parameters.JobId, parameters.FileName, workingFolder).ConfigureAwait(false); String fileName = Path.Combine(Path.GetDirectoryName(pathToFile), parameters.All[JobKeys.FileName]); Logger.DebugFormat("Move blob id {0} to real filename {1}", pathToFile, fileName); if (File.Exists(fileName)) { File.Delete(fileName); } File.Copy(pathToFile, fileName); var converter = new HtmlToPdfConverterFromDiskFile(fileName, base.JobsHostConfiguration) { Logger = Logger }; var pdfConvertedFileName = converter.Run(parameters.TenantId, parameters.JobId); await AddFormatToDocumentFromFile( parameters.TenantId, parameters.JobId, new DocumentFormat(DocumentFormats.Pdf), pdfConvertedFileName, new Dictionary <string, object>()).ConfigureAwait(false); return(ProcessResult.Ok); }
public void correctly_delete_file_after_import() { _queue.DeleteTaskFileAfterImport = true; var descriptor = _queue.LoadTask(_pathToTask); _queue.UploadFile(_pathToTask, descriptor); Assert.That(File.Exists(_pathToTask), Is.False); }
public void Delete(BlobId blobId) { var descriptorLocalFileName = _directoryManager.GetDescriptorFileNameFromBlobId(blobId); if (File.Exists(descriptorLocalFileName)) { File.Delete(descriptorLocalFileName); } }
public void Verify_preview_of_single_html_file() { var tempFile = Path.ChangeExtension(Path.GetTempFileName(), ".html"); File.Copy(TestConfig.PathToSimpleHtmlFile, tempFile); _converter = new HtmlToPdfConverterFromDiskFileOld(tempFile, _config); _converter.Logger = NullLogger.Instance; var result = _converter.Run("jobtest"); Assert.That(File.Exists(result), "Output pdf file not created"); File.Delete(result); }
public void what_happens_if_task_file_cannot_be_deleted() { _queue.DeleteTaskFileAfterImport = true; var attribute = File.GetAttributes(_pathToTask); File.SetAttributes(_pathToTask, attribute | FileAttributes.ReadOnly); _queue.PollFileSystem(); Assert.That(File.Exists(_pathToTask), Is.True); //file cannot be deleted _commandBus.Received(1).Send(Arg.Any <ICommand>(), "import-from-file"); _queue.PollFileSystem(); _commandBus.Received(1).Send(Arg.Any <ICommand>(), "import-from-file"); }
private string GetFileNameFromBlobIdAndRemoveDuplicates(BlobId blobId) { var fileName = _directoryManager.GetFileNameFromBlobId(blobId); if (File.Exists(fileName)) { //Anomaly, we are trying to overwrite the blob Logger.Warn($"Destination file {blobId} already exists for id {blobId}"); //Todo move in another folder ... maybe a lost and found. File.Move(fileName, fileName + "." + Guid.NewGuid().ToString()); } return(fileName); }
public void Verify_sanitize_of_single_html_file() { var tempFile = Path.ChangeExtension(Path.GetTempFileName(), ".html"); File.Copy(TestConfig.PathToSimpleHtmlFile, tempFile); _sanitizer = new SafeHtmlConverter(tempFile) { Logger = NullLogger.Instance }; var result = _sanitizer.Run("jobtest"); Assert.That(File.Exists(result), "HTML file sanitized"); File.Delete(result); }
public string Download(BlobId blobId, string folder) { if (blobId == null) { throw new ArgumentNullException(nameof(blobId)); } if (String.IsNullOrEmpty(folder)) { throw new ArgumentNullException(nameof(folder)); } if (!Directory.Exists(folder)) { throw new ArgumentException($"folder {folder} does not exists", nameof(folder)); } var descriptor = _mongodDbFileSystemBlobDescriptorStorage.FindOneById(blobId); if (descriptor == null) { throw new ArgumentException($"Descriptor for {blobId} not found in {_mongodDbFileSystemBlobDescriptorStorage.GetType().Name}"); } var localFileName = _directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension); if (!File.Exists(localFileName)) { Logger.Error($"Blob {blobId} has descriptor, but blob file {localFileName} not found in the system."); throw new ArgumentException($"Blob {blobId} not found"); } var originalFileName = descriptor.FileNameWithExtension.ToString(); string destinationFileName = Path.Combine(folder, originalFileName); Int32 uniqueId = 1; while (File.Exists(destinationFileName)) { destinationFileName = Path.Combine(folder, Path.GetFileNameWithoutExtension(originalFileName) + $" ({uniqueId++})") + Path.GetExtension(originalFileName); } File.Copy(localFileName, destinationFileName); if (Logger.IsDebugEnabled) { Logger.Debug($"Blob {blobId} downloaded in folder {folder} with name {destinationFileName}"); } return(destinationFileName); }
private static void TestFile( List <PollerTestResult> retValue, CreateImageFromPdfTask task, String fileName, Byte[] fileContent) { var tempFile = Path.Combine(Path.GetTempPath(), fileName); if (File.Exists(tempFile)) { File.Delete(tempFile); } File.WriteAllBytes(tempFile, fileContent); try { var convertParams = new CreatePdfImageTaskParams() { Dpi = 150, FromPage = 1, Pages = 1, Format = CreatePdfImageTaskParams.ImageFormat.Jpg, }; Boolean wasCalled = false; var result = task.Run( tempFile, convertParams, (i, s) => { wasCalled = true; return(Task.FromResult <Boolean>(true)); } ); result.Wait(); if (wasCalled) { retValue.Add(new PollerTestResult(true, "Pdf to Jpg")); } else { retValue.Add(new PollerTestResult(false, "Pdf to Jpg")); } } catch (Exception ex) { retValue.Add(new PollerTestResult(false, "Pdf to Jpg: " + ex.Message)); } }
public string Run(string sourceFile, string outType) { var outputFile = Path.ChangeExtension(sourceFile, outType); Logger.DebugFormat("UNO CONVERSION: Converting: {0} to {1}", sourceFile, outputFile); lock (LockRoot) // -> single runner (todo: more user profiles) { ConvertToPdf(sourceFile, outputFile); } if (!File.Exists(outputFile)) { throw new Exception("Conversion failed"); } return(outputFile); }
static TestConfig() { DocumentsFolder = new DirectoryInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Docs")).FullName; var aDocTestFile = Path.Combine(DocumentsFolder, "A Word Document.docx"); if (!Directory.Exists(DocumentsFolder) || !File.Exists(aDocTestFile)) { //we do not have Docs folder, use doc folder on source. DocumentsFolder = new DirectoryInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..\\..\\Docs")).FullName; } TempFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "temp"); ServerAddress = new Uri("http://localhost:5123"); Tenant = "tests"; DocsTenant = "docs"; DemoTenant = "demo"; QueueFolder = GenerateQueueFolder(); }
public string GetPathToJava() { var javaHome = GetConfigValue("JAVA_HOME"); if (String.IsNullOrEmpty(javaHome)) { throw new Exception("Please set JAVA_HOME in app.config or env variable"); } var pathToJavaExe = Path.Combine(javaHome, "bin\\java.exe"); if (!File.Exists(pathToJavaExe)) { throw new Exception(string.Format("Java not found on {0}", pathToJavaExe)); } return(pathToJavaExe); }
public FileSystemBlobDescriptor FindOneById(BlobId blobId) { if (blobId == null) { throw new ArgumentNullException(nameof(blobId)); } var descriptorLocalFileName = _directoryManager.GetDescriptorFileNameFromBlobId(blobId); if (!File.Exists(descriptorLocalFileName)) { return(null); } var descriptor = JsonConvert.DeserializeObject <FileSystemBlobDescriptor>(File.ReadAllText(descriptorLocalFileName)); descriptor.SetLocalFileName(_directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension)); return(descriptor); }
public void task_file_cannot_be_deleted_but_then_modified_to_retry() { _queue.DeleteTaskFileAfterImport = true; var attribute = File.GetAttributes(_pathToTask); File.SetAttributes(_pathToTask, attribute | FileAttributes.ReadOnly); _queue.PollFileSystem(); Assert.That(File.Exists(_pathToTask), Is.True); //file cannot be deleted _commandBus.Received(1).Send(Arg.Any <ICommand>(), "import-from-file"); _queue.PollFileSystem(); _commandBus.Received(1).Send(Arg.Any <ICommand>(), "import-from-file"); attribute = File.GetAttributes(_pathToTask); File.SetAttributes(_pathToTask, attribute & ~FileAttributes.ReadOnly); File.SetLastWriteTime(_pathToTask, DateTime.UtcNow); _queue.PollFileSystem(); _commandBus.Received(2).Send(Arg.Any <ICommand>(), "import-from-file"); }
public List <PollerTestResult> Execute() { List <PollerTestResult> retValue = new List <PollerTestResult>(); String format = "png"; Int32 secondsOffset = 4; String vlcExecutable = Helper.GetExecutableLocation(); if (vlcExecutable == null) { retValue.Add(new PollerTestResult(false, "Executable location, use app settings vlc_location")); return(retValue); } else { retValue.Add(new PollerTestResult(true, "Executable location, ")); } try { var worker = new VlcCommandLineThumbnailCreator(vlcExecutable, format, NullLogger.Instance); var tempFile = Path.Combine(Path.GetTempPath(), "video.mp4"); if (File.Exists(tempFile)) { File.Delete(tempFile); } File.WriteAllBytes(tempFile, TestFiles.video); var thumb = worker.CreateThumbnail(tempFile, Path.GetTempPath(), 4); retValue.Add(new PollerTestResult( !String.IsNullOrEmpty(tempFile), "video thumb extraction: ")); } catch (Exception ex) { retValue.Add(new PollerTestResult(false, "video thumb extraction: " + ex.Message)); } return(retValue); }
private string GetFileNameFromBlobIdAndRemoveDuplicates(BlobId blobId, String fileName) { var finalFileName = _directoryManager.GetFileNameFromBlobId(blobId, fileName); if (File.Exists(finalFileName)) { //Anomaly, we are trying to overwrite the blob Logger.Warn($"Destination file {blobId} already exists for id {blobId}"); try { //Todo move in another folder ... maybe a lost and found. File.Move(finalFileName, finalFileName + "." + Guid.NewGuid().ToString()); } catch (Exception ex) { throw new Exception($"Error moving file {finalFileName} with guid suffix", ex); } } return(finalFileName); }
public string Run(string sourceFile, string outType) { Logger.DebugFormat("DIRECT SOFFICE.EXE CONVERSION: Starting conversion of blobId {0} to {1}", sourceFile, outType); string pathToLibreOffice = _config.GetPathToLibreOffice(); var outputFile = Path.ChangeExtension(sourceFile, outType); string arguments = string.Format("--headless -convert-to {2} -outdir \"{0}\" \"{1}\" ", Path.GetDirectoryName(sourceFile), sourceFile, outType ); var psi = new ProcessStartInfo(pathToLibreOffice, arguments) { UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true, CreateNoWindow = true, WindowStyle = ProcessWindowStyle.Minimized }; Logger.DebugFormat("Command: {0} {1}", pathToLibreOffice, arguments); using (var p = Process.Start(psi)) { Logger.Debug("Process started"); p.WaitForExit(); Logger.Debug("Process ended"); } if (!File.Exists(outputFile)) { throw new Exception("Conversion failed"); } return(outputFile); }
private string ProcessFile(string pathToFile, string workingFolder) { var extension = Path.GetExtension(pathToFile).ToLower(); if (extension == ".htmlzip" || extension == ".htmzip") { ZipFile.ExtractToDirectory(pathToFile, workingFolder); Logger.DebugFormat("Extracted zip to {0}", workingFolder); var htmlFile = Path.ChangeExtension(pathToFile, "html"); if (File.Exists(htmlFile)) { Logger.DebugFormat("Html file is {0}", htmlFile); return(htmlFile); } htmlFile = Path.ChangeExtension(pathToFile, "htm"); if (File.Exists(htmlFile)) { Logger.DebugFormat("Html file is {0}", htmlFile); return(htmlFile); } Logger.ErrorFormat("Invalid HTMLZIP file, name is {0} but corresponding html file not found after decompression", Path.GetFileName(pathToFile)); } else if (extension == ".mht" || extension == ".mhtml") { MHTMLParser parser = new MHTMLParser(File.ReadAllText(pathToFile)); parser.OutputDirectory = workingFolder; parser.DecodeImageData = false; var html = parser.getHTMLText(); pathToFile = pathToFile + ".html"; File.WriteAllText(pathToFile, html); } return(pathToFile); }
public void should_serialize_document_import_data() { var fname = Path.Combine(TestConfig.TempFolder, "a_file_to_import"); var client = new DocumentStoreServiceClient(new Uri("http://ds"), "test"); var did = client.CreateDocumentImportData(TaskId, "c:\\temp\\a file.docx", "a file.docx", Doc); client.QueueDocumentImport(did, fname); Assert.IsTrue(File.Exists(fname + ".dsimport")); const string expected = @"{ ""TaskId"": ""9a29d730-f57a-41e4-92ba-55b7d99712a2"", ""Uri"": ""c:\\temp\\a file.docx"", ""FileName"": ""a file.docx"", ""Handle"": ""doc"", ""Format"": ""original"", ""Tenant"": ""test"", ""CustomData"": null, ""DeleteAfterImport"": false }"; Assert.AreEqual(expected, File.ReadAllText(fname + ".dsimport")); }
public async Task <Boolean> Run( String pathToFile, CreatePdfImageTaskParams createPdfImageTaskParams, Func <int, Stream, Task <Boolean> > pageWriter) { String tempFileName = null; if (Passwords.Count > 0) { tempFileName = Path.Combine(Path.GetDirectoryName(pathToFile), Path.GetFileNameWithoutExtension(pathToFile) + "_decrypted.pdf"); if (Decryptor.DecryptFile(pathToFile, tempFileName, Passwords)) { pathToFile = tempFileName; } } using (var sourceStream = File.OpenRead(pathToFile)) { var settings = new MagickReadSettings { Density = new PointD(createPdfImageTaskParams.Dpi, createPdfImageTaskParams.Dpi) }; settings.FrameIndex = 0; // First page settings.FrameCount = 1; // Number of pages MagickFormat imageFormat = TranslateFormat(createPdfImageTaskParams.Format); Logger.DebugFormat("Image format is {0}", imageFormat.ToString()); using (var images = new MagickImageCollection()) { bool done = false; if (!_firstDone) { lock (LockForInitializationIssue) { if (!_firstDone) { images.Read(sourceStream, settings); done = true; } } } if (!done) { images.Read(sourceStream, settings); } var lastImage = Math.Min(createPdfImageTaskParams.FromPage - 1 + createPdfImageTaskParams.Pages, images.Count) - 1; for (int page = createPdfImageTaskParams.FromPage - 1; page <= lastImage; page++) { var image = images[page]; image.Format = imageFormat; using (var ms = new MemoryStream()) { image.Write(ms); ms.Seek(0L, SeekOrigin.Begin); await pageWriter(page + 1, ms).ConfigureAwait(false); } } } } if (!String.IsNullOrEmpty(tempFileName) && File.Exists(tempFileName)) { File.Delete(tempFileName); } return(true); }
public string Convert(String jobId, string pathToEml, string workingFolder) { Logger.DebugFormat("Coverting {0} in working folder {1}", pathToEml, workingFolder); var reader = new Reader(); var outFolder = Path.Combine(workingFolder, jobId); Logger.DebugFormat("Creating message working folder is {0}", outFolder); Directory.CreateDirectory(outFolder); Logger.Debug("Extracting files"); var files = reader.ExtractToFolder(pathToEml, outFolder); if (Logger.IsDebugEnabled) { foreach (var file in files) { Logger.DebugFormat("\t{0}", Path.GetFileName(file)); } Logger.DebugFormat("Total files {0}", files.Length); } var htmlFileName = files.FirstOrDefault(x => x.EndsWith(".htm", StringComparison.OrdinalIgnoreCase)) ?? files.FirstOrDefault(x => x.EndsWith(".html", StringComparison.OrdinalIgnoreCase)); if (htmlFileName == null) { var textFile = files.FirstOrDefault(x => x.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)); if (textFile != null) { htmlFileName = textFile + ".html"; var textcontent = File.ReadAllText(textFile); File.WriteAllText(htmlFileName, String.Format("<html><body><pre>{0}</pre></body></html>", textcontent)); } else { htmlFileName = "contentmissing.html"; File.WriteAllText(htmlFileName, "<html>No content found in mail.</html>"); } } var htmlNameWithoutExtension = Path.GetFileNameWithoutExtension(htmlFileName); var htmlContent = File.ReadAllText(htmlFileName); var dirInfoFullName = new DirectoryInfo(outFolder).FullName; htmlContent = Regex.Replace( htmlContent, @"src=""(?<src>.+?)""", new MatchEvaluator((m) => NormalizeImgEvaluator(m, dirInfoFullName)), RegexOptions.IgnoreCase); File.WriteAllText(htmlFileName, htmlContent); var pathToZip = Path.Combine(workingFolder, htmlNameWithoutExtension + ".ezip"); Logger.DebugFormat("New zip file is {0}", pathToZip); if (File.Exists(pathToZip)) { Logger.DebugFormat("Deleting previous file: {0}", pathToZip); File.Delete(pathToZip); } Logger.DebugFormat("Creating new file: {0}", pathToZip); ZipFile.CreateFromDirectory(outFolder, pathToZip); Logger.DebugFormat("Deleting message working folder", outFolder); Directory.Delete(outFolder, true); Logger.DebugFormat( "Convesion done {0} => {1}", pathToEml, pathToZip ); return(pathToZip); }