public async Task attachments_fat_retrieve_nested_attachment()
{
//Upload father
var fatherHandle = new DocumentHandle("father");
await _documentStoreClient.UploadAsync(TestConfig.PathToDocumentPdf, fatherHandle);
// wait background projection polling
await UpdateAndWaitAsync().ConfigureAwait(false);
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToDocumentPng, fatherHandle, "source", Path.GetFileName(TestConfig.PathToDocumentPng));
// wait background projection polling
await UpdateAndWaitAsync().ConfigureAwait(false);
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToExcelDocument, new DocumentHandle("source_1"), "nested", Path.GetFileName(TestConfig.PathToExcelDocument));
// wait background projection polling
await UpdateAndWaitAsync().ConfigureAwait(false);
var attachments = await _documentStoreClient.GetAttachmentsFatAsync(fatherHandle);
Assert.NotNull(attachments);
Assert.That(attachments.Attachments, Has.Count.EqualTo(2));
Assert.That(attachments.Attachments.Select(a => a.FileName), Is.EquivalentTo(new[] {
Path.GetFileName(TestConfig.PathToDocumentPng),
Path.GetFileName(TestConfig.PathToExcelDocument)
}));
Assert.That(attachments.Attachments.Select(a => a.Uri),
Is.EquivalentTo(new[] {
new Uri("http://localhost:5123/tests/documents/source_1"),
new Uri("http://localhost:5123/tests/documents/nested_1")
}));
}
public BlobId Upload(DocumentFormat format, string pathToFile)
{
using (var inStream = File.OpenRead(pathToFile))
{
return(Upload(format, new FileNameWithExtension(Path.GetFileName(pathToFile)), inStream));
}
}
public async Task attachments_not_retrieve_nested_attachment()
{
//Upload father
var fatherHandle = new DocumentHandle("father");
await _documentStoreClient.UploadAsync(TestConfig.PathToDocumentPdf, fatherHandle);
// wait background projection polling
await UpdateAndWaitAsync().ConfigureAwait(false);
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToDocumentPng, fatherHandle, "source", Path.GetFileName(TestConfig.PathToDocumentPng));
// wait background projection polling
await UpdateAndWaitAsync().ConfigureAwait(false);
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToDocumentPng, new DocumentHandle("source_1"), "nested", Path.GetFileName(TestConfig.PathToDocumentPng));
// wait background projection polling
await UpdateAndWaitAsync().ConfigureAwait(false);
var attachments = await _documentStoreClient.GetAttachmentsAsync(fatherHandle);
Assert.NotNull(attachments);
Assert.That(attachments.Attachments.Length, Is.EqualTo(1));
Assert.That(attachments.Attachments[0].RelativePath, Is.EqualTo(Path.GetFileName(TestConfig.PathToDocumentPng)));
Assert.That(attachments.Attachments[0].Handle, Is.EqualTo("http://localhost:5123/tests/documents/source_1"));
}
public IBlobDescriptor Persist(BlobId blobId, string fileName)
{
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
return(Persist(blobId, new FileNameWithExtension(Path.GetFileName(fileName)), fs));
}
}
public void queue_folder()
{
var sourceFolder = @"c:\Downloads\video\";
var taskFolder = @"c:\temp\dsqueue";
var files = Directory.GetFiles(sourceFolder, "*.*", SearchOption.AllDirectories);
var docs = new DocumentStoreServiceClient(new Uri("http://ds"), "docs");
// var demo = new DocumentStoreServiceClient(new Uri("http://ds"), "demo");
var counter = 1;
Parallel.ForEach(files, file =>
{
var handle = "import_" + counter++;
var taskDoc = docs.CreateDocumentImportData(
Guid.NewGuid(),
file,
Path.GetFileName(file),
new DocumentHandle(handle)
);
taskDoc.DeleteAfterImport = false;
var docsFile = Path.Combine(taskFolder, "doc_" + handle + "_" + DateTime.Now.Ticks);
docs.QueueDocumentImport(taskDoc, docsFile);
//var taskDemo = demo.CreateDocumentImportData(
// Guid.NewGuid(),
// file,
// new DocumentHandle(handle)
//);
//taskDemo.DeleteAfterImport = false;
//var demoFile = Path.Combine(taskFolder, "demo_" + handle + "_" + DateTime.Now.Ticks);
//demo.QueueDocumentImport(taskDemo, demoFile);
});
}
void CreateDocument(int id, string handle, string pathToFile)
{
var fname = Path.GetFileName(pathToFile);
var info = new DocumentHandleInfo(new DocumentHandle(handle), new FileNameWithExtension(fname));
_bus.Send(new InitializeDocumentDescriptor(
new DocumentDescriptorId(id),
_filestore.Upload(DocumentFormats.Original, pathToFile),
info,
new FileHash("1234abcd"),
new FileNameWithExtension("a", "file")
));
Thread.Sleep(50);
}
public void Upload_TempDirectory()
{
var files = Directory.GetFiles(@"X:\temp\testupload", "*.*", SearchOption.AllDirectories);
Parallel.ForEach(files, file =>
{
{
if (!Path.GetFileName(file).StartsWith("."))
{
_docs.UploadAsync(file, DocumentHandle.FromString(SanitizeFileName(file))).Wait();
}
}
});
}
BlobId AddFormatToDocument(int id, string handle, DocumentFormat format, PipelineId pipelineId, string pathToFile)
{
var fname = Path.GetFileName(pathToFile);
var info = new DocumentHandleInfo(new DocumentHandle(handle), new FileNameWithExtension(fname));
var blobId = _filestore.Upload(format, pathToFile);
_bus.Send(new AddFormatToDocumentDescriptor(
new DocumentDescriptorId(id),
format,
blobId,
pipelineId
));
Thread.Sleep(50);
return(blobId);
}
public void Upload_simple_html()
{
const string taskFolder = @"c:\temp\dsqueue";
DocumentImportData data = _docs.CreateDocumentImportData(
Guid.NewGuid(),
TestConfig.PathToSimpleHtmlFile,
Path.GetFileName(TestConfig.PathToSimpleHtmlFile),
DocumentHandle.FromString("simple-html-file"));
data.DeleteAfterImport = false;
var docsFile = Path.Combine(taskFolder, "doc_simple-html-file_" + DateTime.Now.Ticks);
_docs.QueueDocumentImport(data, docsFile);
}
public void SetUp()
{
_blobId = new BlobId(_originalFormat, 1);
var fileName = Path.GetFileName(_importFileName);
_pathToTask = Path.Combine(TestConfig.QueueFolder, fileName);
var fileToImport = Path.Combine(TestConfig.QueueFolder, "A Word Document.docx");
ClearQueueTempFolder();
Directory.CreateDirectory(TestConfig.QueueFolder);
File.Copy(Path.Combine(TestConfig.DocumentsFolder, _importFileName), _pathToTask);
File.Copy(TestConfig.PathToWordDocument, fileToImport);
var accessor = Substitute.For <ITenantAccessor>();
var tenant = Substitute.For <ITenant>();
tenant.Id.Returns(new TenantId("tests"));
var container = Substitute.For <IWindsorContainer>();
_commandBus = Substitute.For <ICommandBus>();
var identityGenerator = Substitute.For <IIdentityGenerator>();
_blobstore = Substitute.For <IBlobStore>();
_blobstore.Upload(Arg.Is(_originalFormat), Arg.Any <string>()).Returns(_blobId);
_blobstore.Upload(Arg.Is(_originalFormat), Arg.Any <FileNameWithExtension>(), Arg.Any <Stream>()).Returns(_blobId);
accessor.GetTenant(_testTenant).Returns(tenant);
accessor.Current.Returns(tenant);
tenant.Container.Returns(container);
container.Resolve <IBlobStore>().Returns(_blobstore);
container.Resolve <IIdentityGenerator>().Returns(identityGenerator);
container.Resolve <IMongoDatabase>().Returns(MongoDbTestConnectionProvider.ReadModelDb);
var collection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection <ImportFailure>("sys.importFailures");
collection.Drop();
DocumentStoreTestConfiguration config = new DocumentStoreTestConfiguration(tenantId: "tests");
config.SetFolderToMonitor(TestConfig.QueueFolder);
var sysDb = config.TenantSettings.Single(t => t.TenantId == "tests").Get <IMongoDatabase>("system.db");
sysDb.Drop();
_queue = new ImportFormatFromFileQueue(config, accessor, _commandBus)
{
Logger = new ConsoleLogger()
};
_queue.DeleteTaskFileAfterImport = false;
}
public BlobId Upload(DocumentFormat format, string pathToFile)
{
FileInfo finfo = new FileInfo(pathToFile);
if (!finfo.Exists)
{
throw new ArgumentException($"File {pathToFile} not found");
}
using (var fileStream = new FileStream(pathToFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var descriptor = SaveStream(format, new FileNameWithExtension(Path.GetFileName(pathToFile)), fileStream);
_blobDescriptorCollection.Save(descriptor, descriptor.BlobId);
return(descriptor.BlobId);
}
}
public void Upload_TempDirectory_FLOOD()
{
var files = Directory.GetFiles(@"X:\temp\testupload", "*.*", SearchOption.AllDirectories);
List <Task> taskList = new List <Task>();
foreach (var file in files.Where(f => !Path.GetFileName(f).StartsWith(".")))
{
//NO WAIT. This will FLOOD documentstore
var task = _docs.UploadAsync(file, DocumentHandle.FromString(SanitizeFileName(file)));
taskList.Add(task);
if (taskList.Count > 100)
{
Task.WaitAll(taskList.ToArray());
taskList.Clear();
}
}
}
private async Task <HttpResponseMessage> upload_file(string pathToFile, string documentHandle)
{
using (var stream = new FileStream(pathToFile, FileMode.Open))
{
var multipartFormDataContent = new MultipartFormDataContent("test")
{
{
new StreamContent(stream),
Path.GetFileNameWithoutExtension(pathToFile),
Path.GetFileName(pathToFile)
}
};
Controller.Request.Content = multipartFormDataContent;
return(await Controller.Upload(_tenantId, new DocumentHandle(documentHandle)));
}
}
private String GetRawFileNameFromBlobId(BlobId blobId, String fileName)
{
var id = blobId.Id;
var stringPadded = String.Format("{0:D15}", id / 1000);
StringBuilder directoryName = new StringBuilder(15);
for (int i = 0; i < Math.Min(stringPadded.Length, 15); i++)
{
directoryName.Append(stringPadded[i]);
if (i % _folderPrefixLength == (_folderPrefixLength - 1))
{
directoryName.Append(System.IO.Path.DirectorySeparatorChar);
}
}
var finalDirectory = Path.Combine(BaseDirectory, blobId.Format, directoryName.ToString());
Directory.EnsureDirectory(finalDirectory);
return(finalDirectory + id + "." + Path.GetFileName(fileName));
}
public BlobId Upload(DocumentFormat format, string pathToFile)
{
FileInfo finfo = new FileInfo(pathToFile);
if (!finfo.Exists)
{
throw new ArgumentException($"File {pathToFile} not found");
}
if (Logger.IsDebugEnabled)
{
Logger.Debug($"Upload document format {format}. File: {pathToFile}");
}
using (var fileStream = new FileStream(pathToFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var descriptor = SaveStream(format, new FileNameWithExtension(Path.GetFileName(pathToFile)), fileStream);
if (Logger.IsDebugEnabled)
{
Logger.Debug($"Uploaded document format {format}. File: {pathToFile} with blob Id {descriptor.BlobId}");
}
_mongodDbFileSystemBlobDescriptorStorage.SaveDescriptor(descriptor);
return(descriptor.BlobId);
}
}
public async Task Add_multiple_attachment_to_existing_handle()
{
//Upload father
var fatherHandle = new DocumentHandle("father");
await _documentStoreClient.UploadAsync(TestConfig.PathToDocumentPdf, fatherHandle);
await UpdateAndWaitAsync().ConfigureAwait(false);
//upload attachments
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToDocumentPng, fatherHandle, "Content", Path.GetFileName(TestConfig.PathToDocumentPng)).ConfigureAwait(false);
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToOpenDocumentText, fatherHandle, "Content", Path.GetFileName(TestConfig.PathToOpenDocumentText)).ConfigureAwait(false);
await UpdateAndWaitAsync().ConfigureAwait(false);
var document = _documentDescriptorCollection.Find(Builders <DocumentDescriptorReadModel> .Filter.Eq("Documents", "content_1")).SingleOrDefault();
Assert.That(document, Is.Not.Null, "Document with first child handle was not find.");
document = _documentDescriptorCollection.Find(Builders <DocumentDescriptorReadModel> .Filter.Eq("Documents", "content_2")).SingleOrDefault();
Assert.That(document, Is.Not.Null, "Document with second child handle was not find.");
var handle = _documentDescriptorCollection.Find(Builders <DocumentDescriptorReadModel> .Filter.Eq("Documents", "father")).SingleOrDefault();
Assert.That(handle, Is.Not.Null, "Father Handle Not Find");
Assert.That(handle.Attachments.Select(a => a.Handle), Is.EquivalentTo(new[] { new Core.Model.DocumentHandle("content_1"), new Core.Model.DocumentHandle("content_2") }));
}
public async Task can_add_attachment_to_existing_handle()
{
//Upload father
var fatherHandle = new DocumentHandle("father");
await _documentStoreClient.UploadAsync(TestConfig.PathToDocumentPdf, fatherHandle);
// wait background projection polling
await UpdateAndWaitAsync();
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToDocumentPng, fatherHandle, "Content", Path.GetFileName(TestConfig.PathToDocumentPng));
// wait background projection polling
await UpdateAndWaitAsync();
var document = _documentDescriptorCollection.Find(Builders <DocumentDescriptorReadModel> .Filter.Eq("Documents", "content_1")).SingleOrDefault();
Assert.That(document, Is.Not.Null, "Document with child handle was not find.");
var handle = _documentDescriptorCollection.Find(Builders <DocumentDescriptorReadModel> .Filter.Eq("Documents", "father")).SingleOrDefault();
Assert.That(handle, Is.Not.Null, "Father Handle Not Find");
Assert.That(handle.Attachments.Select(a => a.Handle), Is.EquivalentTo(new[] {
new Jarvis.DocumentStore.Core.Model.DocumentHandle("content_1")
}));
}
public async Task add_multiple_attachment_to_existing_handle_then_delete_handle()
{
//Upload father
var fatherHandle = new DocumentHandle("father");
await _documentStoreClient.UploadAsync(TestConfig.PathToDocumentPdf, fatherHandle);
await UpdateAndWaitAsync().ConfigureAwait(false);
//upload attachments
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToDocumentPng, fatherHandle, "Zip", Path.GetFileName(TestConfig.PathToDocumentPng));
await _documentStoreClient.UploadAttachmentAsync(TestConfig.PathToOpenDocumentText, fatherHandle, "Zip", Path.GetFileName(TestConfig.PathToOpenDocumentText));
await UpdateAndWaitAsync().ConfigureAwait(false);
await _documentStoreClient.DeleteAsync(fatherHandle);
await UpdateAndWaitAsync().ConfigureAwait(false);
Assert.That(_documentDescriptorCollection.AsQueryable().Count(), Is.EqualTo(0), "Attachment should be deleted.");
Assert.That(_documentCollection.AsQueryable().Count(), Is.EqualTo(0), "Attachment should be deleted.");
}
private string ProcessFile(string pathToFile, string workingFolder)
{
var extension = Path.GetExtension(pathToFile).ToLower();
if (extension == ".htmlzip" || extension == ".htmzip")
{
ZipFile.ExtractToDirectory(pathToFile, workingFolder);
Logger.DebugFormat("Extracted zip to {0}", workingFolder);
var htmlFile = Path.ChangeExtension(pathToFile, "html");
if (File.Exists(htmlFile))
{
Logger.DebugFormat("Html file is {0}", htmlFile);
return(htmlFile);
}
htmlFile = Path.ChangeExtension(pathToFile, "htm");
if (File.Exists(htmlFile))
{
Logger.DebugFormat("Html file is {0}", htmlFile);
return(htmlFile);
}
Logger.ErrorFormat("Invalid HTMLZIP file, name is {0} but corresponding html file not found after decompression", Path.GetFileName(pathToFile));
}
else if (extension == ".mht" || extension == ".mhtml")
{
MHTMLParser parser = new MHTMLParser(File.ReadAllText(pathToFile));
parser.OutputDirectory = workingFolder;
parser.DecodeImageData = false;
var html = parser.getHTMLText();
pathToFile = pathToFile + ".html";
File.WriteAllText(pathToFile, html);
}
return(pathToFile);
}
public string Convert(String jobId, string pathToEml, string workingFolder)
{
Logger.DebugFormat("Coverting {0} in working folder {1}", pathToEml, workingFolder);
var reader = new Reader();
var outFolder = Path.Combine(workingFolder, jobId);
Logger.DebugFormat("Creating message working folder is {0}", outFolder);
Directory.CreateDirectory(outFolder);
Logger.Debug("Extracting files");
var files = reader.ExtractToFolder(pathToEml, outFolder);
if (Logger.IsDebugEnabled)
{
foreach (var file in files)
{
Logger.DebugFormat("\t{0}", Path.GetFileName(file));
}
Logger.DebugFormat("Total files {0}", files.Length);
}
var htmlFileName = files.FirstOrDefault(x => x.EndsWith(".htm", StringComparison.OrdinalIgnoreCase)) ??
files.FirstOrDefault(x => x.EndsWith(".html", StringComparison.OrdinalIgnoreCase));
if (htmlFileName == null)
{
var textFile = files.FirstOrDefault(x => x.EndsWith(".txt", StringComparison.OrdinalIgnoreCase));
if (textFile != null)
{
htmlFileName = textFile + ".html";
var textcontent = File.ReadAllText(textFile);
File.WriteAllText(htmlFileName, String.Format("<html><body><pre>{0}</pre></body></html>", textcontent));
}
else
{
htmlFileName = "contentmissing.html";
File.WriteAllText(htmlFileName, "<html>No content found in mail.</html>");
}
}
var htmlNameWithoutExtension = Path.GetFileNameWithoutExtension(htmlFileName);
var htmlContent = File.ReadAllText(htmlFileName);
var dirInfoFullName = new DirectoryInfo(outFolder).FullName;
htmlContent = Regex.Replace(
htmlContent,
@"src=""(?<src>.+?)""",
new MatchEvaluator((m) => NormalizeImgEvaluator(m, dirInfoFullName)),
RegexOptions.IgnoreCase);
File.WriteAllText(htmlFileName, htmlContent);
var pathToZip = Path.Combine(workingFolder, htmlNameWithoutExtension + ".ezip");
Logger.DebugFormat("New zip file is {0}", pathToZip);
if (File.Exists(pathToZip))
{
Logger.DebugFormat("Deleting previous file: {0}", pathToZip);
File.Delete(pathToZip);
}
Logger.DebugFormat("Creating new file: {0}", pathToZip);
ZipFile.CreateFromDirectory(outFolder, pathToZip);
Logger.DebugFormat("Deleting message working folder", outFolder);
Directory.Delete(outFolder, true);
Logger.DebugFormat(
"Convesion done {0} => {1}",
pathToEml,
pathToZip
);
return(pathToZip);
}