public DocumentDescriptorId FindDuplicateDocumentId(
DocumentDescriptorId sourceDocumentId,
FileHash sourceHash,
BlobId sourceBlobId
)
{
if (!_config.IsDeduplicationActive)
return null;
var original = _blobStore.GetDescriptor(sourceBlobId);
var matches = _hashReader.FindDocumentByHash(sourceHash);
Logger.DebugFormat("Deduplicating document {0}", sourceDocumentId);
foreach (var match in matches)
{
if (match.DocumentDescriptorId == sourceDocumentId)
continue;
Logger.DebugFormat("Checking document {0}", match.DocumentDescriptorId);
var candidate = this._blobStore.GetDescriptor(match.BlobId);
// only within same content type!
if (candidate.ContentType != original.ContentType)
{
Logger.DebugFormat("document {0} has different ContentType ({1}), skipping",
match.DocumentDescriptorId, candidate.ContentType
);
continue;
}
// and same length
if (candidate.Length != original.Length)
{
Logger.DebugFormat("document {0} has different length ({1}), skipping",
match.DocumentDescriptorId, candidate.Length
);
continue;
}
// binary check
using (var candidateStream = candidate.OpenRead())
using (var originalStream = original.OpenRead())
{
if (StreamHelper.StreamsContentsAreEqual(candidateStream, originalStream))
{
Logger.DebugFormat("{0} has same content of {1}: match found!",
match.DocumentDescriptorId, sourceDocumentId
);
return match.DocumentDescriptorId;
}
else
{
Logger.DebugFormat("{0} has different content of {1}, skipping",
match.DocumentDescriptorId, sourceDocumentId
);
}
}
}
return null;
}
public void Start(Int32 pollingTimeInMs)
{
var jobDataMap = context.JobDetail.JobDataMap;
PipelineId = new PipelineId(jobDataMap.GetString(JobKeys.PipelineId));
InputDocumentId = new DocumentId(jobDataMap.GetString(JobKeys.DocumentId));
InputBlobId = new BlobId(jobDataMap.GetString(JobKeys.BlobId));
InputDocumentFormat = new DocumentFormat(jobDataMap.GetString(JobKeys.Format));
if (TenantId == null)
throw new Exception("tenant not set!");
_workingFolder = Path.Combine(
ConfigService.GetWorkingFolder(TenantId, InputBlobId),
GetType().Name
);
OnExecute(context);
try
{
if (Directory.Exists(_workingFolder))
Directory.Delete(_workingFolder, true);
}
catch (Exception ex)
{
Logger.ErrorFormat(ex, "Error deleting {0}", _workingFolder);
}
pollingTimer = new Timer(pollingTimeInMs);
pollingTimer.Elapsed += pollingTimer_Elapsed;
pollingTimer.Start();
}
public GridFsBlobDescriptor(BlobId blobId, MongoGridFSFileInfo mongoGridFsFileInfo)
{
if (mongoGridFsFileInfo == null) throw new ArgumentNullException("mongoGridFsFileInfo");
_mongoGridFsFileInfo = mongoGridFsFileInfo;
BlobId = blobId;
FileNameWithExtension = new FileNameWithExtension(_mongoGridFsFileInfo.Name);
}
public DocumentDescriptorInitialized(
BlobId blobId,
DocumentHandleInfo handleInfo,
FileHash hash)
{
Hash = hash;
HandleInfo = handleInfo;
BlobId = blobId;
}
public void should_read_original_file_from_originals_store()
{
var blobId = new BlobId(DocumentFormats.Original, 1);
_originals.Download(blobId, "path/to/nothing").Returns("a.file");
var fname = _manager.Download(blobId, "path/to/nothing");
Assert.AreEqual("a.file", fname);
}
public void should_write_pdf_format_to_original_store()
{
var blobId = new BlobId(DocumentFormats.Pdf, 1);
_artifacts.Upload(Arg.Any<DocumentFormat>(), Arg.Any<string>()).Returns(blobId);
var id = _manager.Upload(DocumentFormats.Pdf, TestConfig.PathToDocumentPdf);
Assert.AreEqual(blobId, id);
}
public DocumentDescriptorDeleted(
BlobId blobId,
BlobId[] formats,
DocumentHandle[] attachments)
{
BlobId = blobId;
BlobFormatsId = formats;
Attachments = attachments;
}
public AddFormatToDocumentDescriptor(
DocumentDescriptorId aggregateId,
DocumentFormat documentFormat,
BlobId blobId,
PipelineId createdById) : base(aggregateId)
{
if (aggregateId == null) throw new ArgumentNullException("aggregateId");
DocumentFormat = documentFormat;
BlobId = blobId;
CreatedBy = createdById;
}
public InitializeDocumentDescriptor(
DocumentDescriptorId aggregateId,
BlobId blobId,
DocumentHandleInfo handleInfo,
FileHash hash,
FileNameWithExtension fileName
) : base(aggregateId)
{
FileName = fileName;
Hash = hash;
BlobId = blobId;
HandleInfo = handleInfo;
}
public Stream CreateNew(BlobId blobId, FileNameWithExtension fname)
{
var gridFs = GetGridFsByBlobId(blobId);
Logger.DebugFormat("Creating file {0} on {1}", blobId, gridFs.DatabaseName);
Delete(blobId);
return gridFs.Create(fname, new MongoGridFSCreateOptions()
{
ContentType = MimeTypes.GetMimeType(fname),
UploadDate = DateTime.UtcNow,
Id = (string)blobId
});
}
public InitializeDocumentDescriptorAsAttach(
DocumentDescriptorId aggregateId,
BlobId blobId,
DocumentHandleInfo handleInfo,
DocumentHandle fatherHandle,
DocumentDescriptorId fatherDocumentDescriptorId,
FileHash hash,
FileNameWithExtension fileName)
: base(aggregateId, blobId, handleInfo, hash, fileName)
{
FatherHandle = fatherHandle;
FatherDocumentDescriptorId = fatherDocumentDescriptorId;
}
public IBlobDescriptor GetDescriptor(BlobId blobId)
{
var gridFs = GetGridFsByBlobId(blobId);
Logger.DebugFormat("GetDescriptor for file {0} on {1}", blobId, gridFs.DatabaseName);
var s = gridFs.FindOneById((string)blobId);
if (s == null)
{
var message = string.Format("Descriptor for file {0} not found!", blobId);
Logger.DebugFormat(message);
throw new Exception(message);
}
return new GridFsBlobDescriptor(blobId, s);
}
public IBlobWriter CreateNew(DocumentFormat format, FileNameWithExtension fname)
{
var blobId = new BlobId(format, _counterService.GetNext(format));
var gridFs = GetGridFsByFormat(format);
Logger.DebugFormat("Creating file {0} on {1}", blobId, gridFs.DatabaseName);
var stream = gridFs.Create(fname, new MongoGridFSCreateOptions()
{
ContentType = MimeTypes.GetMimeType(fname),
UploadDate = DateTime.UtcNow,
Id = (string)blobId
});
return new BlobWriter(blobId, stream, fname);
}
public void SetUp()
{
longFolderName = Path.Combine(Path.GetTempPath(), new String('a', 230));
_blobId = new BlobId(_originalFormat, 1);
_pathToTask = Path.Combine(longFolderName, "File_1.dsimport");
_fileToImport = Path.Combine(longFolderName, "A Word Document.docx");
_fileUri = new Uri(Path.Combine(longFolderName, "A word document.docx"));
ClearQueueTempFolder();
Directory.CreateDirectory(longFolderName);
File.Copy(Path.Combine(TestConfig.DocumentsFolder, "Queue\\File_1.dsimport"), _pathToTask);
File.Copy(TestConfig.PathToWordDocument, _fileToImport);
var accessor = Substitute.For<ITenantAccessor>();
var tenant = Substitute.For<ITenant>();
tenant.Id.Returns(new TenantId("tests"));
var container = Substitute.For<IWindsorContainer>();
_commandBus = Substitute.For<ICommandBus>();
var identityGenerator = Substitute.For<IIdentityGenerator>();
_blobstore = Substitute.For<IBlobStore>();
_blobstore.Upload(Arg.Is(_originalFormat), Arg.Any<string>()).Returns(_blobId);
_blobstore.Upload(Arg.Is(_originalFormat), Arg.Any<FileNameWithExtension>(), Arg.Any<Stream>()).Returns(_blobId);
accessor.GetTenant(_testTenant).Returns(tenant);
accessor.Current.Returns(tenant);
tenant.Container.Returns(container);
container.Resolve<IBlobStore>().Returns(_blobstore);
container.Resolve<IIdentityGenerator>().Returns(identityGenerator);
container.Resolve<IMongoDatabase>().Returns(MongoDbTestConnectionProvider.ReadModelDb);
var collection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection<ImportFailure>("sys.importFailures");
collection.Drop();
DocumentStoreTestConfiguration config = new DocumentStoreTestConfiguration(tenantId : "tests");
config.SetFolderToMonitor(longFolderName);
var sysDb = config.TenantSettings.Single(t => t.TenantId == "tests").Get<IMongoDatabase>("system.db");
sysDb.Drop();
_queue = new ImportFormatFromFileQueue(config, accessor, _commandBus)
{
Logger = new ConsoleLogger()
};
_queue.DeleteTaskFileAfterImport = false;
}
MongoGridFS GetGridFsByBlobId(BlobId id)
{
return GetGridFsByFormat(id.Format);
}
private void CreateDocument(
DocumentDescriptorId documentDescriptorId,
BlobId blobId,
DocumentHandle handle,
DocumentHandle fatherHandle,
DocumentDescriptorId fatherDocumentDescriptorId,
FileNameWithExtension fileName,
DocumentCustomData customData
)
{
var descriptor = _blobStore.GetDescriptor(blobId);
ICommand createDocument;
var handleInfo = new DocumentHandleInfo(handle, fileName, customData);
if (fatherHandle == null)
{
if (Logger.IsDebugEnabled) Logger.DebugFormat("Initialize DocumentDescriptor {0} ", documentDescriptorId);
createDocument = new InitializeDocumentDescriptor(documentDescriptorId, blobId, handleInfo, descriptor.Hash, fileName);
}
else
{
if (Logger.IsDebugEnabled) Logger.DebugFormat("Initialize DocumentDescriptor as attach {0} ", documentDescriptorId);
createDocument = new InitializeDocumentDescriptorAsAttach(
documentDescriptorId,
blobId,
handleInfo,
fatherHandle,
fatherDocumentDescriptorId,
descriptor.Hash, fileName);
}
CommandBus.Send(createDocument, "api");
}
HttpResponseMessage StreamFile(BlobId formatBlobId, FileNameWithExtension fileName = null)
{
var descriptor = _blobStore.GetDescriptor(formatBlobId);
if (descriptor == null)
{
return Request.CreateErrorResponse(
HttpStatusCode.NotFound,
string.Format("File {0} not found", formatBlobId)
);
}
RangeHeaderValue rangeHeader = Request.Headers.Range;
var response = Request.CreateResponse(HttpStatusCode.OK);
response.Headers.AcceptRanges.Add("bytes");
// HEAD?
bool isHead = false;
if (Request.Method == HttpMethod.Head)
{
isHead = true;
rangeHeader = null;
}
// full stream
if (rangeHeader == null || !rangeHeader.Ranges.Any())
{
if (isHead)
{
response.Content = new ByteArrayContent(new byte[0]);
response.Content.Headers.ContentLength = descriptor.Length;
}
else
{
response.Content = new StreamContent(descriptor.OpenRead());
}
response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType);
if (fileName != null)
{
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = fileName
};
}
return response;
}
// range stream
long start = 0, end = 0;
long totalLength = descriptor.Length;
// 1. If the unit is not 'bytes'.
// 2. If there are multiple ranges in header value.
// 3. If start or end position is greater than file length.
if (rangeHeader.Unit != "bytes" || rangeHeader.Ranges.Count > 1 ||
!TryReadRangeItem(rangeHeader.Ranges.First(), totalLength, out start, out end))
{
response.StatusCode = HttpStatusCode.RequestedRangeNotSatisfiable;
response.Content = new StreamContent(Stream.Null); // No content for this status.
response.Content.Headers.ContentRange = new ContentRangeHeaderValue(totalLength);
response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType);
return response;
}
var contentRange = new ContentRangeHeaderValue(start, end, totalLength);
// We are now ready to produce partial content.
response.StatusCode = HttpStatusCode.PartialContent;
response.Content = new PushStreamContent((outputStream, httpContent, transpContext)
=>
{
using (outputStream) // Copy the file to output stream in indicated range.
using (Stream inputStream = descriptor.OpenRead())
CreatePartialContent(inputStream, outputStream, start, end);
}, descriptor.ContentType);
response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType);
response.Content.Headers.ContentLength = end - start + 1;
response.Content.Headers.ContentRange = contentRange;
return response;
}
/// <summary>
/// Upload a file sent in an http request
/// </summary>
/// <param name="httpContent">request's content</param>
/// <returns>Error message or null</returns>
private async Task<String> AddFormatFromHttpContent(HttpContent httpContent, DocumentFormat format)
{
if (httpContent == null || !httpContent.IsMimeMultipartContent())
return "Attachment not found!";
var provider = await httpContent.ReadAsMultipartAsync(
new FormatStoreMultipartStreamProvider(_blobStore, format)
);
if (provider.Filename == null)
return "Attachment not found!";
if (provider.FormData["custom-data"] != null)
{
_customData = JsonConvert.DeserializeObject<DocumentCustomData>(provider.FormData["custom-data"]);
}
_fileName = provider.Filename;
_blobId = provider.BlobId;
return null;
}
public string Download(BlobId blobId, string folder)
{
var gridFs = GetGridFsByBlobId(blobId);
Logger.DebugFormat("Downloading file {0} on {1} to folder {2}", blobId, gridFs.DatabaseName, folder);
if (!Directory.Exists(folder))
Directory.CreateDirectory(folder);
var s = gridFs.FindOneById((string)blobId);
var localFileName = Path.Combine(folder, s.Name);
gridFs.Download(localFileName, s);
return localFileName;
}
public void should_download_original_file()
{
// arrange
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("\"A document.docx\"")
);
var format = new DocumentFormat("original");
var blobId = new BlobId("file_1");
var doc = new DocumentDescriptorReadModel(
1L,
new DocumentDescriptorId(1),
blobId);
SetupDocumentHandle(info, doc.Id);
SetupDocumentModel(doc);
BlobStore
.GetDescriptor(blobId)
.Returns(i => new FsBlobDescriptor(blobId, TestConfig.PathToWordDocument));
// act
using (var response = Controller.GetFormat(_tenantId, info.Handle, format).Result)
{
// assert
response.EnsureSuccessStatusCode();
Assert.AreEqual("\"A document.docx\"", response.Content.Headers.ContentDisposition.FileName);
}
}
public string Download(BlobId blobId, string folder)
{
return ForBlobId(blobId).Download(blobId, folder);
}
public DocumentFormatHasBeenUpdated(DocumentFormat documentFormat, BlobId blobId, PipelineId createdBy)
{
DocumentFormat = documentFormat;
BlobId = blobId;
CreatedBy = createdBy;
}
public void blobId_should_parse_format_from_string()
{
var id = new BlobId("format.100");
Assert.AreEqual(new DocumentFormat("format"), id.Format);
}
public void blobId_should_contain_format_and_number()
{
var id = new BlobId(new DocumentFormat("format"), 1);
Assert.AreEqual("format.1", (string)id);
}
public void should_download_pdf_format()
{
// arrange
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("a.file")
);
var format = new DocumentFormat("pdf");
var pdfBlobId = new BlobId("pdf");
var doc = new DocumentDescriptorReadModel(
1L,
new DocumentDescriptorId(1),
new BlobId("file_1"));
doc.AddFormat(new PipelineId("abc"), format, pdfBlobId);
SetupDocumentHandle(info, doc.Id);
SetupDocumentModel(doc);
BlobStore.GetDescriptor(pdfBlobId).Returns(i => new FsBlobDescriptor(pdfBlobId, TestConfig.PathToDocumentPdf));
// act
using (var response = Controller.GetFormat(_tenantId, info.Handle, format).Result)
{
// assert
response.EnsureSuccessStatusCode();
Assert.AreEqual("application/pdf", response.Content.Headers.ContentType.MediaType);
}
}
HttpResponseMessage StreamFile(BlobId formatBlobId, String fileName = null)
{
var descriptor = _blobStore.GetDescriptor(formatBlobId);
if (descriptor == null)
{
return Request.CreateErrorResponse(
HttpStatusCode.NotFound,
string.Format("File {0} not found", formatBlobId)
);
}
var response = Request.CreateResponse(HttpStatusCode.OK);
response.Content = new StreamContent(descriptor.OpenRead());
response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType);
if (fileName != null)
{
response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = fileName
};
}
return response;
}
public void Delete(BlobId blobId)
{
var gridFs = GetGridFsByBlobId(blobId);
Logger.DebugFormat("Deleting file {0} on {1}", blobId, gridFs.DatabaseName);
gridFs.DeleteById((string)blobId);
}
/// <summary>
/// Upload a file sent in an http request
/// </summary>
/// <param name="httpContent">request's content</param>
/// <returns>Error message or null</returns>
private async Task<String> UploadFromHttpContent(HttpContent httpContent)
{
if (httpContent == null || !httpContent.IsMimeMultipartContent())
return "Attachment not found!";
var provider = await httpContent.ReadAsMultipartAsync(
new FileStoreMultipartStreamProvider(_blobStore, _configService)
);
if (provider.Filename == null)
return "Attachment not found!";
if (provider.IsInvalidFile)
return string.Format("Unsupported file {0}", provider.Filename);
if (provider.FormData["custom-data"] != null)
{
_customData = JsonConvert.DeserializeObject<DocumentCustomData>(provider.FormData["custom-data"]);
}
_fileName = provider.Filename;
_blobId = provider.BlobId;
return null;
}
public void when_file_is_not_found_should_return_404()
{
// arrange
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("a.file")
);
var format = new DocumentFormat("original");
var blobId = new BlobId("file_1");
var doc = new DocumentDescriptorReadModel(
1L,
new DocumentDescriptorId(1),
blobId);
SetupDocumentHandle(info, doc.Id);
SetupDocumentModel(doc);
BlobStore.GetDescriptor(blobId).Returns(i => null);
// act
var response = Controller.GetFormat(_tenantId, info.Handle, format).Result;
// assert
Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode);
Assert.AreEqual("File file_1 not found", response.GetError().Message);
}
public FormatAddedToDocumentDescriptor(DocumentFormat documentFormat, BlobId blobId, PipelineId createdBy)
{
DocumentFormat = documentFormat;
BlobId = blobId;
CreatedBy = createdBy;
}
