public void request_for_missing_format_should_404()
{
// arrange
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("a.file")
);
var format = new DocumentFormat("missing");
var doc = new DocumentDescriptorReadModel(
1L,
new DocumentDescriptorId(1),
new BlobId("file_1")
);
SetupDocumentHandle(info, doc.Id);
SetupDocumentModel(doc);
// act
var response = Controller.GetFormat(_tenantId, info.Handle, format).Result;
// assert
Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode);
Assert.AreEqual("Document doc doesn't have format missing", response.GetError().Message);
}
internal void RemoveFormat(DocumentFormat format)
{
if (this.Formats.ContainsKey(format))
{
this.Formats.Remove(format);
}
}
public void when_file_is_not_found_should_return_404()
{
// arrange
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("a.file")
);
var format = new DocumentFormat("original");
var blobId = new BlobId("file_1");
var doc = new DocumentDescriptorReadModel(
1L,
new DocumentDescriptorId(1),
blobId);
SetupDocumentHandle(info, doc.Id);
SetupDocumentModel(doc);
BlobStore.GetDescriptor(blobId).Returns(i => null);
// act
var response = Controller.GetFormat(_tenantId, info.Handle, format).Result;
// assert
Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode);
Assert.AreEqual("File file_1 not found", response.GetError().Message);
}
public FormatStoreMultipartStreamProvider(
IBlobStore store,
DocumentFormat format
) : base(Path.GetTempPath())
{
_store = store;
_format = format;
}
public HttpResponseMessage GetFormat(
#pragma warning disable RCS1163 // Unused parameter.
#pragma warning disable IDE0060 // Remove unused parameter
TenantId tenantId, //do not remove, even if not used in the method, actually it is used from the infrastructure to determine tenant
#pragma warning restore IDE0060 // Remove unused parameter
#pragma warning restore RCS1163 // Unused parameter.
DocumentHandle handle,
DocumentFormat format,
string fname = null
)
{
var mapping = _handleWriter.FindOneById(handle);
//If mapping is not present return not found
if (mapping == null || mapping.DocumentDescriptorId == null)
{
var deleted = _documentDeletedReader.AllUnsorted.Where(d => d.Handle == handle).ToList();
if (deleted.Count == 0)
{
return(GetFromSecondary(tenantId, handle, format));
}
return(DocumentDeleted(handle, deleted));
}
var document = _documentDescriptorReader.FindOneById(mapping.DocumentDescriptorId);
if (document == null)
{
return(GetFromSecondary(tenantId, handle, format));
}
var fileName = fname != null ? new FileNameWithExtension(fname) : null;
if (format == DocumentFormats.Original)
{
return(StreamFile(
document.GetFormatBlobId(format),
fileName ?? mapping.FileName
));
}
BlobId formatBlobId = document.GetFormatBlobId(format);
if (formatBlobId == BlobId.Null)
{
return(Request.CreateErrorResponse(
HttpStatusCode.NotFound,
string.Format("Document {0} doesn't have format {1}",
handle,
format
)
));
}
return(StreamFile(formatBlobId, fileName));
}
public void request_for_invalid_file_handle_should_404()
{
var documentHandle = new DocumentHandle("not_in_store");
var format = new DocumentFormat("any_format");
DocumentDeletedReader.AllUnsorted.Returns(new List<DocumentDeletedReadModel>().AsQueryable());
var response = Controller.GetFormat(_tenantId, documentHandle, format).Result;
Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode);
Assert.AreEqual("Document not_in_store not found", response.GetError().Message);
}
public async Task <HttpResponseMessage> GetFormat(
TenantId tenantId,
DocumentHandle handle,
DocumentFormat format,
string fname = null
)
{
var mapping = _handleWriter.FindOneById(handle);
//If mapping is not present return not found
if (mapping == null || mapping.DocumentDescriptorId == null)
{
var deleted = _documentDeletedReader.AllUnsorted.Where(d => d.Handle == handle).ToList();
if (deleted.Count == 0)
{
return(DocumentNotFound(handle));
}
return(DocumentDeleted(handle, deleted));
}
var document = _documentDescriptorReader.FindOneById(mapping.DocumentDescriptorId);
if (document == null)
{
return(DocumentNotFound(handle));
}
var fileName = fname != null ? new FileNameWithExtension(fname) : null;
if (format == DocumentFormats.Original)
{
return(StreamFile(
document.GetFormatBlobId(format),
fileName ?? mapping.FileName
));
}
BlobId formatBlobId = document.GetFormatBlobId(format);
if (formatBlobId == BlobId.Null)
{
return(Request.CreateErrorResponse(
HttpStatusCode.NotFound,
string.Format("Document {0} doesn't have format {1}",
handle,
format
)
));
}
return(StreamFile(formatBlobId, fileName));
}
public BlobId GetFormatBlobId(DocumentFormat format)
{
FormatInfo formatInfo = null;
if (Formats.TryGetValue(format, out formatInfo))
{
return(formatInfo.BlobId);
}
return(BlobId.Null);
}
public async Task should_add_format_to_document()
{
CreateDocument(1, "handle", TestConfig.PathToDocumentPng);
var format = new DocumentFormat("tika");
var blobId = AddFormatToDocument(1, "handle", format, new PipelineId("tika"), TestConfig.PathToTextDocument);
await _projections.UpdateAndWait();
var document = _documentReader.AllUnsorted.Single(d => d.Id == new DocumentDescriptorId(1));
Assert.That(document.Formats, Has.Count.EqualTo(2));
Assert.That(document.Formats[format], Is.Not.Null, "Document has not added format");
Assert.That(document.Formats[format].BlobId, Is.EqualTo(blobId), "Wrong BlobId");
}
public IBlobWriter CreateNew(DocumentFormat format, FileNameWithExtension fname)
{
var blobId = new BlobId(format, _counterService.GetNext(format));
var gridFs = GetGridFsByFormat(format);
Logger.DebugFormat("Creating file {0} on {1}", blobId, gridFs.DatabaseName);
var stream = gridFs.Create(fname, new MongoGridFSCreateOptions()
{
ContentType = MimeTypes.GetMimeType(fname),
UploadDate = DateTime.UtcNow,
Id = (string)blobId
});
return new BlobWriter(blobId, stream, fname);
}
BlobId AddFormatToDocument(int id, string handle, DocumentFormat format, PipelineId pipelineId, string pathToFile)
{
var fname = Path.GetFileName(pathToFile);
var info = new DocumentHandleInfo(new DocumentHandle(handle), new FileNameWithExtension(fname));
var blobId = _filestore.Upload(format, pathToFile);
_bus.Send(new AddFormatToDocumentDescriptor(
new DocumentDescriptorId(id),
format,
blobId,
pipelineId
));
Thread.Sleep(50);
return blobId;
}
BlobId AddFormatToDocument(int id, string handle, DocumentFormat format, PipelineId pipelineId, string pathToFile)
{
var fname = Path.GetFileName(pathToFile);
var info = new DocumentHandleInfo(new DocumentHandle(handle), new FileNameWithExtension(fname));
var blobId = _filestore.Upload(format, pathToFile);
_bus.Send(new AddFormatToDocumentDescriptor(
new DocumentDescriptorId(id),
format,
blobId,
pipelineId
));
Thread.Sleep(50);
return(blobId);
}
public void request_for_missing_document_should_404()
{
// arrange
var documentHandle = new DocumentHandle("doc");
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("a.file")
);
var format = new DocumentFormat("any_format");
SetupDocumentHandle(info, new DocumentDescriptorId(1));
// act
var response = Controller.GetFormat(_tenantId, documentHandle, format).Result;
// assert
Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode);
Assert.AreEqual("Document doc not found", response.GetError().Message);
}
public async Task <HttpResponseMessage> DeleteFormatFromDocument(
TenantId tenantId,
DocumentHandle handle,
DocumentFormat format)
{
var documentDescriptor = GetDocumentDescriptorByHandle(handle);
if (documentDescriptor == null)
{
return(DocumentNotFound(handle));
}
CommandBus.Send(new DeleteFormatFromDocumentDescriptor(documentDescriptor.Id, format), "api");
return(Request.CreateResponse(
HttpStatusCode.Accepted,
string.Format("Format {0} marked for deletion for handle {0}", format, handle)
));
}
public HttpResponseMessage DeleteFormatFromDocument(
#pragma warning disable RCS1163 // Unused parameter.
#pragma warning disable IDE0060 // Remove unused parameter
TenantId tenantId, //do not remove, even if not used in the method, actually it is used from the infrastructure to determine tenant
#pragma warning restore IDE0060 // Remove unused parameter
#pragma warning restore RCS1163 // Unused parameter.
DocumentHandle handle,
DocumentFormat format)
{
var documentDescriptor = GetDocumentDescriptorByHandle(handle);
if (documentDescriptor == null)
{
return(DocumentNotFound(handle));
}
CommandBus.Send(new DeleteFormatFromDocumentDescriptor(documentDescriptor.Id, format), "api");
return(Request.CreateResponse(
HttpStatusCode.Accepted,
String.Format("Format {0} marked for deletion for handle {1}", format, handle)
));
}
public async void adding_twice_same_format_overwrite_format()
{
CreateDocument(1, "handle", TestConfig.PathToDocumentPng);
var format = new DocumentFormat("tika");
var blobId1 = AddFormatToDocument(1, "handle", format, new PipelineId("tika"), TestConfig.PathToTextDocument);
var blobId2 = AddFormatToDocument(1, "handle", format, new PipelineId("tika"), TestConfig.PathToHtml);
await _projections.UpdateAndWait();
var document = _documentReader.AllUnsorted.Single(d => d.Id == new DocumentDescriptorId(1));
Assert.That(document.Formats, Has.Count.EqualTo(2));
Assert.That(document.Formats[format], Is.Not.Null, "Document has not added format");
Assert.That(document.Formats[format].BlobId, Is.EqualTo(blobId2), "Wrong BlobId");
}
public async Task<HttpResponseMessage> GetFormat(
TenantId tenantId,
DocumentHandle handle,
DocumentFormat format,
string fname = null
)
{
var mapping = _handleWriter.FindOneById(handle);
//If mapping is not present return not found
if (mapping == null || mapping.DocumentDescriptorId == null)
{
var deleted = _documentDeletedReader.AllUnsorted.Where(d => d.Handle == handle).ToList();
if (deleted.Count == 0)
return DocumentNotFound(handle);
return DocumentDeleted(handle, deleted);
}
var document = _documentDescriptorReader.FindOneById(mapping.DocumentDescriptorId);
if (document == null)
{
return DocumentNotFound(handle);
}
var fileName = fname != null ? new FileNameWithExtension(fname) : null;
if (format == DocumentFormats.Original)
{
return StreamFile(
document.GetFormatBlobId(format),
fileName ?? mapping.FileName
);
}
BlobId formatBlobId = document.GetFormatBlobId(format);
if (formatBlobId == BlobId.Null)
{
return Request.CreateErrorResponse(
HttpStatusCode.NotFound,
string.Format("Document {0} doesn't have format {1}",
handle,
format
)
);
}
return StreamFile(formatBlobId, fileName);
}
/// <summary>
/// Upload a file sent in an http request
/// </summary>
/// <param name="httpContent">request's content</param>
/// <returns>Error message or null</returns>
private async Task<String> AddFormatFromHttpContent(HttpContent httpContent, DocumentFormat format)
{
if (httpContent == null || !httpContent.IsMimeMultipartContent())
return "Attachment not found!";
var provider = await httpContent.ReadAsMultipartAsync(
new FormatStoreMultipartStreamProvider(_blobStore, format)
);
if (provider.Filename == null)
return "Attachment not found!";
if (provider.FormData["custom-data"] != null)
{
_customData = JsonConvert.DeserializeObject<DocumentCustomData>(provider.FormData["custom-data"]);
}
_fileName = provider.Filename;
_blobId = provider.BlobId;
return null;
}
public async Task<HttpResponseMessage> DeleteFormatFromDocument(
TenantId tenantId,
DocumentHandle handle,
DocumentFormat format)
{
var documentDescriptor = GetDocumentDescriptorByHandle(handle);
if (documentDescriptor == null)
return DocumentNotFound(handle);
CommandBus.Send(new DeleteFormatFromDocumentDescriptor(documentDescriptor.Id, format), "api");
return Request.CreateResponse(
HttpStatusCode.Accepted,
string.Format("Format {0} marked for deletion for handle {0}", format, handle)
);
}
public async Task<HttpResponseMessage> AddFormatToDocument(TenantId tenantId, DocumentFormat format)
{
var errorMessage = await AddFormatFromHttpContent(Request.Content, format);
Logger.DebugFormat("File {0} processed with message {1}", _blobId, errorMessage);
if (errorMessage != null)
{
Logger.Error("Error Adding format To Document: " + errorMessage);
return Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
errorMessage
);
}
String queueName = _customData[AddFormatToDocumentParameters.QueueName] as String;
String jobId = _customData[AddFormatToDocumentParameters.JobId] as String;
DocumentDescriptorId documentId;
if (String.IsNullOrEmpty(queueName))
{
//user ask for handle, we need to grab the handle
var documentHandle = new DocumentHandle(_customData[AddFormatToDocumentParameters.DocumentHandle] as String);
var handle = _handleWriter.FindOneById(documentHandle);
documentId = handle.DocumentDescriptorId;
if (documentId == null)
{
Logger.ErrorFormat("Trying to add a format for Handle {0} with a null DocumentId", documentHandle);
return Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
""
);
}
Logger.DebugFormat("Add format {0} to handle {1} and document id {2}", format, handle, documentId);
}
else
{
var job = _queueDispatcher.GetJob(queueName, jobId);
if (job == null)
{
Logger.WarnFormat("Job id {0} not found in queue {1}", jobId, queueName);
return Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
String.Format("Job id {0} not found in queue {1}", jobId, queueName));
}
documentId = job.DocumentDescriptorId;
if (documentId == null)
{
Logger.ErrorFormat("Trying to add a format for Job Id {0} queue {1} - Job has DocumentDescriptorId null", jobId, queueName);
return Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
""
);
}
//need to check if the descriptor is deleted
var exists = _documentDescriptorReader
.AllUnsorted
.Where(d => d.Id == documentId)
.Any();
if (!exists)
{
Logger.ErrorFormat("Trying to add a format for Job Id {0} queue {1} - DocumentDescriptor does not exists or was deleted!", jobId, queueName);
return Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
""
);
}
Logger.DebugFormat("Add format {0} to job id {1} and document id {2}", format, job.Id, documentId);
}
if (format == "null")
{
var formatFromFileName = _documentFormatTranslator.GetFormatFromFileName(_fileName);
if (formatFromFileName == null)
{
String error = "Format not specified and no known format for file: " + _fileName;
Logger.Error(error);
return Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
error
);
}
format = new DocumentFormat(formatFromFileName);
}
var createdById = new PipelineId(_customData[AddFormatToDocumentParameters.CreatedBy] as String);
Logger.DebugFormat("Incoming new format for documentId {0}", documentId);
var command = new AddFormatToDocumentDescriptor(documentId, format, _blobId, createdById);
CommandBus.Send(command, "api");
return Request.CreateResponse(
HttpStatusCode.OK,
new AddFormatToDocumentResponse
{
Result = true,
}
);
}
public void AddFormat(PipelineId pipelineId, DocumentFormat format, BlobId blobId)
{
this.Formats[format] = new FormatInfo(blobId, pipelineId);
}
public async Task <HttpResponseMessage> AddFormatToDocument(TenantId tenantId, DocumentFormat format)
{
var errorMessage = await AddFormatFromHttpContent(Request.Content, format);
Logger.DebugFormat("File {0} processed with message {1}", _blobId, errorMessage);
if (errorMessage != null)
{
Logger.Error("Error Adding format To Document: " + errorMessage);
return(Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
errorMessage
));
}
String queueName = _customData[AddFormatToDocumentParameters.QueueName] as String;
String jobId = _customData[AddFormatToDocumentParameters.JobId] as String;
DocumentDescriptorId documentId;
if (String.IsNullOrEmpty(queueName))
{
//user ask for handle, we need to grab the handle
var documentHandle = new DocumentHandle(_customData[AddFormatToDocumentParameters.DocumentHandle] as String);
var handle = _handleWriter.FindOneById(documentHandle);
documentId = handle.DocumentDescriptorId;
if (documentId == null)
{
Logger.ErrorFormat("Trying to add a format for Handle {0} with a null DocumentId", documentHandle);
return(Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
""
));
}
Logger.DebugFormat("Add format {0} to handle {1} and document id {2}", format, handle, documentId);
}
else
{
var job = _queueDispatcher.GetJob(queueName, jobId);
if (job == null)
{
Logger.WarnFormat("Job id {0} not found in queue {1}", jobId, queueName);
return(Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
String.Format("Job id {0} not found in queue {1}", jobId, queueName)));
}
documentId = job.DocumentDescriptorId;
if (documentId == null)
{
Logger.ErrorFormat("Trying to add a format for Job Id {0} queue {1} - Job has DocumentDescriptorId null", jobId, queueName);
return(Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
""
));
}
//need to check if the descriptor is deleted
var exists = _documentDescriptorReader
.AllUnsorted
.Where(d => d.Id == documentId)
.Any();
if (!exists)
{
Logger.ErrorFormat("Trying to add a format for Job Id {0} queue {1} - DocumentDescriptor does not exists or was deleted!", jobId, queueName);
return(Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
""
));
}
Logger.DebugFormat("Add format {0} to job id {1} and document id {2}", format, job.Id, documentId);
}
if (format == "null")
{
var formatFromFileName = _documentFormatTranslator.GetFormatFromFileName(_fileName);
if (formatFromFileName == null)
{
String error = "Format not specified and no known format for file: " + _fileName;
Logger.Error(error);
return(Request.CreateErrorResponse(
HttpStatusCode.BadRequest,
error
));
}
format = new DocumentFormat(formatFromFileName);
}
var createdById = new PipelineId(_customData[AddFormatToDocumentParameters.CreatedBy] as String);
Logger.DebugFormat("Incoming new format for documentId {0}", documentId);
var command = new AddFormatToDocumentDescriptor(documentId, format, _blobId, createdById);
CommandBus.Send(command, "api");
return(Request.CreateResponse(
HttpStatusCode.OK,
new AddFormatToDocumentResponse
{
Result = true,
}
));
}
public BlobId Upload(DocumentFormat format, FileNameWithExtension fileName, Stream sourceStrem)
{
var gridFs = GetGridFsByFormat(format);
using (var writer = CreateNew(format, fileName))
{
Logger.DebugFormat("Uploading file {0} named {1} on {2}", writer.BlobId, fileName, gridFs.DatabaseName);
sourceStrem.CopyTo(writer.WriteStream);
return writer.BlobId;
}
}
public void should_download_pdf_format()
{
// arrange
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("a.file")
);
var format = new DocumentFormat("pdf");
var pdfBlobId = new BlobId("pdf");
var doc = new DocumentDescriptorReadModel(
1L,
new DocumentDescriptorId(1),
new BlobId("file_1"));
doc.AddFormat(new PipelineId("abc"), format, pdfBlobId);
SetupDocumentHandle(info, doc.Id);
SetupDocumentModel(doc);
BlobStore.GetDescriptor(pdfBlobId).Returns(i => new FsBlobDescriptor(pdfBlobId, TestConfig.PathToDocumentPdf));
// act
using (var response = Controller.GetFormat(_tenantId, info.Handle, format).Result)
{
// assert
response.EnsureSuccessStatusCode();
Assert.AreEqual("application/pdf", response.Content.Headers.ContentType.MediaType);
}
}
public BlobId(DocumentFormat format, long value)
: base(format +"." + value)
{
}
MongoGridFS CreateGridFsForFormat(DocumentFormat format)
{
var settings = new MongoGridFSSettings()
{
Root = format
};
return _database.GetGridFS(settings);
}
/// <summary>
/// Upload a file sent in an http request
/// </summary>
/// <param name="httpContent">request's content</param>
/// <returns>Error message or null</returns>
private async Task <String> AddFormatFromHttpContent(HttpContent httpContent, DocumentFormat format)
{
if (httpContent == null || !httpContent.IsMimeMultipartContent())
{
return("Attachment not found!");
}
var provider = await httpContent.ReadAsMultipartAsync(
new FormatStoreMultipartStreamProvider(_blobStore, format)
);
if (provider.Filename == null)
{
return("Attachment not found!");
}
if (provider.FormData["custom-data"] != null)
{
_customData = JsonConvert.DeserializeObject <DocumentCustomData>(provider.FormData["custom-data"]);
}
_fileName = provider.Filename;
_blobId = provider.BlobId;
return(null);
}
public BlobId Upload(DocumentFormat format, string pathToFile)
{
using (var inStream = File.OpenRead(pathToFile))
{
return Upload(format, new FileNameWithExtension(Path.GetFileName(pathToFile)), inStream);
}
}
public bool HasFormat(DocumentFormat documentFormat)
{
return Formats.ContainsKey(documentFormat);
}
MongoGridFS GetGridFsByFormat(DocumentFormat format)
{
return _fs.GetOrAdd(format, CreateGridFsForFormat);
}
private HttpResponseMessage GetFromSecondary(String tenantId, DocumentHandle handle, DocumentFormat documentFormat)
{
if (_configService.SecondaryAddressConfigured)
{
var secondaryUrl = $"{_configService.SecondaryDocumentStoreAddress}/{tenantId}/documents/{handle}/{documentFormat}";
var response = Request.CreateResponse(HttpStatusCode.Moved);
response.Headers.Location = new Uri(secondaryUrl);
return(response);
}
return(Request.CreateErrorResponse(
HttpStatusCode.NotFound,
string.Format("Document {0} not found", handle)
));
}
public void should_download_original_file()
{
// arrange
var info = new DocumentHandleInfo(
new DocumentHandle("doc"),
new FileNameWithExtension("\"A document.docx\"")
);
var format = new DocumentFormat("original");
var blobId = new BlobId("file_1");
var doc = new DocumentDescriptorReadModel(
1L,
new DocumentDescriptorId(1),
blobId);
SetupDocumentHandle(info, doc.Id);
SetupDocumentModel(doc);
BlobStore
.GetDescriptor(blobId)
.Returns(i => new FsBlobDescriptor(blobId, TestConfig.PathToWordDocument));
// act
using (var response = Controller.GetFormat(_tenantId, info.Handle, format).Result)
{
// assert
response.EnsureSuccessStatusCode();
Assert.AreEqual("\"A document.docx\"", response.Content.Headers.ContentDisposition.FileName);
}
}
public bool HasFormat(DocumentFormat documentFormat)
{
return(Formats.ContainsKey(documentFormat));
}