public DocumentDescriptorId FindDuplicateDocumentId( DocumentDescriptorId sourceDocumentId, FileHash sourceHash, BlobId sourceBlobId ) { if (!_config.IsDeduplicationActive) return null; var original = _blobStore.GetDescriptor(sourceBlobId); var matches = _hashReader.FindDocumentByHash(sourceHash); Logger.DebugFormat("Deduplicating document {0}", sourceDocumentId); foreach (var match in matches) { if (match.DocumentDescriptorId == sourceDocumentId) continue; Logger.DebugFormat("Checking document {0}", match.DocumentDescriptorId); var candidate = this._blobStore.GetDescriptor(match.BlobId); // only within same content type! if (candidate.ContentType != original.ContentType) { Logger.DebugFormat("document {0} has different ContentType ({1}), skipping", match.DocumentDescriptorId, candidate.ContentType ); continue; } // and same length if (candidate.Length != original.Length) { Logger.DebugFormat("document {0} has different length ({1}), skipping", match.DocumentDescriptorId, candidate.Length ); continue; } // binary check using (var candidateStream = candidate.OpenRead()) using (var originalStream = original.OpenRead()) { if (StreamHelper.StreamsContentsAreEqual(candidateStream, originalStream)) { Logger.DebugFormat("{0} has same content of {1}: match found!", match.DocumentDescriptorId, sourceDocumentId ); return match.DocumentDescriptorId; } else { Logger.DebugFormat("{0} has different content of {1}, skipping", match.DocumentDescriptorId, sourceDocumentId ); } } } return null; }
public void Start(Int32 pollingTimeInMs) { var jobDataMap = context.JobDetail.JobDataMap; PipelineId = new PipelineId(jobDataMap.GetString(JobKeys.PipelineId)); InputDocumentId = new DocumentId(jobDataMap.GetString(JobKeys.DocumentId)); InputBlobId = new BlobId(jobDataMap.GetString(JobKeys.BlobId)); InputDocumentFormat = new DocumentFormat(jobDataMap.GetString(JobKeys.Format)); if (TenantId == null) throw new Exception("tenant not set!"); _workingFolder = Path.Combine( ConfigService.GetWorkingFolder(TenantId, InputBlobId), GetType().Name ); OnExecute(context); try { if (Directory.Exists(_workingFolder)) Directory.Delete(_workingFolder, true); } catch (Exception ex) { Logger.ErrorFormat(ex, "Error deleting {0}", _workingFolder); } pollingTimer = new Timer(pollingTimeInMs); pollingTimer.Elapsed += pollingTimer_Elapsed; pollingTimer.Start(); }
public GridFsBlobDescriptor(BlobId blobId, MongoGridFSFileInfo mongoGridFsFileInfo) { if (mongoGridFsFileInfo == null) throw new ArgumentNullException("mongoGridFsFileInfo"); _mongoGridFsFileInfo = mongoGridFsFileInfo; BlobId = blobId; FileNameWithExtension = new FileNameWithExtension(_mongoGridFsFileInfo.Name); }
public DocumentDescriptorInitialized( BlobId blobId, DocumentHandleInfo handleInfo, FileHash hash) { Hash = hash; HandleInfo = handleInfo; BlobId = blobId; }
public void should_read_original_file_from_originals_store() { var blobId = new BlobId(DocumentFormats.Original, 1); _originals.Download(blobId, "path/to/nothing").Returns("a.file"); var fname = _manager.Download(blobId, "path/to/nothing"); Assert.AreEqual("a.file", fname); }
public void should_write_pdf_format_to_original_store() { var blobId = new BlobId(DocumentFormats.Pdf, 1); _artifacts.Upload(Arg.Any<DocumentFormat>(), Arg.Any<string>()).Returns(blobId); var id = _manager.Upload(DocumentFormats.Pdf, TestConfig.PathToDocumentPdf); Assert.AreEqual(blobId, id); }
public DocumentDescriptorDeleted( BlobId blobId, BlobId[] formats, DocumentHandle[] attachments) { BlobId = blobId; BlobFormatsId = formats; Attachments = attachments; }
public AddFormatToDocumentDescriptor( DocumentDescriptorId aggregateId, DocumentFormat documentFormat, BlobId blobId, PipelineId createdById) : base(aggregateId) { if (aggregateId == null) throw new ArgumentNullException("aggregateId"); DocumentFormat = documentFormat; BlobId = blobId; CreatedBy = createdById; }
public InitializeDocumentDescriptor( DocumentDescriptorId aggregateId, BlobId blobId, DocumentHandleInfo handleInfo, FileHash hash, FileNameWithExtension fileName ) : base(aggregateId) { FileName = fileName; Hash = hash; BlobId = blobId; HandleInfo = handleInfo; }
public Stream CreateNew(BlobId blobId, FileNameWithExtension fname) { var gridFs = GetGridFsByBlobId(blobId); Logger.DebugFormat("Creating file {0} on {1}", blobId, gridFs.DatabaseName); Delete(blobId); return gridFs.Create(fname, new MongoGridFSCreateOptions() { ContentType = MimeTypes.GetMimeType(fname), UploadDate = DateTime.UtcNow, Id = (string)blobId }); }
public InitializeDocumentDescriptorAsAttach( DocumentDescriptorId aggregateId, BlobId blobId, DocumentHandleInfo handleInfo, DocumentHandle fatherHandle, DocumentDescriptorId fatherDocumentDescriptorId, FileHash hash, FileNameWithExtension fileName) : base(aggregateId, blobId, handleInfo, hash, fileName) { FatherHandle = fatherHandle; FatherDocumentDescriptorId = fatherDocumentDescriptorId; }
public IBlobDescriptor GetDescriptor(BlobId blobId) { var gridFs = GetGridFsByBlobId(blobId); Logger.DebugFormat("GetDescriptor for file {0} on {1}", blobId, gridFs.DatabaseName); var s = gridFs.FindOneById((string)blobId); if (s == null) { var message = string.Format("Descriptor for file {0} not found!", blobId); Logger.DebugFormat(message); throw new Exception(message); } return new GridFsBlobDescriptor(blobId, s); }
public IBlobWriter CreateNew(DocumentFormat format, FileNameWithExtension fname) { var blobId = new BlobId(format, _counterService.GetNext(format)); var gridFs = GetGridFsByFormat(format); Logger.DebugFormat("Creating file {0} on {1}", blobId, gridFs.DatabaseName); var stream = gridFs.Create(fname, new MongoGridFSCreateOptions() { ContentType = MimeTypes.GetMimeType(fname), UploadDate = DateTime.UtcNow, Id = (string)blobId }); return new BlobWriter(blobId, stream, fname); }
public void SetUp() { longFolderName = Path.Combine(Path.GetTempPath(), new String('a', 230)); _blobId = new BlobId(_originalFormat, 1); _pathToTask = Path.Combine(longFolderName, "File_1.dsimport"); _fileToImport = Path.Combine(longFolderName, "A Word Document.docx"); _fileUri = new Uri(Path.Combine(longFolderName, "A word document.docx")); ClearQueueTempFolder(); Directory.CreateDirectory(longFolderName); File.Copy(Path.Combine(TestConfig.DocumentsFolder, "Queue\\File_1.dsimport"), _pathToTask); File.Copy(TestConfig.PathToWordDocument, _fileToImport); var accessor = Substitute.For<ITenantAccessor>(); var tenant = Substitute.For<ITenant>(); tenant.Id.Returns(new TenantId("tests")); var container = Substitute.For<IWindsorContainer>(); _commandBus = Substitute.For<ICommandBus>(); var identityGenerator = Substitute.For<IIdentityGenerator>(); _blobstore = Substitute.For<IBlobStore>(); _blobstore.Upload(Arg.Is(_originalFormat), Arg.Any<string>()).Returns(_blobId); _blobstore.Upload(Arg.Is(_originalFormat), Arg.Any<FileNameWithExtension>(), Arg.Any<Stream>()).Returns(_blobId); accessor.GetTenant(_testTenant).Returns(tenant); accessor.Current.Returns(tenant); tenant.Container.Returns(container); container.Resolve<IBlobStore>().Returns(_blobstore); container.Resolve<IIdentityGenerator>().Returns(identityGenerator); container.Resolve<IMongoDatabase>().Returns(MongoDbTestConnectionProvider.ReadModelDb); var collection = MongoDbTestConnectionProvider.ReadModelDb.GetCollection<ImportFailure>("sys.importFailures"); collection.Drop(); DocumentStoreTestConfiguration config = new DocumentStoreTestConfiguration(tenantId : "tests"); config.SetFolderToMonitor(longFolderName); var sysDb = config.TenantSettings.Single(t => t.TenantId == "tests").Get<IMongoDatabase>("system.db"); sysDb.Drop(); _queue = new ImportFormatFromFileQueue(config, accessor, _commandBus) { Logger = new ConsoleLogger() }; _queue.DeleteTaskFileAfterImport = false; }
MongoGridFS GetGridFsByBlobId(BlobId id) { return GetGridFsByFormat(id.Format); }
private void CreateDocument( DocumentDescriptorId documentDescriptorId, BlobId blobId, DocumentHandle handle, DocumentHandle fatherHandle, DocumentDescriptorId fatherDocumentDescriptorId, FileNameWithExtension fileName, DocumentCustomData customData ) { var descriptor = _blobStore.GetDescriptor(blobId); ICommand createDocument; var handleInfo = new DocumentHandleInfo(handle, fileName, customData); if (fatherHandle == null) { if (Logger.IsDebugEnabled) Logger.DebugFormat("Initialize DocumentDescriptor {0} ", documentDescriptorId); createDocument = new InitializeDocumentDescriptor(documentDescriptorId, blobId, handleInfo, descriptor.Hash, fileName); } else { if (Logger.IsDebugEnabled) Logger.DebugFormat("Initialize DocumentDescriptor as attach {0} ", documentDescriptorId); createDocument = new InitializeDocumentDescriptorAsAttach( documentDescriptorId, blobId, handleInfo, fatherHandle, fatherDocumentDescriptorId, descriptor.Hash, fileName); } CommandBus.Send(createDocument, "api"); }
HttpResponseMessage StreamFile(BlobId formatBlobId, FileNameWithExtension fileName = null) { var descriptor = _blobStore.GetDescriptor(formatBlobId); if (descriptor == null) { return Request.CreateErrorResponse( HttpStatusCode.NotFound, string.Format("File {0} not found", formatBlobId) ); } RangeHeaderValue rangeHeader = Request.Headers.Range; var response = Request.CreateResponse(HttpStatusCode.OK); response.Headers.AcceptRanges.Add("bytes"); // HEAD? bool isHead = false; if (Request.Method == HttpMethod.Head) { isHead = true; rangeHeader = null; } // full stream if (rangeHeader == null || !rangeHeader.Ranges.Any()) { if (isHead) { response.Content = new ByteArrayContent(new byte[0]); response.Content.Headers.ContentLength = descriptor.Length; } else { response.Content = new StreamContent(descriptor.OpenRead()); } response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType); if (fileName != null) { response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = fileName }; } return response; } // range stream long start = 0, end = 0; long totalLength = descriptor.Length; // 1. If the unit is not 'bytes'. // 2. If there are multiple ranges in header value. // 3. If start or end position is greater than file length. if (rangeHeader.Unit != "bytes" || rangeHeader.Ranges.Count > 1 || !TryReadRangeItem(rangeHeader.Ranges.First(), totalLength, out start, out end)) { response.StatusCode = HttpStatusCode.RequestedRangeNotSatisfiable; response.Content = new StreamContent(Stream.Null); // No content for this status. response.Content.Headers.ContentRange = new ContentRangeHeaderValue(totalLength); response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType); return response; } var contentRange = new ContentRangeHeaderValue(start, end, totalLength); // We are now ready to produce partial content. response.StatusCode = HttpStatusCode.PartialContent; response.Content = new PushStreamContent((outputStream, httpContent, transpContext) => { using (outputStream) // Copy the file to output stream in indicated range. using (Stream inputStream = descriptor.OpenRead()) CreatePartialContent(inputStream, outputStream, start, end); }, descriptor.ContentType); response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType); response.Content.Headers.ContentLength = end - start + 1; response.Content.Headers.ContentRange = contentRange; return response; }
/// <summary> /// Upload a file sent in an http request /// </summary> /// <param name="httpContent">request's content</param> /// <returns>Error message or null</returns> private async Task<String> AddFormatFromHttpContent(HttpContent httpContent, DocumentFormat format) { if (httpContent == null || !httpContent.IsMimeMultipartContent()) return "Attachment not found!"; var provider = await httpContent.ReadAsMultipartAsync( new FormatStoreMultipartStreamProvider(_blobStore, format) ); if (provider.Filename == null) return "Attachment not found!"; if (provider.FormData["custom-data"] != null) { _customData = JsonConvert.DeserializeObject<DocumentCustomData>(provider.FormData["custom-data"]); } _fileName = provider.Filename; _blobId = provider.BlobId; return null; }
public string Download(BlobId blobId, string folder) { var gridFs = GetGridFsByBlobId(blobId); Logger.DebugFormat("Downloading file {0} on {1} to folder {2}", blobId, gridFs.DatabaseName, folder); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); var s = gridFs.FindOneById((string)blobId); var localFileName = Path.Combine(folder, s.Name); gridFs.Download(localFileName, s); return localFileName; }
public void should_download_original_file() { // arrange var info = new DocumentHandleInfo( new DocumentHandle("doc"), new FileNameWithExtension("\"A document.docx\"") ); var format = new DocumentFormat("original"); var blobId = new BlobId("file_1"); var doc = new DocumentDescriptorReadModel( 1L, new DocumentDescriptorId(1), blobId); SetupDocumentHandle(info, doc.Id); SetupDocumentModel(doc); BlobStore .GetDescriptor(blobId) .Returns(i => new FsBlobDescriptor(blobId, TestConfig.PathToWordDocument)); // act using (var response = Controller.GetFormat(_tenantId, info.Handle, format).Result) { // assert response.EnsureSuccessStatusCode(); Assert.AreEqual("\"A document.docx\"", response.Content.Headers.ContentDisposition.FileName); } }
public string Download(BlobId blobId, string folder) { return ForBlobId(blobId).Download(blobId, folder); }
public DocumentFormatHasBeenUpdated(DocumentFormat documentFormat, BlobId blobId, PipelineId createdBy) { DocumentFormat = documentFormat; BlobId = blobId; CreatedBy = createdBy; }
public void blobId_should_parse_format_from_string() { var id = new BlobId("format.100"); Assert.AreEqual(new DocumentFormat("format"), id.Format); }
public void blobId_should_contain_format_and_number() { var id = new BlobId(new DocumentFormat("format"), 1); Assert.AreEqual("format.1", (string)id); }
public void should_download_pdf_format() { // arrange var info = new DocumentHandleInfo( new DocumentHandle("doc"), new FileNameWithExtension("a.file") ); var format = new DocumentFormat("pdf"); var pdfBlobId = new BlobId("pdf"); var doc = new DocumentDescriptorReadModel( 1L, new DocumentDescriptorId(1), new BlobId("file_1")); doc.AddFormat(new PipelineId("abc"), format, pdfBlobId); SetupDocumentHandle(info, doc.Id); SetupDocumentModel(doc); BlobStore.GetDescriptor(pdfBlobId).Returns(i => new FsBlobDescriptor(pdfBlobId, TestConfig.PathToDocumentPdf)); // act using (var response = Controller.GetFormat(_tenantId, info.Handle, format).Result) { // assert response.EnsureSuccessStatusCode(); Assert.AreEqual("application/pdf", response.Content.Headers.ContentType.MediaType); } }
HttpResponseMessage StreamFile(BlobId formatBlobId, String fileName = null) { var descriptor = _blobStore.GetDescriptor(formatBlobId); if (descriptor == null) { return Request.CreateErrorResponse( HttpStatusCode.NotFound, string.Format("File {0} not found", formatBlobId) ); } var response = Request.CreateResponse(HttpStatusCode.OK); response.Content = new StreamContent(descriptor.OpenRead()); response.Content.Headers.ContentType = new MediaTypeHeaderValue(descriptor.ContentType); if (fileName != null) { response.Content.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment") { FileName = fileName }; } return response; }
public void Delete(BlobId blobId) { var gridFs = GetGridFsByBlobId(blobId); Logger.DebugFormat("Deleting file {0} on {1}", blobId, gridFs.DatabaseName); gridFs.DeleteById((string)blobId); }
/// <summary> /// Upload a file sent in an http request /// </summary> /// <param name="httpContent">request's content</param> /// <returns>Error message or null</returns> private async Task<String> UploadFromHttpContent(HttpContent httpContent) { if (httpContent == null || !httpContent.IsMimeMultipartContent()) return "Attachment not found!"; var provider = await httpContent.ReadAsMultipartAsync( new FileStoreMultipartStreamProvider(_blobStore, _configService) ); if (provider.Filename == null) return "Attachment not found!"; if (provider.IsInvalidFile) return string.Format("Unsupported file {0}", provider.Filename); if (provider.FormData["custom-data"] != null) { _customData = JsonConvert.DeserializeObject<DocumentCustomData>(provider.FormData["custom-data"]); } _fileName = provider.Filename; _blobId = provider.BlobId; return null; }
public void when_file_is_not_found_should_return_404() { // arrange var info = new DocumentHandleInfo( new DocumentHandle("doc"), new FileNameWithExtension("a.file") ); var format = new DocumentFormat("original"); var blobId = new BlobId("file_1"); var doc = new DocumentDescriptorReadModel( 1L, new DocumentDescriptorId(1), blobId); SetupDocumentHandle(info, doc.Id); SetupDocumentModel(doc); BlobStore.GetDescriptor(blobId).Returns(i => null); // act var response = Controller.GetFormat(_tenantId, info.Handle, format).Result; // assert Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode); Assert.AreEqual("File file_1 not found", response.GetError().Message); }
public FormatAddedToDocumentDescriptor(DocumentFormat documentFormat, BlobId blobId, PipelineId createdBy) { DocumentFormat = documentFormat; BlobId = blobId; CreatedBy = createdBy; }