Пример #1
0
        void VerifyMapDAOEntryIsCreated()
        {
            var mapper = new Mapper();
            var parts  = new List <BlobId>();
            var blob   = new BlobId();

            parts.Add(blob);

            var fileId = mapper.AddMap(parts);

            Assert.True(mapper.GetFileMap.Any(f => f.FileId.Equals(fileId)));
            Assert.True(mapper.GetFileMap.Any(f => f.BlobParts.Contains(blob)));
        }
Пример #2
0
 public InitializeDocumentDescriptor(
     DocumentDescriptorId aggregateId,
     BlobId blobId,
     DocumentHandleInfo handleInfo,
     FileHash hash,
     FileNameWithExtension fileName
     ) : base(aggregateId)
 {
     FileName   = fileName;
     Hash       = hash;
     BlobId     = blobId;
     HandleInfo = handleInfo;
 }
Пример #3
0
 public InitializeDocumentDescriptorAsAttach(
     DocumentDescriptorId aggregateId,
     BlobId blobId,
     DocumentHandleInfo handleInfo,
     DocumentHandle fatherHandle,
     DocumentDescriptorId fatherDocumentDescriptorId,
     FileHash hash,
     FileNameWithExtension fileName)
     : base(aggregateId, blobId, handleInfo, hash, fileName)
 {
     FatherHandle = fatherHandle;
     FatherDocumentDescriptorId = fatherDocumentDescriptorId;
 }
Пример #4
0
 public DocumentDescriptorReadModel(
     Int64 sequenceNumber,
     DocumentDescriptorId id,
     BlobId blobId)
 {
     this.Formats        = new Dictionary <DocumentFormat, FormatInfo>();
     this.Documents      = new HashSet <DocumentHandle>();
     this.Id             = id;
     this.SequenceNumber = sequenceNumber;
     AddFormat(PipelineId.Null, new DocumentFormat(DocumentFormats.Original), blobId);
     Attachments = new HashSet <DocumentAttachmentReadModel>(
         DocumentAttachmentReadModel.Comparer.Default);
 }
        public bool BlobExists(BlobId blobId)
        {
            if (blobId == null)
            {
                throw new ArgumentNullException(nameof(blobId));
            }

            Logger.DebugFormat($"BlobExists for blobid {blobId}");

            var descriptor = _mongodDbFileSystemBlobDescriptorStorage.FindOneById(blobId);

            return(descriptor != null);
        }
Пример #6
0
        public static FileMetadatas ToFileMetadata(this BlobId blobId)
        {
            string nameWithoutExt = blobId.FullPath.Remove(blobId.FullPath.LastIndexOf(".", StringComparison.InvariantCultureIgnoreCase));
            string parent         = Path.GetFileName(Path.GetDirectoryName(blobId.FullPath));

            return(new FileMetadatas
            {
                Name = blobId.Id,
                Parent = parent,
                FullPath = blobId.FullPath,
                NameWithoutExt = nameWithoutExt,
                FolderPath = blobId.FolderPath
            });
        }
Пример #7
0
        private string GetFileNameFromBlobIdAndRemoveDuplicates(BlobId blobId)
        {
            var fileName = _directoryManager.GetFileNameFromBlobId(blobId);

            if (File.Exists(fileName))
            {
                //Anomaly, we are trying to overwrite the blob
                Logger.Warn($"Destination file {blobId} already exists for id {blobId}");
                //Todo move in another folder ... maybe a lost and found.
                File.Move(fileName, fileName + "." + Guid.NewGuid().ToString());
            }

            return(fileName);
        }
Пример #8
0
        public async Task adding_two_time_same_format_overwrite_older()
        {
            //Upload original
            var handle = new DocumentHandle("Add_Format_Test");
            await _documentStoreClient.UploadAsync(TestConfig.PathToOpenDocumentText, handle);

            // wait background projection polling
            await UpdateAndWaitAsync();

            //now add format to document.
            AddFormatFromFileToDocumentModel model = new AddFormatFromFileToDocumentModel();

            model.DocumentHandle = handle;
            model.PathToFile     = TestConfig.PathToTextDocument;
            model.CreatedById    = "tika";
            model.Format         = new DocumentFormat("tika");
            await _documentStoreClient.AddFormatToDocument(model, new Dictionary <String, Object>());

            // wait background projection polling
            await UpdateAndWaitAsync();

            //get blobId of the original format
            var    descriptor = _documentDescriptorCollection.FindAll().Single();
            BlobId blobId     = descriptor.Formats
                                .Single(f => f.Key == new DocumentFormat("tika"))
                                .Value.BlobId;

            //now add same format with different content.
            model = new AddFormatFromFileToDocumentModel();
            model.DocumentHandle = handle;
            model.PathToFile     = TestConfig.PathToHtml;
            model.CreatedById    = "tika";
            model.Format         = new DocumentFormat("tika");
            await _documentStoreClient.AddFormatToDocument(model, new Dictionary <String, Object>());

            // wait background projection polling
            await UpdateAndWaitAsync();

            var formats = await _documentStoreClient.GetFormatsAsync(handle);

            Assert.NotNull(formats);
            Assert.IsTrue(formats.HasFormat(new DocumentFormat("original")));
            Assert.IsTrue(formats.HasFormat(new DocumentFormat("tika")));
            Assert.That(formats, Has.Count.EqualTo(2));

            await CompareDownloadedStreamToFile(TestConfig.PathToHtml, _documentStoreClient.OpenRead(handle, new DocumentFormat("tika")));

            //verify old blob storage was deleted
            //Assert.Throws<Exception>(() => _blobStore.GetDescriptor(blobId), "Blob Id for artifact is not deleted");
        }
 public AddFormatToDocumentDescriptor(
     DocumentDescriptorId aggregateId,
     DocumentFormat documentFormat,
     BlobId blobId,
     PipelineId createdById) : base(aggregateId)
 {
     if (aggregateId == null)
     {
         throw new ArgumentNullException("aggregateId");
     }
     DocumentFormat = documentFormat;
     BlobId         = blobId;
     CreatedBy      = createdById;
 }
        public string Download(BlobId blobId, string folder)
        {
            if (blobId == null)
            {
                throw new ArgumentNullException(nameof(blobId));
            }

            if (String.IsNullOrEmpty(folder))
            {
                throw new ArgumentNullException(nameof(folder));
            }

            if (!Directory.Exists(folder))
            {
                throw new ArgumentException($"folder {folder} does not exists", nameof(folder));
            }

            var descriptor = _mongodDbFileSystemBlobDescriptorStorage.FindOneById(blobId);

            if (descriptor == null)
            {
                throw new ArgumentException($"Descriptor for {blobId} not found in {_mongodDbFileSystemBlobDescriptorStorage.GetType().Name}");
            }

            var localFileName = _directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension);

            if (!File.Exists(localFileName))
            {
                Logger.Error($"Blob {blobId} has descriptor, but blob file {localFileName} not found in the system.");
                throw new ArgumentException($"Blob {blobId} not found");
            }

            var    originalFileName    = descriptor.FileNameWithExtension.ToString();
            string destinationFileName = Path.Combine(folder, originalFileName);
            Int32  uniqueId            = 1;

            while (File.Exists(destinationFileName))
            {
                destinationFileName = Path.Combine(folder, Path.GetFileNameWithoutExtension(originalFileName) + $" ({uniqueId++})") + Path.GetExtension(originalFileName);
            }

            File.Copy(localFileName, destinationFileName);

            if (Logger.IsDebugEnabled)
            {
                Logger.Debug($"Blob {blobId} downloaded in folder {folder} with name {destinationFileName}");
            }
            return(destinationFileName);
        }
        public IBlobWriter CreateNew(DocumentFormat format, FileNameWithExtension fname)
        {
            var blobId = new BlobId(format, _counterService.GetNext(format));
            var gridFs = GetGridFsByFormat(format);

            Logger.DebugFormat("Creating file {0} on {1}", blobId, gridFs.DatabaseName);
            var stream = gridFs.Create(fname, new MongoGridFSCreateOptions()
            {
                ContentType = MimeTypes.GetMimeType(fname),
                UploadDate  = DateTime.UtcNow,
                Id          = (string)blobId
            });

            return(new BlobWriter(blobId, stream, fname));
        }
Пример #12
0
        public static void BlobId_Compare()
        {
            var blobId1 = new BlobId(s_hasher.HashData("abc"));
            var blobId2 = new BlobId(s_hasher.HashData("abc"));
            var blobId3 = new BlobId(s_hasher.HashData("def"));

            Assert.True(blobId1.CompareTo(blobId2) == 0);
            Assert.True(blobId1.CompareTo(blobId3) != 0);

            BlobId[] list = new[] { blobId1, blobId2, blobId3 };
            Array.Sort(list);

            Assert.True(list[0] <= list[1]);
            Assert.True(list[2] >= list[1]);
        }
Пример #13
0
      public async Task List_FilesInFolder_NonRecursive()
      {
         string id = RandomGenerator.RandomString;

         await _storage.WriteTextAsync(id, RandomGenerator.RandomString);

         List<BlobId> items = (await _storage.ListAsync(new ListOptions { Recurse = false })).ToList();

         Assert.True(items.Count > 0);

         BlobId tid = items.Where(i => i.Id == id).FirstOrDefault();
         Assert.NotNull(tid);
         Assert.Equal(StoragePath.RootFolderPath, tid.FolderPath);
         Assert.Equal(id, tid.Id);
      }
Пример #14
0
        async Task TestNumberOfParts()
        {
            var blobId        = new BlobId();
            var numberParts   = 2;
            var blobStoreMock = new Mock <IBlobStore>();

            blobStoreMock.Setup(store => store.Put(It.IsAny <Stream>())).Returns(blobId);
            var splitter   = new FileSplitter(blobStoreMock.Object);
            var fileSystem = TestUtilities.CreateFilesystem();
            var fileStream = fileSystem.FileStream.Create(@"c:\myfile.txt", FileMode.Open);

            var blobids = await splitter.SplitData(numberParts, fileStream);

            Assert.Equal(numberParts, blobids.Count());
        }
        public IBlobWriter CreateNew(DocumentFormat format, FileNameWithExtension fname)
        {
            var blobId = new BlobId(format, _counterService.GetNext(format));

            if (Logger.IsDebugEnabled)
            {
                Logger.Debug($"CreateNew blob for format {format} with file {fname} - assigned blobId: {blobId}");
            }
            return(new FileSystemBlobWriter(
                       blobId,
                       fname,
                       GetFileNameFromBlobIdAndRemoveDuplicates(blobId, fname),
                       _mongodDbFileSystemBlobDescriptorStorage,
                       Logger));
        }
        public void Delete(BlobId blobId)
        {
            var descriptor = _mongodDbFileSystemBlobDescriptorStorage.FindOneById(blobId);

            if (descriptor == null)
            {
                throw new ArgumentException($"Descriptor for {blobId} not found in {_mongodDbFileSystemBlobDescriptorStorage.GetType().Name}");
            }

            var fileName = _directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension);

            File.Delete(fileName);

            _mongodDbFileSystemBlobDescriptorStorage.Delete(blobId);
        }
Пример #17
0
 async Task Upload(DropboxClient dbx, Stream content, BlobId id)
 {
     try
     {
         var task = dbx.Files.UploadAsync(
             "/" + id.GetBlobId.ToString(),
             WriteMode.Overwrite.Instance,
             body: content);
         Console.WriteLine("Upload to Dropbox completed", Color.GreenYellow);
         await task;
     }
     catch (Exception e)
     {
         Console.WriteLine(e);
     }
 }
Пример #18
0
        public async Task List_FilesInFolder_NonRecursive()
        {
            string id = RandomBlobId();

            await _storage.WriteTextAsync(id, RandomGenerator.RandomString);

            List <BlobId> items = (await _storage.ListAsync(new ListOptions {
                FolderPath = _blobPrefix, Recurse = false
            })).ToList();

            Assert.True(items.Count > 0);

            BlobId tid = items.FirstOrDefault(i => i.FullPath == id);

            Assert.NotNull(tid);
        }
Пример #19
0
        private BlobId ToBlobItem(string fullPath, BlobItemKind kind, bool includeMeta)
        {
            fullPath = fullPath.Substring(_directory.FullName.Length);
            fullPath = fullPath.Replace(Path.DirectorySeparatorChar, StoragePath.PathSeparator);
            fullPath = fullPath.Trim(StoragePath.PathSeparator);
            fullPath = StoragePath.PathStrSeparator + fullPath;

            var blobId = new BlobId(fullPath, kind);

            if (includeMeta)
            {
                blobId.Meta = BlobMetaFromId(blobId.FullPath);
            }

            return(blobId);
        }
Пример #20
0
        public static void BlobId_Compare()
        {
            var comparer = BlobIdComparer.Default;

            var blobId1 = new BlobId(Sha1.Hash("abc"));
            var blobId2 = new BlobId(Sha1.Hash("abc"));
            var blobId3 = new BlobId(Sha1.Hash("def"));
            var list    = new[] { blobId1, blobId2, blobId3 };

            Assert.True(blobId1.CompareTo(blobId2) == 0);
            Assert.True(blobId1.CompareTo(blobId3) != 0);

            Array.Sort(list, comparer.Compare);

            Assert.True(list[0] <= list[1]);
            Assert.True(list[2] >= list[1]);
        }
        public async Task <CoCStatus> Upload(BlobId fileId)
        {
            var file = this._store.GetBlob(fileId);

            var hoe = new DropBoxCrackHoe();

            try
            {
                await hoe.Magic(file, fileId);
            }
            catch (Exception)
            {
                return(CoCStatus.FAILURE);
            }

            return(CoCStatus.SUCCESS);
        }
Пример #22
0
        public async Task <IReadOnlyCollection <BlobId> > ListAsync(ListOptions options = null, CancellationToken cancellationToken = default)
        {
            FtpClient client = await GetClientAsync();

            if (options == null)
            {
                options = new ListOptions();
            }

            FtpListItem[] items = await client.GetListingAsync(options.FolderPath);

            var results = new List <BlobId>();

            foreach (FtpListItem item in items)
            {
                if (options.FilePrefix != null && !item.Name.StartsWith(options.FilePrefix))
                {
                    continue;
                }

                BlobId bid = ToBlobId(item);
                if (bid == null)
                {
                    continue;
                }

                if (options.BrowseFilter != null)
                {
                    bool include = options.BrowseFilter(bid);
                    if (!include)
                    {
                        continue;
                    }
                }

                results.Add(bid);

                if (options.MaxResults != null && results.Count >= options.MaxResults.Value)
                {
                    break;
                }
            }

            return(results);
        }
Пример #23
0
        private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
        {
            CloudBlobDirectory dir = GetCloudBlobDirectory(path);

            BlobContinuationToken token = null;

            var batch = new List <BlobId>();

            do
            {
                BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
                    false, BlobListingDetails.None, null, token, null, null, cancellationToken);

                token = segment.ContinuationToken;

                foreach (IListBlobItem blob in segment.Results)
                {
                    BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown);

                    if (options.IsMatch(id))
                    {
                        batch.Add(id);
                    }
                }
            }while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));

            batch = batch.Where(options.IsMatch).ToList();
            if (options.Add(container, batch))
            {
                return;
            }

            if (options.Recurse)
            {
                List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
                foreach (BlobId folderId in folderIds)
                {
                    await ListFolderAsync(
                        container,
                        StoragePath.Combine(path, folderId.Id),
                        options,
                        cancellationToken);
                }
            }
        }
 public void RawStore(BlobId blobId, IBlobDescriptor descriptor)
 {
     Logger.InfoFormat("Ask for raw storage of blob {0} for file {1}", blobId, descriptor.FileNameWithExtension);
     using (var stream = descriptor.OpenRead())
     {
         InnerPersistOfBlob(blobId, descriptor.FileNameWithExtension, stream);
         FileSystemBlobDescriptor newDescriptor = new FileSystemBlobDescriptor()
         {
             BlobId                = blobId,
             ContentType           = descriptor.ContentType,
             FileNameWithExtension = descriptor.FileNameWithExtension,
             Length                = descriptor.Length,
             Md5       = descriptor.Hash.ToString(),
             Timestamp = DateTime.UtcNow,
         };
         _mongodDbFileSystemBlobDescriptorStorage.SaveDescriptor(newDescriptor);
     }
 }
Пример #25
0
        /// <summary>
        /// Returns all of the blob names in a container
        /// </summary>
        /// <returns>All of the blob names in a container</returns>
        /// <remarks>This does not scale, for scalability usitlize the pagaing functionaltiy
        /// to page through the blobs in t</remarks>
        public async Task <List <BlobId> > GetListOfBlobs()
        {
            BlobContainerClient blobContainerClient = GetBlobContainerClient(ConfigSettings.UPLOADEDIMAGES_CONTAINERNAME);
            var blobs = blobContainerClient.GetBlobsAsync();

            List <BlobId> blobNames = new List <BlobId>();

            await foreach (var blobPage in blobs.AsPages())
            {
                foreach (var blobItem in blobPage.Values)
                {
                    var blobject = new BlobId();
                    blobject.id = blobItem.Name;
                    blobNames.Add(blobject);
                }
            }
            return(blobNames);
        }
        public FileSystemBlobDescriptor FindOneById(BlobId blobId)
        {
            if (blobId == null)
            {
                throw new ArgumentNullException(nameof(blobId));
            }

            var descriptorLocalFileName = _directoryManager.GetDescriptorFileNameFromBlobId(blobId);

            if (!File.Exists(descriptorLocalFileName))
            {
                return(null);
            }

            var descriptor = JsonConvert.DeserializeObject <FileSystemBlobDescriptor>(File.ReadAllText(descriptorLocalFileName));

            descriptor.SetLocalFileName(_directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension));
            return(descriptor);
        }