void VerifyMapDAOEntryIsCreated()
{
var mapper = new Mapper();
var parts = new List <BlobId>();
var blob = new BlobId();
parts.Add(blob);
var fileId = mapper.AddMap(parts);
Assert.True(mapper.GetFileMap.Any(f => f.FileId.Equals(fileId)));
Assert.True(mapper.GetFileMap.Any(f => f.BlobParts.Contains(blob)));
}
public InitializeDocumentDescriptor(
DocumentDescriptorId aggregateId,
BlobId blobId,
DocumentHandleInfo handleInfo,
FileHash hash,
FileNameWithExtension fileName
) : base(aggregateId)
{
FileName = fileName;
Hash = hash;
BlobId = blobId;
HandleInfo = handleInfo;
}
public InitializeDocumentDescriptorAsAttach(
DocumentDescriptorId aggregateId,
BlobId blobId,
DocumentHandleInfo handleInfo,
DocumentHandle fatherHandle,
DocumentDescriptorId fatherDocumentDescriptorId,
FileHash hash,
FileNameWithExtension fileName)
: base(aggregateId, blobId, handleInfo, hash, fileName)
{
FatherHandle = fatherHandle;
FatherDocumentDescriptorId = fatherDocumentDescriptorId;
}
public DocumentDescriptorReadModel(
Int64 sequenceNumber,
DocumentDescriptorId id,
BlobId blobId)
{
this.Formats = new Dictionary <DocumentFormat, FormatInfo>();
this.Documents = new HashSet <DocumentHandle>();
this.Id = id;
this.SequenceNumber = sequenceNumber;
AddFormat(PipelineId.Null, new DocumentFormat(DocumentFormats.Original), blobId);
Attachments = new HashSet <DocumentAttachmentReadModel>(
DocumentAttachmentReadModel.Comparer.Default);
}
public bool BlobExists(BlobId blobId)
{
if (blobId == null)
{
throw new ArgumentNullException(nameof(blobId));
}
Logger.DebugFormat($"BlobExists for blobid {blobId}");
var descriptor = _mongodDbFileSystemBlobDescriptorStorage.FindOneById(blobId);
return(descriptor != null);
}
public static FileMetadatas ToFileMetadata(this BlobId blobId)
{
string nameWithoutExt = blobId.FullPath.Remove(blobId.FullPath.LastIndexOf(".", StringComparison.InvariantCultureIgnoreCase));
string parent = Path.GetFileName(Path.GetDirectoryName(blobId.FullPath));
return(new FileMetadatas
{
Name = blobId.Id,
Parent = parent,
FullPath = blobId.FullPath,
NameWithoutExt = nameWithoutExt,
FolderPath = blobId.FolderPath
});
}
private string GetFileNameFromBlobIdAndRemoveDuplicates(BlobId blobId)
{
var fileName = _directoryManager.GetFileNameFromBlobId(blobId);
if (File.Exists(fileName))
{
//Anomaly, we are trying to overwrite the blob
Logger.Warn($"Destination file {blobId} already exists for id {blobId}");
//Todo move in another folder ... maybe a lost and found.
File.Move(fileName, fileName + "." + Guid.NewGuid().ToString());
}
return(fileName);
}
public async Task adding_two_time_same_format_overwrite_older()
{
//Upload original
var handle = new DocumentHandle("Add_Format_Test");
await _documentStoreClient.UploadAsync(TestConfig.PathToOpenDocumentText, handle);
// wait background projection polling
await UpdateAndWaitAsync();
//now add format to document.
AddFormatFromFileToDocumentModel model = new AddFormatFromFileToDocumentModel();
model.DocumentHandle = handle;
model.PathToFile = TestConfig.PathToTextDocument;
model.CreatedById = "tika";
model.Format = new DocumentFormat("tika");
await _documentStoreClient.AddFormatToDocument(model, new Dictionary <String, Object>());
// wait background projection polling
await UpdateAndWaitAsync();
//get blobId of the original format
var descriptor = _documentDescriptorCollection.FindAll().Single();
BlobId blobId = descriptor.Formats
.Single(f => f.Key == new DocumentFormat("tika"))
.Value.BlobId;
//now add same format with different content.
model = new AddFormatFromFileToDocumentModel();
model.DocumentHandle = handle;
model.PathToFile = TestConfig.PathToHtml;
model.CreatedById = "tika";
model.Format = new DocumentFormat("tika");
await _documentStoreClient.AddFormatToDocument(model, new Dictionary <String, Object>());
// wait background projection polling
await UpdateAndWaitAsync();
var formats = await _documentStoreClient.GetFormatsAsync(handle);
Assert.NotNull(formats);
Assert.IsTrue(formats.HasFormat(new DocumentFormat("original")));
Assert.IsTrue(formats.HasFormat(new DocumentFormat("tika")));
Assert.That(formats, Has.Count.EqualTo(2));
await CompareDownloadedStreamToFile(TestConfig.PathToHtml, _documentStoreClient.OpenRead(handle, new DocumentFormat("tika")));
//verify old blob storage was deleted
//Assert.Throws<Exception>(() => _blobStore.GetDescriptor(blobId), "Blob Id for artifact is not deleted");
}
public AddFormatToDocumentDescriptor(
DocumentDescriptorId aggregateId,
DocumentFormat documentFormat,
BlobId blobId,
PipelineId createdById) : base(aggregateId)
{
if (aggregateId == null)
{
throw new ArgumentNullException("aggregateId");
}
DocumentFormat = documentFormat;
BlobId = blobId;
CreatedBy = createdById;
}
public string Download(BlobId blobId, string folder)
{
if (blobId == null)
{
throw new ArgumentNullException(nameof(blobId));
}
if (String.IsNullOrEmpty(folder))
{
throw new ArgumentNullException(nameof(folder));
}
if (!Directory.Exists(folder))
{
throw new ArgumentException($"folder {folder} does not exists", nameof(folder));
}
var descriptor = _mongodDbFileSystemBlobDescriptorStorage.FindOneById(blobId);
if (descriptor == null)
{
throw new ArgumentException($"Descriptor for {blobId} not found in {_mongodDbFileSystemBlobDescriptorStorage.GetType().Name}");
}
var localFileName = _directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension);
if (!File.Exists(localFileName))
{
Logger.Error($"Blob {blobId} has descriptor, but blob file {localFileName} not found in the system.");
throw new ArgumentException($"Blob {blobId} not found");
}
var originalFileName = descriptor.FileNameWithExtension.ToString();
string destinationFileName = Path.Combine(folder, originalFileName);
Int32 uniqueId = 1;
while (File.Exists(destinationFileName))
{
destinationFileName = Path.Combine(folder, Path.GetFileNameWithoutExtension(originalFileName) + $" ({uniqueId++})") + Path.GetExtension(originalFileName);
}
File.Copy(localFileName, destinationFileName);
if (Logger.IsDebugEnabled)
{
Logger.Debug($"Blob {blobId} downloaded in folder {folder} with name {destinationFileName}");
}
return(destinationFileName);
}
public IBlobWriter CreateNew(DocumentFormat format, FileNameWithExtension fname)
{
var blobId = new BlobId(format, _counterService.GetNext(format));
var gridFs = GetGridFsByFormat(format);
Logger.DebugFormat("Creating file {0} on {1}", blobId, gridFs.DatabaseName);
var stream = gridFs.Create(fname, new MongoGridFSCreateOptions()
{
ContentType = MimeTypes.GetMimeType(fname),
UploadDate = DateTime.UtcNow,
Id = (string)blobId
});
return(new BlobWriter(blobId, stream, fname));
}
public static void BlobId_Compare()
{
var blobId1 = new BlobId(s_hasher.HashData("abc"));
var blobId2 = new BlobId(s_hasher.HashData("abc"));
var blobId3 = new BlobId(s_hasher.HashData("def"));
Assert.True(blobId1.CompareTo(blobId2) == 0);
Assert.True(blobId1.CompareTo(blobId3) != 0);
BlobId[] list = new[] { blobId1, blobId2, blobId3 };
Array.Sort(list);
Assert.True(list[0] <= list[1]);
Assert.True(list[2] >= list[1]);
}
public async Task List_FilesInFolder_NonRecursive()
{
string id = RandomGenerator.RandomString;
await _storage.WriteTextAsync(id, RandomGenerator.RandomString);
List<BlobId> items = (await _storage.ListAsync(new ListOptions { Recurse = false })).ToList();
Assert.True(items.Count > 0);
BlobId tid = items.Where(i => i.Id == id).FirstOrDefault();
Assert.NotNull(tid);
Assert.Equal(StoragePath.RootFolderPath, tid.FolderPath);
Assert.Equal(id, tid.Id);
}
async Task TestNumberOfParts()
{
var blobId = new BlobId();
var numberParts = 2;
var blobStoreMock = new Mock <IBlobStore>();
blobStoreMock.Setup(store => store.Put(It.IsAny <Stream>())).Returns(blobId);
var splitter = new FileSplitter(blobStoreMock.Object);
var fileSystem = TestUtilities.CreateFilesystem();
var fileStream = fileSystem.FileStream.Create(@"c:\myfile.txt", FileMode.Open);
var blobids = await splitter.SplitData(numberParts, fileStream);
Assert.Equal(numberParts, blobids.Count());
}
public IBlobWriter CreateNew(DocumentFormat format, FileNameWithExtension fname)
{
var blobId = new BlobId(format, _counterService.GetNext(format));
if (Logger.IsDebugEnabled)
{
Logger.Debug($"CreateNew blob for format {format} with file {fname} - assigned blobId: {blobId}");
}
return(new FileSystemBlobWriter(
blobId,
fname,
GetFileNameFromBlobIdAndRemoveDuplicates(blobId, fname),
_mongodDbFileSystemBlobDescriptorStorage,
Logger));
}
public void Delete(BlobId blobId)
{
var descriptor = _mongodDbFileSystemBlobDescriptorStorage.FindOneById(blobId);
if (descriptor == null)
{
throw new ArgumentException($"Descriptor for {blobId} not found in {_mongodDbFileSystemBlobDescriptorStorage.GetType().Name}");
}
var fileName = _directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension);
File.Delete(fileName);
_mongodDbFileSystemBlobDescriptorStorage.Delete(blobId);
}
async Task Upload(DropboxClient dbx, Stream content, BlobId id)
{
try
{
var task = dbx.Files.UploadAsync(
"/" + id.GetBlobId.ToString(),
WriteMode.Overwrite.Instance,
body: content);
Console.WriteLine("Upload to Dropbox completed", Color.GreenYellow);
await task;
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public async Task List_FilesInFolder_NonRecursive()
{
string id = RandomBlobId();
await _storage.WriteTextAsync(id, RandomGenerator.RandomString);
List <BlobId> items = (await _storage.ListAsync(new ListOptions {
FolderPath = _blobPrefix, Recurse = false
})).ToList();
Assert.True(items.Count > 0);
BlobId tid = items.FirstOrDefault(i => i.FullPath == id);
Assert.NotNull(tid);
}
private BlobId ToBlobItem(string fullPath, BlobItemKind kind, bool includeMeta)
{
fullPath = fullPath.Substring(_directory.FullName.Length);
fullPath = fullPath.Replace(Path.DirectorySeparatorChar, StoragePath.PathSeparator);
fullPath = fullPath.Trim(StoragePath.PathSeparator);
fullPath = StoragePath.PathStrSeparator + fullPath;
var blobId = new BlobId(fullPath, kind);
if (includeMeta)
{
blobId.Meta = BlobMetaFromId(blobId.FullPath);
}
return(blobId);
}
public static void BlobId_Compare()
{
var comparer = BlobIdComparer.Default;
var blobId1 = new BlobId(Sha1.Hash("abc"));
var blobId2 = new BlobId(Sha1.Hash("abc"));
var blobId3 = new BlobId(Sha1.Hash("def"));
var list = new[] { blobId1, blobId2, blobId3 };
Assert.True(blobId1.CompareTo(blobId2) == 0);
Assert.True(blobId1.CompareTo(blobId3) != 0);
Array.Sort(list, comparer.Compare);
Assert.True(list[0] <= list[1]);
Assert.True(list[2] >= list[1]);
}
public async Task <CoCStatus> Upload(BlobId fileId)
{
var file = this._store.GetBlob(fileId);
var hoe = new DropBoxCrackHoe();
try
{
await hoe.Magic(file, fileId);
}
catch (Exception)
{
return(CoCStatus.FAILURE);
}
return(CoCStatus.SUCCESS);
}
public async Task <IReadOnlyCollection <BlobId> > ListAsync(ListOptions options = null, CancellationToken cancellationToken = default)
{
FtpClient client = await GetClientAsync();
if (options == null)
{
options = new ListOptions();
}
FtpListItem[] items = await client.GetListingAsync(options.FolderPath);
var results = new List <BlobId>();
foreach (FtpListItem item in items)
{
if (options.FilePrefix != null && !item.Name.StartsWith(options.FilePrefix))
{
continue;
}
BlobId bid = ToBlobId(item);
if (bid == null)
{
continue;
}
if (options.BrowseFilter != null)
{
bool include = options.BrowseFilter(bid);
if (!include)
{
continue;
}
}
results.Add(bid);
if (options.MaxResults != null && results.Count >= options.MaxResults.Value)
{
break;
}
}
return(results);
}
private async Task ListFolderAsync(List <BlobId> container, string path, ListOptions options, CancellationToken cancellationToken)
{
CloudBlobDirectory dir = GetCloudBlobDirectory(path);
BlobContinuationToken token = null;
var batch = new List <BlobId>();
do
{
BlobResultSegment segment = await dir.ListBlobsSegmentedAsync(
false, BlobListingDetails.None, null, token, null, null, cancellationToken);
token = segment.ContinuationToken;
foreach (IListBlobItem blob in segment.Results)
{
BlobId id = ToBlobId(blob, options.IncludeMetaWhenKnown);
if (options.IsMatch(id))
{
batch.Add(id);
}
}
}while (token != null && ((options.MaxResults == null) || (container.Count + batch.Count < options.MaxResults.Value)));
batch = batch.Where(options.IsMatch).ToList();
if (options.Add(container, batch))
{
return;
}
if (options.Recurse)
{
List <BlobId> folderIds = batch.Where(r => r.Kind == BlobItemKind.Folder).ToList();
foreach (BlobId folderId in folderIds)
{
await ListFolderAsync(
container,
StoragePath.Combine(path, folderId.Id),
options,
cancellationToken);
}
}
}
public void RawStore(BlobId blobId, IBlobDescriptor descriptor)
{
Logger.InfoFormat("Ask for raw storage of blob {0} for file {1}", blobId, descriptor.FileNameWithExtension);
using (var stream = descriptor.OpenRead())
{
InnerPersistOfBlob(blobId, descriptor.FileNameWithExtension, stream);
FileSystemBlobDescriptor newDescriptor = new FileSystemBlobDescriptor()
{
BlobId = blobId,
ContentType = descriptor.ContentType,
FileNameWithExtension = descriptor.FileNameWithExtension,
Length = descriptor.Length,
Md5 = descriptor.Hash.ToString(),
Timestamp = DateTime.UtcNow,
};
_mongodDbFileSystemBlobDescriptorStorage.SaveDescriptor(newDescriptor);
}
}
/// <summary>
/// Returns all of the blob names in a container
/// </summary>
/// <returns>All of the blob names in a container</returns>
/// <remarks>This does not scale, for scalability usitlize the pagaing functionaltiy
/// to page through the blobs in t</remarks>
public async Task <List <BlobId> > GetListOfBlobs()
{
BlobContainerClient blobContainerClient = GetBlobContainerClient(ConfigSettings.UPLOADEDIMAGES_CONTAINERNAME);
var blobs = blobContainerClient.GetBlobsAsync();
List <BlobId> blobNames = new List <BlobId>();
await foreach (var blobPage in blobs.AsPages())
{
foreach (var blobItem in blobPage.Values)
{
var blobject = new BlobId();
blobject.id = blobItem.Name;
blobNames.Add(blobject);
}
}
return(blobNames);
}
public FileSystemBlobDescriptor FindOneById(BlobId blobId)
{
if (blobId == null)
{
throw new ArgumentNullException(nameof(blobId));
}
var descriptorLocalFileName = _directoryManager.GetDescriptorFileNameFromBlobId(blobId);
if (!File.Exists(descriptorLocalFileName))
{
return(null);
}
var descriptor = JsonConvert.DeserializeObject <FileSystemBlobDescriptor>(File.ReadAllText(descriptorLocalFileName));
descriptor.SetLocalFileName(_directoryManager.GetFileNameFromBlobId(blobId, descriptor.FileNameWithExtension));
return(descriptor);
}
