public async Task <Tuple <string, byte[]> > GetFile(string guid)
{
var id = ObjectId.Parse(guid);
var query = await gridFs.FindAsync(Builders <GridFSFileInfo> .Filter.Eq("_id", id));
var file = await query.SingleOrDefaultAsync();
if (file == null)
{
throw new Exception(string.Format("File not found (id: {0})", guid));
}
var array = await gridFs.DownloadAsBytesAsync(id);
return(Tuple.Create(file.Filename, array));
}
private async Task MigratePortraits(CancellationToken cancellationToken)
{
var source = new GridFSBucket(_source, new GridFSBucketOptions
{
BucketName = "portraits"
});
using (var cursor = await source.FindAsync(Builders <GridFSFileInfo> .Filter.Empty))
{
var files = await cursor.ToListAsync();
var destination = new GridFSBucket(_destination, new GridFSBucketOptions
{
BucketName = "portraits"
});
foreach (var file in files)
{
using (var stream = await source.OpenDownloadStreamByNameAsync(file.Filename, null, cancellationToken))
{
await destination.UploadFromStreamAsync(file.Filename, stream, null, cancellationToken);
}
}
}
}
public async Task DownloadFileMongo()
{
var client = new MongoClient("mongodb://localhost:27017");
var database = client.GetDatabase("BookstoreDb");
var gridFsBucketOptions = new GridFSBucketOptions()
{
BucketName = "images",
ChunkSizeBytes = 1048576, // 1МБ
};
var bucket = new GridFSBucket(database, gridFsBucketOptions);
var filter = Builders <GridFSFileInfo <ObjectId> >
.Filter.Eq(x => x.Filename, "123.png");
var searchResult = await bucket.FindAsync(filter);
var fileEntry = searchResult.FirstOrDefault();
byte[] content = await bucket.DownloadAsBytesAsync(fileEntry.Id);
File.WriteAllBytes("123.png", content);
}
public async Task <List <GridFSFileInfo> > FindfileByPostId(string postId, bool flag = true)
{
var gridFsBucket = new GridFSBucket(this.database);
var filter = Builders <GridFSFileInfo> .Filter.And(
Builders <GridFSFileInfo> .Filter.Eq(x => x.Metadata["PostId"], postId)
);
if (flag)
{
filter = Builders <GridFSFileInfo> .Filter.And(
Builders <GridFSFileInfo> .Filter.Eq(x => x.Metadata["PostId"], postId),
Builders <GridFSFileInfo> .Filter.Eq(x => x.Metadata["Type"], 1));
}
var sort = Builders <GridFSFileInfo> .Sort.Descending(x => x.UploadDateTime);
var options = new GridFSFindOptions
{
Sort = sort
};
using (var cursor = await gridFsBucket.FindAsync(filter, options))
{
return(await cursor.ToListAsync());
}
}
public async Task <Picture?> Get(ObjectId pictureId)
{
var pictures = await _pictures.FindAsync(p => p.Id == pictureId);
var pictureEntities = pictures.Current.ToList();
if (!pictureEntities.Any())
{
return(null);
}
_logger.LogDebug($"Got pictures from mongo: {string.Concat(pictureEntities.Select(e => e.ToString()))}");
var picture = pictures.First();
var filter = Builders <GridFSFileInfo> .Filter.Eq("_id", picture.GridFsIds.Last());
var fileInfos = await _gridFs.FindAsync(filter);
var fileInfo = fileInfos.FirstOrDefault();
if (fileInfo == null)
{
return(null);
}
var pic = await _gridFs.DownloadAsBytesAsync(picture.Id);
_logger.LogInformation($"Got picture data for {pictureId} with length {pic.Length}");
return(new Picture(pic, picture.Filename, picture.Id, picture.Width, picture.Height));
}
public async Task <FileInfoModel> FindAsync(string id, bool includeSource = false)
{
FilterDefinition <GridFSFileInfo> filter = Builders <GridFSFileInfo> .Filter.Eq(x => x.Id, id.ToObjectId());
IAsyncCursor <GridFSFileInfo> fileInfoCursor = await GridFSBucket.FindAsync(filter);
fileInfoCursor.MoveNext();
GridFSFileInfo fileInfo = fileInfoCursor.Current.Single();
byte[] source = null;
if (includeSource)
{
source = await Download(id);
}
return(new FileInfoModel
{
DocumentId = fileInfo.Id.ToString(),
FileName = fileInfo.Filename,
UploadedDateTime = fileInfo.UploadDateTime,
CreateUserId = fileInfo.Metadata[nameof(FileInfoModel.CreateUserId)].AsString,
Source = source
});
}
public async Task <string> UploadFile(string filename, byte[] fileBytes, bool shouldOverride = true)
{
if (shouldOverride)
{
var filter = Builders <GridFSFileInfo <ObjectId> > .Filter.Eq(x => x.Filename, filename);
var currentFile = await(await _gridFS.FindAsync(filter)).FirstOrDefaultAsync();
if (currentFile != null)
{
await _gridFS.DeleteAsync(currentFile.Id);
}
}
var file = await _gridFS.UploadFromBytesAsync(filename, fileBytes);
return(file.ToString());
}
public async Task ThenExpirationIsNotSet()
{
var cursor = await _bucket.FindAsync(Builders <GridFSFileInfo> .Filter.Eq(x => x.Filename, _filename));
var list = await cursor.ToListAsync();
var doc = list.Single();
Assert.That(doc.Metadata.Contains("expiration"), Is.False);
}
private static async Task <bool> HasGridFs(IMongoDatabase database)
{
var bucket = new GridFSBucket(database);
var filter = Builders <GridFSFileInfo> .Filter.Empty;
using (var cursor = await bucket.FindAsync(filter))
{
return(await cursor.MoveNextAsync());
}
}
async Task <GridFSFileInfo> GetGridFsFileInfo(string id)
{
var filter = new ExpressionFilterDefinition <GridFSFileInfo>(f => f.Filename == id);
using (var cursor = await _bucket.FindAsync(filter))
{
var result = await cursor.FirstOrDefaultAsync();
return(result);
}
}
public async Task CreateFileFromStreamAsync(string path, Stream inputStream, bool overwrite = false)
{
path = GetPhysicalPath(path);
var filter = Builders <GridFSFileInfo> .Filter.And(Builders <GridFSFileInfo> .Filter.Eq(x => x.Filename, path));
using (var cursor = await _bucket.FindAsync(filter))
{
if (cursor.Any())
{
throw new FileStoreException($"Cannot create file '{path}' because it already exists.");
}
}
var response = await _bucket.UploadFromStreamAsync(path, inputStream);
if (response == null)
{
throw new FileStoreException($"Cannot create file '{path}'.");
}
}
public async Task ThenExpirationSetAsExpected()
{
var cursor = await _bucket.FindAsync(Builders <GridFSFileInfo> .Filter.Eq("_id", _id));
var list = await cursor.ToListAsync();
var doc = list.Single();
var expiration = doc.Metadata["expiration"].ToUniversalTime();
Assert.That(expiration, Is.EqualTo(_now.Add(_expectedTtl)).Within(1).Milliseconds);
}
static async Task Main(string[] args)
{
var mongoSettings = MongoClientSettings.FromConnectionString(args[0]);
var client = new MongoClient(mongoSettings);
var mongoDatabase = client.GetDatabase("dr-move-public-api");
var bucket = new GridFSBucket(mongoDatabase, new GridFSBucketOptions
{
BucketName = "packages",
});
if (!int.TryParse(args[1], out var numberOfDays))
{
Console.WriteLine($"Could not parse {args[1]} to an int.");
return;
}
if (numberOfDays < 30)
{
Console.WriteLine("Minimum number of days = 30");
return;
}
var mongoFilter =
Builders <GridFSFileInfo <ObjectId> > .Filter.Lt(x => x.UploadDateTime, DateTime.UtcNow.AddDays(-numberOfDays));
var options = new GridFSFindOptions
{
BatchSize = 50,
};
var batchnum = 0;
var numFiles = 0;
using (var cursor = await bucket.FindAsync(mongoFilter, options))
{
while (await cursor.MoveNextAsync())
{
var batch = cursor.Current;
Console.WriteLine($"Deleting from batch {++batchnum}");
foreach (var item in batch)
{
numFiles++;
await bucket.DeleteAsync(item.Id);
}
}
}
Console.WriteLine($"Deleted {numFiles} number of files");
}
private static async Task MigrateGridFS(
IMongoDatabase database,
JsonTextWriter jsonTextWriter,
StreamWriter streamWriter)
{
var bucket = new GridFSBucket(database);
var collectionName = bucket.Options.BucketName == "fs" ? "Files" : bucket.Options.BucketName;
var filter = Builders <GridFSFileInfo> .Filter.Empty;
var attachmentNumber = 0;
using (var cursor = await bucket.FindAsync(filter))
{
while (await cursor.MoveNextAsync())
{
foreach (var fileInfo in cursor.Current)
{
var document = ToExpandoObject(fileInfo.Metadata);
var dictionary = (IDictionary <string, object>)document;
var documentId = fileInfo.Id.ToString();
var totalSize = fileInfo.Length;
var contentType = GetContentType(fileInfo, dictionary);
using (var stream = new MemoryStream())
{
//can't OpenDownloadStreamAsync because it's not a seeakble stream
await bucket.DownloadToStreamAsync(fileInfo.Id, stream);
stream.Position = 0;
attachmentNumber++;
var attachmentInfo = await MigrationHelpers.WriteAttachment(
stream, totalSize, documentId, collectionName, contentType,
attachmentNumber, jsonTextWriter, streamWriter);
MigrationHelpers.WriteDocument(
document,
documentId,
collectionName,
null,
jsonTextWriter,
new List <Dictionary <string, object> >
{
attachmentInfo
});
}
}
}
}
}
public async Task <byte[]> LoadPdfAsync(string fileName)
{
var gridFsBucket = new GridFSBucket(_database);
var filter = Builders <GridFSFileInfo> .Filter.Eq(x => x.Filename, fileName);
var finData = await gridFsBucket.FindAsync(filter);
var firstData = finData.FirstAsync().Result;
var bsonId = firstData.Id;
byte[] content = await gridFsBucket.DownloadAsBytesAsync(firstData.Id);
// var dataStream = await gridFsBucket.OpenDownloadStreamAsync(bsonId);
return(content);
}
public static async Task DownloadFileAsBytes()
{
var database = DatabaseHelper.GetDatabaseReference("localhost", "file_store");
IGridFSBucket bucket = new GridFSBucket(database);
var filter = Builders <GridFSFileInfo <ObjectId> > .Filter.Eq(x => x.Filename, "sample2.pdf");
var searchResult = await bucket.FindAsync(filter);
var fileEntry = searchResult.FirstOrDefault();
byte[] content = await bucket.DownloadAsBytesAsync(fileEntry.Id);
File.WriteAllBytes("C:\\temp\\sample3.pdf", content);
//System.Diagnostics.Process.Start("C:\\temp\\sample2.pdf");
}
/// <summary>
/// 下载文件
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public async Task <MongoDbFileDoc> DownLoadFile(string id)
{
var model = new MongoDbFileDoc();
var filter = Builders <GridFSFileInfo <ObjectId> > .Filter.Eq(x => x.Id, new ObjectId(id));
var bucket = new GridFSBucket(GetMongoDatabase(), new GridFSBucketOptions());
var fileInfo = await bucket.FindAsync(filter);
var file = await fileInfo.FirstOrDefaultAsync();
if (file != null)
{
model.FileName = file.Filename;
model.FileId = id;
model.FileBytes = await bucket.DownloadAsBytesAsync(new ObjectId(id));
}
return(model);
}
public async Task <File> GetGif(dynamic id)
{
var fid = new ObjectId(id);
var filter = Builders <GridFSFileInfo> .Filter.Eq("_id", fid);
using var cursor = await _DMIGifs.FindAsync(filter);
var metadata = await cursor.FirstOrDefaultAsync();
return(metadata != null
? new File()
{
FileName = metadata.Filename,
UploadDate = metadata.UploadDateTime,
Length = metadata.Length,
_id = metadata.Id
}
: null);
}
public static async Task DownloadFileToStream()
{
var database = DatabaseHelper.GetDatabaseReference("localhost", "file_store");
IGridFSBucket bucket = new GridFSBucket(database);
var filter = Builders <GridFSFileInfo <ObjectId> > .Filter.Eq(x => x.Filename, "sample2.pdf");
var searchResult = await bucket.FindAsync(filter);
var fileEntry = searchResult.FirstOrDefault();
var fileName = "c:\\temp\\mystream.pdf";
using (Stream fileStream = new FileStream(fileName, FileMode.CreateNew, FileAccess.Write))
{
await bucket.DownloadToStreamAsync(fileEntry.Id, fileStream);
fileStream.Close();
}
}
private async Task <ObjectId> GetFileId(string fileName)
{
var filter = Builders <GridFSFileInfo> .Filter.Eq(x => x.Filename, fileName);
var sort = Builders <GridFSFileInfo> .Sort.Descending(x => x.UploadDateTime);
var options = new GridFSFindOptions
{
Limit = 1,
Sort = sort
};
using (var cursor = await bucket.FindAsync(filter, options))
{
if (cursor != null)
{
var fileInfo = cursor.ToList().FirstOrDefault();
return(fileInfo.Id);
}
}
return(new ObjectId());
}
public async Task <bool> DownloadFileAsync(ObjectId id, Stream OutStrem, string bucketname)
{
var bucket = new GridFSBucket(database, new GridFSBucketOptions
{
ChunkSizeBytes = 64512,
BucketName = bucketname
});
var filter = Builders <GridFSFileInfo <ObjectId> > .Filter.Where(x => x.Id == id);
var cursor = await bucket.FindAsync(filter);
var result = await cursor.ToListAsync();
if (result.Count == 0)
{
return(false);
}
await bucket.DownloadToStreamAsync(id, OutStrem);
return(true);
}
protected override async Task Handle(DeletePlayerCommand request, CancellationToken cancellationToken)
{
var collection = _database.GetCollection <Party>("players");
await collection.DeleteOneAsync(x => x.Name == request.Name, cancellationToken);
var bucket = new GridFSBucket(_database, new GridFSBucketOptions
{
BucketName = "portraits"
});
var filter = Builders <GridFSFileInfo> .Filter.And(
Builders <GridFSFileInfo> .Filter.Eq(x => x.Filename, request.Name));
var files = await bucket.FindAsync(filter, null, cancellationToken);
var file = await files.FirstOrDefaultAsync(cancellationToken);
if (file != null)
{
await bucket.DeleteAsync(file.Id, cancellationToken);
}
}
public async Task <bool> DownloadFileAsync(ObjectId id, string outpath, string bucketname)
{
var bucket = new GridFSBucket(database, new GridFSBucketOptions
{
ChunkSizeBytes = 64512,
BucketName = bucketname
});
var filter = Builders <GridFSFileInfo <ObjectId> > .Filter.Where(x => x.Id == id);
var cursor = await bucket.FindAsync(filter);
var result = await cursor.ToListAsync();
if (result.Count == 0)
{
return(false);
}
using (var outStream = new FileStream(outpath, FileMode.OpenOrCreate, FileAccess.Write))
{
await bucket.DownloadToStreamAsync(id, outStream);
}
return(true);
}
public async void UploadFileRealTest()
{
//prepare
var(gridFs, fileAgent) = Configure();
var byteData = new byte[] { 2, 0, 2, 0, 0, 4, 1, 8, 1, 0, 3, 8 };
var testId = new Guid().ToString("N");
var f = await gridFs.FindAsync(
new ExpressionFilterDefinition <GridFSFileInfo <ObjectId> >(x => x.Filename == testId));
var existFile = await f.FirstOrDefaultAsync();
if (existFile != null)
{
await gridFs.DeleteAsync(existFile.Id);
}
//execute
await fileAgent.ProcessMessageAsync(byteData, new Dictionary <string, string>
{
["Task"] = testId
});
//checks
f = await gridFs.FindAsync(
new ExpressionFilterDefinition <GridFSFileInfo <ObjectId> >(x => x.Filename == testId));
var fileInFs = await f.FirstOrDefaultAsync();
Assert.NotNull(fileInFs);
Assert.Equal(testId, fileInFs.Filename);
Assert.Equal(byteData.Length, fileInFs.Length);
// executeGet
var data = await fileAgent.ProcessRpcAsync(new AgentMessage <RpcRequest>
{
MessageType = MessageType.FileRequest,
Data = new RpcRequest
{
Args = new[] { testId }
}
});
var resBytes = data.To <byte[]>().Data;
//checks
Assert.Equal(resBytes, byteData);
//clear
await fileAgent.ProcessMessageAsync(new AgentMessage
{
MessageType = MessageType.DeleteFile,
Data = testId
});
//checks
f = await gridFs.FindAsync(
new ExpressionFilterDefinition <GridFSFileInfo <ObjectId> >(x => x.Filename == testId));
Assert.False(await f.AnyAsync());
}
/// <summary>
/// Resolves a token to a FileReference, if the token exists for this user.
/// The action should be called once the calling code can ensure the file is handled ok.
/// </summary>
/// <param name="mongoWrapper">For accessing the database</param>
/// <param name="tokenId">The token ID to search for</param>
/// <param name="userId">The user ID to correlate with the token</param>
/// <returns>A Tuple of the FileReference and an async action to expire the token</returns>
public static async Task <(FileReference, Func <Task>)> GetFileForReferenceToken(MongoWrapper mongoWrapper, string tokenId, ObjectId userId)
{
var tokenCollection = mongoWrapper.Database.GetCollection <DataReferenceToken <ConsumableData <ObjectId> > >(nameof(ReferenceToken));
var tokenFilterBuilder = new FilterDefinitionBuilder <DataReferenceToken <ConsumableData <ObjectId> > >();
var tokenFilter = tokenFilterBuilder.And
(
GeneralUtils.NotDeactivated(tokenFilterBuilder),
tokenFilterBuilder.Eq(t => t._id, tokenId),
tokenFilterBuilder.Eq(t => t.UserId, userId)
);
var tokenUpdateBuilder = new UpdateDefinitionBuilder <DataReferenceToken <ConsumableData <ObjectId> > >();
var tokenUpdate = tokenUpdateBuilder
.Set(t => t.DeactivationDate, DateTime.UtcNow)
.Set(t => t.AdditionalData.IsConsumed, true);
DataReferenceToken <ConsumableData <ObjectId> > token = (await tokenCollection.FindAsync(tokenFilter, new FindOptions <DataReferenceToken <ConsumableData <ObjectId> >, DataReferenceToken <ConsumableData <ObjectId> > >
{
Limit = 1,
AllowPartialResults = false,
})).SingleOrDefault();
if (token == null)
{
throw new ValidationException("Token de Arquivo não encontrado!");
}
var gridFsBucket = new GridFSBucket <ObjectId>(mongoWrapper.Database);
var fileFilterBuilder = new FilterDefinitionBuilder <GridFSFileInfo <ObjectId> >();
var fileFilter = fileFilterBuilder.Eq(finfo => finfo.Id, token.AdditionalData.Data);
var fileInfo = (await gridFsBucket.FindAsync(fileFilter, new GridFSFindOptions <ObjectId>
{
Limit = 1,
})).SingleOrDefault();
if (fileInfo == null)
{
throw new ApplicationException("Arquivo não encontrado, mas havia um token ativado referenciando-o!");
}
var fileReference = new FileReference
{
FileInfo = new Models.FileInfo
{
FileMetadata = BsonSerializer.Deserialize <FileMetadata>(fileInfo.Metadata),
Size = fileInfo.Length
},
_id = token.AdditionalData.Data
};
return
(
fileReference,
async() =>
{
var tokenUpdateTask = tokenCollection.UpdateOneAsync
(
tokenFilter,
tokenUpdate
);
var userCollection = mongoWrapper.Database.GetCollection <User>(nameof(User));
var userFilterBuilder = new FilterDefinitionBuilder <User>();
var userFilter = userFilterBuilder.And
(
GeneralUtils.NotDeactivated(userFilterBuilder),
userFilterBuilder.Eq(u => u._id, userId)
);
var userUpdateBuilder = new UpdateDefinitionBuilder <User>();
var userUpdate = userUpdateBuilder.Inc(u => u.FileBytesOccupied, fileReference.FileInfo.Size);
var userUpdateTask = userCollection.UpdateOneAsync
(
userFilter,
userUpdate
);
await tokenUpdateTask;
await userUpdateTask;
}
);
}
/// <summary>
/// Finds matching entries from the files collection.
/// </summary>
/// <param name="filter"></param>
/// <param name="options"></param>
/// <returns></returns>
public Task <IAsyncCursor <GridFSFileInfo> > FindAsync(FilterDefinition <GridFSFileInfo> filter, GridFSFindOptions options = null)
{
var bucket = new GridFSBucket(base.GetDatabase(), BucketOptions);
return(bucket.FindAsync(filter, options));
}