public void constructor_should_use_default_options_when_options_is_null()
{
var database = Substitute.For<IMongoDatabase>();
var result = new GridFSBucket(database, null);
result.Options.Should().BeSameAs(ImmutableGridFSBucketOptions.Defaults);
}
public void constructor_should_use_default_options_when_options_is_null()
{
var database = (new Mock<IMongoDatabase> { DefaultValue = DefaultValue.Mock }).Object;
var result = new GridFSBucket(database, null);
result.Options.Should().BeSameAs(ImmutableGridFSBucketOptions.Defaults);
}
public void Database_get_should_return_the_expected_result()
{
var database = (new Mock<IMongoDatabase> { DefaultValue = DefaultValue.Mock }).Object;
var subject = new GridFSBucket(database, null);
var result = subject.Database;
result.Should().BeSameAs(database);
}
public void Database_get_should_return_the_expected_result()
{
var database = Substitute.For<IMongoDatabase>();
var subject = new GridFSBucket(database, null);
var result = subject.Database;
result.Should().BeSameAs(database);
}
public void constructor_should_initialize_instance()
{
var database = Substitute.For<IMongoDatabase>();
var options = new ImmutableGridFSBucketOptions();
var result = new GridFSBucket(database, options);
result.Database.Should().BeSameAs(database);
result.Options.Should().BeSameAs(options);
}
// protected methods
protected override void Act(GridFSBucket bucket, bool async)
{
if (async)
{
InvokeMethodAsync(bucket).GetAwaiter().GetResult();
}
else
{
InvokeMethod(bucket);
}
}
public void constructor_should_initialize_instance()
{
var database = Substitute.For<IMongoDatabase>();
var bucket = new GridFSBucket(database);
var binding = Substitute.For<IReadBinding>();
var fileInfo = new GridFSFileInfo(new BsonDocument());
var result = new GridFSSeekableDownloadStream(bucket, binding, fileInfo);
result.Position.Should().Be(0);
result._chunk().Should().BeNull();
result._n().Should().Be(-1);
}
public void constructor_should_initialize_instance()
{
var database = (new Mock<IMongoDatabase> { DefaultValue = DefaultValue.Mock }).Object;
var options = new GridFSBucketOptions();
var result = new GridFSBucket(database, options);
result.Database.Should().BeSameAs(database);
result.Options.BucketName.Should().Be(options.BucketName);
result.Options.ChunkSizeBytes.Should().Be(options.ChunkSizeBytes);
result.Options.ReadPreference.Should().Be(options.ReadPreference);
result.Options.WriteConcern.Should().Be(options.WriteConcern);
}
public void constructor_should_initialize_instance()
{
var database = Substitute.For<IMongoDatabase>();
var options = new GridFSBucketOptions();
var result = new GridFSBucket(database, options);
result.Database.Should().BeSameAs(database);
result.Options.BucketName.Should().Be(options.BucketName);
result.Options.ChunkSizeBytes.Should().Be(options.ChunkSizeBytes);
result.Options.ReadPreference.Should().Be(options.ReadPreference);
result.Options.WriteConcern.Should().Be(options.WriteConcern);
}
protected virtual void Assert(GridFSBucket bucket)
{
var actualFilesCollectionDocuments = GetActualFilesCollectionDocuments(bucket);
var actualChunks = GetActualChunks(bucket);
InitializeExpectedFilesCollection(bucket);
InitializeExpectedChunksCollection(bucket);
RunAssertCommands(bucket, actualFilesCollectionDocuments, actualChunks);
var expectedFilesCollectionDocuments = GetExpectedFilesCollectionDocuments(bucket);
var expectedChunks = GetExpectedChunks(bucket);
Assert(actualFilesCollectionDocuments, actualChunks, expectedFilesCollectionDocuments, expectedChunks);
}
protected virtual async Task AssertAsync(GridFSBucket bucket)
{
var actualFilesCollectionDocuments = await GetActualFilesCollectionDocumentsAsync(bucket);
var actualChunks = await GetActualChunksAsync(bucket);
await InitializeExpectedFilesCollectionAsync(bucket);
await InitializeExpectedChunksCollectionAsync(bucket);
await RunAssertCommandsAsync(bucket, actualFilesCollectionDocuments, actualChunks);
var expectedFilesCollectionDocuments = await GetExpectedFilesCollectionDocumentsAsync(bucket);
var expectedChunks = await GetExpectedChunksAsync(bucket);
Assert(actualFilesCollectionDocuments, actualChunks, expectedFilesCollectionDocuments, expectedChunks);
}
public async Task RunTestAsync(BsonDocument data, BsonDocument testDefinition)
{
var test = GridFSTestFactory.CreateTest(data, testDefinition);
string reason;
if (!test.CanRun(out reason))
{
Assert.Ignore(reason);
return;
}
var client = DriverTestConfiguration.Client;
var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName);
var bucket = new GridFSBucket(database);
await test.RunAsync(bucket);
}
public MongoDistributedFIleSystem(IOptions <MongoDbOption> opts)
{
client = new MongoClient(opts.Value.ToString());
database = client.GetDatabase(opts.Value.DbName);
fs = new GridFSBucket(database);
}
protected virtual async Task ArrangeAsync(GridFSBucket bucket)
{
await InitializeFilesCollectionAsync(bucket);
await InitializeChunksCollectionAsync(bucket);
await RunArrangeCommandsAsync(bucket);
}
public GridFileSystem(string connectionString, string collectionName)
{
_files = MongoUtil.FromConnectionString <GridFile>(connectionString, collectionName);
_bucket = new GridFSBucket(_files.Database);
}
// protected methods
protected Task InvokeMethodAsync(GridFSBucket bucket)
{
return(bucket.DeleteAsync(_id));
}
public async Task RunAsync(GridFSBucket bucket)
{
await ArrangeAsync(bucket);
await ActAsync(bucket);
await AssertAsync(bucket);
}
protected Task<ObjectId> InvokeMethodAsync(GridFSBucket bucket)
{
return bucket.UploadFromBytesAsync(_filename, _source, _options);
}
protected override Task AssertAsync(GridFSBucket bucket)
{
_result.Should().Equal(_expectedResult);
return(Task.FromResult(true)); // don't call base.AssertAsync
}
public void Log141Adapter_AddToStore_Can_Add_Data_Chunk_Exceeds_MongoDb_Document_Size()
{
var response = _devKit.Add <WellList, Well>(_well);
// Adjust Points and Nodes for large file
WitsmlSettings.LogMaxDataPointsAdd = 5000000;
WitsmlSettings.LogMaxDataNodesAdd = 15000;
WitsmlSettings.LogMaxDataPointsGet = 5000000;
WitsmlSettings.LogMaxDataNodesGet = 15000;
_wellbore.UidWell = response.SuppMsgOut;
response = _devKit.Add <WellboreList, Wellbore>(_wellbore);
var xmlfile = Path.Combine(_testDataDir, string.Format(_exceedFileFormat, "log"));
var xmlin = File.ReadAllText(xmlfile);
var logList = EnergisticsConverter.XmlToObject <LogList>(xmlin);
Assert.IsNotNull(logList);
var log = logList.Log.FirstOrDefault();
Assert.IsNotNull(log);
log.Uid = null;
log.UidWell = _wellbore.UidWell;
log.UidWellbore = response.SuppMsgOut;
log.NameWell = _well.Name;
log.NameWellbore = _wellbore.Name;
var logDataAdded = log.LogData.FirstOrDefault();
Assert.IsNotNull(logDataAdded);
response = _devKit.Add <LogList, Log>(log);
Assert.AreEqual((short)ErrorCodes.Success, response.Result);
var uidLog = response.SuppMsgOut;
log.Uid = uidLog;
var uri = log.GetUri();
// Query Channel Data Chunk
var filter = MongoDbUtility.BuildFilter <ChannelDataChunk>("Uri", uri.ToString());
var database = _provider.GetDatabase();
var collection = database.GetCollection <ChannelDataChunk>("channelDataChunk");
var chunks = collection.Find(filter).ToList();
Assert.IsTrue(chunks.Count > 0);
// Query Mongo File
var fileChunks = chunks.Where(c => string.IsNullOrEmpty(c.Data)).ToList();
Assert.IsTrue(fileChunks.Count > 0);
var bucket = new GridFSBucket(database, new GridFSBucketOptions
{
BucketName = ChannelDataChunkAdapter.BucketName,
ChunkSizeBytes = WitsmlSettings.ChunkSizeBytes
});
foreach (var fc in fileChunks)
{
Assert.IsNull(fc.Data);
var mongoFileFilter = Builders <GridFSFileInfo> .Filter.Eq(fi => fi.Metadata[ChannelDataChunkAdapter.FileName], fc.Uid);
var mongoFile = bucket.Find(mongoFileFilter).FirstOrDefault();
Assert.IsNotNull(mongoFile);
}
// Query Log
var query = new Log
{
Uid = uidLog,
UidWell = log.UidWell,
UidWellbore = log.UidWellbore
};
var results = _devKit.Query <LogList, Log>(query, optionsIn: OptionsIn.ReturnElements.All);
Assert.AreEqual(1, results.Count);
var result = results.First();
Assert.IsNotNull(result);
var logDataReturned = result.LogData.FirstOrDefault();
Assert.IsNotNull(logDataReturned);
Assert.AreEqual(logDataAdded.Data.Count, logDataReturned.Data.Count);
}
public static void AddSquidexAssetInfrastructure(this IServiceCollection services, IConfiguration config)
{
config.ConfigureByOption("assetStore:type", new Alternatives
{
["Default"] = () =>
{
services.AddSingletonAs <NoopAssetStore>()
.AsOptional <IAssetStore>();
},
["Folder"] = () =>
{
var path = config.GetRequiredValue("assetStore:folder:path");
services.AddSingletonAs(c => new FolderAssetStore(path, c.GetRequiredService <ILogger <FolderAssetStore> >()))
.As <IAssetStore>();
},
["GoogleCloud"] = () =>
{
var bucketName = config.GetRequiredValue("assetStore:googleCloud:bucket");
services.AddSingletonAs(c => new GoogleCloudAssetStore(bucketName))
.As <IAssetStore>();
},
["AzureBlob"] = () =>
{
var connectionString = config.GetRequiredValue("assetStore:azureBlob:connectionString");
var containerName = config.GetRequiredValue("assetStore:azureBlob:containerName");
services.AddSingletonAs(c => new AzureBlobAssetStore(connectionString, containerName))
.As <IAssetStore>();
},
["AmazonS3"] = () =>
{
var amazonS3Options = config.GetSection("assetStore:amazonS3").Get <AmazonS3Options>();
services.AddSingletonAs(c => new AmazonS3AssetStore(amazonS3Options))
.As <IAssetStore>();
},
["MongoDb"] = () =>
{
var mongoConfiguration = config.GetRequiredValue("assetStore:mongoDb:configuration");
var mongoDatabaseName = config.GetRequiredValue("assetStore:mongoDb:database");
var mongoGridFsBucketName = config.GetRequiredValue("assetStore:mongoDb:bucket");
services.AddSingletonAs(c =>
{
var mongoClient = Singletons <IMongoClient> .GetOrAdd(mongoConfiguration, s => new MongoClient(s));
var mongoDatabase = mongoClient.GetDatabase(mongoDatabaseName);
var gridFsbucket = new GridFSBucket <string>(mongoDatabase, new GridFSBucketOptions
{
BucketName = mongoGridFsBucketName
});
return(new MongoGridFsAssetStore(gridFsbucket));
})
.As <IAssetStore>();
},
["Ftp"] = () =>
{
var serverHost = config.GetRequiredValue("assetStore:ftp:serverHost");
var serverPort = config.GetOptionalValue <int>("assetStore:ftp:serverPort", 21);
var username = config.GetRequiredValue("assetStore:ftp:username");
var password = config.GetRequiredValue("assetStore:ftp:password");
var path = config.GetOptionalValue("assetStore:ftp:path", "/");
services.AddSingletonAs(c =>
{
var factory = new Func <FtpClient>(() => new FtpClient(serverHost, serverPort, username, password));
return(new FTPAssetStore(factory, path, c.GetRequiredService <ILogger <FTPAssetStore> >()));
})
.As <IAssetStore>();
}
});
services.AddSingletonAs <ImageSharpAssetThumbnailGenerator>()
.As <IAssetThumbnailGenerator>();
services.AddSingletonAs(c => new DelegateInitializer(
c.GetRequiredService <IAssetStore>().GetType().Name,
c.GetRequiredService <IAssetStore>().InitializeAsync))
.As <IInitializable>();
}
public byte[] Get(string fileReference)
{
var fs = new GridFSBucket(_context.Database);
return(fs.DownloadAsBytes(new ObjectId(fileReference)));
}
public async Task <byte[]> GetAsync(string fileReference)
{
var fs = new GridFSBucket(_context.Database);
return(await fs.DownloadAsBytesAsync(new ObjectId(fileReference)));
}
public async Task <dynamic> Put()
{
var contentType = Request.Headers["Content-Type"];
FileType?fileType = FileTypeFromMime(contentType);
if (!fileType.HasValue)
{
throw new ValidationException("Tipo de arquivo desconhecido! Tipo: " + contentType);
}
var generatedFileId = ObjectId.GenerateNewId();
var gridFsBucket = new GridFSBucket <ObjectId>(MongoWrapper.Database);
var uploadTask = gridFsBucket.UploadFromStreamAsync(generatedFileId, generatedFileId.ToString(), Request.Body, new GridFSUploadOptions
{
Metadata = new FileMetadata
{
ContentType = contentType,
FileType = fileType.Value
}
.ToBsonDocument()
});
var id = this.GetCurrentUserId();
var confirmationCollection = MongoWrapper.Database.GetCollection <ReferenceToken>(typeof(ReferenceToken).Name);
var generatedToken = await GeneralUtils.GenerateRandomBase64(256);
var token = new DataReferenceToken <ConsumableData <ObjectId> >()
{
UserId = new ObjectId(id),
TokenType = TokenType.FileUpload,
_id = generatedToken,
AdditionalData = new ConsumableData <ObjectId>
{
Data = generatedFileId,
IsConsumed = false,
},
// Auto-Deletion allowed 1 day from now.
// TODO: Implement job that does auto-deletion
// TODO: How to check for FileReference? Can't delete file if it is already used,
// in those cases we should delete only the token!
// TODO: Maybe a Boolean in AdditionalData which says if this token is being used or not:
// If it is, the job will delete only the token, and not the file as well. Sounds good.
// Boolean is implemented, only the job is left.
DeactivationDate = DateTime.UtcNow + TimeSpan.FromHours(8)
};
var insertTokenTask = confirmationCollection.InsertOneAsync(token);
using (var session = await MongoWrapper.MongoClient.StartSessionAsync())
{
session.StartTransaction();
try
{
await insertTokenTask;
await uploadTask;
}
catch (Exception e)
{
Logger.LogError("Error while uploading user resource", e);
await session.AbortTransactionAsync();
throw;
}
await session.CommitTransactionAsync();
}
return(new ResponseBody
{
Code = ResponseCode.GenericSuccess,
Data = generatedToken,
Message = "Arquivo enviado com sucesso!",
Success = true
});
}
public static IVirtualFileSystem CreateMongoDb(this VirtualFileFactory factory, GridFSBucket bucket) => new GridFSVistualFileSystem(bucket);
// protected methods
protected override async Task ActAsync(GridFSBucket bucket)
{
await InvokeMethodAsync(bucket);
}
public MongoRepository(MongoClientContext clienthelper, IOptions <MongoOption> option)
{
_client = clienthelper.GetMongoClient();
_dataBase = _client.GetDatabase(option.Value.DefaultDataBase);
_bucket = new GridFSBucket(_dataBase);
}
// protected methods
protected Task <byte[]> InvokeMethodAsync(GridFSBucket bucket)
{
return(bucket.DownloadAsBytesByNameAsync(_filename, _options));
}
protected override void Assert(GridFSBucket bucket)
{
_result.Timestamp.Should().BeInRange(_referenceObjectId.Timestamp, _referenceObjectId.Timestamp + 1);
_result.Machine.Should().Be(_referenceObjectId.Machine);
_result.Pid.Should().Be(_referenceObjectId.Pid);
base.Assert(bucket);
}
protected override Task AssertAsync(GridFSBucket bucket)
{
_action.ShouldThrow <TException>();
return(Task.FromResult(true)); // don't call base.AssertAsync
}
/// <summary>
/// Resolves a token to a FileReference, if the token exists for this user.
/// The action should be called once the calling code can ensure the file is handled ok.
/// </summary>
/// <param name="mongoWrapper">For accessing the database</param>
/// <param name="tokenId">The token ID to search for</param>
/// <param name="userId">The user ID to correlate with the token</param>
/// <returns>A Tuple of the FileReference and an async action to expire the token</returns>
public static async Task <(FileReference, Func <Task>)> GetFileForReferenceToken(MongoWrapper mongoWrapper, string tokenId, ObjectId userId)
{
var tokenCollection = mongoWrapper.Database.GetCollection <DataReferenceToken <ConsumableData <ObjectId> > >(nameof(ReferenceToken));
var tokenFilterBuilder = new FilterDefinitionBuilder <DataReferenceToken <ConsumableData <ObjectId> > >();
var tokenFilter = tokenFilterBuilder.And
(
GeneralUtils.NotDeactivated(tokenFilterBuilder),
tokenFilterBuilder.Eq(t => t._id, tokenId),
tokenFilterBuilder.Eq(t => t.UserId, userId)
);
var tokenUpdateBuilder = new UpdateDefinitionBuilder <DataReferenceToken <ConsumableData <ObjectId> > >();
var tokenUpdate = tokenUpdateBuilder
.Set(t => t.DeactivationDate, DateTime.UtcNow)
.Set(t => t.AdditionalData.IsConsumed, true);
DataReferenceToken <ConsumableData <ObjectId> > token = (await tokenCollection.FindAsync(tokenFilter, new FindOptions <DataReferenceToken <ConsumableData <ObjectId> >, DataReferenceToken <ConsumableData <ObjectId> > >
{
Limit = 1,
AllowPartialResults = false,
})).SingleOrDefault();
if (token == null)
{
throw new ValidationException("Token de Arquivo não encontrado!");
}
var gridFsBucket = new GridFSBucket <ObjectId>(mongoWrapper.Database);
var fileFilterBuilder = new FilterDefinitionBuilder <GridFSFileInfo <ObjectId> >();
var fileFilter = fileFilterBuilder.Eq(finfo => finfo.Id, token.AdditionalData.Data);
var fileInfo = (await gridFsBucket.FindAsync(fileFilter, new GridFSFindOptions <ObjectId>
{
Limit = 1,
})).SingleOrDefault();
if (fileInfo == null)
{
throw new ApplicationException("Arquivo não encontrado, mas havia um token ativado referenciando-o!");
}
var fileReference = new FileReference
{
FileInfo = new Models.FileInfo
{
FileMetadata = BsonSerializer.Deserialize <FileMetadata>(fileInfo.Metadata),
Size = fileInfo.Length
},
_id = token.AdditionalData.Data
};
return
(
fileReference,
async() =>
{
var tokenUpdateTask = tokenCollection.UpdateOneAsync
(
tokenFilter,
tokenUpdate
);
var userCollection = mongoWrapper.Database.GetCollection <User>(nameof(User));
var userFilterBuilder = new FilterDefinitionBuilder <User>();
var userFilter = userFilterBuilder.And
(
GeneralUtils.NotDeactivated(userFilterBuilder),
userFilterBuilder.Eq(u => u._id, userId)
);
var userUpdateBuilder = new UpdateDefinitionBuilder <User>();
var userUpdate = userUpdateBuilder.Inc(u => u.FileBytesOccupied, fileReference.FileInfo.Size);
var userUpdateTask = userCollection.UpdateOneAsync
(
userFilter,
userUpdate
);
await tokenUpdateTask;
await userUpdateTask;
}
);
}
public MediaController(MediaService media, GridFSBucket gridFS)
{
_media = media;
_gridFS = gridFS;
}
// protected methods
protected byte[] InvokeMethod(GridFSBucket bucket)
{
return bucket.DownloadAsBytes(_id, _options);
}
public static async Task UploadPhoto(this ControllerBase controller, string foto, GridFSBucket <ObjectId> gridFsBucket, User user, DateTime creationDate)
{
if (string.IsNullOrWhiteSpace(foto))
{
user.Avatar = null;
return;
}
var photo = ImageUtils.FromBytes(Convert.FromBase64String(foto));
if (photo == null)
{
controller.Response.StatusCode = (int)HttpStatusCode.BadRequest;
throw new ResultException
(
new ObjectResult
(
new ResponseBody()
{
Code = ResponseCode.InvalidImage,
Success = false,
Message = "A imagem enviada é inválida!",
}
)
);
}
photo = ImageUtils.GuaranteeMaxSize(photo, 1000);
var photoStream = ImageUtils.ToStream(photo);
var fileId = ObjectId.GenerateNewId(creationDate);
var fileInfo = new FileInfo
{
FileMetadata = new FileMetadata
{
ContentType = "image/jpeg",
FileType = FileType.Image
},
Size = photoStream.Length,
};
user.Avatar = new FileReference
{
_id = fileId,
FileInfo = fileInfo
};
Task photoTask = gridFsBucket.UploadFromStreamAsync(
fileId,
fileId.ToString(),
photoStream,
new GridFSUploadOptions()
{
Metadata = fileInfo.FileMetadata.ToBsonDocument()
}
);
var streamCloseTask = photoTask.ContinueWith(tsk => photoStream.Close(), TaskContinuationOptions.ExecuteSynchronously);
await photoTask;
}
private Task RunUpdateCommandAsync(GridFSBucket bucket, BsonDocument command)
{
var collectionName = command["update"].AsString;
var collection = bucket.Database.GetCollection<BsonDocument>(collectionName);
var requests = new List<WriteModel<BsonDocument>>();
foreach (BsonDocument updateStatement in command["updates"].AsBsonArray)
{
var filter = updateStatement["q"].AsBsonDocument;
var update = updateStatement["u"].AsBsonDocument;
var upsert = updateStatement.GetValue("upsert", false).ToBoolean();
var multi = updateStatement.GetValue("multi", false).ToBoolean();
WriteModel<BsonDocument> request;
if (multi)
{
request = new UpdateManyModel<BsonDocument>(filter, update) { IsUpsert = upsert };
}
else
{
request = new UpdateOneModel<BsonDocument>(filter, update) { IsUpsert = upsert };
}
requests.Add(request);
}
return collection.BulkWriteAsync(requests);
}
public FileAccessor(MongoContext context) : base(context)
{
_bucket = new GridFSBucket(context.Database);
}
// protected methods protected abstract Task ActAsync(GridFSBucket bucket);
[AllowAnonymous] // No authorization required for Register Request, obviously
public async Task <dynamic> Post([FromBody] TBody requestBody)
{
this.EnsureModelValidation();
var userCollection = MongoWrapper.Database.GetCollection <Models.User>(nameof(Models.User));
await RegistrarUtils.TestUserExists(this, userCollection, requestBody.Email);
ResponseBody responseBody = new ResponseBody();
DateTime creationDate = DateTime.UtcNow;
var user = await BindUser(requestBody, creationDate);
// 300MB file storage limit default
user.FileBytesLimit = 300_000_000;
var gridFsBucket = new GridFSBucket <ObjectId>(MongoWrapper.Database);
Task photoTask = this.UploadPhoto(requestBody.Foto, gridFsBucket, user, creationDate);
var insertTask = userCollection.InsertOneAsync(user);
try
{
await insertTask;
}
catch (Exception e)
{
Logger.LogError("Error while registering user", e);
try
{
await photoTask;
ObjectId?photoId = user?.Avatar?._id;
if (photoId.HasValue)
{
var deleteLingeringPhotoTask = gridFsBucket.DeleteAsync(photoId.Value);
}
}
catch
{
// Ignore, no need to erase photo or log anything: User's insert has failed anyways.
}
throw;
}
// Não esperamos o e-mail ser enviado, apenas disparamos. Caso não tenha sido enviado vai ter um botão na home depois enchendo o saco pro usuário confirmar o e-mail dele.
var sendEmailTask = EmailUtils.SendConfirmationEmail(MongoWrapper, SmtpConfiguration, user);
var(tokenCreationDate, tokenExpiryDate, token) = await AuthenticationUtils.GenerateJwtTokenForUser(user._id.ToString(), TokenConfigurations, SigningConfigurations);
responseBody.Message = "Registro efetuado com sucesso. Não se esqueça de confirmar seu e-mail!";
responseBody.Code = ResponseCode.GenericSuccess;
responseBody.Success = true;
responseBody.Data = new
{
tokenData = new
{
created = tokenCreationDate,
expiration = tokenExpiryDate,
accessToken = token,
}
};
Response.StatusCode = (int)HttpStatusCode.OK;
return(responseBody);
}
// protected methods
protected override Task ActAsync(GridFSBucket bucket)
{
_action = () => InvokeMethodAsync(bucket);
return(Task.FromResult(true));
}
public MongoReadContentProvider(GridFSBucket bucket, ObjectId id)
{
this.stream = bucket.OpenDownloadStream(id);
}
public static void AddSquidexStoreServices(this IServiceCollection services, IConfiguration config)
{
config.ConfigureByOption("store:type", new Alternatives
{
["MongoDB"] = () =>
{
var mongoConfiguration = config.GetRequiredValue("store:mongoDb:configuration");
var mongoDatabaseName = config.GetRequiredValue("store:mongoDb:database");
var mongoContentDatabaseName = config.GetOptionalValue("store:mongoDb:contentDatabase", mongoDatabaseName);
services.AddSingleton(typeof(ISnapshotStore <,>), typeof(MongoSnapshotStore <,>));
services.AddSingletonAs(c => GetClient(mongoConfiguration))
.As <IMongoClient>();
services.AddSingletonAs(c => GetDatabase(c, mongoDatabaseName))
.As <IMongoDatabase>();
services.AddTransientAs(c => new DeleteContentCollections(GetDatabase(c, mongoContentDatabaseName)))
.As <IMigration>();
services.AddTransientAs(c => new RestructureContentCollection(GetDatabase(c, mongoContentDatabaseName)))
.As <IMigration>();
services.AddSingletonAs <MongoMigrationStatus>()
.As <IMigrationStatus>();
services.AddTransientAs <ConvertOldSnapshotStores>()
.As <IMigration>();
services.AddTransientAs <ConvertRuleEventsJson>()
.As <IMigration>();
services.AddTransientAs <RenameAssetSlugField>()
.As <IMigration>();
services.AddTransientAs <RenameAssetMetadata>()
.As <IMigration>();
services.AddHealthChecks()
.AddCheck <MongoDBHealthCheck>("MongoDB", tags: new[] { "node" });
services.AddSingletonAs <MongoRequestLogRepository>()
.As <IRequestLogRepository>();
services.AddSingletonAs <MongoUsageRepository>()
.As <IUsageRepository>();
services.AddSingletonAs <MongoRuleEventRepository>()
.As <IRuleEventRepository>();
services.AddSingletonAs <MongoHistoryEventRepository>()
.As <IHistoryEventRepository>();
services.AddSingletonAs <MongoRoleStore>()
.As <IRoleStore <IdentityRole> >();
services.AddSingletonAs <MongoUserStore>()
.As <IUserStore <IdentityUser> >().As <IUserFactory>();
services.AddSingletonAs <MongoKeyStore>()
.As <ISigningCredentialStore>().As <IValidationKeysStore>();
services.AddSingletonAs <MongoAssetRepository>()
.As <IAssetRepository>().As <ISnapshotStore <AssetState, Guid> >();
services.AddSingletonAs <MongoAssetFolderRepository>()
.As <IAssetFolderRepository>().As <ISnapshotStore <AssetFolderState, Guid> >();
services.AddSingletonAs(c => ActivatorUtilities.CreateInstance <MongoContentRepository>(c, GetDatabase(c, mongoContentDatabaseName)))
.As <IContentRepository>().As <ISnapshotStore <ContentState, Guid> >();
services.AddSingletonAs <MongoTextIndexerState>()
.AsSelf();
services.AddSingletonAs(c => new CachingTextIndexerState(c.GetRequiredService <MongoTextIndexerState>()))
.As <ITextIndexerState>();
var registration = services.FirstOrDefault(x => x.ServiceType == typeof(IPersistedGrantStore));
if (registration == null || registration.ImplementationType == typeof(InMemoryPersistedGrantStore))
{
services.AddSingletonAs <MongoPersistedGrantStore>()
.As <IPersistedGrantStore>();
}
services.AddSingletonAs(c =>
{
var database = c.GetRequiredService <IMongoDatabase>();
var mongoBucket = new GridFSBucket <string>(database, new GridFSBucketOptions
{
BucketName = "fullText"
});
return(new MongoIndexStorage(mongoBucket));
}).As <IIndexStorage>();
}
});
services.AddSingleton(typeof(IStore <>), typeof(Store <>));
}
public MongoWriteContentProvider(GridFSBucket bucket, string label)
{
this.stream = bucket.OpenUploadStream(label);
}
private static Task StartRadioFeeder(RadioCastServer radioCastServer, MongoWrapper mongoWrapper)
{
var userCollection = mongoWrapper.Database.GetCollection <Models.Musician>(nameof(Models.User));
var songSortBuilder = new SortDefinitionBuilder <ProjectedMusicianSong>();
var songSort = songSortBuilder
.Descending(nameof(ProjectedMusicianSong.Score));
var userFilterBuilder = new FilterDefinitionBuilder <Models.Musician>();
var userFilter = userFilterBuilder.And
(
GeneralUtils.NotDeactivated(userFilterBuilder),
userFilterBuilder.AnyEq("_t", nameof(Models.Musician))
);
var songFilterBuilder = new FilterDefinitionBuilder <ProjectedMusicianSong>();
var songFilter = songFilterBuilder.And
(
GeneralUtils.NotDeactivated(songFilterBuilder, s => s.Song),
songFilterBuilder.Eq(s => s.Song.RadioAuthorized, true),
songFilterBuilder.Eq(s => s.Song.Original, true)
);
var projectionBuilder = new ProjectionDefinitionBuilder <Models.Musician>();
var projection = projectionBuilder.Include(m => m.FullName).Include(m => m.Songs);
var fsBucket = new GridFSBucket <ObjectId>(mongoWrapper.Database);
var trackHistory = new List <(IAudioSource, ProjectedMusicianSong)>();
var onTrackChangedTE = new ManualResetEvent(true);
radioCastServer.OnTrackChanged += async(s, e) =>
{
List <(IAudioSource, ProjectedMusicianSong)> myTrackHistory;
lock (trackHistory)
{
myTrackHistory = trackHistory.ToList();
}
RadioInfoController.CurrentSong = myTrackHistory.Where(th => th.Item1.Equals(e.NewTrack)).Select(th => th.Item2).LastOrDefault();
LOGGER.Info("Now playing: {}", JsonConvert.SerializeObject(RadioInfoController.CurrentSong));
onTrackChangedTE.Set();
if (e.OldTrack == null)
{
return;
}
var oldTrack = myTrackHistory.Where(th => th.Item1.Equals(e.OldTrack)).Select(th => th.Item2).LastOrDefault();
var oldMusicianId = oldTrack._id;
var oldTrackId = oldTrack.Song._id;
var musSongFilterBuilder = new FilterDefinitionBuilder <Models.Musician>();
var musSongFilter = musSongFilterBuilder.And
(
musSongFilterBuilder.Eq(m => m._id, oldMusicianId),
musSongFilterBuilder.ElemMatch(m => m.Songs, sg => sg._id == oldTrackId)
);
var musSongUpdate = new UpdateDefinitionBuilder <Models.Musician>()
.Inc($"{nameof(Models.Musician.Songs)}.$.{nameof(Song.TimesPlayedRadio)}", 1);
await userCollection.UpdateOneAsync(musSongFilter, musSongUpdate);
// Remove oldest, only keep 5 in history
if (myTrackHistory.Count > 5)
{
lock (trackHistory)
{
trackHistory.RemoveAt(0);
}
}
};
return(Task.Run(async() =>
{
while (true)
{
List <(IAudioSource, ProjectedMusicianSong)> myTrackHistory;
lock (trackHistory)
{
myTrackHistory = trackHistory.ToList();
}
var lookupStageRandomArr = $@"
{{
$lookup:
{{
from: ""randomNumbers"",
pipeline:
[
{{ $sample: {{ size: 2 }} }}
],
as: ""RandomArr""
}}
}}
";
// OK Vezes totais que a música foi tocada * 0.5
// OK Vezes totais que a música foi tocada na rádio * -1
// OK Se música está presente na lista das últimas 5 tocadas, -100
// OK Se o autor da música está presente na lista das últimas 5 tocadas, -50
// OK Pontuação aleatória para cada música, entre - 10 e + 10
// OK Número de dias desde o cadastramento da música * -1
// OK Há uma chance de 5% de multiplicar a pontuação resultante por -1 (efeito nostalgia)
var scoreStage = $@"
{{
$addFields:
{{
""Score"":
{{
$add:
[
{{
$multiply: [ ""$Song.timesPlayed"", 0.5 ]
}},
{{
$multiply: [ ""$Song.timesPlayedRadio"", -1 ]
}},
{{
$cond:
{{
if:
{{
$in: [ ""$_id"", [ {myTrackHistory.Select(th => $"ObjectId(\"{th.Item2._id.ToString()}\")").DefaultIfEmpty("").Aggregate((s1, s2) => $"{s1.TrimEnd(',')},{s2.TrimEnd(',')},").TrimEnd(',')} ] ]
}},
then: -50,
else: 0
}}
}},
{{
$cond:
{{
if:
{{
$in: [ ""$Song._id"", [ {myTrackHistory.Select(th => $"ObjectId(\"{th.Item2.Song._id.ToString()}\")").DefaultIfEmpty("").Aggregate((s1, s2) => $"{s1.TrimEnd(',')},{s2.TrimEnd(',')},").TrimEnd(',')} ] ]
}},
then: -100,
else: 0
}}
}},
{{
$add:
[
{{
$multiply:
[
{{ $toDecimal: {{ $arrayElemAt: [""$RandomArr.decimal"", 0] }} }},
21
]
}},
-10
]
}},
{{
$multiply:
[
{{
$divide:
[
{{
$subtract:
[
{{ $toDate: ""{DateTime.UtcNow.ToString("o")}"" }},
{{ $toDate: ""$Song._id"" }}
]
}},
NumberLong(""86400000"")
]
}},
{{
$cond:
{{
if:
{{
$gt:
[
{{ $toDecimal: {{ $arrayElemAt: [""$RandomArr.decimal"", 1] }} }},
NumberDecimal(""0.1"")
]
}},
then: -1,
else: 1
}}
}}
]
}}
]
}}
}}
}}";
LOGGER.Info("Score stage generated MongoDB query: {}", scoreStage);
var pipeline = PipelineDefinitionBuilder
.For <Models.Musician>()
.Match(userFilter)
.Unwind(m => m.Songs, new AggregateUnwindOptions <Models.Musician>
{
PreserveNullAndEmptyArrays = false,
IncludeArrayIndex = null,
})
.Project(m => new ProjectedMusicianSong
{
_id = m._id,
Song = (Song)m.Songs,
Score = 1,
})
.Match(songFilter)
.AppendStage <Musician, ProjectedMusicianSong, ProjectedMusicianSong>(lookupStageRandomArr)
.AppendStage <Musician, ProjectedMusicianSong, ProjectedMusicianSong>(scoreStage)
.Sort(songSort)
.Limit(1);
if (LOGGER.IsDebugEnabled)
{
LOGGER.Debug("Pipeline generated MongoDB query for song: {}", pipeline.ToString());
}
var findTask = userCollection.AggregateAsync(pipeline, new AggregateOptions
{
AllowDiskUse = true,
BatchSize = 1,
UseCursor = true,
Comment = "Radio Aggregate Query",
TranslationOptions = new ExpressionTranslationOptions
{
StringTranslationMode = AggregateStringTranslationMode.CodePoints
}
});
var findResult = await findTask;
var firstSong = findResult.SingleOrDefault();
// If no songs, wait a minute before checking again
// Mostly not to strain the CPU on development environments
if (firstSong?.Song == null)
{
Thread.Sleep(TimeSpan.FromMinutes(1));
continue;
}
LOGGER.Info("Next selected song: {}", JsonConvert.SerializeObject(firstSong));
var audioRef = firstSong.Song.AudioReference;
var gridId = audioRef._id;
var fileStreamTask = fsBucket.OpenDownloadStreamAsync(gridId, new GridFSDownloadOptions
{
Seekable = true,
CheckMD5 = false,
});
var audioSource = new Mp3FileAudioSource(await fileStreamTask, firstSong.Song.Name);
// Wait for the radio to need more songs before we add the track we have on our hands
while (radioCastServer.TrackCount > 1)
{
onTrackChangedTE.WaitOne();
onTrackChangedTE.Reset();
}
lock (trackHistory)
{
trackHistory.Add((audioSource, firstSong));
}
radioCastServer.AddTrack(audioSource);
}
}));
}
public FilesRepo(GridFSBucket gridFs)
{
_gridFs = gridFs;
}
// protected methods
protected override void Act(GridFSBucket bucket, bool async)
{
_referenceObjectId = ObjectId.GenerateNewId();
_startTime = DateTime.UtcNow;
if (async)
{
_result = InvokeMethodAsync(bucket).GetAwaiter().GetResult();
}
else
{
_result = InvokeMethod(bucket);
}
}
public void Options_get_should_return_the_expected_result()
{
var database = Substitute.For<IMongoDatabase>();
var options = new GridFSBucketOptions();
var subject = new GridFSBucket(database, options);
var result = subject.Options;
result.BucketName.Should().Be(options.BucketName);
result.ChunkSizeBytes.Should().Be(options.ChunkSizeBytes);
result.ReadPreference.Should().Be(options.ReadPreference);
result.WriteConcern.Should().Be(options.WriteConcern);
}
// protected methods
protected ObjectId InvokeMethod(GridFSBucket bucket)
{
return bucket.UploadFromBytes(_filename, _source, _options);
}
protected virtual async Task ArrangeAsync(GridFSBucket bucket)
{
await InitializeFilesCollectionAsync(bucket);
await InitializeChunksCollectionAsync(bucket);
await RunArrangeCommandsAsync(bucket);
}
public async Task DeleteFileAsync(string id)
{
var gridFsBucket = new GridFSBucket(this.database);
await gridFsBucket.DeleteAsync(new ObjectId(id));
}
public async Task <IActionResult> Generate(ObjectId memoryId)
{
var client = new MongoDB.Driver.MongoClient("mongodb://*****:*****@localhost/?safe=true");
var db = client.GetDatabase("memoryvault");
var coll = db.GetCollection <MemoryItemModel>("memories");
var founds = coll.Find(p => p.Id == memoryId).FirstOrDefault();
if (founds == null)
{
return(NotFound("File not found."));
}
GridFSBucketOptions opts = new GridFSBucketOptions();
opts.BucketName = "memoryfiles";
var bucket = new GridFSBucket(db, opts);
bool isChanged = false;
List <ThumbnailItemModel> lstThumbs = new List <ThumbnailItemModel>();
foreach (ThumbnailSizes size in Enum.GetValues(typeof(ThumbnailSizes)))
{
using (var stream = bucket.OpenDownloadStream(founds.ItemId))
{
var dimension = GetDimension(size);
using (Image <Rgba32> bg = new Image <Rgba32>(dimension, dimension))
{
try
{
using (Image <Rgba32> image = Image.Load(stream)) //open the file and detect the file type and decode it
{
int iWidth;
int iHeight;
int margin = 3;
float ratio = (float)image.Width / (float)image.Height;
if (ratio > 1)
{
iWidth = dimension - 2 * margin;
iHeight = (int)(iWidth / ratio);
}
else
{
iHeight = dimension - 2 * margin;
iWidth = (int)(iHeight * ratio);
}
// image is now in a file format agnostic structure in memory as a series of Rgba32 pixels
image.Mutate(ctx => ctx.Resize(iWidth, iHeight)); // resize the image in place and return it for chaining
bg.Mutate(ctx => ctx.Fill(Rgba32.Black));
int pX = (int)((dimension - iWidth) / 2);
int pY = (int)((dimension - iHeight) / 2);
bg.Mutate(ctx => ctx.DrawImage(image, PixelBlenderMode.Normal, 1, new SixLabors.Primitives.Point(pX, pY)));
//var thPath = $"c:/temp/thumbs/th.{memoryId}.{DateTime.Now.ToString("MMDD_HHmmsss")}.{(ThumbnailSizes)size}.png";
//using (var fs = new System.IO.FileStream(thPath, FileMode.CreateNew))
//{
// bg.SaveAsPng(fs); // based on the file extension pick an encoder then encode and write the data to disk
// await fs.FlushAsync();
// fs.Close();
//}
using (var ms = new MemoryStream())
{
bg.SaveAsPng(ms);
lstThumbs.Add(new ThumbnailItemModel
{
CreateTime = DateTime.Now,
Size = size,
Data = ms.ToArray()
});
isChanged = true;
}
}
}
catch (Exception exc)
{
Debug.WriteLine(exc.Message);
}
}
}
}
if (isChanged)
{
await coll.UpdateOneAsync(
p => p.Id == memoryId
, Builders <MemoryItemModel> .Update.AddToSetEach(s => s.Thumbnails, lstThumbs)
);
}
return(Ok());
}
private GridFSSeekableDownloadStream CreateSubject(long? length = null)
{
var database = Substitute.For<IMongoDatabase>();
var bucket = new GridFSBucket(database);
var binding = Substitute.For<IReadBinding>();
var fileInfoDocument = new BsonDocument
{
{ "length", () => length.Value, length.HasValue }
};
var fileInfo = new GridFSFileInfo(fileInfoDocument);
return new GridFSSeekableDownloadStream(bucket, binding, fileInfo);
}
private Task RunDeleteCommandAsync(GridFSBucket bucket, BsonDocument command)
{
var collectionName = command["delete"].AsString;
var collection = bucket.Database.GetCollection<BsonDocument>(collectionName);
var requests = new List<WriteModel<BsonDocument>>();
foreach (BsonDocument deleteStatement in command["deletes"].AsBsonArray)
{
var filter = deleteStatement["q"].AsBsonDocument;
var limit = deleteStatement["limit"].ToInt32();
WriteModel<BsonDocument> request;
if (limit == 1)
{
request = new DeleteOneModel<BsonDocument>(filter);
}
else
{
request = new DeleteManyModel<BsonDocument>(filter);
}
requests.Add(request);
}
return collection.BulkWriteAsync(requests);
}
protected override void Assert(GridFSBucket bucket)
{
_result.Should().Equal(_expectedResult);
// don't call base.Assert
}
private Task RunInsertCommandAsync(GridFSBucket bucket, BsonDocument command)
{
var collectionName = command["insert"].AsString;
var collection = bucket.Database.GetCollection<BsonDocument>(collectionName);
var requests = new List<WriteModel<BsonDocument>>();
foreach (BsonDocument document in command["documents"].AsBsonArray)
{
var request = new InsertOneModel<BsonDocument>(document);
requests.Add(request);
}
return collection.BulkWriteAsync(requests);
}
protected Task<byte[]> InvokeMethodAsync(GridFSBucket bucket)
{
return bucket.DownloadAsBytesAsync(_id, _options);
}
public AppQueryHandler(IMongoCollection <AppData> apps, GridFSBucket files, DataTransferManager dataTransfer, IActorRef changeTracker)
{
Receive <QueryChangeSource>(changeTracker.Forward);
MakeQueryCall <QueryApps, AppList>("QueryApps", (query, _)
=> new AppList(apps.AsQueryable().Select(ad => ad.ToInfo()).ToImmutableList()));
MakeQueryCall <QueryApp, AppInfo>("QueryApp", (query, reporter) =>
{
var data = apps.AsQueryable().FirstOrDefault(e => e.Name == query.AppName);
if (data != null)
{
return(data.ToInfo());
}
reporter.Compled(OperationResult.Failure(BuildErrorCodes.QueryAppNotFound));
return(null);
});
MakeQueryCall <QueryBinarys, FileTransactionId>("QueryBinaries", (query, reporter) =>
{
if (query.Manager == null)
{
return(null);
}
var data = apps.AsQueryable().FirstOrDefault(e => e.Name == query.AppName);
if (data == null)
{
reporter.Compled(OperationResult.Failure(BuildErrorCodes.QueryAppNotFound));
return(null);
}
var targetVersion = query.AppVersion != -1 ? query.AppVersion : data.Last;
var file = data.Versions.FirstOrDefault(f => f.Version == targetVersion);
if (file == null)
{
reporter.Compled(OperationResult.Failure(BuildErrorCodes.QueryFileNotFound));
return(null);
}
var request = DataTransferRequest.FromStream(() => files.OpenDownloadStream(file.File), query.Manager, query.AppName);
dataTransfer.Request(request);
return(new FileTransactionId(request.OperationId));
});
MakeQueryCall <QueryBinaryInfo, BinaryList>("QueryBinaryInfo", (binarys, reporter) =>
{
var data = apps.AsQueryable().FirstOrDefault(ad => ad.Name == binarys.AppName);
if (data != null)
{
return(new BinaryList(data.Versions.Select(i => new AppBinary(data.Name, i.Version, i.CreationTime, i.Deleted, i.Commit, data.Repository)).ToImmutableList()));
}
reporter.Compled(OperationResult.Failure(BuildErrorCodes.QueryAppNotFound));
return(null);
});
}
private Task RunCommandAsync(GridFSBucket bucket, BsonDocument command)
{
var commandName = command.Names.First();
switch (commandName)
{
case "delete": return RunDeleteCommandAsync(bucket, command);
case "insert": return RunInsertCommandAsync(bucket, command);
case "update": return RunUpdateCommandAsync(bucket, command);
default: throw new ArgumentException("Unexpected command.");
}
}
private async Task RunCommandsAsync(GridFSBucket bucket, IEnumerable<BsonDocument> commands)
{
foreach (var command in commands)
{
await RunCommandAsync(bucket, command);
}
}
private async Task RunAssertCommandsAsync(GridFSBucket bucket, List<BsonDocument> actualFilesCollectionDocuments, List<BsonDocument> actualChunks)
{
var commands = GetAssertCommands();
PreprocessAssertCommands(commands, actualFilesCollectionDocuments, actualChunks);
await RunCommandsAsync(bucket, commands);
}
public async Task DeleteAsync(string id)
{
var gridFs = new GridFSBucket(_database);
await gridFs.DeleteAsync(new ObjectId(id));
}
