public void TestCallGetDatabasesHasContent()
{
var vfs = new WHIPVFS(Constants.VFS_PATH);
var dbs = vfs.GetDatabases();
Assert.That(dbs.Any, $"No DBs found?!");
}
public void Setup()
{
var vfs = new WHIPVFS(Constants.VFS_PATH);
_dbs = vfs.GetDatabases();
}
public void TestCallGetDatabasesNoExceptions()
{
var vfs = new WHIPVFS(Constants.VFS_PATH);
Assert.DoesNotThrow(() => vfs.GetDatabases());
}
private static async Task Execute(IConfigSource configSource)
{
var startupConfig = configSource.Configs["Startup"];
var vfs = new WHIPVFS(startupConfig.GetString("inputWhipDbFolder", string.Empty));
// Using the Pipeline concept: https://msdn.microsoft.com/en-us/library/ff963548.aspx
var taskFactory = new TaskFactory(TaskCreationOptions.LongRunning, TaskContinuationOptions.None);
var dataBases = vfs.GetDatabases();
var assetIndexRecordBuffer = new BlockingCollection <AssetIndexRecord>(4096); // Magic numbers that are pure guesswork. See https://msdn.microsoft.com/en-us/library/ff963548.aspx for how to guess them better.
var assetIndexRecordBufferFiltered = new BlockingCollection <AssetIndexRecord>(2048);
var assetBuffer = new BlockingCollection <StratusAsset>(1024);
var cfConfig = configSource.Configs["CFSettings"];
var username = cfConfig.GetString("Username", string.Empty);
var apiKey = cfConfig.GetString("APIKey", string.Empty);
var useInternalURL = cfConfig.GetBoolean("UseInternalURL", true);
var defaultRegion = cfConfig.GetString("DefaultRegion", string.Empty);
var containerPrefix = cfConfig.GetString("ContainerPrefix", string.Empty);
using (var cloudFiles = new CloudFiles.AssetServer("default", username, apiKey, defaultRegion, useInternalURL, containerPrefix))
using (var cts = new CancellationTokenSource()) {
var cancellationToken = cts.Token;
var readAssetIndexRecords = await taskFactory.StartNew(async() => {
try {
foreach (var db in dataBases)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
using (var indexReader = db.CreateIndexReader(AssetScope.Global)) {
var records = await indexReader.GetAllAssetRecordsAsync(cancellationToken);
foreach (var record in records)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (!record.Deleted) // Deleted records are ignorable.
{
assetIndexRecordBuffer.Add(record, cancellationToken);
}
}
}
using (var indexReader = db.CreateIndexReader(AssetScope.Local)) {
var records = await indexReader.GetAllAssetRecordsAsync(cancellationToken);
foreach (var record in records)
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
if (!record.Deleted) // Deleted records are ignorable.
{
assetIndexRecordBuffer.Add(record, cancellationToken);
}
}
}
}
}
catch (Exception e) {
// If an exception occurs, notify all other pipeline stages.
cts.Cancel();
if (!(e is OperationCanceledException))
{
throw;
}
}
finally {
assetIndexRecordBuffer.CompleteAdding();
}
});
var filterAssetIndexRecords = taskFactory.StartNew(() => {
try {
foreach (var assetIndexRecord in assetIndexRecordBuffer.GetConsumingEnumerable())
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
// Check CF for the asset ID. If it exists, move on.
if (!cloudFiles.VerifyAssetIdSync(assetIndexRecord.Id))
{
assetIndexRecordBufferFiltered.Add(assetIndexRecord, cancellationToken);
}
}
}
catch (Exception e) {
// If an exception occurs, notify all other pipeline stages.
cts.Cancel();
if (!(e is OperationCanceledException))
{
throw;
}
}
finally {
assetIndexRecordBuffer.CompleteAdding();
}
});
var readAssets = await taskFactory.StartNew(async() => {
try {
foreach (var assetIndexRecord in assetIndexRecordBufferFiltered.GetConsumingEnumerable())
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
var prefix = assetIndexRecord.getPrefix();
var database = dataBases.First(db => db.Prefix == prefix);
using (var dataReader = database.CreateDataFileReader(assetIndexRecord.Scope)) {
var asset = await dataReader.GetAssetAsync(assetIndexRecord, cancellationToken);
var stratusAsset = new StratusAsset()
{
CreateTime = asset.GetCreateTime().DateTime,
Data = asset.GetAssetData(),
Description = asset.GetDescription(),
Id = assetIndexRecord.Id,
Local = asset.IsLocal(),
Name = asset.GetName(),
StorageFlags = 0, // From Halcyon CloudFilesAssetClient.cs::StoreAsset(AssetBase) near line 535: for now we're not going to use compression etc, so set to zero
Temporary = asset.IsTemporary(),
Type = (sbyte)asset.GetAssetType(),
};
assetBuffer.Add(stratusAsset, cancellationToken);
}
}
}
catch (Exception e) {
// If an exception occurs, notify all other pipeline stages.
cts.Cancel();
if (!(e is OperationCanceledException))
{
throw;
}
}
finally {
assetBuffer.CompleteAdding();
}
});
var uploadAssets = taskFactory.StartNew(() => {
try {
foreach (var asset in assetBuffer.GetConsumingEnumerable())
{
if (cancellationToken.IsCancellationRequested)
{
break;
}
cloudFiles.StoreAssetSync(asset);
}
}
catch (Exception e) {
// If an exception occurs, notify all other pipeline stages.
cts.Cancel();
if (!(e is OperationCanceledException))
{
throw;
}
}
});
Task.WaitAll(readAssetIndexRecords, filterAssetIndexRecords, readAssets, uploadAssets);
}
}
