コード例 #1
0
        public void TestCallGetDatabasesHasContent()
        {
            var vfs = new WHIPVFS(Constants.VFS_PATH);

            var dbs = vfs.GetDatabases();

            Assert.That(dbs.Any, $"No DBs found?!");
        }
コード例 #2
0
        public void Setup()
        {
            var vfs = new WHIPVFS(Constants.VFS_PATH);

            _dbs = vfs.GetDatabases();
        }
コード例 #3
0
        public void TestCallGetDatabasesNoExceptions()
        {
            var vfs = new WHIPVFS(Constants.VFS_PATH);

            Assert.DoesNotThrow(() => vfs.GetDatabases());
        }
コード例 #4
0
        private static async Task Execute(IConfigSource configSource)
        {
            var startupConfig = configSource.Configs["Startup"];
            var vfs           = new WHIPVFS(startupConfig.GetString("inputWhipDbFolder", string.Empty));

            // Using the Pipeline concept: https://msdn.microsoft.com/en-us/library/ff963548.aspx

            var taskFactory = new TaskFactory(TaskCreationOptions.LongRunning, TaskContinuationOptions.None);

            var dataBases = vfs.GetDatabases();
            var assetIndexRecordBuffer         = new BlockingCollection <AssetIndexRecord>(4096);    // Magic numbers that are pure guesswork. See https://msdn.microsoft.com/en-us/library/ff963548.aspx for how to guess them better.
            var assetIndexRecordBufferFiltered = new BlockingCollection <AssetIndexRecord>(2048);
            var assetBuffer = new BlockingCollection <StratusAsset>(1024);

            var cfConfig        = configSource.Configs["CFSettings"];
            var username        = cfConfig.GetString("Username", string.Empty);
            var apiKey          = cfConfig.GetString("APIKey", string.Empty);
            var useInternalURL  = cfConfig.GetBoolean("UseInternalURL", true);
            var defaultRegion   = cfConfig.GetString("DefaultRegion", string.Empty);
            var containerPrefix = cfConfig.GetString("ContainerPrefix", string.Empty);

            using (var cloudFiles = new CloudFiles.AssetServer("default", username, apiKey, defaultRegion, useInternalURL, containerPrefix))
                using (var cts = new CancellationTokenSource()) {
                    var cancellationToken = cts.Token;

                    var readAssetIndexRecords = await taskFactory.StartNew(async() => {
                        try {
                            foreach (var db in dataBases)
                            {
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    break;
                                }

                                using (var indexReader = db.CreateIndexReader(AssetScope.Global)) {
                                    var records = await indexReader.GetAllAssetRecordsAsync(cancellationToken);

                                    foreach (var record in records)
                                    {
                                        if (cancellationToken.IsCancellationRequested)
                                        {
                                            break;
                                        }

                                        if (!record.Deleted)                                   // Deleted records are ignorable.
                                        {
                                            assetIndexRecordBuffer.Add(record, cancellationToken);
                                        }
                                    }
                                }

                                using (var indexReader = db.CreateIndexReader(AssetScope.Local)) {
                                    var records = await indexReader.GetAllAssetRecordsAsync(cancellationToken);

                                    foreach (var record in records)
                                    {
                                        if (cancellationToken.IsCancellationRequested)
                                        {
                                            break;
                                        }

                                        if (!record.Deleted)                                   // Deleted records are ignorable.
                                        {
                                            assetIndexRecordBuffer.Add(record, cancellationToken);
                                        }
                                    }
                                }
                            }
                        }
                        catch (Exception e) {
                            // If an exception occurs, notify all other pipeline stages.
                            cts.Cancel();
                            if (!(e is OperationCanceledException))
                            {
                                throw;
                            }
                        }
                        finally {
                            assetIndexRecordBuffer.CompleteAdding();
                        }
                    });

                    var filterAssetIndexRecords = taskFactory.StartNew(() => {
                        try {
                            foreach (var assetIndexRecord in assetIndexRecordBuffer.GetConsumingEnumerable())
                            {
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    break;
                                }

                                // Check CF for the asset ID.  If it exists, move on.
                                if (!cloudFiles.VerifyAssetIdSync(assetIndexRecord.Id))
                                {
                                    assetIndexRecordBufferFiltered.Add(assetIndexRecord, cancellationToken);
                                }
                            }
                        }
                        catch (Exception e) {
                            // If an exception occurs, notify all other pipeline stages.
                            cts.Cancel();
                            if (!(e is OperationCanceledException))
                            {
                                throw;
                            }
                        }
                        finally {
                            assetIndexRecordBuffer.CompleteAdding();
                        }
                    });

                    var readAssets = await taskFactory.StartNew(async() => {
                        try {
                            foreach (var assetIndexRecord in assetIndexRecordBufferFiltered.GetConsumingEnumerable())
                            {
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    break;
                                }

                                var prefix = assetIndexRecord.getPrefix();

                                var database = dataBases.First(db => db.Prefix == prefix);

                                using (var dataReader = database.CreateDataFileReader(assetIndexRecord.Scope)) {
                                    var asset = await dataReader.GetAssetAsync(assetIndexRecord, cancellationToken);

                                    var stratusAsset = new StratusAsset()
                                    {
                                        CreateTime   = asset.GetCreateTime().DateTime,
                                        Data         = asset.GetAssetData(),
                                        Description  = asset.GetDescription(),
                                        Id           = assetIndexRecord.Id,
                                        Local        = asset.IsLocal(),
                                        Name         = asset.GetName(),
                                        StorageFlags = 0,                                 // From Halcyon CloudFilesAssetClient.cs::StoreAsset(AssetBase) near line 535: for now we're not going to use compression etc, so set to zero
                                        Temporary    = asset.IsTemporary(),
                                        Type         = (sbyte)asset.GetAssetType(),
                                    };

                                    assetBuffer.Add(stratusAsset, cancellationToken);
                                }
                            }
                        }
                        catch (Exception e) {
                            // If an exception occurs, notify all other pipeline stages.
                            cts.Cancel();
                            if (!(e is OperationCanceledException))
                            {
                                throw;
                            }
                        }
                        finally {
                            assetBuffer.CompleteAdding();
                        }
                    });

                    var uploadAssets = taskFactory.StartNew(() => {
                        try {
                            foreach (var asset in assetBuffer.GetConsumingEnumerable())
                            {
                                if (cancellationToken.IsCancellationRequested)
                                {
                                    break;
                                }

                                cloudFiles.StoreAssetSync(asset);
                            }
                        }
                        catch (Exception e) {
                            // If an exception occurs, notify all other pipeline stages.
                            cts.Cancel();
                            if (!(e is OperationCanceledException))
                            {
                                throw;
                            }
                        }
                    });

                    Task.WaitAll(readAssetIndexRecords, filterAssetIndexRecords, readAssets, uploadAssets);
                }
        }