public Task IsValidName() { if (Enum.TryParse(GetQueryStringValueAndAssertIfSingleAndNotEmpty("type").Trim(), out ItemType elementType) == false) { throw new ArgumentException($"Type {elementType} is not supported"); } var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name").Trim(); var path = GetStringQueryString("dataPath", false); using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context)) { bool isValid = true; string errorMessage = null; switch (elementType) { case ItemType.Database: isValid = ResourceNameValidator.IsValidResourceName(name, path, out errorMessage); break; case ItemType.Index: isValid = IndexStore.IsValidIndexName(name, isStatic: true, out errorMessage); break; } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { context.Write(writer, new DynamicJsonValue { [nameof(NameValidation.IsValid)] = isValid, [nameof(NameValidation.ErrorMessage)] = errorMessage }); writer.Flush(); } } return(Task.CompletedTask); }
public Global(string dataDir, Config config, UiConfig uiConfig, WalletManager walletManager) { using (BenchmarkLogger.Measure()) { StoppingCts = new CancellationTokenSource(); DataDir = dataDir; Config = config; UiConfig = uiConfig; TorSettings = new TorSettings(DataDir, distributionFolderPath: EnvironmentHelpers.GetFullBaseDirectory(), Config.TerminateTorOnExit, Environment.ProcessId); HostedServices = new HostedServices(); WalletManager = walletManager; WalletManager.OnDequeue += WalletManager_OnDequeue; WalletManager.WalletRelevantTransactionProcessed += WalletManager_WalletRelevantTransactionProcessed; var networkWorkFolderPath = Path.Combine(DataDir, "BitcoinStore", Network.ToString()); var transactionStore = new AllTransactionStore(networkWorkFolderPath, Network); var indexStore = new IndexStore(Path.Combine(networkWorkFolderPath, "IndexStore"), Network, new SmartHeaderChain()); var mempoolService = new MempoolService(); var blocks = new FileSystemBlockRepository(Path.Combine(networkWorkFolderPath, "Blocks"), Network); BitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService, blocks); if (Config.UseTor) { BackendHttpClientFactory = new HttpClientFactory(TorSettings.SocksEndpoint, backendUriGetter: () => Config.GetCurrentBackendUri()); ExternalHttpClientFactory = new HttpClientFactory(TorSettings.SocksEndpoint, backendUriGetter: null); } else { BackendHttpClientFactory = new HttpClientFactory(torEndPoint: null, backendUriGetter: () => Config.GetFallbackBackendUri()); ExternalHttpClientFactory = new HttpClientFactory(torEndPoint: null, backendUriGetter: null); } Synchronizer = new WasabiSynchronizer(BitcoinStore, BackendHttpClientFactory); LegalChecker = new(DataDir); TransactionBroadcaster = new TransactionBroadcaster(Network, BitcoinStore, BackendHttpClientFactory, WalletManager); } }
public Global(string dataDir, string torLogsFile, Config config, UiConfig uiConfig, WalletManager walletManager) { using (BenchmarkLogger.Measure()) { CrashReporter = new CrashReporter(); StoppingCts = new CancellationTokenSource(); DataDir = dataDir; Config = config; UiConfig = uiConfig; TorLogsFile = torLogsFile; Logger.InitializeDefaults(Path.Combine(DataDir, "Logs.txt")); HostedServices = new HostedServices(); WalletManager = walletManager; LegalDocuments = LegalDocuments.TryLoadAgreed(DataDir); WalletManager.OnDequeue += WalletManager_OnDequeue; WalletManager.WalletRelevantTransactionProcessed += WalletManager_WalletRelevantTransactionProcessed; var networkWorkFolderPath = Path.Combine(DataDir, "BitcoinStore", Network.ToString()); var transactionStore = new AllTransactionStore(networkWorkFolderPath, Network); var indexStore = new IndexStore(Path.Combine(networkWorkFolderPath, "IndexStore"), Network, new SmartHeaderChain()); var mempoolService = new MempoolService(); BitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService); SingleInstanceChecker = new SingleInstanceChecker(Network); if (Config.UseTor) { Synchronizer = new WasabiSynchronizer(Network, BitcoinStore, () => Config.GetCurrentBackendUri(), Config.TorSocks5EndPoint); } else { Synchronizer = new WasabiSynchronizer(Network, BitcoinStore, Config.GetFallbackBackendUri(), null); } } }
public void InMemoryStoreCanPut1MillionObjectsWithoutCrashing() { var index = new IndexStore("1M"); var fs = new FileDataStore("1M", index); var journal = new FileJournal("1M", fs); var store = new InMemoryDataStore(fs, journal.DirectoryPath); try { Assert.DoesNotThrow(() => { for (int i = 0; i < 1000000; i++) { store.PutObject(new Sword("Exo-Sword " + i, 1), i); } }); } finally { Directory.Delete("1M", true); } }
public void GetReturnsPutObjects() { var index = new IndexStore("GetAndPut"); var fs = new FileDataStore("GetAndPut", index); var journal = new FileJournal("GetAndPut", fs); var store = new InMemoryDataStore(fs, journal.DirectoryPath); try { var sword = new Sword("Lego Sword", 13); var car = new Car("Kia", "Eeep", "Black"); store.PutObject(sword, 1); store.PutObject(car, 3); Assert.AreEqual(sword, store.GetObject <Sword>(1)); Assert.AreEqual(car, store.GetObject <Car>(3)); } finally { Directory.Delete("GetAndPut", true); } }
public override void Init(RequestHandlerContext context) { base.Init(context); Database = context.Database; ContextPool = Database.DocumentsStorage.ContextPool; IndexStore = context.Database.IndexStore; Logger = LoggingSource.Instance.GetLogger(Database.Name, GetType().FullName); var topologyEtag = GetLongFromHeaders(Constants.Headers.TopologyEtag); if (topologyEtag.HasValue && Database.HasTopologyChanged(topologyEtag.Value)) { context.HttpContext.Response.Headers[Constants.Headers.RefreshTopology] = "true"; } var clientConfigurationEtag = GetLongFromHeaders(Constants.Headers.ClientConfigurationEtag); if (clientConfigurationEtag.HasValue && Database.HasClientConfigurationChanged(clientConfigurationEtag.Value)) { context.HttpContext.Response.Headers[Constants.Headers.RefreshClientConfiguration] = "true"; } }
public Global(string dataDir, string torLogsFile, Config config, UiConfig uiConfig, WalletManager walletManager) { using (BenchmarkLogger.Measure()) { StoppingCts = new CancellationTokenSource(); DataDir = dataDir; Config = config; UiConfig = uiConfig; TorSettings = new TorSettings(DataDir, torLogsFile, distributionFolderPath: EnvironmentHelpers.GetFullBaseDirectory()); Logger.InitializeDefaults(Path.Combine(DataDir, "Logs.txt")); HostedServices = new HostedServices(); WalletManager = walletManager; LegalDocuments = LegalDocuments.TryLoadAgreed(DataDir); WalletManager.OnDequeue += WalletManager_OnDequeue; WalletManager.WalletRelevantTransactionProcessed += WalletManager_WalletRelevantTransactionProcessed; var networkWorkFolderPath = Path.Combine(DataDir, "BitcoinStore", Network.ToString()); var transactionStore = new AllTransactionStore(networkWorkFolderPath, Network); var indexStore = new IndexStore(Path.Combine(networkWorkFolderPath, "IndexStore"), Network, new SmartHeaderChain()); var mempoolService = new MempoolService(); var blocks = new FileSystemBlockRepository(Path.Combine(networkWorkFolderPath, "Blocks"), Network); BitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService, blocks); SingleInstanceChecker = new SingleInstanceChecker(Network); WasabiClientFactory wasabiClientFactory = Config.UseTor ? new WasabiClientFactory(Config.TorSocks5EndPoint, backendUriGetter: () => Config.GetCurrentBackendUri()) : new WasabiClientFactory(torEndPoint: null, backendUriGetter: () => Config.GetFallbackBackendUri()); Synchronizer = new WasabiSynchronizer(Network, BitcoinStore, wasabiClientFactory); } }
public DynamicJsonValue GenerateDatabaseInfo() { var envs = GetAllStoragesEnvironment().ToList(); if (envs.Any(x => x.Environment == null)) { return(null); } var size = new Size(envs.Sum(env => env.Environment.Stats().AllocatedDataFileSizeInBytes)); var databaseInfo = new DynamicJsonValue { [nameof(DatabaseInfo.HasRevisionsConfiguration)] = DocumentsStorage.RevisionsStorage.Configuration != null, [nameof(DatabaseInfo.HasExpirationConfiguration)] = ExpiredDocumentsCleaner != null, [nameof(DatabaseInfo.IsAdmin)] = true, //TODO: implement me! [nameof(DatabaseInfo.IsEncrypted)] = DocumentsStorage.Environment.Options.EncryptionEnabled, [nameof(DatabaseInfo.Name)] = Name, [nameof(DatabaseInfo.Disabled)] = false, //TODO: this value should be overwritten by the studio since it is cached [nameof(DatabaseInfo.TotalSize)] = new DynamicJsonValue { [nameof(Size.HumaneSize)] = size.HumaneSize, [nameof(Size.SizeInBytes)] = size.SizeInBytes }, [nameof(DatabaseInfo.IndexingErrors)] = IndexStore.GetIndexes().Sum(index => index.GetErrorCount()), [nameof(DatabaseInfo.Alerts)] = NotificationCenter.GetAlertCount(), [nameof(DatabaseInfo.UpTime)] = null, //it is shutting down [nameof(DatabaseInfo.BackupInfo)] = PeriodicBackupRunner?.GetBackupInfo(), [nameof(DatabaseInfo.MountPointsUsage)] = new DynamicJsonArray(GetMountPointsUsage().Select(x => x.ToJson())), [nameof(DatabaseInfo.DocumentsCount)] = DocumentsStorage.GetNumberOfDocuments(), [nameof(DatabaseInfo.IndexesCount)] = IndexStore.GetIndexes().Count(), [nameof(DatabaseInfo.RejectClients)] = false, //TODO: implement me! [nameof(DatabaseInfo.IndexingStatus)] = IndexStore.Status.ToString(), ["CachedDatabaseInfo"] = true }; return(databaseInfo); }
private void AddToIndexStore(List <string> words, string document, int lineNumber, string blackListItems) { try { lock (thisLock) { foreach (var word in words) { string inputWord = word; //remove null characters from words inputWord = inputWord.Trim().Replace("\0", string.Empty); //Check blacklist items and single letter words if (!blackListItems.Split(',').Contains(inputWord) && inputWord.Length > 1) { if (IndexStore.ContainsKey(inputWord)) { IndexStore[inputWord].Add(document + "|" + lineNumber.ToString()); } else { WordList.Add(inputWord); IndexStore.Add(inputWord, new List <string> { document + "|" + lineNumber.ToString() }); } } } } } catch (Exception ex) { Logger.Log(ex); } }
public void Dispose() { //before we dispose of the database we take its latest info to be displayed in the studio var databaseInfo = GenerateDatabaseInfo(); DatabaseInfoCache?.InsertDatabaseInfo(databaseInfo, Name); _databaseShutdown.Cancel(); // we'll wait for 1 minute to drain all the requests // from the database for (int i = 0; i < 60; i++) { if (Interlocked.Read(ref _usages) == 0) { break; } _waitForUsagesOnDisposal.Wait(100); } var exceptionAggregator = new ExceptionAggregator(_logger, $"Could not dispose {nameof(DocumentDatabase)}"); foreach (var connection in RunningTcpConnections) { exceptionAggregator.Execute(() => { connection.Dispose(); }); } exceptionAggregator.Execute(() => { TxMerger.Dispose(); }); exceptionAggregator.Execute(() => { DocumentReplicationLoader.Dispose(); }); if (_indexStoreTask != null) { exceptionAggregator.Execute(() => { _indexStoreTask.Wait(DatabaseShutdown); _indexStoreTask = null; }); } if (_transformerStoreTask != null) { exceptionAggregator.Execute(() => { _transformerStoreTask.Wait(DatabaseShutdown); _transformerStoreTask = null; }); } exceptionAggregator.Execute(() => { IndexStore?.Dispose(); IndexStore = null; }); exceptionAggregator.Execute(() => { BundleLoader?.Dispose(); BundleLoader = null; }); exceptionAggregator.Execute(() => { DocumentTombstoneCleaner?.Dispose(); DocumentTombstoneCleaner = null; }); exceptionAggregator.Execute(() => { DocumentReplicationLoader?.Dispose(); DocumentReplicationLoader = null; }); exceptionAggregator.Execute(() => { SqlReplicationLoader?.Dispose(); SqlReplicationLoader = null; }); exceptionAggregator.Execute(() => { Operations?.Dispose(exceptionAggregator); Operations = null; }); exceptionAggregator.Execute(() => { SubscriptionStorage?.Dispose(); }); exceptionAggregator.Execute(() => { ConfigurationStorage?.Dispose(); }); exceptionAggregator.Execute(() => { DocumentsStorage?.Dispose(); DocumentsStorage = null; }); exceptionAggregator.ThrowIfNeeded(); }
public async Task TestServicesAsync(string networkString) { await RuntimeParams.LoadAsync(); var network = Network.GetNetwork(networkString); var blocksToDownload = new List <uint256>(); if (network == Network.Main) { blocksToDownload.Add(new uint256("00000000000000000037c2de35bd85f3e57f14ddd741ce6cee5b28e51473d5d0")); blocksToDownload.Add(new uint256("000000000000000000115315a43cb0cdfc4ea54a0e92bed127f4e395e718d8f9")); blocksToDownload.Add(new uint256("00000000000000000011b5b042ad0522b69aae36f7de796f563c895714bbd629")); } else if (network == Network.TestNet) { blocksToDownload.Add(new uint256("0000000097a664c4084b49faa6fd4417055cb8e5aac480abc31ddc57a8208524")); blocksToDownload.Add(new uint256("000000009ed5b82259ecd2aa4cd1f119db8da7a70e7ea78d9c9f603e01f93bcc")); blocksToDownload.Add(new uint256("00000000e6da8c2da304e9f5ad99c079df2c3803b49efded3061ecaf206ddc66")); } else { throw new NotSupportedNetworkException(network); } var dataDir = Common.GetWorkDir(); var indexStore = new IndexStore(Path.Combine(dataDir, "indexStore"), network, new SmartHeaderChain()); await using var transactionStore = new AllTransactionStore(Path.Combine(dataDir, "transactionStore"), network); var mempoolService = new MempoolService(); var blocks = new FileSystemBlockRepository(Path.Combine(dataDir, "blocks"), network); await using BitcoinStore bitcoinStore = new(indexStore, transactionStore, mempoolService, blocks); await bitcoinStore.InitializeAsync(); var addressManagerFolderPath = Path.Combine(dataDir, "AddressManager"); var addressManagerFilePath = Path.Combine(addressManagerFolderPath, $"AddressManager{network}.dat"); var connectionParameters = new NodeConnectionParameters(); AddressManager addressManager; try { addressManager = await NBitcoinHelpers.LoadAddressManagerFromPeerFileAsync(addressManagerFilePath); Logger.LogInfo($"Loaded {nameof(AddressManager)} from `{addressManagerFilePath}`."); } catch (DirectoryNotFoundException) { addressManager = new AddressManager(); } catch (FileNotFoundException) { addressManager = new AddressManager(); } catch (OverflowException) { File.Delete(addressManagerFilePath); addressManager = new AddressManager(); } catch (FormatException) { File.Delete(addressManagerFilePath); addressManager = new AddressManager(); } connectionParameters.TemplateBehaviors.Add(new AddressManagerBehavior(addressManager)); connectionParameters.TemplateBehaviors.Add(bitcoinStore.CreateUntrustedP2pBehavior()); using var nodes = new NodesGroup(network, connectionParameters, requirements: Constants.NodeRequirements); KeyManager keyManager = KeyManager.CreateNew(out _, "password"); HttpClientFactory httpClientFactory = new(Common.TorSocks5Endpoint, backendUriGetter : () => new Uri("http://localhost:12345")); WasabiSynchronizer syncer = new(network, bitcoinStore, httpClientFactory); ServiceConfiguration serviceConfig = new(MixUntilAnonymitySet.PrivacyLevelStrong.ToString(), 2, 21, 50, new IPEndPoint(IPAddress.Loopback, network.DefaultPort), Money.Coins(Constants.DefaultDustThreshold)); CachedBlockProvider blockProvider = new( new P2pBlockProvider(nodes, null, httpClientFactory, serviceConfig, network), bitcoinStore.BlockRepository); using Wallet wallet = Wallet.CreateAndRegisterServices( network, bitcoinStore, keyManager, syncer, nodes, dataDir, new ServiceConfiguration(MixUntilAnonymitySet.PrivacyLevelStrong.ToString(), 2, 21, 50, new IPEndPoint(IPAddress.Loopback, network.DefaultPort), Money.Coins(Constants.DefaultDustThreshold)), syncer, blockProvider); Assert.True(Directory.Exists(blocks.BlocksFolderPath)); try { var mempoolTransactionAwaiter = new EventsAwaiter <SmartTransaction>( h => bitcoinStore.MempoolService.TransactionReceived += h, h => bitcoinStore.MempoolService.TransactionReceived -= h, 3); var nodeConnectionAwaiter = new EventsAwaiter <NodeEventArgs>( h => nodes.ConnectedNodes.Added += h, h => nodes.ConnectedNodes.Added -= h, 3); nodes.Connect(); var downloadTasks = new List <Task <Block> >(); using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(4)); foreach (var hash in blocksToDownload) { downloadTasks.Add(blockProvider.GetBlockAsync(hash, cts.Token)); } await nodeConnectionAwaiter.WaitAsync(TimeSpan.FromMinutes(3)); var i = 0; var hashArray = blocksToDownload.ToArray(); foreach (var block in await Task.WhenAll(downloadTasks)) { Assert.True(File.Exists(Path.Combine(blocks.BlocksFolderPath, hashArray[i].ToString()))); i++; } await mempoolTransactionAwaiter.WaitAsync(TimeSpan.FromMinutes(1)); } finally { // So next test will download the block. foreach (var hash in blocksToDownload) { await blockProvider.BlockRepository.RemoveAsync(hash, CancellationToken.None); } if (wallet is { })
public void For(IndexStore indices, Action <int, int, int> action) { Parallel.Invoke( () => { for (int i = indices.Lower; i < indices.ILength / 2; i++) { for (int j = indices.Lower; j < indices.JLength / 2; j++) { for (int k = indices.Lower; k < indices.KLength / 2; k++) { action(i, j, k); } } } }, () => { for (int i = indices.Lower; i < indices.ILength / 2; i++) { for (int j = indices.JLength / 2; j < indices.JLength; j++) { for (int k = indices.Lower; k < indices.KLength / 2; k++) { action(i, j, k); } } } }, () => { for (int i = indices.Lower; i < indices.ILength / 2; i++) { for (int j = indices.Lower; j < indices.JLength / 2; j++) { for (int k = indices.KLength / 2; k < indices.KLength; k++) { action(i, j, k); } } } }, () => { for (int i = indices.Lower; i < indices.ILength / 2; i++) { for (int j = indices.JLength / 2; j < indices.JLength; j++) { for (int k = indices.KLength / 2; k < indices.KLength; k++) { action(i, j, k); } } } }, () => { for (int i = indices.ILength / 2; i < indices.ILength; i++) { for (int j = indices.Lower; j < indices.JLength / 2; j++) { for (int k = indices.Lower; k < indices.KLength / 2; k++) { action(i, j, k); } } } }, () => { for (int i = indices.ILength / 2; i < indices.ILength; i++) { for (int j = indices.JLength / 2; j < indices.JLength; j++) { for (int k = indices.Lower; k < indices.KLength / 2; k++) { action(i, j, k); } } } }, () => { for (int i = indices.ILength / 2; i < indices.ILength; i++) { for (int j = indices.Lower; j < indices.JLength / 2; j++) { for (int k = indices.KLength / 2; k < indices.KLength; k++) { action(i, j, k); } } } }, () => { for (int i = indices.ILength / 2; i < indices.ILength; i++) { for (int j = indices.JLength / 2; j < indices.JLength; j++) { for (int k = indices.KLength / 2; k < indices.KLength; k++) { action(i, j, k); } } } }); }
public bool ContainsHash(byte[] hash) { return(IndexStore.GetRecord(hash) != null); }
/// <summary> /// Initializes a new instance of the <see cref="PmlBoundary" /> class. /// </summary> /// <param name="pmlLength">Length of the PML.</param> /// <param name="store">The store.</param> public PmlBoundary(int pmlLength, IndexStore store) { this.Length = pmlLength; this.boundaryConditionsCalc(store); }
public DynamicQueryToIndexMatcher(IndexStore indexStore) { _indexStore = indexStore; }
private void DeleteIndexMetadataForRemovedIndexesAndTransformers(Transaction tx, TransactionOperationContext context, IndexStore indexStore, TransformerStore transformerStore) { var toRemove = new List <IndexEntryMetadata>(); var table = tx.OpenTable(IndexesTableSchema, SchemaNameConstants.IndexMetadataTable); foreach (var tvr in table.SeekForwardFrom(IndexesTableSchema.FixedSizeIndexes[EtagIndexName], 0)) { var metadata = TableValueToMetadata(tvr, context, true); if (metadata == null) //noting to do if it is a tombstone { continue; } if (metadata.Type == IndexEntryType.Index && indexStore.GetIndex(metadata.Id) == null) { toRemove.Add(metadata); } if (metadata.Type == IndexEntryType.Transformer && transformerStore.GetTransformer(metadata.Id) == null) { toRemove.Add(metadata); } } foreach (var metadata in toRemove) { WriteEntry(tx, metadata.Name, metadata.Type, -1, context); } }
public void Initialize(InitializeOptions options = InitializeOptions.None) { try { _addToInitLog("Initializing NotificationCenter"); NotificationCenter.Initialize(this); _addToInitLog("Initializing DocumentStorage"); DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId); _addToInitLog("Starting Transaction Merger"); TxMerger.Start(); _addToInitLog("Initializing ConfigurationStorage"); ConfigurationStorage.Initialize(); if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord) { return; } _addToInitLog("Loading Database"); long index; DatabaseRecord record; using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) record = _serverStore.Cluster.ReadDatabase(context, Name, out index); if (record == null) { DatabaseDoesNotExistException.Throw(Name); } PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore); _addToInitLog("Initializing IndexStore (async)"); _indexStoreTask = IndexStore.InitializeAsync(record); _addToInitLog("Initializing Replication"); ReplicationLoader?.Initialize(record); _addToInitLog("Initializing ETL"); EtlLoader.Initialize(record); DocumentTombstoneCleaner.Start(); try { _indexStoreTask.Wait(DatabaseShutdown); } finally { _addToInitLog("Initializing IndexStore completed"); _indexStoreTask = null; } SubscriptionStorage.Initialize(); _addToInitLog("Initializing SubscriptionStorage completed"); TaskExecutor.Execute((state) => { try { NotifyFeaturesAboutStateChange(record, index); } catch { // We ignore the exception since it was caught in the function itself } }, null); } catch (Exception) { Dispose(); throw; } }
public override void Finish() { ActionSet.AddAction(IndexStore.ModifyVariable(Operation.Add, 1)); base.Finish(); }
public TextSearchIndex(IndexStore <Metadata> indexStore) { this.indexStore = indexStore; }
public override void Setup() { ActionSet.AddAction(IndexStore.SetVariable(0)); base.Setup(); }
private void DoWork(CommerceInstance instance) { IndexStore store = null; try { var rebuildDirectory = IndexStores.GetDirectory(Context.Instance, Context.Culture, Context.ModelType, true); var liveDirectory = IndexStores.GetDirectory(Context.Instance, Context.Culture, Context.ModelType, false); // Ensure temp folder are deleted (last rebuild might encounter errors when deleting the temp folder) Kooboo.IO.IOUtility.DeleteDirectory(rebuildDirectory, true); Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory + "-tmp", true); var total = _source.Count(instance, Context.Culture); var totalRebuilt = 0; Progress = 0; store = new IndexStore(Context.ModelType, FSDirectory.Open(rebuildDirectory), Analyzers.GetAnalyzer(Context.Culture)); foreach (var data in _source.Enumerate(instance, Context.Culture)) { if (_cancelling) { break; } store.Index(data); totalRebuilt++; Progress = (int)Math.Round(totalRebuilt * 100 / (double)total); } if (_cancelling) { store.Dispose(); UpdateTaskInfo(info => { info.ClearError(); info.LastRebuildStatus = RebuildStatus.Cancelled; }); _cancelling = false; _cancelledEvent.Set(); Status = RebuildStatus.Cancelled; } else { store.Commit(); store.Dispose(); UpdateTaskInfo(info => { info.ClearError(); info.LastRebuildStatus = RebuildStatus.Success; info.LastSucceededRebuildTimeUtc = DateTime.UtcNow; }); // Replace old index files with the new ones IndexStores.Close(Context.Instance, Context.Culture, Context.ModelType); var liveDirectoryExists = System.IO.Directory.Exists(liveDirectory); if (liveDirectoryExists) { System.IO.Directory.Move(liveDirectory, liveDirectory + "-tmp"); Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory, true); } System.IO.Directory.Move(rebuildDirectory, liveDirectory); if (liveDirectoryExists) { Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory + "-tmp", true); } Status = RebuildStatus.Success; } Progress = 0; } catch (Exception ex) { if (store != null) { store.Dispose(); } UpdateTaskInfo(info => { info.LastRebuildStatus = RebuildStatus.Failed; info.LastRebuildError = ex.Message; info.LastRebuildErrorDetail = ex.StackTrace; }); Status = RebuildStatus.Failed; } }
public void ForExceptK(IndexStore indices, Action <int, int> action) { throw new NotImplementedException(); }
public DynamicQueryRunner(DocumentDatabase database) : base(database) { _indexStore = database.IndexStore; }
public TextSearchIndex(IndexStore<Metadata> indexStore) { this.indexStore = indexStore; }
public void Finish() { ActionSet.AddAction(IndexStore.ModifyVariable(Operation.Add, 1)); ActionSet.AddAction(new A_End()); }
/// <summary> /// Creates the array with dimensions set in index store and initialize with defaults. /// </summary> /// <typeparam name="T">The type of underlying values.</typeparam> /// <param name="indices">The indices.</param> /// <returns>The new three dimensional array.</returns> public static T[,,] CreateArray <T>(this IndexStore indices) { var array = new T[indices.ILength, indices.JLength, indices.KLength]; return(array); }
public unsafe void Dispose() { if (_databaseShutdown.IsCancellationRequested) { return; // double dispose? } lock (this) { if (_databaseShutdown.IsCancellationRequested) { return; // double dispose? } //before we dispose of the database we take its latest info to be displayed in the studio try { var databaseInfo = GenerateDatabaseInfo(); if (databaseInfo != null) { DatabaseInfoCache?.InsertDatabaseInfo(databaseInfo, Name); } } catch (Exception e) { // if we encountered a catastrophic failure we might not be able to retrieve database info if (_logger.IsInfoEnabled) { _logger.Info("Failed to generate and store database info", e); } } _databaseShutdown.Cancel(); // we'll wait for 1 minute to drain all the requests // from the database var sp = Stopwatch.StartNew(); while (sp.ElapsedMilliseconds < 60 * 1000) { if (Interlocked.Read(ref _usages) == 0) { break; } if (_waitForUsagesOnDisposal.Wait(1000)) { _waitForUsagesOnDisposal.Reset(); } } var exceptionAggregator = new ExceptionAggregator(_logger, $"Could not dispose {nameof(DocumentDatabase)} {Name}"); foreach (var connection in RunningTcpConnections) { exceptionAggregator.Execute(() => { connection.Dispose(); }); } exceptionAggregator.Execute(() => { TxMerger?.Dispose(); }); if (_indexStoreTask != null) { exceptionAggregator.Execute(() => { _indexStoreTask.Wait(DatabaseShutdown); }); } exceptionAggregator.Execute(() => { IndexStore?.Dispose(); }); exceptionAggregator.Execute(() => { ExpiredDocumentsCleaner?.Dispose(); }); exceptionAggregator.Execute(() => { PeriodicBackupRunner?.Dispose(); }); exceptionAggregator.Execute(() => { DocumentTombstoneCleaner?.Dispose(); }); exceptionAggregator.Execute(() => { ReplicationLoader?.Dispose(); }); exceptionAggregator.Execute(() => { EtlLoader?.Dispose(); }); exceptionAggregator.Execute(() => { Operations?.Dispose(exceptionAggregator); }); exceptionAggregator.Execute(() => { NotificationCenter?.Dispose(); }); exceptionAggregator.Execute(() => { SubscriptionStorage?.Dispose(); }); exceptionAggregator.Execute(() => { ConfigurationStorage?.Dispose(); }); exceptionAggregator.Execute(() => { DocumentsStorage?.Dispose(); }); exceptionAggregator.Execute(() => { _databaseShutdown.Dispose(); }); exceptionAggregator.Execute(() => { if (MasterKey == null) { return; } fixed(byte *pKey = MasterKey) { Sodium.sodium_memzero(pKey, (UIntPtr)MasterKey.Length); } }); exceptionAggregator.ThrowIfNeeded(); } }
public async Task MempoolNotifiesAsync() { using var services = new HostedServices(); CoreNode coreNode = await TestNodeBuilder.CreateAsync(services); await services.StartAllAsync(); BitcoinStore?bitcoinStore = null; using var node = await coreNode.CreateNewP2pNodeAsync(); try { string dir = Common.GetWorkDir(); var network = coreNode.Network; var rpc = coreNode.RpcClient; var indexStore = new IndexStore(Path.Combine(dir, "indexStore"), network, new SmartHeaderChain()); var transactionStore = new AllTransactionStore(Path.Combine(dir, "transactionStore"), network); var mempoolService = new MempoolService(); var blocks = new FileSystemBlockRepository(Path.Combine(dir, "blocks"), network); // Construct BitcoinStore. bitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService, blocks); await bitcoinStore.InitializeAsync(); await rpc.GenerateAsync(blockCount : 101); node.Behaviors.Add(bitcoinStore.CreateUntrustedP2pBehavior()); node.VersionHandshake(); BitcoinWitPubKeyAddress address = new Key().PubKey.GetSegwitAddress(network); // Number of transactions to send. const int TransactionsCount = 10; var eventAwaiter = new EventsAwaiter <SmartTransaction>( subscribe: h => mempoolService.TransactionReceived += h, unsubscribe: h => mempoolService.TransactionReceived -= h, count: TransactionsCount); var txHashesList = new List <Task <uint256> >(); IRPCClient rpcBatch = rpc.PrepareBatch(); // Add to the batch 10 RPC commands: Send 1 coin to the same address. for (int i = 0; i < TransactionsCount; i++) { txHashesList.Add(rpcBatch.SendToAddressAsync(address, Money.Coins(1))); } // Publish the RPC batch. Task rpcBatchTask = rpcBatch.SendBatchAsync(); // Wait until the mempool service receives all the sent transactions. IEnumerable <SmartTransaction> mempoolSmartTxs = await eventAwaiter.WaitAsync(TimeSpan.FromSeconds(30)); await rpcBatchTask; // Collect all the transaction hashes of the sent transactions. uint256[] hashes = await Task.WhenAll(txHashesList); // Check that all the received transaction hashes are in the set of sent transaction hashes. foreach (SmartTransaction tx in mempoolSmartTxs) { Assert.Contains(tx.GetHash(), hashes); } } finally { if (bitcoinStore is { } store) { await store.DisposeAsync(); } await services.StopAllAsync(); node.Disconnect(); await coreNode.TryStopAsync(); } }
public DocumentDatabase(string name, RavenConfiguration configuration, ServerStore serverStore, Action <string> addToInitLog) { Name = name; _logger = LoggingSource.Instance.GetLogger <DocumentDatabase>(Name); _serverStore = serverStore; _addToInitLog = addToInitLog; StartTime = Time.GetUtcNow(); LastAccessTime = Time.GetUtcNow(); Configuration = configuration; Scripts = new ScriptRunnerCache(this, Configuration); try { using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) using (ctx.OpenReadTransaction()) { MasterKey = serverStore.GetSecretKey(ctx, Name); var databaseRecord = _serverStore.Cluster.ReadDatabase(ctx, Name); if (databaseRecord != null) { // can happen when we are in the process of restoring a database if (databaseRecord.Encrypted && MasterKey == null) { throw new InvalidOperationException($"Attempt to create encrypted db {Name} without supplying the secret key"); } if (databaseRecord.Encrypted == false && MasterKey != null) { throw new InvalidOperationException($"Attempt to create a non-encrypted db {Name}, but a secret key exists for this db."); } } } QueryMetadataCache = new QueryMetadataCache(); IoChanges = new IoChangesNotifications(); Changes = new DocumentsChanges(); DocumentTombstoneCleaner = new DocumentTombstoneCleaner(this); DocumentsStorage = new DocumentsStorage(this, addToInitLog); IndexStore = new IndexStore(this, serverStore); QueryRunner = new QueryRunner(this); EtlLoader = new EtlLoader(this, serverStore); ReplicationLoader = new ReplicationLoader(this, serverStore); SubscriptionStorage = new SubscriptionStorage(this, serverStore); Metrics = new MetricCounters(); TxMerger = new TransactionOperationsMerger(this, DatabaseShutdown); HugeDocuments = new HugeDocuments(configuration.PerformanceHints.HugeDocumentsCollectionSize, configuration.PerformanceHints.HugeDocumentSize.GetValue(SizeUnit.Bytes)); ConfigurationStorage = new ConfigurationStorage(this); NotificationCenter = new NotificationCenter.NotificationCenter(ConfigurationStorage.NotificationsStorage, Name, _databaseShutdown.Token); Operations = new Operations.Operations(Name, ConfigurationStorage.OperationsStorage, NotificationCenter, Changes); DatabaseInfoCache = serverStore.DatabaseInfoCache; RachisLogIndexNotifications = new RachisLogIndexNotifications(DatabaseShutdown); CatastrophicFailureNotification = new CatastrophicFailureNotification((environmentId, e) => { serverStore.DatabasesLandlord.CatastrophicFailureHandler.Execute(name, e, environmentId); }); } catch (Exception) { Dispose(); throw; } }
public void Initialize(StorageEnvironment environment, TransactionContextPool contextPool, IndexStore indexStore, TransformerStore transformerStore) { _environment = environment; _contextPool = contextPool; TransactionOperationContext context; using (contextPool.AllocateOperationContext(out context)) using (var tx = context.OpenWriteTransaction()) { IndexesTableSchema.Create(tx.InnerTransaction, SchemaNameConstants.IndexMetadataTable, 16); ConflictsTableSchema.Create(tx.InnerTransaction, SchemaNameConstants.ConflictMetadataTable, 16); tx.InnerTransaction.CreateTree(SchemaNameConstants.GlobalChangeVectorTree); tx.InnerTransaction.CreateTree(SchemaNameConstants.LastReplicatedEtagsTree); DeleteIndexMetadataForRemovedIndexesAndTransformers(tx.InnerTransaction, context, indexStore, transformerStore); tx.Commit(); } IsInitialized = true; }
public void Initialize(IndexStore indexStore, TransformerStore transformerStore) { _contextPool = new TransactionContextPool(Environment); AlertsStorage.Initialize(Environment, _contextPool); IndexesEtagsStorage.Initialize(Environment, _contextPool, indexStore, transformerStore); }