public Task IsValidName()
{
if (Enum.TryParse(GetQueryStringValueAndAssertIfSingleAndNotEmpty("type").Trim(), out ItemType elementType) == false)
{
throw new ArgumentException($"Type {elementType} is not supported");
}
var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name").Trim();
var path = GetStringQueryString("dataPath", false);
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
bool isValid = true;
string errorMessage = null;
switch (elementType)
{
case ItemType.Database:
isValid = ResourceNameValidator.IsValidResourceName(name, path, out errorMessage);
break;
case ItemType.Index:
isValid = IndexStore.IsValidIndexName(name, isStatic: true, out errorMessage);
break;
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(NameValidation.IsValid)] = isValid,
[nameof(NameValidation.ErrorMessage)] = errorMessage
});
writer.Flush();
}
}
return(Task.CompletedTask);
}
public Global(string dataDir, Config config, UiConfig uiConfig, WalletManager walletManager)
{
using (BenchmarkLogger.Measure())
{
StoppingCts = new CancellationTokenSource();
DataDir = dataDir;
Config = config;
UiConfig = uiConfig;
TorSettings = new TorSettings(DataDir, distributionFolderPath: EnvironmentHelpers.GetFullBaseDirectory(), Config.TerminateTorOnExit, Environment.ProcessId);
HostedServices = new HostedServices();
WalletManager = walletManager;
WalletManager.OnDequeue += WalletManager_OnDequeue;
WalletManager.WalletRelevantTransactionProcessed += WalletManager_WalletRelevantTransactionProcessed;
var networkWorkFolderPath = Path.Combine(DataDir, "BitcoinStore", Network.ToString());
var transactionStore = new AllTransactionStore(networkWorkFolderPath, Network);
var indexStore = new IndexStore(Path.Combine(networkWorkFolderPath, "IndexStore"), Network, new SmartHeaderChain());
var mempoolService = new MempoolService();
var blocks = new FileSystemBlockRepository(Path.Combine(networkWorkFolderPath, "Blocks"), Network);
BitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService, blocks);
if (Config.UseTor)
{
BackendHttpClientFactory = new HttpClientFactory(TorSettings.SocksEndpoint, backendUriGetter: () => Config.GetCurrentBackendUri());
ExternalHttpClientFactory = new HttpClientFactory(TorSettings.SocksEndpoint, backendUriGetter: null);
}
else
{
BackendHttpClientFactory = new HttpClientFactory(torEndPoint: null, backendUriGetter: () => Config.GetFallbackBackendUri());
ExternalHttpClientFactory = new HttpClientFactory(torEndPoint: null, backendUriGetter: null);
}
Synchronizer = new WasabiSynchronizer(BitcoinStore, BackendHttpClientFactory);
LegalChecker = new(DataDir);
TransactionBroadcaster = new TransactionBroadcaster(Network, BitcoinStore, BackendHttpClientFactory, WalletManager);
}
}
public Global(string dataDir, string torLogsFile, Config config, UiConfig uiConfig, WalletManager walletManager)
{
using (BenchmarkLogger.Measure())
{
CrashReporter = new CrashReporter();
StoppingCts = new CancellationTokenSource();
DataDir = dataDir;
Config = config;
UiConfig = uiConfig;
TorLogsFile = torLogsFile;
Logger.InitializeDefaults(Path.Combine(DataDir, "Logs.txt"));
HostedServices = new HostedServices();
WalletManager = walletManager;
LegalDocuments = LegalDocuments.TryLoadAgreed(DataDir);
WalletManager.OnDequeue += WalletManager_OnDequeue;
WalletManager.WalletRelevantTransactionProcessed += WalletManager_WalletRelevantTransactionProcessed;
var networkWorkFolderPath = Path.Combine(DataDir, "BitcoinStore", Network.ToString());
var transactionStore = new AllTransactionStore(networkWorkFolderPath, Network);
var indexStore = new IndexStore(Path.Combine(networkWorkFolderPath, "IndexStore"), Network, new SmartHeaderChain());
var mempoolService = new MempoolService();
BitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService);
SingleInstanceChecker = new SingleInstanceChecker(Network);
if (Config.UseTor)
{
Synchronizer = new WasabiSynchronizer(Network, BitcoinStore, () => Config.GetCurrentBackendUri(), Config.TorSocks5EndPoint);
}
else
{
Synchronizer = new WasabiSynchronizer(Network, BitcoinStore, Config.GetFallbackBackendUri(), null);
}
}
}
public void InMemoryStoreCanPut1MillionObjectsWithoutCrashing()
{
var index = new IndexStore("1M");
var fs = new FileDataStore("1M", index);
var journal = new FileJournal("1M", fs);
var store = new InMemoryDataStore(fs, journal.DirectoryPath);
try
{
Assert.DoesNotThrow(() =>
{
for (int i = 0; i < 1000000; i++)
{
store.PutObject(new Sword("Exo-Sword " + i, 1), i);
}
});
}
finally
{
Directory.Delete("1M", true);
}
}
public void GetReturnsPutObjects()
{
var index = new IndexStore("GetAndPut");
var fs = new FileDataStore("GetAndPut", index);
var journal = new FileJournal("GetAndPut", fs);
var store = new InMemoryDataStore(fs, journal.DirectoryPath);
try
{
var sword = new Sword("Lego Sword", 13);
var car = new Car("Kia", "Eeep", "Black");
store.PutObject(sword, 1);
store.PutObject(car, 3);
Assert.AreEqual(sword, store.GetObject <Sword>(1));
Assert.AreEqual(car, store.GetObject <Car>(3));
}
finally
{
Directory.Delete("GetAndPut", true);
}
}
public override void Init(RequestHandlerContext context)
{
base.Init(context);
Database = context.Database;
ContextPool = Database.DocumentsStorage.ContextPool;
IndexStore = context.Database.IndexStore;
Logger = LoggingSource.Instance.GetLogger(Database.Name, GetType().FullName);
var topologyEtag = GetLongFromHeaders(Constants.Headers.TopologyEtag);
if (topologyEtag.HasValue && Database.HasTopologyChanged(topologyEtag.Value))
{
context.HttpContext.Response.Headers[Constants.Headers.RefreshTopology] = "true";
}
var clientConfigurationEtag = GetLongFromHeaders(Constants.Headers.ClientConfigurationEtag);
if (clientConfigurationEtag.HasValue && Database.HasClientConfigurationChanged(clientConfigurationEtag.Value))
{
context.HttpContext.Response.Headers[Constants.Headers.RefreshClientConfiguration] = "true";
}
}
public Global(string dataDir, string torLogsFile, Config config, UiConfig uiConfig, WalletManager walletManager)
{
using (BenchmarkLogger.Measure())
{
StoppingCts = new CancellationTokenSource();
DataDir = dataDir;
Config = config;
UiConfig = uiConfig;
TorSettings = new TorSettings(DataDir, torLogsFile, distributionFolderPath: EnvironmentHelpers.GetFullBaseDirectory());
Logger.InitializeDefaults(Path.Combine(DataDir, "Logs.txt"));
HostedServices = new HostedServices();
WalletManager = walletManager;
LegalDocuments = LegalDocuments.TryLoadAgreed(DataDir);
WalletManager.OnDequeue += WalletManager_OnDequeue;
WalletManager.WalletRelevantTransactionProcessed += WalletManager_WalletRelevantTransactionProcessed;
var networkWorkFolderPath = Path.Combine(DataDir, "BitcoinStore", Network.ToString());
var transactionStore = new AllTransactionStore(networkWorkFolderPath, Network);
var indexStore = new IndexStore(Path.Combine(networkWorkFolderPath, "IndexStore"), Network, new SmartHeaderChain());
var mempoolService = new MempoolService();
var blocks = new FileSystemBlockRepository(Path.Combine(networkWorkFolderPath, "Blocks"), Network);
BitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService, blocks);
SingleInstanceChecker = new SingleInstanceChecker(Network);
WasabiClientFactory wasabiClientFactory = Config.UseTor
? new WasabiClientFactory(Config.TorSocks5EndPoint, backendUriGetter: () => Config.GetCurrentBackendUri())
: new WasabiClientFactory(torEndPoint: null, backendUriGetter: () => Config.GetFallbackBackendUri());
Synchronizer = new WasabiSynchronizer(Network, BitcoinStore, wasabiClientFactory);
}
}
public DynamicJsonValue GenerateDatabaseInfo()
{
var envs = GetAllStoragesEnvironment().ToList();
if (envs.Any(x => x.Environment == null))
{
return(null);
}
var size = new Size(envs.Sum(env => env.Environment.Stats().AllocatedDataFileSizeInBytes));
var databaseInfo = new DynamicJsonValue
{
[nameof(DatabaseInfo.HasRevisionsConfiguration)] = DocumentsStorage.RevisionsStorage.Configuration != null,
[nameof(DatabaseInfo.HasExpirationConfiguration)] = ExpiredDocumentsCleaner != null,
[nameof(DatabaseInfo.IsAdmin)] = true, //TODO: implement me!
[nameof(DatabaseInfo.IsEncrypted)] = DocumentsStorage.Environment.Options.EncryptionEnabled,
[nameof(DatabaseInfo.Name)] = Name,
[nameof(DatabaseInfo.Disabled)] = false, //TODO: this value should be overwritten by the studio since it is cached
[nameof(DatabaseInfo.TotalSize)] = new DynamicJsonValue
{
[nameof(Size.HumaneSize)] = size.HumaneSize,
[nameof(Size.SizeInBytes)] = size.SizeInBytes
},
[nameof(DatabaseInfo.IndexingErrors)] = IndexStore.GetIndexes().Sum(index => index.GetErrorCount()),
[nameof(DatabaseInfo.Alerts)] = NotificationCenter.GetAlertCount(),
[nameof(DatabaseInfo.UpTime)] = null, //it is shutting down
[nameof(DatabaseInfo.BackupInfo)] = PeriodicBackupRunner?.GetBackupInfo(),
[nameof(DatabaseInfo.MountPointsUsage)] = new DynamicJsonArray(GetMountPointsUsage().Select(x => x.ToJson())),
[nameof(DatabaseInfo.DocumentsCount)] = DocumentsStorage.GetNumberOfDocuments(),
[nameof(DatabaseInfo.IndexesCount)] = IndexStore.GetIndexes().Count(),
[nameof(DatabaseInfo.RejectClients)] = false, //TODO: implement me!
[nameof(DatabaseInfo.IndexingStatus)] = IndexStore.Status.ToString(),
["CachedDatabaseInfo"] = true
};
return(databaseInfo);
}
private void AddToIndexStore(List <string> words, string document, int lineNumber, string blackListItems)
{
try
{
lock (thisLock)
{
foreach (var word in words)
{
string inputWord = word;
//remove null characters from words
inputWord = inputWord.Trim().Replace("\0", string.Empty);
//Check blacklist items and single letter words
if (!blackListItems.Split(',').Contains(inputWord) && inputWord.Length > 1)
{
if (IndexStore.ContainsKey(inputWord))
{
IndexStore[inputWord].Add(document + "|" + lineNumber.ToString());
}
else
{
WordList.Add(inputWord);
IndexStore.Add(inputWord, new List <string> {
document + "|" + lineNumber.ToString()
});
}
}
}
}
}
catch (Exception ex)
{
Logger.Log(ex);
}
}
public void Dispose()
{
//before we dispose of the database we take its latest info to be displayed in the studio
var databaseInfo = GenerateDatabaseInfo();
DatabaseInfoCache?.InsertDatabaseInfo(databaseInfo, Name);
_databaseShutdown.Cancel();
// we'll wait for 1 minute to drain all the requests
// from the database
for (int i = 0; i < 60; i++)
{
if (Interlocked.Read(ref _usages) == 0)
{
break;
}
_waitForUsagesOnDisposal.Wait(100);
}
var exceptionAggregator = new ExceptionAggregator(_logger, $"Could not dispose {nameof(DocumentDatabase)}");
foreach (var connection in RunningTcpConnections)
{
exceptionAggregator.Execute(() =>
{
connection.Dispose();
});
}
exceptionAggregator.Execute(() =>
{
TxMerger.Dispose();
});
exceptionAggregator.Execute(() =>
{
DocumentReplicationLoader.Dispose();
});
if (_indexStoreTask != null)
{
exceptionAggregator.Execute(() =>
{
_indexStoreTask.Wait(DatabaseShutdown);
_indexStoreTask = null;
});
}
if (_transformerStoreTask != null)
{
exceptionAggregator.Execute(() =>
{
_transformerStoreTask.Wait(DatabaseShutdown);
_transformerStoreTask = null;
});
}
exceptionAggregator.Execute(() =>
{
IndexStore?.Dispose();
IndexStore = null;
});
exceptionAggregator.Execute(() =>
{
BundleLoader?.Dispose();
BundleLoader = null;
});
exceptionAggregator.Execute(() =>
{
DocumentTombstoneCleaner?.Dispose();
DocumentTombstoneCleaner = null;
});
exceptionAggregator.Execute(() =>
{
DocumentReplicationLoader?.Dispose();
DocumentReplicationLoader = null;
});
exceptionAggregator.Execute(() =>
{
SqlReplicationLoader?.Dispose();
SqlReplicationLoader = null;
});
exceptionAggregator.Execute(() =>
{
Operations?.Dispose(exceptionAggregator);
Operations = null;
});
exceptionAggregator.Execute(() =>
{
SubscriptionStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ConfigurationStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
DocumentsStorage?.Dispose();
DocumentsStorage = null;
});
exceptionAggregator.ThrowIfNeeded();
}
public async Task TestServicesAsync(string networkString)
{
await RuntimeParams.LoadAsync();
var network = Network.GetNetwork(networkString);
var blocksToDownload = new List <uint256>();
if (network == Network.Main)
{
blocksToDownload.Add(new uint256("00000000000000000037c2de35bd85f3e57f14ddd741ce6cee5b28e51473d5d0"));
blocksToDownload.Add(new uint256("000000000000000000115315a43cb0cdfc4ea54a0e92bed127f4e395e718d8f9"));
blocksToDownload.Add(new uint256("00000000000000000011b5b042ad0522b69aae36f7de796f563c895714bbd629"));
}
else if (network == Network.TestNet)
{
blocksToDownload.Add(new uint256("0000000097a664c4084b49faa6fd4417055cb8e5aac480abc31ddc57a8208524"));
blocksToDownload.Add(new uint256("000000009ed5b82259ecd2aa4cd1f119db8da7a70e7ea78d9c9f603e01f93bcc"));
blocksToDownload.Add(new uint256("00000000e6da8c2da304e9f5ad99c079df2c3803b49efded3061ecaf206ddc66"));
}
else
{
throw new NotSupportedNetworkException(network);
}
var dataDir = Common.GetWorkDir();
var indexStore = new IndexStore(Path.Combine(dataDir, "indexStore"), network, new SmartHeaderChain());
await using var transactionStore = new AllTransactionStore(Path.Combine(dataDir, "transactionStore"), network);
var mempoolService = new MempoolService();
var blocks = new FileSystemBlockRepository(Path.Combine(dataDir, "blocks"), network);
await using BitcoinStore bitcoinStore = new(indexStore, transactionStore, mempoolService, blocks);
await bitcoinStore.InitializeAsync();
var addressManagerFolderPath = Path.Combine(dataDir, "AddressManager");
var addressManagerFilePath = Path.Combine(addressManagerFolderPath, $"AddressManager{network}.dat");
var connectionParameters = new NodeConnectionParameters();
AddressManager addressManager;
try
{
addressManager = await NBitcoinHelpers.LoadAddressManagerFromPeerFileAsync(addressManagerFilePath);
Logger.LogInfo($"Loaded {nameof(AddressManager)} from `{addressManagerFilePath}`.");
}
catch (DirectoryNotFoundException)
{
addressManager = new AddressManager();
}
catch (FileNotFoundException)
{
addressManager = new AddressManager();
}
catch (OverflowException)
{
File.Delete(addressManagerFilePath);
addressManager = new AddressManager();
}
catch (FormatException)
{
File.Delete(addressManagerFilePath);
addressManager = new AddressManager();
}
connectionParameters.TemplateBehaviors.Add(new AddressManagerBehavior(addressManager));
connectionParameters.TemplateBehaviors.Add(bitcoinStore.CreateUntrustedP2pBehavior());
using var nodes = new NodesGroup(network, connectionParameters, requirements: Constants.NodeRequirements);
KeyManager keyManager = KeyManager.CreateNew(out _, "password");
HttpClientFactory httpClientFactory = new(Common.TorSocks5Endpoint, backendUriGetter : () => new Uri("http://localhost:12345"));
WasabiSynchronizer syncer = new(network, bitcoinStore, httpClientFactory);
ServiceConfiguration serviceConfig = new(MixUntilAnonymitySet.PrivacyLevelStrong.ToString(), 2, 21, 50, new IPEndPoint(IPAddress.Loopback, network.DefaultPort), Money.Coins(Constants.DefaultDustThreshold));
CachedBlockProvider blockProvider = new(
new P2pBlockProvider(nodes, null, httpClientFactory, serviceConfig, network),
bitcoinStore.BlockRepository);
using Wallet wallet = Wallet.CreateAndRegisterServices(
network,
bitcoinStore,
keyManager,
syncer,
nodes,
dataDir,
new ServiceConfiguration(MixUntilAnonymitySet.PrivacyLevelStrong.ToString(), 2, 21, 50, new IPEndPoint(IPAddress.Loopback, network.DefaultPort), Money.Coins(Constants.DefaultDustThreshold)),
syncer,
blockProvider);
Assert.True(Directory.Exists(blocks.BlocksFolderPath));
try
{
var mempoolTransactionAwaiter = new EventsAwaiter <SmartTransaction>(
h => bitcoinStore.MempoolService.TransactionReceived += h,
h => bitcoinStore.MempoolService.TransactionReceived -= h,
3);
var nodeConnectionAwaiter = new EventsAwaiter <NodeEventArgs>(
h => nodes.ConnectedNodes.Added += h,
h => nodes.ConnectedNodes.Added -= h,
3);
nodes.Connect();
var downloadTasks = new List <Task <Block> >();
using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(4));
foreach (var hash in blocksToDownload)
{
downloadTasks.Add(blockProvider.GetBlockAsync(hash, cts.Token));
}
await nodeConnectionAwaiter.WaitAsync(TimeSpan.FromMinutes(3));
var i = 0;
var hashArray = blocksToDownload.ToArray();
foreach (var block in await Task.WhenAll(downloadTasks))
{
Assert.True(File.Exists(Path.Combine(blocks.BlocksFolderPath, hashArray[i].ToString())));
i++;
}
await mempoolTransactionAwaiter.WaitAsync(TimeSpan.FromMinutes(1));
}
finally
{
// So next test will download the block.
foreach (var hash in blocksToDownload)
{
await blockProvider.BlockRepository.RemoveAsync(hash, CancellationToken.None);
}
if (wallet is { })
public void For(IndexStore indices, Action <int, int, int> action)
{
Parallel.Invoke(
() =>
{
for (int i = indices.Lower; i < indices.ILength / 2; i++)
{
for (int j = indices.Lower; j < indices.JLength / 2; j++)
{
for (int k = indices.Lower; k < indices.KLength / 2; k++)
{
action(i, j, k);
}
}
}
},
() =>
{
for (int i = indices.Lower; i < indices.ILength / 2; i++)
{
for (int j = indices.JLength / 2; j < indices.JLength; j++)
{
for (int k = indices.Lower; k < indices.KLength / 2; k++)
{
action(i, j, k);
}
}
}
},
() =>
{
for (int i = indices.Lower; i < indices.ILength / 2; i++)
{
for (int j = indices.Lower; j < indices.JLength / 2; j++)
{
for (int k = indices.KLength / 2; k < indices.KLength; k++)
{
action(i, j, k);
}
}
}
},
() =>
{
for (int i = indices.Lower; i < indices.ILength / 2; i++)
{
for (int j = indices.JLength / 2; j < indices.JLength; j++)
{
for (int k = indices.KLength / 2; k < indices.KLength; k++)
{
action(i, j, k);
}
}
}
},
() =>
{
for (int i = indices.ILength / 2; i < indices.ILength; i++)
{
for (int j = indices.Lower; j < indices.JLength / 2; j++)
{
for (int k = indices.Lower; k < indices.KLength / 2; k++)
{
action(i, j, k);
}
}
}
},
() =>
{
for (int i = indices.ILength / 2; i < indices.ILength; i++)
{
for (int j = indices.JLength / 2; j < indices.JLength; j++)
{
for (int k = indices.Lower; k < indices.KLength / 2; k++)
{
action(i, j, k);
}
}
}
},
() =>
{
for (int i = indices.ILength / 2; i < indices.ILength; i++)
{
for (int j = indices.Lower; j < indices.JLength / 2; j++)
{
for (int k = indices.KLength / 2; k < indices.KLength; k++)
{
action(i, j, k);
}
}
}
},
() =>
{
for (int i = indices.ILength / 2; i < indices.ILength; i++)
{
for (int j = indices.JLength / 2; j < indices.JLength; j++)
{
for (int k = indices.KLength / 2; k < indices.KLength; k++)
{
action(i, j, k);
}
}
}
});
}
public bool ContainsHash(byte[] hash)
{
return(IndexStore.GetRecord(hash) != null);
}
/// <summary>
/// Initializes a new instance of the <see cref="PmlBoundary" /> class.
/// </summary>
/// <param name="pmlLength">Length of the PML.</param>
/// <param name="store">The store.</param>
public PmlBoundary(int pmlLength, IndexStore store)
{
this.Length = pmlLength;
this.boundaryConditionsCalc(store);
}
public DynamicQueryToIndexMatcher(IndexStore indexStore)
{
_indexStore = indexStore;
}
private void DeleteIndexMetadataForRemovedIndexesAndTransformers(Transaction tx, TransactionOperationContext context, IndexStore indexStore,
TransformerStore transformerStore)
{
var toRemove = new List <IndexEntryMetadata>();
var table = tx.OpenTable(IndexesTableSchema,
SchemaNameConstants.IndexMetadataTable);
foreach (var tvr in table.SeekForwardFrom(IndexesTableSchema.FixedSizeIndexes[EtagIndexName], 0))
{
var metadata = TableValueToMetadata(tvr, context, true);
if (metadata == null) //noting to do if it is a tombstone
{
continue;
}
if (metadata.Type == IndexEntryType.Index &&
indexStore.GetIndex(metadata.Id) == null)
{
toRemove.Add(metadata);
}
if (metadata.Type == IndexEntryType.Transformer &&
transformerStore.GetTransformer(metadata.Id) == null)
{
toRemove.Add(metadata);
}
}
foreach (var metadata in toRemove)
{
WriteEntry(tx, metadata.Name, metadata.Type, -1, context);
}
}
public void Initialize(InitializeOptions options = InitializeOptions.None)
{
try
{
_addToInitLog("Initializing NotificationCenter");
NotificationCenter.Initialize(this);
_addToInitLog("Initializing DocumentStorage");
DocumentsStorage.Initialize((options & InitializeOptions.GenerateNewDatabaseId) == InitializeOptions.GenerateNewDatabaseId);
_addToInitLog("Starting Transaction Merger");
TxMerger.Start();
_addToInitLog("Initializing ConfigurationStorage");
ConfigurationStorage.Initialize();
if ((options & InitializeOptions.SkipLoadingDatabaseRecord) == InitializeOptions.SkipLoadingDatabaseRecord)
{
return;
}
_addToInitLog("Loading Database");
long index;
DatabaseRecord record;
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
record = _serverStore.Cluster.ReadDatabase(context, Name, out index);
if (record == null)
{
DatabaseDoesNotExistException.Throw(Name);
}
PeriodicBackupRunner = new PeriodicBackupRunner(this, _serverStore);
_addToInitLog("Initializing IndexStore (async)");
_indexStoreTask = IndexStore.InitializeAsync(record);
_addToInitLog("Initializing Replication");
ReplicationLoader?.Initialize(record);
_addToInitLog("Initializing ETL");
EtlLoader.Initialize(record);
DocumentTombstoneCleaner.Start();
try
{
_indexStoreTask.Wait(DatabaseShutdown);
}
finally
{
_addToInitLog("Initializing IndexStore completed");
_indexStoreTask = null;
}
SubscriptionStorage.Initialize();
_addToInitLog("Initializing SubscriptionStorage completed");
TaskExecutor.Execute((state) =>
{
try
{
NotifyFeaturesAboutStateChange(record, index);
}
catch
{
// We ignore the exception since it was caught in the function itself
}
}, null);
}
catch (Exception)
{
Dispose();
throw;
}
}
public override void Finish()
{
ActionSet.AddAction(IndexStore.ModifyVariable(Operation.Add, 1));
base.Finish();
}
public TextSearchIndex(IndexStore <Metadata> indexStore)
{
this.indexStore = indexStore;
}
public override void Setup()
{
ActionSet.AddAction(IndexStore.SetVariable(0));
base.Setup();
}
private void DoWork(CommerceInstance instance)
{
IndexStore store = null;
try
{
var rebuildDirectory = IndexStores.GetDirectory(Context.Instance, Context.Culture, Context.ModelType, true);
var liveDirectory = IndexStores.GetDirectory(Context.Instance, Context.Culture, Context.ModelType, false);
// Ensure temp folder are deleted (last rebuild might encounter errors when deleting the temp folder)
Kooboo.IO.IOUtility.DeleteDirectory(rebuildDirectory, true);
Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory + "-tmp", true);
var total = _source.Count(instance, Context.Culture);
var totalRebuilt = 0;
Progress = 0;
store = new IndexStore(Context.ModelType, FSDirectory.Open(rebuildDirectory), Analyzers.GetAnalyzer(Context.Culture));
foreach (var data in _source.Enumerate(instance, Context.Culture))
{
if (_cancelling)
{
break;
}
store.Index(data);
totalRebuilt++;
Progress = (int)Math.Round(totalRebuilt * 100 / (double)total);
}
if (_cancelling)
{
store.Dispose();
UpdateTaskInfo(info =>
{
info.ClearError();
info.LastRebuildStatus = RebuildStatus.Cancelled;
});
_cancelling = false;
_cancelledEvent.Set();
Status = RebuildStatus.Cancelled;
}
else
{
store.Commit();
store.Dispose();
UpdateTaskInfo(info =>
{
info.ClearError();
info.LastRebuildStatus = RebuildStatus.Success;
info.LastSucceededRebuildTimeUtc = DateTime.UtcNow;
});
// Replace old index files with the new ones
IndexStores.Close(Context.Instance, Context.Culture, Context.ModelType);
var liveDirectoryExists = System.IO.Directory.Exists(liveDirectory);
if (liveDirectoryExists)
{
System.IO.Directory.Move(liveDirectory, liveDirectory + "-tmp");
Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory, true);
}
System.IO.Directory.Move(rebuildDirectory, liveDirectory);
if (liveDirectoryExists)
{
Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory + "-tmp", true);
}
Status = RebuildStatus.Success;
}
Progress = 0;
}
catch (Exception ex)
{
if (store != null)
{
store.Dispose();
}
UpdateTaskInfo(info =>
{
info.LastRebuildStatus = RebuildStatus.Failed;
info.LastRebuildError = ex.Message;
info.LastRebuildErrorDetail = ex.StackTrace;
});
Status = RebuildStatus.Failed;
}
}
public void ForExceptK(IndexStore indices, Action <int, int> action)
{
throw new NotImplementedException();
}
public DynamicQueryRunner(DocumentDatabase database) : base(database)
{
_indexStore = database.IndexStore;
}
public TextSearchIndex(IndexStore<Metadata> indexStore)
{
this.indexStore = indexStore;
}
public void Finish()
{
ActionSet.AddAction(IndexStore.ModifyVariable(Operation.Add, 1));
ActionSet.AddAction(new A_End());
}
/// <summary>
/// Creates the array with dimensions set in index store and initialize with defaults.
/// </summary>
/// <typeparam name="T">The type of underlying values.</typeparam>
/// <param name="indices">The indices.</param>
/// <returns>The new three dimensional array.</returns>
public static T[,,] CreateArray <T>(this IndexStore indices)
{
var array = new T[indices.ILength, indices.JLength, indices.KLength];
return(array);
}
public unsafe void Dispose()
{
if (_databaseShutdown.IsCancellationRequested)
{
return; // double dispose?
}
lock (this)
{
if (_databaseShutdown.IsCancellationRequested)
{
return; // double dispose?
}
//before we dispose of the database we take its latest info to be displayed in the studio
try
{
var databaseInfo = GenerateDatabaseInfo();
if (databaseInfo != null)
{
DatabaseInfoCache?.InsertDatabaseInfo(databaseInfo, Name);
}
}
catch (Exception e)
{
// if we encountered a catastrophic failure we might not be able to retrieve database info
if (_logger.IsInfoEnabled)
{
_logger.Info("Failed to generate and store database info", e);
}
}
_databaseShutdown.Cancel();
// we'll wait for 1 minute to drain all the requests
// from the database
var sp = Stopwatch.StartNew();
while (sp.ElapsedMilliseconds < 60 * 1000)
{
if (Interlocked.Read(ref _usages) == 0)
{
break;
}
if (_waitForUsagesOnDisposal.Wait(1000))
{
_waitForUsagesOnDisposal.Reset();
}
}
var exceptionAggregator = new ExceptionAggregator(_logger, $"Could not dispose {nameof(DocumentDatabase)} {Name}");
foreach (var connection in RunningTcpConnections)
{
exceptionAggregator.Execute(() =>
{
connection.Dispose();
});
}
exceptionAggregator.Execute(() =>
{
TxMerger?.Dispose();
});
if (_indexStoreTask != null)
{
exceptionAggregator.Execute(() =>
{
_indexStoreTask.Wait(DatabaseShutdown);
});
}
exceptionAggregator.Execute(() =>
{
IndexStore?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ExpiredDocumentsCleaner?.Dispose();
});
exceptionAggregator.Execute(() =>
{
PeriodicBackupRunner?.Dispose();
});
exceptionAggregator.Execute(() =>
{
DocumentTombstoneCleaner?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ReplicationLoader?.Dispose();
});
exceptionAggregator.Execute(() =>
{
EtlLoader?.Dispose();
});
exceptionAggregator.Execute(() =>
{
Operations?.Dispose(exceptionAggregator);
});
exceptionAggregator.Execute(() =>
{
NotificationCenter?.Dispose();
});
exceptionAggregator.Execute(() =>
{
SubscriptionStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
ConfigurationStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
DocumentsStorage?.Dispose();
});
exceptionAggregator.Execute(() =>
{
_databaseShutdown.Dispose();
});
exceptionAggregator.Execute(() =>
{
if (MasterKey == null)
{
return;
}
fixed(byte *pKey = MasterKey)
{
Sodium.sodium_memzero(pKey, (UIntPtr)MasterKey.Length);
}
});
exceptionAggregator.ThrowIfNeeded();
}
}
public async Task MempoolNotifiesAsync()
{
using var services = new HostedServices();
CoreNode coreNode = await TestNodeBuilder.CreateAsync(services);
await services.StartAllAsync();
BitcoinStore?bitcoinStore = null;
using var node = await coreNode.CreateNewP2pNodeAsync();
try
{
string dir = Common.GetWorkDir();
var network = coreNode.Network;
var rpc = coreNode.RpcClient;
var indexStore = new IndexStore(Path.Combine(dir, "indexStore"), network, new SmartHeaderChain());
var transactionStore = new AllTransactionStore(Path.Combine(dir, "transactionStore"), network);
var mempoolService = new MempoolService();
var blocks = new FileSystemBlockRepository(Path.Combine(dir, "blocks"), network);
// Construct BitcoinStore.
bitcoinStore = new BitcoinStore(indexStore, transactionStore, mempoolService, blocks);
await bitcoinStore.InitializeAsync();
await rpc.GenerateAsync(blockCount : 101);
node.Behaviors.Add(bitcoinStore.CreateUntrustedP2pBehavior());
node.VersionHandshake();
BitcoinWitPubKeyAddress address = new Key().PubKey.GetSegwitAddress(network);
// Number of transactions to send.
const int TransactionsCount = 10;
var eventAwaiter = new EventsAwaiter <SmartTransaction>(
subscribe: h => mempoolService.TransactionReceived += h,
unsubscribe: h => mempoolService.TransactionReceived -= h,
count: TransactionsCount);
var txHashesList = new List <Task <uint256> >();
IRPCClient rpcBatch = rpc.PrepareBatch();
// Add to the batch 10 RPC commands: Send 1 coin to the same address.
for (int i = 0; i < TransactionsCount; i++)
{
txHashesList.Add(rpcBatch.SendToAddressAsync(address, Money.Coins(1)));
}
// Publish the RPC batch.
Task rpcBatchTask = rpcBatch.SendBatchAsync();
// Wait until the mempool service receives all the sent transactions.
IEnumerable <SmartTransaction> mempoolSmartTxs = await eventAwaiter.WaitAsync(TimeSpan.FromSeconds(30));
await rpcBatchTask;
// Collect all the transaction hashes of the sent transactions.
uint256[] hashes = await Task.WhenAll(txHashesList);
// Check that all the received transaction hashes are in the set of sent transaction hashes.
foreach (SmartTransaction tx in mempoolSmartTxs)
{
Assert.Contains(tx.GetHash(), hashes);
}
}
finally
{
if (bitcoinStore is { } store)
{
await store.DisposeAsync();
}
await services.StopAllAsync();
node.Disconnect();
await coreNode.TryStopAsync();
}
}
public DocumentDatabase(string name, RavenConfiguration configuration, ServerStore serverStore, Action <string> addToInitLog)
{
Name = name;
_logger = LoggingSource.Instance.GetLogger <DocumentDatabase>(Name);
_serverStore = serverStore;
_addToInitLog = addToInitLog;
StartTime = Time.GetUtcNow();
LastAccessTime = Time.GetUtcNow();
Configuration = configuration;
Scripts = new ScriptRunnerCache(this, Configuration);
try
{
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
using (ctx.OpenReadTransaction())
{
MasterKey = serverStore.GetSecretKey(ctx, Name);
var databaseRecord = _serverStore.Cluster.ReadDatabase(ctx, Name);
if (databaseRecord != null)
{
// can happen when we are in the process of restoring a database
if (databaseRecord.Encrypted && MasterKey == null)
{
throw new InvalidOperationException($"Attempt to create encrypted db {Name} without supplying the secret key");
}
if (databaseRecord.Encrypted == false && MasterKey != null)
{
throw new InvalidOperationException($"Attempt to create a non-encrypted db {Name}, but a secret key exists for this db.");
}
}
}
QueryMetadataCache = new QueryMetadataCache();
IoChanges = new IoChangesNotifications();
Changes = new DocumentsChanges();
DocumentTombstoneCleaner = new DocumentTombstoneCleaner(this);
DocumentsStorage = new DocumentsStorage(this, addToInitLog);
IndexStore = new IndexStore(this, serverStore);
QueryRunner = new QueryRunner(this);
EtlLoader = new EtlLoader(this, serverStore);
ReplicationLoader = new ReplicationLoader(this, serverStore);
SubscriptionStorage = new SubscriptionStorage(this, serverStore);
Metrics = new MetricCounters();
TxMerger = new TransactionOperationsMerger(this, DatabaseShutdown);
HugeDocuments = new HugeDocuments(configuration.PerformanceHints.HugeDocumentsCollectionSize,
configuration.PerformanceHints.HugeDocumentSize.GetValue(SizeUnit.Bytes));
ConfigurationStorage = new ConfigurationStorage(this);
NotificationCenter = new NotificationCenter.NotificationCenter(ConfigurationStorage.NotificationsStorage, Name, _databaseShutdown.Token);
Operations = new Operations.Operations(Name, ConfigurationStorage.OperationsStorage, NotificationCenter, Changes);
DatabaseInfoCache = serverStore.DatabaseInfoCache;
RachisLogIndexNotifications = new RachisLogIndexNotifications(DatabaseShutdown);
CatastrophicFailureNotification = new CatastrophicFailureNotification((environmentId, e) =>
{
serverStore.DatabasesLandlord.CatastrophicFailureHandler.Execute(name, e, environmentId);
});
}
catch (Exception)
{
Dispose();
throw;
}
}
private void DoWork(CommerceInstance instance)
{
IndexStore store = null;
try
{
var rebuildDirectory = IndexStores.GetDirectory(Context.Instance, Context.Culture, Context.ModelType, true);
var liveDirectory = IndexStores.GetDirectory(Context.Instance, Context.Culture, Context.ModelType, false);
// Ensure temp folder are deleted (last rebuild might encounter errors when deleting the temp folder)
Kooboo.IO.IOUtility.DeleteDirectory(rebuildDirectory, true);
Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory + "-tmp", true);
var total = _source.Count(instance, Context.Culture);
var totalRebuilt = 0;
Progress = 0;
store = new IndexStore(Context.ModelType, FSDirectory.Open(rebuildDirectory), Analyzers.GetAnalyzer(Context.Culture));
foreach (var data in _source.Enumerate(instance, Context.Culture))
{
if (_cancelling)
{
break;
}
store.Index(data);
totalRebuilt++;
Progress = (int)Math.Round(totalRebuilt * 100 / (double)total);
}
if (_cancelling)
{
store.Dispose();
UpdateTaskInfo(info =>
{
info.ClearError();
info.LastRebuildStatus = RebuildStatus.Cancelled;
});
_cancelling = false;
_cancelledEvent.Set();
Status = RebuildStatus.Cancelled;
}
else
{
store.Commit();
store.Dispose();
UpdateTaskInfo(info =>
{
info.ClearError();
info.LastRebuildStatus = RebuildStatus.Success;
info.LastSucceededRebuildTimeUtc = DateTime.UtcNow;
});
// Replace old index files with the new ones
IndexStores.Close(Context.Instance, Context.Culture, Context.ModelType);
var liveDirectoryExists = System.IO.Directory.Exists(liveDirectory);
if (liveDirectoryExists)
{
System.IO.Directory.Move(liveDirectory, liveDirectory + "-tmp");
Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory, true);
}
System.IO.Directory.Move(rebuildDirectory, liveDirectory);
if (liveDirectoryExists)
{
Kooboo.IO.IOUtility.DeleteDirectory(liveDirectory + "-tmp", true);
}
Status = RebuildStatus.Success;
}
Progress = 0;
}
catch (Exception ex)
{
if (store != null)
{
store.Dispose();
}
UpdateTaskInfo(info =>
{
info.LastRebuildStatus = RebuildStatus.Failed;
info.LastRebuildError = ex.Message;
info.LastRebuildErrorDetail = ex.StackTrace;
});
Status = RebuildStatus.Failed;
}
}
public void Initialize(StorageEnvironment environment, TransactionContextPool contextPool, IndexStore indexStore, TransformerStore transformerStore)
{
_environment = environment;
_contextPool = contextPool;
TransactionOperationContext context;
using (contextPool.AllocateOperationContext(out context))
using (var tx = context.OpenWriteTransaction())
{
IndexesTableSchema.Create(tx.InnerTransaction, SchemaNameConstants.IndexMetadataTable, 16);
ConflictsTableSchema.Create(tx.InnerTransaction, SchemaNameConstants.ConflictMetadataTable, 16);
tx.InnerTransaction.CreateTree(SchemaNameConstants.GlobalChangeVectorTree);
tx.InnerTransaction.CreateTree(SchemaNameConstants.LastReplicatedEtagsTree);
DeleteIndexMetadataForRemovedIndexesAndTransformers(tx.InnerTransaction, context, indexStore, transformerStore);
tx.Commit();
}
IsInitialized = true;
}
public void Initialize(IndexStore indexStore, TransformerStore transformerStore)
{
_contextPool = new TransactionContextPool(Environment);
AlertsStorage.Initialize(Environment, _contextPool);
IndexesEtagsStorage.Initialize(Environment, _contextPool, indexStore, transformerStore);
}
