public override int Execute(DocumentsOperationContext context) { if (string.IsNullOrEmpty(context.LastDatabaseChangeVector)) { context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } var status = ChangeVectorUtils.GetConflictStatus(_replicationBatchReply.DatabaseChangeVector, context.LastDatabaseChangeVector); if (status != ConflictStatus.AlreadyMerged) { return(0); } var res = ChangeVectorUtils.TryUpdateChangeVector(_replicationBatchReply.NodeTag, _dbId, _replicationBatchReply.CurrentEtag, ref context.LastDatabaseChangeVector) ? 1 : 0; if (res == 1) { context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += _ => { try { _trigger.Set(); } catch { // } }; } return(res); }
private (long, int) GetSizeAndOperationsCount(BatchRequestParser.CommandData commandData) { long size = 0; switch (commandData.Type) { case CommandType.PUT: return(commandData.Document.Size, 1); case CommandType.Counters: foreach (var operation in commandData.Counters.Operations) { size += operation.CounterName.Length + sizeof(long) // etag + sizeof(long) // counter value + GetChangeVectorSizeInternal() // estimated change vector size + 10; // estimated collection name size } return(size, commandData.Counters.Operations.Count); case CommandType.AttachmentPUT: return(commandData.ContentLength, 1); case CommandType.TimeSeries: case CommandType.TimeSeriesBulkInsert: // we don't know the size of the change so we are just estimating foreach (var append in commandData.TimeSeries.Appends) { size += sizeof(long); // DateTime if (string.IsNullOrWhiteSpace(append.Tag) == false) { size += append.Tag.Length; } size += append.Values.Length * sizeof(double); } return(size, commandData.TimeSeries.Appends.Count); default: throw new ArgumentOutOfRangeException($"'{commandData.Type}' isn't supported"); } int GetChangeVectorSizeInternal() { if (_changeVectorSize.HasValue) { return(_changeVectorSize.Value); } using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); _changeVectorSize = Encoding.UTF8.GetBytes(databaseChangeVector).Length; return(_changeVectorSize.Value); } } }
public Task Stats() { using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) using (context.OpenReadTransaction()) { var indexes = Database.IndexStore.GetIndexes().ToList(); var stats = new DatabaseStatistics { LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction), CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context), CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context) }; var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context); stats.CountOfAttachments = attachments.AttachmentCount; stats.CountOfUniqueAttachments = attachments.StreamsCount; stats.CountOfIndexes = indexes.Count; var statsDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); stats.DatabaseChangeVector = statsDatabaseChangeVector; stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id; stats.Is64Bit = IntPtr.Size == sizeof(long); stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString(); stats.Indexes = new IndexInformation[indexes.Count]; for (var i = 0; i < indexes.Count; i++) { var index = indexes[i]; stats.Indexes[i] = new IndexInformation { State = index.State, IsStale = index.IsStale(context), Name = index.Name, Etag = index.Etag, LockMode = index.Definition.LockMode, Priority = index.Definition.Priority, Type = index.Type, LastIndexingTime = index.LastIndexingTime }; if (stats.LastIndexingTime.HasValue) { stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime; } else { stats.LastIndexingTime = index.LastIndexingTime; } } writer.WriteDatabaseStatistics(context, stats); } return(Task.CompletedTask); }
private async Task GetDocumentsAsync(DocumentsOperationContext context, bool metadataOnly) { var sw = Stopwatch.StartNew(); // everything here operates on all docs var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); if (GetStringFromHeaders("If-None-Match") == databaseChangeVector) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified; return; } HttpContext.Response.Headers["ETag"] = "\"" + databaseChangeVector + "\""; var etag = GetLongQueryString("etag", false); var start = GetStart(); var pageSize = GetPageSize(); var isStartsWith = HttpContext.Request.Query.ContainsKey("startsWith"); IEnumerable <Document> documents; if (etag != null) { documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, start, pageSize); } else if (isStartsWith) { documents = Database.DocumentsStorage.GetDocumentsStartingWith(context, HttpContext.Request.Query["startsWith"], HttpContext.Request.Query["matches"], HttpContext.Request.Query["exclude"], HttpContext.Request.Query["startAfter"], start, pageSize); } else // recent docs { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize); } int numberOfResults; long totalDocumentsSizeInBytes; using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream(), Database.DatabaseShutdown)) { writer.WriteStartObject(); writer.WritePropertyName("Results"); (numberOfResults, totalDocumentsSizeInBytes) = await writer.WriteDocumentsAsync(context, documents, metadataOnly); writer.WriteEndObject(); await writer.OuterFlushAsync(); } AddPagingPerformanceHint(PagingOperationType.Documents, isStartsWith ? nameof(DocumentsStorage.GetDocumentsStartingWith) : nameof(GetDocumentsAsync), HttpContext.Request.QueryString.Value, numberOfResults, pageSize, sw.ElapsedMilliseconds, totalDocumentsSizeInBytes); }
public override int Execute(DocumentsOperationContext context) { if (string.IsNullOrEmpty(context.LastDatabaseChangeVector)) { context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } var status = ChangeVectorUtils.GetConflictStatus(_replicationBatchReply.DatabaseChangeVector, context.LastDatabaseChangeVector); if (status != ConflictStatus.AlreadyMerged) { return(0); } var result = ChangeVectorUtils.TryUpdateChangeVector(_replicationBatchReply.NodeTag, _dbId, _replicationBatchReply.CurrentEtag, context.LastDatabaseChangeVector); if (result.IsValid) { if (context.LastReplicationEtagFrom == null) { context.LastReplicationEtagFrom = new Dictionary <string, long>(); } if (context.LastReplicationEtagFrom.ContainsKey(_replicationBatchReply.DatabaseId) == false) { context.LastReplicationEtagFrom[_replicationBatchReply.DatabaseId] = _replicationBatchReply.CurrentEtag; } context.LastDatabaseChangeVector = result.ChangeVector; context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += _ => { try { _trigger.Set(); } catch { // } }; } return(result.IsValid ? 1 : 0); }
protected async Task<bool> WaitForChangeVectorInClusterAsync(List<RavenServer> nodes, string database, int timeout = 15000) { return await WaitForValueAsync(async () => { var cvs = new List<string>(); foreach (var server in nodes) { var storage = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database); using (storage.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { cvs.Add(DocumentsStorage.GetDatabaseChangeVector(context)); } } return cvs.Any(x => x != cvs.FirstOrDefault()) == false; }, true, timeout: timeout, interval: 333); }
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups) || options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) || options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions) || options.OperateOnTypes.HasFlag(DatabaseItemType.ReplicationHubCertificates)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name)) { LastRaftIndex = rawRecord.EtagForBackup; } } var disposable = new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); }); return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.Build))); }
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion) { _currentTypeIndex = 0; if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) || options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) || options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) || options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) || options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups)) { _returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context); _disposeTransaction = _context.OpenReadTransaction(); LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction); LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction); } if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) || options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) || options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones)) { _returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext); _disposeServerTransaction = _serverContext.OpenReadTransaction(); using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name)) { LastRaftIndex = rawRecord.GetEtagForBackup(); } } buildVersion = ServerVersion.Build; return(new DisposableAction(() => { _disposeServerTransaction?.Dispose(); _returnServerContext?.Dispose(); _disposeTransaction?.Dispose(); _returnContext?.Dispose(); })); }
private static void FillDocumentsInfo(DatabaseStatusReport prevDatabaseReport, DocumentDatabase dbInstance, DatabaseStatusReport report, DocumentsOperationContext context, DocumentsStorage documentsStorage) { if (prevDatabaseReport?.LastTransactionId != null && prevDatabaseReport.LastTransactionId == dbInstance.LastTransactionId) { report.LastEtag = prevDatabaseReport.LastEtag; report.LastTombstoneEtag = prevDatabaseReport.LastTombstoneEtag; report.NumberOfConflicts = prevDatabaseReport.NumberOfConflicts; report.NumberOfDocuments = prevDatabaseReport.NumberOfDocuments; report.DatabaseChangeVector = prevDatabaseReport.DatabaseChangeVector; } else { using (var tx = context.OpenReadTransaction()) { report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction); report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount; report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context); report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); } } }
protected async Task can_backup_and_restore_internal() { var defaultS3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "oren" }, "users/1"); session.CountersFor("users/1").Increment("likes", 100); await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, S3Settings = defaultS3Settings, IncrementalBackupFrequency = "0 0 1 1 *" }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); PeriodicBackupStatus status = null; var value = WaitForValue(() => { status = store.Maintenance.Send(operation).Status; return(status?.LastEtag); }, expectedVal: 4, timeout: 30_000); Assert.True(4 == value, $"gotStatus? {status != null}, Status Error: {status?.Error?.Exception}," + $" S3 Error: {status?.UploadToS3?.Exception}, LocalBackup Exception: {status?.LocalBackup?.Exception}"); Assert.True(status.LastOperationId != null, $"status.LastOperationId != null, Got status: {status != null}, exception: {status?.Error?.Exception}"); var backupOperation = store.Maintenance.Send(new GetOperationStateOperation(status.LastOperationId.Value)); var backupResult = backupOperation.Result as BackupResult; Assert.True(backupResult != null && backupResult.Counters.Processed, "backupResult != null && backupResult.Counters.Processed"); Assert.True(1 == backupResult.Counters.ReadCount, "1 == backupResult.Counters.ReadCount"); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "ayende" }, "users/2"); session.CountersFor("users/2").Increment("downloads", 200); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => { status = store.Maintenance.Send(operation).Status; return(status?.LastEtag); }, expectedVal: lastEtag, timeout: 30_000); Assert.True(lastEtag == value, $"gotStatus? {status != null}, Status Error: {status?.Error?.Exception}," + $" S3 Error: {status?.UploadToS3?.Exception}, LocalBackup Exception: {status?.LocalBackup?.Exception}"); // restore the database with a different name var databaseName = $"restored_database-{Guid.NewGuid()}"; var subfolderS3Settings = GetS3Settings(status.FolderName); using (RestoreDatabaseFromCloud( store, new RestoreFromS3Configuration { DatabaseName = databaseName, Settings = subfolderS3Settings, DisableOngoingTasks = true }, TimeSpan.FromSeconds(60))) { using (var session = store.OpenAsyncSession(databaseName)) { var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" }); Assert.True(users.Any(x => x.Value.Name == "oren")); Assert.True(users.Any(x => x.Value.Name == "ayende")); var val = await session.CountersFor("users/1").GetAsync("likes"); Assert.Equal(100, val); val = await session.CountersFor("users/2").GetAsync("downloads"); Assert.Equal(200, val); } var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName); using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Equal($"A:7-{originalDatabase.DbBase64Id}, A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector); } } } }
private void ReplicateToDestination() { try { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate); NativeMemory.EnsureRegistered(); if (_log.IsInfoEnabled) { _log.Info($"Will replicate to {Destination.FromString()} via {_connectionInfo.Url}"); } using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) using (context.OpenReadTransaction()) { var record = _parent.LoadDatabaseRecord(); if (record == null) { throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!"); } if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false) { throw new InvalidOperationException( $"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}"); } } var task = TcpUtils.ConnectSocketAsync(_connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log); task.Wait(CancellationToken); using (Interlocked.Exchange(ref _tcpClient, task.Result)) { var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, _connectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Engine.TcpConnectionTimeout); wrapSsl.Wait(CancellationToken); using (_stream = wrapSsl.Result) // note that _stream is being disposed by the interruptible read using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream)) using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance()) { var documentSender = new ReplicationDocumentSender(_stream, this, _log); WriteHeaderToRemotePeer(); //handle initial response to last etag and staff try { var response = HandleServerResponse(getFullResponse: true); switch (response.ReplyType) { //The first time we start replication we need to register the destination current CV case ReplicationMessageReply.ReplyType.Ok: LastAcceptedChangeVector = response.Reply.DatabaseChangeVector; break; case ReplicationMessageReply.ReplyType.Error: var exception = new InvalidOperationException(response.Reply.Exception); if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) || response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException))) { AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist"); DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}"); throw exception; } } catch (DatabaseDoesNotExistException e) { var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " + "on the other end. "; if (_external) { msg += "In order for the replication to work, a database with the same name needs to be created at the destination"; } var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30; if (young) { msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly."; } if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); // won't add an alert on young connections // because it may take a few seconds for the other side to be notified by // the cluster that it has this db. if (young == false) { AddAlertOnFailureToReachOtherSide(msg, e); } throw; } catch (OperationCanceledException e) { const string msg = "Got operation canceled notification while opening outgoing replication channel. " + "Aborting and closing the channel."; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); throw; } catch (Exception e) { var msg = $"{OutgoingReplicationThreadName} got an unexpected exception during initial handshake"; if (_log.IsInfoEnabled) { _log.Info(msg, e); } AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg); AddAlertOnFailureToReachOtherSide(msg, e); throw; } DateTime nextReplicateAt = default(DateTime); while (_cts.IsCancellationRequested == false) { while (_database.Time.GetUtcNow() > nextReplicateAt) { if (_parent.DebugWaitAndRunReplicationOnce != null) { _parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token); _parent.DebugWaitAndRunReplicationOnce.Reset(); } var sp = Stopwatch.StartNew(); var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats); AddReplicationPerformance(stats); AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin); try { using (var scope = stats.CreateScope()) { try { if (Destination is InternalReplication dest) { _parent.EnsureNotDeleted(dest.NodeTag); } var didWork = documentSender.ExecuteReplicationOnce(scope, ref nextReplicateAt); if (didWork == false) { break; } if (Destination is ExternalReplication externalReplication) { var taskId = externalReplication.TaskId; UpdateExternalReplicationInfo(taskId); } DocumentsSend?.Invoke(this); if (sp.ElapsedMilliseconds > 60 * 1000) { _waitForChanges.Set(); break; } } catch (OperationCanceledException) { // cancellation is not an actual error, // it is a "notification" that we need to cancel current operation const string msg = "Operation was canceled."; AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg); throw; } catch (Exception e) { AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message); scope.AddError(e); throw; } } } finally { stats.Complete(); AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd); } } //if this returns false, this means either timeout or canceled token is activated while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false) { //If we got cancelled we need to break right away if (_cts.IsCancellationRequested) { break; } // open tx // read current change vector compare to last sent // if okay, send cv using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (var tx = ctx.OpenReadTransaction()) { var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); if (etag == _lastSentDocumentEtag) { SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx)); _parent.CompleteDeletionIfNeeded(); } else if (nextReplicateAt > DateTime.UtcNow) { SendHeartbeat(null); } else { //Send a heartbeat first so we will get an updated CV of the destination var currentChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); SendHeartbeat(null); //If our previous CV is already merged to the destination wait a bit more if (ChangeVectorUtils.GetConflictStatus(LastAcceptedChangeVector, currentChangeVector) == ConflictStatus.AlreadyMerged) { continue; } // we have updates that we need to send to the other side // let's do that.. // this can happen if we got replication from another node // that we need to send to it. Note that we typically // will wait for the other node to send the data directly to // our destination, but if it doesn't, we'll step in. // In this case, we try to limit congestion in the network and // only send updates that we have gotten from someone else after // a certain time, to let the other side tell us that it already // got it. Note that this is merely an optimization to reduce network // traffic. It is fine to have the same data come from different sources. break; } } } _waitForChanges.Reset(); } } } } catch (AggregateException e) { if (e.InnerExceptions.Count == 1) { if (e.InnerException is OperationCanceledException oce) { HandleOperationCancelException(oce); } if (e.InnerException is IOException ioe) { HandleIOException(ioe); } } HandleException(e); } catch (OperationCanceledException e) { HandleOperationCancelException(e); } catch (IOException e) { HandleIOException(e); } catch (Exception e) { HandleException(e); } void HandleOperationCancelException(OperationCanceledException e) { if (_log.IsInfoEnabled) { _log.Info($"Operation canceled on replication thread ({FromToString}). " + $"This is not necessary due to an issue. Stopped the thread."); } if (_cts.IsCancellationRequested == false) { Failed?.Invoke(this, e); } } void HandleIOException(IOException e) { if (_log.IsInfoEnabled) { if (e.InnerException is SocketException) { _log.Info($"SocketException was thrown from the connection to remote node ({FromToString}). " + $"This might mean that the remote node is done or there is a network issue.", e); } else { _log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e); } } Failed?.Invoke(this, e); } void HandleException(Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Unexpected exception occurred on replication thread ({FromToString}). " + $"Replication stopped (will be retried later).", e); } Failed?.Invoke(this, e); } }
public Task GetCollectionFields() { var collection = GetStringQueryString("collection", required: false); var prefix = GetStringQueryString("prefix", required: false); using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { long totalResults; string changeVector; string etag = null; if (string.IsNullOrEmpty(collection)) { changeVector = DocumentsStorage.GetDatabaseChangeVector(context); totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context); etag = $"{changeVector}/{totalResults}"; } else { changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, collection); totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count; if (changeVector != null) { etag = $"{changeVector}/{totalResults}"; } } if (etag != null && GetStringFromHeaders("If-None-Match") == etag) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified; return(Task.CompletedTask); } HttpContext.Response.Headers["ETag"] = "\"" + etag + "\""; var fields = new Dictionary <LazyStringValue, FieldType>(LazyStringValueComparer.Instance); if (string.IsNullOrEmpty(collection)) { foreach (var collectionStats in Database.DocumentsStorage.GetCollections(context)) { FetchFieldsForCollection(context, collectionStats.Name, prefix, fields); } } else { FetchFieldsForCollection(context, collection, prefix, fields); } using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); var first = true; foreach (var field in fields) { if (first == false) { writer.WriteComma(); } first = false; writer.WritePropertyName(field.Key); writer.WriteString(field.Value.ToString()); } writer.WriteEndObject(); } return(Task.CompletedTask); } }
public async Task can_backup_and_restore() { using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "oren" }, "users/1"); session.CountersFor("users/1").Increment("likes", 100); await session.SaveChangesAsync(); } var googleCloudSettings = GetGoogleCloudSettings(); var config = Backup.CreateBackupConfiguration(googleCloudSettings: googleCloudSettings); var backupTaskId = Backup.UpdateConfigAndRunBackup(Server, config, store); var backupResult = (BackupResult)store.Maintenance.Send(new GetOperationStateOperation(Backup.GetBackupOperationId(store, backupTaskId))).Result; Assert.NotNull(backupResult); Assert.True(backupResult.Counters.Processed); Assert.Equal(1, backupResult.Counters.ReadCount); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "ayende" }, "users/2"); session.CountersFor("users/2").Increment("downloads", 200); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; var status = Backup.RunBackupAndReturnStatus(Server, backupTaskId, store, isFullBackup: false, expectedEtag: lastEtag); // restore the database with a different name var databaseName = $"restored_database-{Guid.NewGuid()}"; var subfolderGoogleCloudSettings = GetGoogleCloudSettings(status.FolderName); var restoreFromGoogleCloudConfiguration = new RestoreFromGoogleCloudConfiguration { DatabaseName = databaseName, Settings = subfolderGoogleCloudSettings, DisableOngoingTasks = true }; using (Backup.RestoreDatabaseFromCloud(store, restoreFromGoogleCloudConfiguration, TimeSpan.FromSeconds(30))) { using (var session = store.OpenAsyncSession(databaseName)) { var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" }); Assert.True(users.Any(x => x.Value.Name == "oren")); Assert.True(users.Any(x => x.Value.Name == "ayende")); var val = await session.CountersFor("users/1").GetAsync("likes"); Assert.Equal(100, val); val = await session.CountersFor("users/2").GetAsync("downloads"); Assert.Equal(200, val); } var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName); using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Contains($"A:7-{originalDatabase.DbBase64Id}", databaseChangeVector); Assert.Contains($"A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector); } } } }
public async Task can_backup_and_restore_snapshot() { using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "oren" }, "users/1"); await session.SaveChangesAsync(); } using (var session = store.OpenAsyncSession()) { await session .Query <User>() .Where(x => x.Name == "oren") .ToListAsync(); // create an index to backup await session .Query <Order>() .Where(x => x.Freight > 20) .ToListAsync(); // create an index to backup session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup await session.SaveChangesAsync(); } var googleCloudSettings = GetGoogleCloudSettings(); var config = Backup.CreateBackupConfiguration(backupType: BackupType.Snapshot, googleCloudSettings: googleCloudSettings); var backupTaskId = Backup.UpdateConfigAndRunBackup(Server, config, store); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "ayende" }, "users/2"); await session.StoreAsync(new User { Name = "ayende" }, "users/3"); session.CountersFor("users/2").Increment("downloads", 200); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; var status = Backup.RunBackupAndReturnStatus(Server, backupTaskId, store, isFullBackup: false, expectedEtag: lastEtag); // restore the database with a different name string databaseName = $"restored_database_snapshot-{Guid.NewGuid()}"; var subfolderGoogleCloudSettings = GetGoogleCloudSettings(status.FolderName); var restoreFromGoogleCloudConfiguration = new RestoreFromGoogleCloudConfiguration { DatabaseName = databaseName, Settings = subfolderGoogleCloudSettings }; using (Backup.RestoreDatabaseFromCloud(store, restoreFromGoogleCloudConfiguration, TimeSpan.FromSeconds(300))) { using (var session = store.OpenAsyncSession(databaseName)) { var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" }); Assert.NotNull(users["users/1"]); Assert.NotNull(users["users/2"]); Assert.True(users.Any(x => x.Value.Name == "oren")); Assert.True(users.Any(x => x.Value.Name == "ayende")); var val = await session.CountersFor("users/1").GetAsync("likes"); Assert.Equal(100, val); val = await session.CountersFor("users/2").GetAsync("downloads"); Assert.Equal(200, val); } var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation()); Assert.Equal(2, stats.CountOfIndexes); var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName); using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Contains($"A:8-{originalDatabase.DbBase64Id}", databaseChangeVector); Assert.Contains($"A:10-{restoredDatabase.DbBase64Id}", databaseChangeVector); } } } }
public bool ExecuteReplicationOnce(TcpConnectionOptions tcpConnectionOptions, OutgoingReplicationStatsScope stats, ref long next) { EnsureValidStats(stats); var wasInterrupted = false; var delay = GetDelayReplication(); var currentNext = next; using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext)) using (documentsContext.OpenReadTransaction()) { try { // we scan through the documents to send to the other side, we need to be careful about // filtering a lot of documents, because we need to let the other side know about this, and // at the same time, we need to send a heartbeat to keep the tcp connection alive _lastEtag = _parent._lastSentDocumentEtag; _parent.CancellationToken.ThrowIfCancellationRequested(); var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo(); long prevLastEtag = _lastEtag; var replicationState = new ReplicationState { BatchSize = _parent._database.Configuration.Replication.MaxItemsCount, MaxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend, CurrentNext = currentNext, Delay = delay, Context = documentsContext, LastTransactionMarker = -1, NumberOfItemsSent = 0, Size = 0L, MissingTxMarkers = new HashSet <short>() }; using (_stats.Storage.Start()) { foreach (var item in GetReplicationItems(_parent._database, documentsContext, _lastEtag, _stats, _parent.SupportedFeatures.Replication.CaseInsensitiveCounters)) { _parent.CancellationToken.ThrowIfCancellationRequested(); if (replicationState.LastTransactionMarker != item.TransactionMarker) { replicationState.Item = item; if (CanContinueBatch(replicationState, ref next) == false) { wasInterrupted = true; break; } replicationState.LastTransactionMarker = item.TransactionMarker; } _stats.Storage.RecordInputAttempt(); // here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundary if (MissingAttachmentsInLastBatch && item.Type == ReplicationBatchItem.ReplicationItemType.Document && item is DocumentReplicationItem docItem && docItem.Flags.Contain(DocumentFlags.HasAttachments)) { var type = (docItem.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision: AttachmentType.Document; foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, docItem.Id, docItem.ChangeVector)) { // we need to filter attachments that are been sent in the same batch as the document if (attachment.Etag >= prevLastEtag) { if (attachment.TransactionMarker != item.TransactionMarker) { replicationState.MissingTxMarkers.Add(attachment.TransactionMarker); } continue; } var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash); attachment.Stream = stream; var attachmentItem = AttachmentReplicationItem.From(documentsContext, attachment); AddReplicationItemToBatch(attachmentItem, _stats.Storage, skippedReplicationItemsInfo); replicationState.Size += attachmentItem.Size; } } _lastEtag = item.Etag; if (AddReplicationItemToBatch(item, _stats.Storage, skippedReplicationItemsInfo) == false) { // this item won't be needed anymore item.Dispose(); continue; } replicationState.Size += item.Size; replicationState.NumberOfItemsSent++; } } if (_log.IsInfoEnabled) { if (skippedReplicationItemsInfo.SkippedItems > 0) { var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector); _log.Info(message); } var msg = $"Found {_orderedReplicaItems.Count:#,#;;0} documents " + $"and {_replicaAttachmentStreams.Count} attachment's streams " + $"to replicate to {_parent.Node.FromString()}, "; var encryptionSize = documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes); if (encryptionSize > 0) { msg += $"encryption buffer overhead size is {new Size(encryptionSize, SizeUnit.Bytes)}, "; } msg += $"total size: {new Size(replicationState.Size + encryptionSize, SizeUnit.Bytes)}"; _log.Info(msg); } if (_orderedReplicaItems.Count == 0) { var hasModification = _lastEtag != _parent._lastSentDocumentEtag; // ensure that the other server is aware that we skipped // on (potentially a lot of) documents to send, and we update // the last etag they have from us on the other side _parent._lastSentDocumentEtag = _lastEtag; _parent._lastDocumentSentTime = DateTime.UtcNow; var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext); _parent.SendHeartbeat(changeVector); return(hasModification); } _parent.CancellationToken.ThrowIfCancellationRequested(); try { using (_stats.Network.Start()) { SendDocumentsBatch(documentsContext, _stats.Network); tcpConnectionOptions._lastEtagSent = _lastEtag; tcpConnectionOptions.RegisterBytesSent(replicationState.Size); if (MissingAttachmentsInLastBatch) { return(false); } } } catch (OperationCanceledException) { if (_log.IsInfoEnabled) { _log.Info("Received cancellation notification while sending document replication batch."); } throw; } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send document replication batch", e); } throw; } MissingAttachmentsInLastBatch = false; return(true); } finally { foreach (var item in _orderedReplicaItems) { item.Value.Dispose(); } _orderedReplicaItems.Clear(); _replicaAttachmentStreams.Clear(); } } }
public bool ExecuteReplicationOnce(OutgoingReplicationStatsScope stats, ref DateTime next) { EnsureValidStats(stats); var wasInterrupted = false; var delay = GetDelayReplication(); using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext)) using (documentsContext.OpenReadTransaction()) { try { // we scan through the documents to send to the other side, we need to be careful about // filtering a lot of documents, because we need to let the other side know about this, and // at the same time, we need to send a heartbeat to keep the tcp connection alive _lastEtag = _parent._lastSentDocumentEtag; _parent.CancellationToken.ThrowIfCancellationRequested(); var batchSize = _parent._database.Configuration.Replication.MaxItemsCount; var maxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend; long size = 0; var numberOfItemsSent = 0; var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo(); short lastTransactionMarker = -1; long prevLastEtag = _lastEtag; using (_stats.Storage.Start()) { foreach (var item in GetReplicationItems(documentsContext, _lastEtag, _stats)) { if (lastTransactionMarker != item.TransactionMarker) { if (delay.Ticks > 0) { var nextReplication = item.LastModifiedTicks + delay.Ticks; if (_parent._database.Time.GetUtcNow().Ticks < nextReplication) { next = new DateTime(nextReplication); wasInterrupted = true; break; } } lastTransactionMarker = item.TransactionMarker; if (_parent.SupportedFeatures.Replication.Counters == false) { AssertNotCounterForLegacyReplication(item); } if (_parent.SupportedFeatures.Replication.ClusterTransaction == false) { AssertNotClusterTransactionDocumentForLegacyReplication(item); } // Include the attachment's document which is right after its latest attachment. if ((item.Type == ReplicationBatchItem.ReplicationItemType.Document || item.Type == ReplicationBatchItem.ReplicationItemType.DocumentTombstone) && // We want to limit batch sizes to reasonable limits. ((maxSizeToSend.HasValue && size > maxSizeToSend.Value.GetValue(SizeUnit.Bytes)) || (batchSize.HasValue && numberOfItemsSent > batchSize.Value))) { wasInterrupted = true; break; } if (_stats.Storage.CurrentStats.InputCount % 16384 == 0) { // ReSharper disable once PossibleLossOfFraction if ((_parent._parent.MinimalHeartbeatInterval / 2) < _stats.Storage.Duration.TotalMilliseconds) { wasInterrupted = true; break; } } } _stats.Storage.RecordInputAttempt(); //Here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundry if (MissingAttachmentsInLastBatch && item.Type == ReplicationBatchItem.ReplicationItemType.Document && (item.Flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { var type = (item.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision: AttachmentType.Document; foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, item.Id)) { //We need to filter attachments that are been sent in the same batch as the document if (attachment.Etag >= prevLastEtag) { continue; } var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash); attachment.Stream = stream; AddReplicationItemToBatch(ReplicationBatchItem.From(attachment), _stats.Storage, skippedReplicationItemsInfo); size += attachment.Stream.Length; } } _lastEtag = item.Etag; if (item.Data != null) { size += item.Data.Size; } else if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment) { size += item.Stream.Length; } if (AddReplicationItemToBatch(item, _stats.Storage, skippedReplicationItemsInfo) == false) { continue; } numberOfItemsSent++; } } if (_log.IsInfoEnabled) { if (skippedReplicationItemsInfo.SkippedItems > 0) { var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector); _log.Info(message); } _log.Info($"Found {_orderedReplicaItems.Count:#,#;;0} documents and {_replicaAttachmentStreams.Count} attachment's streams to replicate to {_parent.Node.FromString()}."); } if (_orderedReplicaItems.Count == 0) { var hasModification = _lastEtag != _parent._lastSentDocumentEtag; // ensure that the other server is aware that we skipped // on (potentially a lot of) documents to send, and we update // the last etag they have from us on the other side _parent._lastSentDocumentEtag = _lastEtag; _parent._lastDocumentSentTime = DateTime.UtcNow; var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext); _parent.SendHeartbeat(changeVector); return(hasModification); } _parent.CancellationToken.ThrowIfCancellationRequested(); try { using (_stats.Network.Start()) { SendDocumentsBatch(documentsContext, _stats.Network); if (MissingAttachmentsInLastBatch) { return(false); } } } catch (OperationCanceledException) { if (_log.IsInfoEnabled) { _log.Info("Received cancellation notification while sending document replication batch."); } throw; } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send document replication batch", e); } throw; } MissingAttachmentsInLastBatch = false; return(true); } finally { foreach (var item in _orderedReplicaItems) { var value = item.Value; if (value.Type == ReplicationBatchItem.ReplicationItemType.Attachment) { value.Stream.Dispose(); } else { value.Data?.Dispose(); //item.Value.Data is null if tombstone } } _orderedReplicaItems.Clear(); _replicaAttachmentStreams.Clear(); } } }
public async Task AddGlobalChangeVectorToNewDocument(bool useSsl) { var clusterSize = 3; var databaseName = GetDatabaseName(); var leader = await CreateRaftClusterAndGetLeader(clusterSize, true, 0, useSsl : useSsl); X509Certificate2 clientCertificate = null; X509Certificate2 adminCertificate = null; if (useSsl) { var certificates = GenerateAndSaveSelfSignedCertificate(); adminCertificate = RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate1.Value, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader); clientCertificate = RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate2.Value, new Dictionary <string, DatabaseAccess> { [databaseName] = DatabaseAccess.Admin }, server: leader); } var doc = new DatabaseRecord(databaseName); using (var store = new DocumentStore() { Urls = new[] { leader.WebUrl }, Database = databaseName, Certificate = adminCertificate, Conventions = { DisableTopologyUpdates = true } }.Initialize()) { var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize)); var topology = databaseResult.Topology; Assert.Equal(clusterSize, topology.AllNodes.Count()); foreach (var server in Servers) { await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex); } foreach (var server in Servers) { await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName); } using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "Karmel" }, "users/1"); await session.SaveChangesAsync(); } // we need to wait for database change vector to be updated // which means that we need to wait for replication to do a full mesh propagation Assert.True(await WaitForValueOnGroupAsync(topology, serverStore => { var database = serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName).Result; using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var cv = DocumentsStorage.GetDatabaseChangeVector(context); return(cv != null && cv.Contains("A:1-") && cv.Contains("B:1-") && cv.Contains("C:1-")); } }, expected: true, timeout: 60000));
public async Task can_create_azure_snapshot_and_restore_using_restore_point() { using (var holder = new Azure.AzureClientHolder(AzureFactAttribute.AzureSettings)) { using (var store = GetDocumentStore()) { using (var session = store.OpenSession()) { session.Store(new User { Name = "oren" }, "users/1"); session.CountersFor("users/1").Increment("likes", 100); session.SaveChanges(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Snapshot, AzureSettings = holder.Settings, IncrementalBackupFrequency = "0 0 1 1 *" }; var backupTaskId = (store.Maintenance.Send(new UpdatePeriodicBackupOperation(config))).TaskId; store.Maintenance.Send(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); PeriodicBackupStatus status = null; var value = WaitForValue(() => { status = store.Maintenance.Send(operation).Status; return(status?.LastEtag); }, 4); Assert.True(4 == value, $"4 == value, Got status: {status != null}, exception: {status?.Error?.Exception}"); Assert.True(status.LastOperationId != null, $"status.LastOperationId != null, Got status: {status != null}, exception: {status?.Error?.Exception}"); var client = store.GetRequestExecutor().HttpClient; var data = new StringContent(JsonConvert.SerializeObject(holder.Settings), Encoding.UTF8, "application/json"); var response = await client.PostAsync(store.Urls.First() + "/admin/restore/points?type=Azure ", data); string result = response.Content.ReadAsStringAsync().Result; var restorePoints = JsonConvert.DeserializeObject <RestorePoints>(result); Assert.Equal(1, restorePoints.List.Count); var point = restorePoints.List.First(); var databaseName = $"restored_database-{Guid.NewGuid()}"; holder.Settings.RemoteFolderName = holder.Settings.RemoteFolderName + "/" + status.FolderName; var restoreOperation = await store.Maintenance.Server.SendAsync(new RestoreBackupOperation(new RestoreFromAzureConfiguration() { DatabaseName = databaseName, Settings = holder.Settings, DisableOngoingTasks = true, LastFileNameToRestore = point.FileName, })); await restoreOperation.WaitForCompletionAsync(TimeSpan.FromSeconds(60)); using (var store2 = GetDocumentStore(new Options() { CreateDatabase = false, ModifyDatabaseName = s => databaseName })) { using (var session = store2.OpenSession(databaseName)) { var users = session.Load <User>(new[] { "users/1", "users/2" }); Assert.True(users.Any(x => x.Value.Name == "oren")); var val = session.CountersFor("users/1").Get("likes"); Assert.Equal(100, val); } var originalDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database).Result; var restoredDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName).Result; using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Equal($"A:4-{originalDatabase.DbBase64Id}", databaseChangeVector); } } } } }
public async Task can_backup_and_restore() { using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "oren" }, "users/1"); session.CountersFor("users/1").Increment("likes", 100); await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, GoogleCloudSettings = GoogleCloudFact.GoogleCloudSettings, IncrementalBackupFrequency = "* * * * *" //every minute }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); var value = WaitForValue(() => { var status = store.Maintenance.Send(operation).Status; return(status?.LastEtag); }, 4); Assert.Equal(4, value); var backupStatus = store.Maintenance.Send(operation); var backupOperationId = backupStatus.Status.LastOperationId; var backupOperation = store.Maintenance.Send(new GetOperationStateOperation(backupOperationId.Value)); var backupResult = backupOperation.Result as BackupResult; Assert.True(backupResult.Counters.Processed); Assert.Equal(1, backupResult.Counters.ReadCount); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "ayende" }, "users/2"); session.CountersFor("users/2").Increment("downloads", 200); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); // restore the database with a different name var databaseName = $"restored_database-{Guid.NewGuid()}"; GoogleCloudFact.GoogleCloudSettings.RemoteFolderName = $"{backupStatus.Status.FolderName}"; var restoreFromGoogleCloudConfiguration = new RestoreFromGoogleCloudConfiguration() { DatabaseName = databaseName, Settings = GoogleCloudFact.GoogleCloudSettings }; var googleCloudOperation = new RestoreBackupOperation(restoreFromGoogleCloudConfiguration); var restoreOperation = store.Maintenance.Server.Send(googleCloudOperation); restoreOperation.WaitForCompletion(TimeSpan.FromSeconds(30)); { using (var session = store.OpenAsyncSession(databaseName)) { var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" }); Assert.True(users.Any(x => x.Value.Name == "oren")); Assert.True(users.Any(x => x.Value.Name == "ayende")); var val = await session.CountersFor("users/1").GetAsync("likes"); Assert.Equal(100, val); val = await session.CountersFor("users/2").GetAsync("downloads"); Assert.Equal(200, val); } var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName); using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Equal($"A:7-{originalDatabase.DbBase64Id}, A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector); } } } }
public void AcceptIncomingConnection(TcpConnectionOptions tcpConnectionOptions) { ReplicationLatestEtagRequest getLatestEtagMessage; using (tcpConnectionOptions.ContextPool.AllocateOperationContext(out JsonOperationContext context)) using (var readerObject = context.ParseToMemory( tcpConnectionOptions.Stream, "IncomingReplication/get-last-etag-message read", BlittableJsonDocumentBuilder.UsageMode.None, tcpConnectionOptions.PinnedBuffer)) { getLatestEtagMessage = JsonDeserializationServer.ReplicationLatestEtagRequest(readerObject); if (_log.IsInfoEnabled) { _log.Info( $"GetLastEtag: {getLatestEtagMessage.SourceTag}({getLatestEtagMessage.SourceMachineName}) / {getLatestEtagMessage.SourceDatabaseName} ({getLatestEtagMessage.SourceDatabaseId}) - {getLatestEtagMessage.SourceUrl}"); } } var connectionInfo = IncomingConnectionInfo.FromGetLatestEtag(getLatestEtagMessage); try { AssertValidConnection(connectionInfo); } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info($"Connection from [{connectionInfo}] is rejected.", e); } var incomingConnectionRejectionInfos = _incomingRejectionStats.GetOrAdd(connectionInfo, _ => new ConcurrentQueue <IncomingConnectionRejectionInfo>()); incomingConnectionRejectionInfos.Enqueue(new IncomingConnectionRejectionInfo { Reason = e.ToString() }); try { tcpConnectionOptions.Dispose(); } catch { // do nothing } throw; } try { using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsOperationContext)) using (Database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext)) using (var writer = new BlittableJsonTextWriter(documentsOperationContext, tcpConnectionOptions.Stream)) using (documentsOperationContext.OpenReadTransaction()) using (configurationContext.OpenReadTransaction()) { var changeVector = DocumentsStorage.GetDatabaseChangeVector(documentsOperationContext); var lastEtagFromSrc = Database.DocumentsStorage.GetLastReplicateEtagFrom( documentsOperationContext, getLatestEtagMessage.SourceDatabaseId); if (_log.IsInfoEnabled) { _log.Info($"GetLastEtag response, last etag: {lastEtagFromSrc}"); } var response = new DynamicJsonValue { [nameof(ReplicationMessageReply.Type)] = "Ok", [nameof(ReplicationMessageReply.MessageType)] = ReplicationMessageType.Heartbeat, [nameof(ReplicationMessageReply.LastEtagAccepted)] = lastEtagFromSrc, [nameof(ReplicationMessageReply.NodeTag)] = _server.NodeTag, [nameof(ReplicationMessageReply.DatabaseChangeVector)] = changeVector }; documentsOperationContext.Write(writer, response); writer.Flush(); } } catch (Exception) { try { tcpConnectionOptions.Dispose(); } catch (Exception) { // do nothing } throw; } var newIncoming = new IncomingReplicationHandler( tcpConnectionOptions, getLatestEtagMessage, this); newIncoming.Failed += OnIncomingReceiveFailed; newIncoming.DocumentsReceived += OnIncomingReceiveSucceeded; if (_log.IsInfoEnabled) { _log.Info( $"Initialized document replication connection from {connectionInfo.SourceDatabaseName} located at {connectionInfo.SourceUrl}"); } // need to safeguard against two concurrent connection attempts var newConnection = _incoming.GetOrAdd(newIncoming.ConnectionInfo.SourceDatabaseId, newIncoming); if (newConnection == newIncoming) { newIncoming.Start(); IncomingReplicationAdded?.Invoke(newIncoming); ForceTryReconnectAll(); } else { newIncoming.Dispose(); } }
protected async Task can_backup_and_restore_snapshot_internal() { var defaultS3Settings = GetS3Settings(); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "oren" }, "users/1"); await session.SaveChangesAsync(); } using (var session = store.OpenAsyncSession()) { await session .Query <User>() .Where(x => x.Name == "oren") .ToListAsync(); // create an index to backup await session .Query <Order>() .Where(x => x.Freight > 20) .ToListAsync(); // create an index to backup session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Snapshot, S3Settings = defaultS3Settings, IncrementalBackupFrequency = "0 0 1 1 *" }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); PeriodicBackupStatus backupStatus = null; var value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: 4, timeout: 30_000); Assert.True(4 == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "ayende" }, "users/2"); await session.StoreAsync(new User { Name = "ayende" }, "users/3"); session.CountersFor("users/2").Increment("downloads", 200); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => { backupStatus = store.Maintenance.Send(operation).Status; return(backupStatus?.LastEtag); }, expectedVal: lastEtag, timeout: 30_000); Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," + $" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}"); // restore the database with a different name string restoredDatabaseName = $"restored_database_snapshot-{Guid.NewGuid()}"; var subfolderS3Settings = GetS3Settings(backupStatus.FolderName); using (RestoreDatabaseFromCloud(store, new RestoreFromS3Configuration { DatabaseName = restoredDatabaseName, Settings = subfolderS3Settings }, TimeSpan.FromSeconds(60))) { using (var session = store.OpenAsyncSession(restoredDatabaseName)) { var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" }); Assert.NotNull(users["users/1"]); Assert.NotNull(users["users/2"]); Assert.True(users.Any(x => x.Value.Name == "oren")); Assert.True(users.Any(x => x.Value.Name == "ayende")); var val = await session.CountersFor("users/1").GetAsync("likes"); Assert.Equal(100, val); val = await session.CountersFor("users/2").GetAsync("downloads"); Assert.Equal(200, val); } var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation()); Assert.Equal(2, stats.CountOfIndexes); var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(restoredDatabaseName); using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Equal($"A:8-{originalDatabase.DbBase64Id}, A:10-{restoredDatabase.DbBase64Id}", databaseChangeVector); } } } }
public async Task DatabaseChangeVectorIsUpdatedCorrectly() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new CreateSampleDataOperation()); var documentDatabase = (await Databases.GetDocumentDatabaseInstanceFor(store)); var documentsStorage = documentDatabase.DocumentsStorage; using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var lastAttachmentChangeVector = string.Empty; var lastRevisionChangeVector = string.Empty; var attachmentStorage = documentDatabase.DocumentsStorage.AttachmentsStorage; foreach (var document in documentsStorage.GetDocumentsFrom(context, etag: 0)) { VerifyAttachments(); VerifyRevisions(); void VerifyAttachments() { if (document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) == false) { return; } if (metadata.TryGet(Constants.Documents.Metadata.Attachments, out BlittableJsonReaderArray attachments) == false) { return; } foreach (BlittableJsonReaderObject attachmentFromDocument in attachments) { attachmentFromDocument.TryGet(nameof(AttachmentName.Name), out string attachmentName); var attachment = attachmentStorage.GetAttachment(context, document.Id, attachmentName, AttachmentType.Document, null); Assert.True(document.Etag > attachment.Etag); var conflictStatus = ChangeVectorUtils.GetConflictStatus(document.ChangeVector, attachment.ChangeVector); Assert.Equal(ConflictStatus.Update, conflictStatus); var attachmentConflictStatus = ChangeVectorUtils.GetConflictStatus(attachment.ChangeVector, lastAttachmentChangeVector); if (attachmentConflictStatus == ConflictStatus.Update) { lastAttachmentChangeVector = attachment.ChangeVector; } } } void VerifyRevisions() { var revisions = documentsStorage.RevisionsStorage.GetRevisions(context, document.Id, 0, int.MaxValue); foreach (var revision in revisions.Revisions) { var conflictStatus = ChangeVectorUtils.GetConflictStatus(revision.ChangeVector, lastRevisionChangeVector); if (conflictStatus == ConflictStatus.Update) { lastRevisionChangeVector = revision.ChangeVector; } } } } var lastChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); var lastCvAttachmentConflictStatus = ChangeVectorUtils.GetConflictStatus(lastAttachmentChangeVector, lastChangeVector); Assert.Equal(ConflictStatus.AlreadyMerged, lastCvAttachmentConflictStatus); var lastCvRevisionConflictStatus = ChangeVectorUtils.GetConflictStatus(lastRevisionChangeVector, lastChangeVector); Assert.Equal(ConflictStatus.AlreadyMerged, lastCvRevisionConflictStatus); } } }
public bool ExecuteReplicationOnce(OutgoingReplicationStatsScope stats) { EnsureValidStats(stats); using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext)) using (documentsContext.OpenReadTransaction()) { try { // we scan through the documents to send to the other side, we need to be careful about // filtering a lot of documents, because we need to let the other side know about this, and // at the same time, we need to send a heartbeat to keep the tcp connection alive _lastEtag = _parent._lastSentDocumentEtag; _parent.CancellationToken.ThrowIfCancellationRequested(); var batchSize = _parent._database.Configuration.Replication.MaxItemsCount; var maxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend; long size = 0; int numberOfItemsSent = 0; short lastTransactionMarker = -1; using (_stats.Storage.Start()) { foreach (var item in GetDocsConflictsTombstonesRevisionsAndAttachmentsAfter(documentsContext, _lastEtag, _stats)) { if (lastTransactionMarker != item.TransactionMarker) { lastTransactionMarker = item.TransactionMarker; // Include the attachment's document which is right after its latest attachment. if ((item.Type == ReplicationBatchItem.ReplicationItemType.Document || item.Type == ReplicationBatchItem.ReplicationItemType.DocumentTombstone) && // We want to limit batch sizes to reasonable limits. ((maxSizeToSend.HasValue && size > maxSizeToSend.Value.GetValue(SizeUnit.Bytes)) || (batchSize.HasValue && numberOfItemsSent > batchSize.Value))) { break; } } _stats.Storage.RecordInputAttempt(); _lastEtag = item.Etag; if (item.Data != null) { size += item.Data.Size; } else if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment) { size += item.Stream.Length; } if (AddReplicationItemToBatch(item, _stats.Storage)) { numberOfItemsSent++; } } } if (_log.IsInfoEnabled) { _log.Info($"Found {_orderedReplicaItems.Count:#,#;;0} documents and {_replicaAttachmentStreams.Count} attachment's streams to replicate to {_parent.Node.FromString()}."); } if (_orderedReplicaItems.Count == 0) { var hasModification = _lastEtag != _parent._lastSentDocumentEtag; // ensure that the other server is aware that we skipped // on (potentially a lot of) documents to send, and we update // the last etag they have from us on the other side _parent._lastSentDocumentEtag = _lastEtag; _parent._lastDocumentSentTime = DateTime.UtcNow; _parent.SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(documentsContext)); return(hasModification); } _parent.CancellationToken.ThrowIfCancellationRequested(); try { using (_stats.Network.Start()) { SendDocumentsBatch(documentsContext, _stats.Network); } } catch (OperationCanceledException) { if (_log.IsInfoEnabled) { _log.Info("Received cancellation notification while sending document replication batch."); } throw; } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send document replication batch", e); } throw; } return(true); } finally { foreach (var item in _orderedReplicaItems) { var value = item.Value; if (value.Type == ReplicationBatchItem.ReplicationItemType.Attachment) { // TODO: Why are we disposing here? // Shouldn't the all context be disposed here? // If not, should we dispose all strings here? value.Stream.Dispose(); } else { value.Data?.Dispose(); //item.Value.Data is null if tombstone } } _orderedReplicaItems.Clear(); _replicaAttachmentStreams.Clear(); } } }
public async Task can_backup_and_restore_snapshot() { var backupPath = NewDataPath(suffix: "BackupFolder"); using (var store = GetDocumentStore()) { using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "oren" }, "users/1"); await session.SaveChangesAsync(); } using (var session = store.OpenAsyncSession()) { await session .Query <User>() .Where(x => x.Name == "oren") .ToListAsync(); // create an index to backup await session .Query <User>() .Where(x => x.Age > 20) .ToListAsync(); // create an index to backup session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup await session.SaveChangesAsync(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Snapshot, LocalSettings = new LocalSettings { FolderPath = backupPath }, IncrementalBackupFrequency = "* * * * *" //every minute }; var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId; await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); var value = WaitForValue(() => { var status = store.Maintenance.Send(operation).Status; return(status?.LastEtag); }, 3); Assert.Equal(3, value); using (var session = store.OpenAsyncSession()) { await session.StoreAsync(new User { Name = "ayende" }, "users/2"); await session.StoreAsync(new User { Name = "ayende" }, "users/3"); session.CountersFor("users/2").Increment("downloads", 200); await session.SaveChangesAsync(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); // restore the database with a different name string restoredDatabaseName = $"restored_database_snapshot-{Guid.NewGuid()}"; using (RestoreDatabase(store, new RestoreBackupConfiguration { BackupLocation = Directory.GetDirectories(backupPath).First(), DatabaseName = restoredDatabaseName })) { using (var session = store.OpenAsyncSession(restoredDatabaseName)) { var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" }); Assert.NotNull(users["users/1"]); Assert.NotNull(users["users/2"]); Assert.True(users.Any(x => x.Value.Name == "oren")); Assert.True(users.Any(x => x.Value.Name == "ayende")); var val = await session.CountersFor("users/1").GetAsync("likes"); Assert.Equal(100, val); val = await session.CountersFor("users/2").GetAsync("downloads"); Assert.Equal(200, val); } var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation()); Assert.Equal(2, stats.CountOfIndexes); var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database); var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(restoredDatabaseName); using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Equal($"A:3-{originalDatabase.DbBase64Id}, A:9-{restoredDatabase.DbBase64Id}", databaseChangeVector); } } } }
private void FillDatabaseStatistics(DatabaseStatistics stats, DocumentsOperationContext context) { using (context.OpenReadTransaction()) { var indexes = Database.IndexStore.GetIndexes().ToList(); var size = Database.GetSizeOnDisk(); stats.LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction); stats.CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context); stats.CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context); stats.CountOfDocumentsConflicts = Database.DocumentsStorage.ConflictsStorage.GetNumberOfDocumentsConflicts(context); stats.CountOfTombstones = Database.DocumentsStorage.GetNumberOfTombstones(context); stats.CountOfConflicts = Database.DocumentsStorage.ConflictsStorage.ConflictsCount; stats.SizeOnDisk = size.Data; stats.NumberOfTransactionMergerQueueOperations = Database.TxMerger.NumberOfQueuedOperations; stats.TempBuffersSizeOnDisk = size.TempBuffers; stats.CountOfCounterEntries = Database.DocumentsStorage.CountersStorage.GetNumberOfCounterEntries(context); var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context); stats.CountOfAttachments = attachments.AttachmentCount; stats.CountOfUniqueAttachments = attachments.StreamsCount; stats.CountOfIndexes = indexes.Count; var statsDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); stats.DatabaseChangeVector = statsDatabaseChangeVector; stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id; stats.Is64Bit = !Database.DocumentsStorage.Environment.Options.ForceUsing32BitsPager && IntPtr.Size == sizeof(long); stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString(); stats.Indexes = new IndexInformation[indexes.Count]; for (var i = 0; i < indexes.Count; i++) { var index = indexes[i]; bool isStale; try { isStale = index.IsStale(context); } catch (OperationCanceledException) { // if the index has just been removed, let us consider it stale // until it can be safely removed from the list of indexes in the // database isStale = true; } stats.Indexes[i] = new IndexInformation { State = index.State, IsStale = isStale, Name = index.Name, LockMode = index.Definition.LockMode, Priority = index.Definition.Priority, Type = index.Type, LastIndexingTime = index.LastIndexingTime }; if (stats.LastIndexingTime.HasValue) { stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime; } else { stats.LastIndexingTime = index.LastIndexingTime; } } } }
protected override async Task DoWork() { await WaitOrThrowOperationCanceled(_notificationCenter.Options.DatabaseStatsThrottle); Stats current; DateTime?lastIndexingErrorTime = null; using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { var indexes = _database.IndexStore.GetIndexes().ToList(); var staleIndexes = 0; var countOfIndexingErrors = 0L; // ReSharper disable once LoopCanBeConvertedToQuery foreach (var index in indexes) { if (index.IsStale(context)) { staleIndexes++; } var errorCount = index.GetErrorCount(); if (errorCount > 0) { var lastError = index.GetLastIndexingErrorTime(); if (lastError != null) { if (lastIndexingErrorTime == null || lastError > lastIndexingErrorTime) { lastIndexingErrorTime = lastError; } } } countOfIndexingErrors += errorCount; } current = new Stats { CountOfConflicts = _database.DocumentsStorage.ConflictsStorage.GetCountOfDocumentsConflicts(context), CountOfDocuments = _database.DocumentsStorage.GetNumberOfDocuments(context), CountOfIndexes = indexes.Count, CountOfStaleIndexes = staleIndexes, CountOfIndexingErrors = countOfIndexingErrors, LastEtag = DocumentsStorage.ReadLastEtag(context.Transaction.InnerTransaction), GlobalChangeVector = DocumentsStorage.GetDatabaseChangeVector(context) }; current.Collections = _database.DocumentsStorage.GetCollections(context) .ToDictionary(x => x.Name, x => new DatabaseStatsChanged.ModifiedCollection(x.Name, x.Count, _database.DocumentsStorage.GetLastDocumentChangeVector(context, x.Name))); } if (_latest != null && _latest.Equals(current)) { return; } var modifiedCollections = _latest == null?current.Collections.Values.ToList() : ExtractModifiedCollections(current); _notificationCenter.Add(DatabaseStatsChanged.Create(current.CountOfConflicts, current.CountOfDocuments, current.CountOfIndexes, current.CountOfStaleIndexes, current.GlobalChangeVector, current.LastEtag, current.CountOfIndexingErrors, lastIndexingErrorTime, modifiedCollections)); _latest = current; }
public async Task PreviewCollection() { var start = GetStart(); var pageSize = GetPageSize(); var collection = GetStringQueryString("collection", required: false); var bindings = GetStringValuesQueryString("binding", required: false); var fullBindings = GetStringValuesQueryString("fullBinding", required: false); using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (context.OpenReadTransaction()) { Document[] documents; HashSet <LazyStringValue> availableColumns; HashSet <string> propertiesPreviewToSend; HashSet <string> fullPropertiesToSend = new HashSet <string>(fullBindings); long totalResults; string changeVector; string etag = null; if (string.IsNullOrEmpty(collection)) { changeVector = DocumentsStorage.GetDatabaseChangeVector(context); totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context); etag = $"{changeVector}/{totalResults}"; } else { changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, collection); totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count; if (changeVector != null) { etag = $"{changeVector}/{totalResults}"; } } if (etag != null && GetStringFromHeaders("If-None-Match") == etag) { HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified; return; } HttpContext.Response.Headers["ETag"] = "\"" + etag + "\""; if (string.IsNullOrEmpty(collection)) { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize).ToArray(); availableColumns = ExtractColumnNames(documents, context); propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : new HashSet <string>(); } else { documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, collection, start, pageSize).ToArray(); availableColumns = ExtractColumnNames(documents, context); propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : availableColumns.Take(ColumnsSamplingLimit).Select(x => x.ToString()).ToHashSet(); } await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream())) { writer.WriteStartObject(); writer.WritePropertyName("Results"); writer.WriteStartArray(); var first = true; foreach (var document in documents) { if (first == false) { writer.WriteComma(); } first = false; using (document.Data) { WriteDocument(writer, context, document, propertiesPreviewToSend, fullPropertiesToSend); } } writer.WriteEndArray(); writer.WriteComma(); writer.WritePropertyName("TotalResults"); writer.WriteInteger(totalResults); writer.WriteComma(); writer.WriteArray("AvailableColumns", availableColumns); writer.WriteEndObject(); } } }
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx) { foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx)) { if (_token.IsCancellationRequested) { yield break; } var report = new DatabaseStatusReport { Name = dbName, NodeName = _server.NodeTag }; if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false) { var recorod = _server.Cluster.ReadDatabase(ctx, dbName); if (recorod == null || recorod.Topology.RelevantFor(_server.NodeTag) == false) { continue; // Database does not exists in this server } report.Status = DatabaseStatus.Unloaded; yield return(dbName, report); continue; } if (dbTask.IsFaulted) { var extractSingleInnerException = dbTask.Exception.ExtractSingleInnerException(); if (Equals(extractSingleInnerException.Data[DatabasesLandlord.DoNotRemove], true)) { report.Status = DatabaseStatus.Unloaded; yield return(dbName, report); continue; } } if (dbTask.IsCanceled || dbTask.IsFaulted) { report.Status = DatabaseStatus.Faulted; report.Error = dbTask.Exception.ToString(); yield return(dbName, report); continue; } if (dbTask.IsCompleted == false) { report.Status = DatabaseStatus.Loading; yield return(dbName, report); continue; } var dbInstance = dbTask.Result; var documentsStorage = dbInstance.DocumentsStorage; var indexStorage = dbInstance.IndexStore; if (dbInstance.DatabaseShutdown.IsCancellationRequested) { report.Status = DatabaseStatus.Shutdown; yield return(dbName, report); continue; } report.Status = DatabaseStatus.Loaded; try { using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var tx = context.OpenReadTransaction()) { report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction); report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount; report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context); report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); foreach (var outgoing in dbInstance.ReplicationLoader.OutgoingHandlers) { var node = outgoing.GetNode(); if (node != null) { report.LastSentEtag.Add(node, outgoing._lastSentDocumentEtag); } } if (indexStorage != null) { foreach (var index in indexStorage.GetIndexes()) { var stats = index.GetIndexStats(context); //We might have old version of this index with the same name report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus { LastIndexedEtag = stats.LastProcessedEtag, IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase), IsStale = stats.IsStale, State = index.State }; } } } } catch (Exception e) { report.Error = e.ToString(); } yield return(dbName, report); } }
public void can_backup_and_restore() { var azureSettings = GenerateAzureSettings(); InitContainer(azureSettings); using (var store = GetDocumentStore()) { using (var session = store.OpenSession()) { session.Store(new User { Name = "oren" }, "users/1"); session.CountersFor("users/1").Increment("likes", 100); session.SaveChanges(); } var config = new PeriodicBackupConfiguration { BackupType = BackupType.Backup, AzureSettings = azureSettings, IncrementalBackupFrequency = "0 0 1 1 *" }; var backupTaskId = (store.Maintenance.Send(new UpdatePeriodicBackupOperation(config))).TaskId; store.Maintenance.Send(new StartBackupOperation(true, backupTaskId)); var operation = new GetPeriodicBackupStatusOperation(backupTaskId); PeriodicBackupStatus status = null; var value = WaitForValue(() => { status = store.Maintenance.Send(operation).Status; return(status?.LastEtag); }, 4); Assert.True(4 == value, $"4 == value, Got status: {status != null}, exception: {status?.Error?.Exception}"); Assert.True(status.LastOperationId != null, $"status.LastOperationId != null, Got status: {status != null}, exception: {status?.Error?.Exception}"); var backupOperation = store.Maintenance.Send(new GetOperationStateOperation(status.LastOperationId.Value)); var backupResult = backupOperation.Result as BackupResult; Assert.True(backupResult != null && backupResult.Counters.Processed, "backupResult != null && backupResult.Counters.Processed"); Assert.True(1 == backupResult.Counters.ReadCount, "1 == backupResult.Counters.ReadCount"); using (var session = store.OpenSession()) { session.Store(new User { Name = "ayende" }, "users/2"); session.CountersFor("users/2").Increment("downloads", 200); session.SaveChanges(); } var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag; store.Maintenance.Send(new StartBackupOperation(false, backupTaskId)); value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag); Assert.Equal(lastEtag, value); // restore the database with a different name var databaseName = $"restored_database-{Guid.NewGuid()}"; azureSettings.RemoteFolderName = status.FolderName; var restoreFromGoogleCloudConfiguration = new RestoreFromAzureConfiguration() { DatabaseName = databaseName, Settings = azureSettings, DisableOngoingTasks = true }; var googleCloudOperation = new RestoreBackupOperation(restoreFromGoogleCloudConfiguration); var restoreOperation = store.Maintenance.Server.Send(googleCloudOperation); restoreOperation.WaitForCompletion(TimeSpan.FromSeconds(30)); using (var store2 = GetDocumentStore(new Options() { CreateDatabase = false, ModifyDatabaseName = s => databaseName })) { using (var session = store2.OpenSession(databaseName)) { var users = session.Load <User>(new[] { "users/1", "users/2" }); Assert.True(users.Any(x => x.Value.Name == "oren")); Assert.True(users.Any(x => x.Value.Name == "ayende")); var val = session.CountersFor("users/1").Get("likes"); Assert.Equal(100, val); val = session.CountersFor("users/2").Get("downloads"); Assert.Equal(200, val); } var originalDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database).Result; var restoredDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName).Result; using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) using (ctx.OpenReadTransaction()) { var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx); Assert.Equal($"A:7-{originalDatabase.DbBase64Id}, A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector); } } } }
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx) { foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx)) { if (_token.IsCancellationRequested) { yield break; } if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false) { continue; // Database does not exists in this server } var report = new DatabaseStatusReport { Name = dbName, NodeName = _server.NodeTag }; if (dbTask.IsCanceled || dbTask.IsFaulted) { report.Status = DatabaseStatus.Faulted; report.Error = dbTask.Exception.ToString(); yield return(dbName, report); continue; } if (dbTask.IsCompleted == false) { report.Status = DatabaseStatus.Loading; yield return(dbName, report); continue; } var dbInstance = dbTask.Result; var documentsStorage = dbInstance.DocumentsStorage; var indexStorage = dbInstance.IndexStore; if (dbInstance.DatabaseShutdown.IsCancellationRequested) { report.Status = DatabaseStatus.Shutdown; yield return(dbName, report); continue; } report.Status = DatabaseStatus.Loaded; try { using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var tx = context.OpenReadTransaction()) { report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction); report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction); report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount; report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context); report.LastChangeVector = DocumentsStorage.GetDatabaseChangeVector(context); if (indexStorage != null) { foreach (var index in indexStorage.GetIndexes()) { var stats = index.GetIndexStats(context); //We might have old version of this index with the same name report.LastIndexStats.Add(index.Name, new DatabaseStatusReport.ObservedIndexStatus { LastIndexedEtag = stats.LastProcessedEtag, IsSideBySide = false, // TODO: fix this so it get whatever this has side by side or not IsStale = stats.IsStale }); } } } } catch (Exception e) { report.Error = e.ToString(); } yield return(dbName, report); } }