public override int Execute(DocumentsOperationContext context)
{
if (string.IsNullOrEmpty(context.LastDatabaseChangeVector))
{
context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
}
var status = ChangeVectorUtils.GetConflictStatus(_replicationBatchReply.DatabaseChangeVector,
context.LastDatabaseChangeVector);
if (status != ConflictStatus.AlreadyMerged)
{
return(0);
}
var res = ChangeVectorUtils.TryUpdateChangeVector(_replicationBatchReply.NodeTag, _dbId, _replicationBatchReply.CurrentEtag, ref context.LastDatabaseChangeVector) ? 1 : 0;
if (res == 1)
{
context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += _ =>
{
try
{
_trigger.Set();
}
catch
{
//
}
};
}
return(res);
}
private (long, int) GetSizeAndOperationsCount(BatchRequestParser.CommandData commandData)
{
long size = 0;
switch (commandData.Type)
{
case CommandType.PUT:
return(commandData.Document.Size, 1);
case CommandType.Counters:
foreach (var operation in commandData.Counters.Operations)
{
size += operation.CounterName.Length
+ sizeof(long) // etag
+ sizeof(long) // counter value
+ GetChangeVectorSizeInternal() // estimated change vector size
+ 10; // estimated collection name size
}
return(size, commandData.Counters.Operations.Count);
case CommandType.AttachmentPUT:
return(commandData.ContentLength, 1);
case CommandType.TimeSeries:
case CommandType.TimeSeriesBulkInsert:
// we don't know the size of the change so we are just estimating
foreach (var append in commandData.TimeSeries.Appends)
{
size += sizeof(long); // DateTime
if (string.IsNullOrWhiteSpace(append.Tag) == false)
{
size += append.Tag.Length;
}
size += append.Values.Length * sizeof(double);
}
return(size, commandData.TimeSeries.Appends.Count);
default:
throw new ArgumentOutOfRangeException($"'{commandData.Type}' isn't supported");
}
int GetChangeVectorSizeInternal()
{
if (_changeVectorSize.HasValue)
{
return(_changeVectorSize.Value);
}
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
_changeVectorSize = Encoding.UTF8.GetBytes(databaseChangeVector).Length;
return(_changeVectorSize.Value);
}
}
}
public Task Stats()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
using (context.OpenReadTransaction())
{
var indexes = Database.IndexStore.GetIndexes().ToList();
var stats = new DatabaseStatistics
{
LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction),
CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context),
CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context)
};
var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context);
stats.CountOfAttachments = attachments.AttachmentCount;
stats.CountOfUniqueAttachments = attachments.StreamsCount;
stats.CountOfIndexes = indexes.Count;
var statsDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
stats.DatabaseChangeVector = statsDatabaseChangeVector;
stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id;
stats.Is64Bit = IntPtr.Size == sizeof(long);
stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString();
stats.Indexes = new IndexInformation[indexes.Count];
for (var i = 0; i < indexes.Count; i++)
{
var index = indexes[i];
stats.Indexes[i] = new IndexInformation
{
State = index.State,
IsStale = index.IsStale(context),
Name = index.Name,
Etag = index.Etag,
LockMode = index.Definition.LockMode,
Priority = index.Definition.Priority,
Type = index.Type,
LastIndexingTime = index.LastIndexingTime
};
if (stats.LastIndexingTime.HasValue)
{
stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime;
}
else
{
stats.LastIndexingTime = index.LastIndexingTime;
}
}
writer.WriteDatabaseStatistics(context, stats);
}
return(Task.CompletedTask);
}
private async Task GetDocumentsAsync(DocumentsOperationContext context, bool metadataOnly)
{
var sw = Stopwatch.StartNew();
// everything here operates on all docs
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
if (GetStringFromHeaders("If-None-Match") == databaseChangeVector)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return;
}
HttpContext.Response.Headers["ETag"] = "\"" + databaseChangeVector + "\"";
var etag = GetLongQueryString("etag", false);
var start = GetStart();
var pageSize = GetPageSize();
var isStartsWith = HttpContext.Request.Query.ContainsKey("startsWith");
IEnumerable <Document> documents;
if (etag != null)
{
documents = Database.DocumentsStorage.GetDocumentsFrom(context, etag.Value, start, pageSize);
}
else if (isStartsWith)
{
documents = Database.DocumentsStorage.GetDocumentsStartingWith(context,
HttpContext.Request.Query["startsWith"],
HttpContext.Request.Query["matches"],
HttpContext.Request.Query["exclude"],
HttpContext.Request.Query["startAfter"],
start,
pageSize);
}
else // recent docs
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize);
}
int numberOfResults;
long totalDocumentsSizeInBytes;
using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream(), Database.DatabaseShutdown))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
(numberOfResults, totalDocumentsSizeInBytes) = await writer.WriteDocumentsAsync(context, documents, metadataOnly);
writer.WriteEndObject();
await writer.OuterFlushAsync();
}
AddPagingPerformanceHint(PagingOperationType.Documents, isStartsWith ? nameof(DocumentsStorage.GetDocumentsStartingWith) : nameof(GetDocumentsAsync), HttpContext.Request.QueryString.Value, numberOfResults, pageSize, sw.ElapsedMilliseconds, totalDocumentsSizeInBytes);
}
public override int Execute(DocumentsOperationContext context)
{
if (string.IsNullOrEmpty(context.LastDatabaseChangeVector))
{
context.LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
}
var status = ChangeVectorUtils.GetConflictStatus(_replicationBatchReply.DatabaseChangeVector,
context.LastDatabaseChangeVector);
if (status != ConflictStatus.AlreadyMerged)
{
return(0);
}
var result = ChangeVectorUtils.TryUpdateChangeVector(_replicationBatchReply.NodeTag, _dbId, _replicationBatchReply.CurrentEtag, context.LastDatabaseChangeVector);
if (result.IsValid)
{
if (context.LastReplicationEtagFrom == null)
{
context.LastReplicationEtagFrom = new Dictionary <string, long>();
}
if (context.LastReplicationEtagFrom.ContainsKey(_replicationBatchReply.DatabaseId) == false)
{
context.LastReplicationEtagFrom[_replicationBatchReply.DatabaseId] = _replicationBatchReply.CurrentEtag;
}
context.LastDatabaseChangeVector = result.ChangeVector;
context.Transaction.InnerTransaction.LowLevelTransaction.OnDispose += _ =>
{
try
{
_trigger.Set();
}
catch
{
//
}
};
}
return(result.IsValid ? 1 : 0);
}
protected async Task<bool> WaitForChangeVectorInClusterAsync(List<RavenServer> nodes, string database, int timeout = 15000)
{
return await WaitForValueAsync(async () =>
{
var cvs = new List<string>();
foreach (var server in nodes)
{
var storage = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
using (storage.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
cvs.Add(DocumentsStorage.GetDatabaseChangeVector(context));
}
}
return cvs.Any(x => x != cvs.FirstOrDefault()) == false;
}, true, timeout: timeout, interval: 333);
}
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.ReplicationHubCertificates))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name))
{
LastRaftIndex = rawRecord.EtagForBackup;
}
}
var disposable = new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
});
return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.Build)));
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name))
{
LastRaftIndex = rawRecord.GetEtagForBackup();
}
}
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
}));
}
private static void FillDocumentsInfo(DatabaseStatusReport prevDatabaseReport, DocumentDatabase dbInstance, DatabaseStatusReport report,
DocumentsOperationContext context, DocumentsStorage documentsStorage)
{
if (prevDatabaseReport?.LastTransactionId != null && prevDatabaseReport.LastTransactionId == dbInstance.LastTransactionId)
{
report.LastEtag = prevDatabaseReport.LastEtag;
report.LastTombstoneEtag = prevDatabaseReport.LastTombstoneEtag;
report.NumberOfConflicts = prevDatabaseReport.NumberOfConflicts;
report.NumberOfDocuments = prevDatabaseReport.NumberOfDocuments;
report.DatabaseChangeVector = prevDatabaseReport.DatabaseChangeVector;
}
else
{
using (var tx = context.OpenReadTransaction())
{
report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction);
report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount;
report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context);
report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
}
}
}
protected async Task can_backup_and_restore_internal()
{
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
session.CountersFor("users/1").Increment("likes", 100);
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 0 1 1 *"
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus status = null;
var value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, expectedVal: 4, timeout: 30_000);
Assert.True(4 == value, $"gotStatus? {status != null}, Status Error: {status?.Error?.Exception}," +
$" S3 Error: {status?.UploadToS3?.Exception}, LocalBackup Exception: {status?.LocalBackup?.Exception}");
Assert.True(status.LastOperationId != null, $"status.LastOperationId != null, Got status: {status != null}, exception: {status?.Error?.Exception}");
var backupOperation = store.Maintenance.Send(new GetOperationStateOperation(status.LastOperationId.Value));
var backupResult = backupOperation.Result as BackupResult;
Assert.True(backupResult != null && backupResult.Counters.Processed, "backupResult != null && backupResult.Counters.Processed");
Assert.True(1 == backupResult.Counters.ReadCount, "1 == backupResult.Counters.ReadCount");
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, expectedVal: lastEtag, timeout: 30_000);
Assert.True(lastEtag == value, $"gotStatus? {status != null}, Status Error: {status?.Error?.Exception}," +
$" S3 Error: {status?.UploadToS3?.Exception}, LocalBackup Exception: {status?.LocalBackup?.Exception}");
// restore the database with a different name
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(status.FolderName);
using (RestoreDatabaseFromCloud(
store,
new RestoreFromS3Configuration {
DatabaseName = databaseName, Settings = subfolderS3Settings, DisableOngoingTasks = true
},
TimeSpan.FromSeconds(60)))
{
using (var session = store.OpenAsyncSession(databaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:7-{originalDatabase.DbBase64Id}, A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
private void ReplicateToDestination()
{
try
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiate);
NativeMemory.EnsureRegistered();
if (_log.IsInfoEnabled)
{
_log.Info($"Will replicate to {Destination.FromString()} via {_connectionInfo.Url}");
}
using (_parent._server.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (context.OpenReadTransaction())
{
var record = _parent.LoadDatabaseRecord();
if (record == null)
{
throw new InvalidOperationException($"The database record for {_parent.Database.Name} does not exist?!");
}
if (record.Encrypted && Destination.Url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false)
{
throw new InvalidOperationException(
$"{record.DatabaseName} is encrypted, and require HTTPS for replication, but had endpoint with url {Destination.Url} to database {Destination.Database}");
}
}
var task = TcpUtils.ConnectSocketAsync(_connectionInfo, _parent._server.Engine.TcpConnectionTimeout, _log);
task.Wait(CancellationToken);
using (Interlocked.Exchange(ref _tcpClient, task.Result))
{
var wrapSsl = TcpUtils.WrapStreamWithSslAsync(_tcpClient, _connectionInfo, _parent._server.Server.Certificate.Certificate, _parent._server.Engine.TcpConnectionTimeout);
wrapSsl.Wait(CancellationToken);
using (_stream = wrapSsl.Result) // note that _stream is being disposed by the interruptible read
using (_interruptibleRead = new InterruptibleRead(_database.DocumentsStorage.ContextPool, _stream))
using (_buffer = JsonOperationContext.ManagedPinnedBuffer.LongLivedInstance())
{
var documentSender = new ReplicationDocumentSender(_stream, this, _log);
WriteHeaderToRemotePeer();
//handle initial response to last etag and staff
try
{
var response = HandleServerResponse(getFullResponse: true);
switch (response.ReplyType)
{
//The first time we start replication we need to register the destination current CV
case ReplicationMessageReply.ReplyType.Ok:
LastAcceptedChangeVector = response.Reply.DatabaseChangeVector;
break;
case ReplicationMessageReply.ReplyType.Error:
var exception = new InvalidOperationException(response.Reply.Exception);
if (response.Reply.Exception.Contains(nameof(DatabaseDoesNotExistException)) ||
response.Reply.Exception.Contains(nameof(DatabaseNotRelevantException)))
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, "Database does not exist");
DatabaseDoesNotExistException.ThrowWithMessageAndException(Destination.Database, response.Reply.Message, exception);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, $"Got error: {response.Reply.Exception}");
throw exception;
}
}
catch (DatabaseDoesNotExistException e)
{
var msg = $"Failed to parse initial server replication response, because there is no database named {_database.Name} " +
"on the other end. ";
if (_external)
{
msg += "In order for the replication to work, a database with the same name needs to be created at the destination";
}
var young = (DateTime.UtcNow - _startedAt).TotalSeconds < 30;
if (young)
{
msg += "This can happen if the other node wasn't yet notified about being assigned this database and should be resolved shortly.";
}
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
// won't add an alert on young connections
// because it may take a few seconds for the other side to be notified by
// the cluster that it has this db.
if (young == false)
{
AddAlertOnFailureToReachOtherSide(msg, e);
}
throw;
}
catch (OperationCanceledException e)
{
const string msg = "Got operation canceled notification while opening outgoing replication channel. " +
"Aborting and closing the channel.";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
throw;
}
catch (Exception e)
{
var msg = $"{OutgoingReplicationThreadName} got an unexpected exception during initial handshake";
if (_log.IsInfoEnabled)
{
_log.Info(msg, e);
}
AddReplicationPulse(ReplicationPulseDirection.OutgoingInitiateError, msg);
AddAlertOnFailureToReachOtherSide(msg, e);
throw;
}
DateTime nextReplicateAt = default(DateTime);
while (_cts.IsCancellationRequested == false)
{
while (_database.Time.GetUtcNow() > nextReplicateAt)
{
if (_parent.DebugWaitAndRunReplicationOnce != null)
{
_parent.DebugWaitAndRunReplicationOnce.Wait(_cts.Token);
_parent.DebugWaitAndRunReplicationOnce.Reset();
}
var sp = Stopwatch.StartNew();
var stats = _lastStats = new OutgoingReplicationStatsAggregator(_parent.GetNextReplicationStatsId(), _lastStats);
AddReplicationPerformance(stats);
AddReplicationPulse(ReplicationPulseDirection.OutgoingBegin);
try
{
using (var scope = stats.CreateScope())
{
try
{
if (Destination is InternalReplication dest)
{
_parent.EnsureNotDeleted(dest.NodeTag);
}
var didWork = documentSender.ExecuteReplicationOnce(scope, ref nextReplicateAt);
if (didWork == false)
{
break;
}
if (Destination is ExternalReplication externalReplication)
{
var taskId = externalReplication.TaskId;
UpdateExternalReplicationInfo(taskId);
}
DocumentsSend?.Invoke(this);
if (sp.ElapsedMilliseconds > 60 * 1000)
{
_waitForChanges.Set();
break;
}
}
catch (OperationCanceledException)
{
// cancellation is not an actual error,
// it is a "notification" that we need to cancel current operation
const string msg = "Operation was canceled.";
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, msg);
throw;
}
catch (Exception e)
{
AddReplicationPulse(ReplicationPulseDirection.OutgoingError, e.Message);
scope.AddError(e);
throw;
}
}
}
finally
{
stats.Complete();
AddReplicationPulse(ReplicationPulseDirection.OutgoingEnd);
}
}
//if this returns false, this means either timeout or canceled token is activated
while (WaitForChanges(_parent.MinimalHeartbeatInterval, _cts.Token) == false)
{
//If we got cancelled we need to break right away
if (_cts.IsCancellationRequested)
{
break;
}
// open tx
// read current change vector compare to last sent
// if okay, send cv
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (var tx = ctx.OpenReadTransaction())
{
var etag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
if (etag == _lastSentDocumentEtag)
{
SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(ctx));
_parent.CompleteDeletionIfNeeded();
}
else if (nextReplicateAt > DateTime.UtcNow)
{
SendHeartbeat(null);
}
else
{
//Send a heartbeat first so we will get an updated CV of the destination
var currentChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
SendHeartbeat(null);
//If our previous CV is already merged to the destination wait a bit more
if (ChangeVectorUtils.GetConflictStatus(LastAcceptedChangeVector, currentChangeVector) ==
ConflictStatus.AlreadyMerged)
{
continue;
}
// we have updates that we need to send to the other side
// let's do that..
// this can happen if we got replication from another node
// that we need to send to it. Note that we typically
// will wait for the other node to send the data directly to
// our destination, but if it doesn't, we'll step in.
// In this case, we try to limit congestion in the network and
// only send updates that we have gotten from someone else after
// a certain time, to let the other side tell us that it already
// got it. Note that this is merely an optimization to reduce network
// traffic. It is fine to have the same data come from different sources.
break;
}
}
}
_waitForChanges.Reset();
}
}
}
}
catch (AggregateException e)
{
if (e.InnerExceptions.Count == 1)
{
if (e.InnerException is OperationCanceledException oce)
{
HandleOperationCancelException(oce);
}
if (e.InnerException is IOException ioe)
{
HandleIOException(ioe);
}
}
HandleException(e);
}
catch (OperationCanceledException e)
{
HandleOperationCancelException(e);
}
catch (IOException e)
{
HandleIOException(e);
}
catch (Exception e)
{
HandleException(e);
}
void HandleOperationCancelException(OperationCanceledException e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Operation canceled on replication thread ({FromToString}). " +
$"This is not necessary due to an issue. Stopped the thread.");
}
if (_cts.IsCancellationRequested == false)
{
Failed?.Invoke(this, e);
}
}
void HandleIOException(IOException e)
{
if (_log.IsInfoEnabled)
{
if (e.InnerException is SocketException)
{
_log.Info($"SocketException was thrown from the connection to remote node ({FromToString}). " +
$"This might mean that the remote node is done or there is a network issue.", e);
}
else
{
_log.Info($"IOException was thrown from the connection to remote node ({FromToString}).", e);
}
}
Failed?.Invoke(this, e);
}
void HandleException(Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Unexpected exception occurred on replication thread ({FromToString}). " +
$"Replication stopped (will be retried later).", e);
}
Failed?.Invoke(this, e);
}
}
public Task GetCollectionFields()
{
var collection = GetStringQueryString("collection", required: false);
var prefix = GetStringQueryString("prefix", required: false);
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
long totalResults;
string changeVector;
string etag = null;
if (string.IsNullOrEmpty(collection))
{
changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
etag = $"{changeVector}/{totalResults}";
}
else
{
changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, collection);
totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;
if (changeVector != null)
{
etag = $"{changeVector}/{totalResults}";
}
}
if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return(Task.CompletedTask);
}
HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";
var fields = new Dictionary <LazyStringValue, FieldType>(LazyStringValueComparer.Instance);
if (string.IsNullOrEmpty(collection))
{
foreach (var collectionStats in Database.DocumentsStorage.GetCollections(context))
{
FetchFieldsForCollection(context, collectionStats.Name, prefix, fields);
}
}
else
{
FetchFieldsForCollection(context, collection, prefix, fields);
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
var first = true;
foreach (var field in fields)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
writer.WritePropertyName(field.Key);
writer.WriteString(field.Value.ToString());
}
writer.WriteEndObject();
}
return(Task.CompletedTask);
}
}
public async Task can_backup_and_restore()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
session.CountersFor("users/1").Increment("likes", 100);
await session.SaveChangesAsync();
}
var googleCloudSettings = GetGoogleCloudSettings();
var config = Backup.CreateBackupConfiguration(googleCloudSettings: googleCloudSettings);
var backupTaskId = Backup.UpdateConfigAndRunBackup(Server, config, store);
var backupResult = (BackupResult)store.Maintenance.Send(new GetOperationStateOperation(Backup.GetBackupOperationId(store, backupTaskId))).Result;
Assert.NotNull(backupResult);
Assert.True(backupResult.Counters.Processed);
Assert.Equal(1, backupResult.Counters.ReadCount);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
var status = Backup.RunBackupAndReturnStatus(Server, backupTaskId, store, isFullBackup: false, expectedEtag: lastEtag);
// restore the database with a different name
var databaseName = $"restored_database-{Guid.NewGuid()}";
var subfolderGoogleCloudSettings = GetGoogleCloudSettings(status.FolderName);
var restoreFromGoogleCloudConfiguration = new RestoreFromGoogleCloudConfiguration
{
DatabaseName = databaseName,
Settings = subfolderGoogleCloudSettings,
DisableOngoingTasks = true
};
using (Backup.RestoreDatabaseFromCloud(store, restoreFromGoogleCloudConfiguration, TimeSpan.FromSeconds(30)))
{
using (var session = store.OpenAsyncSession(databaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Contains($"A:7-{originalDatabase.DbBase64Id}", databaseChangeVector);
Assert.Contains($"A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
public async Task can_backup_and_restore_snapshot()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
await session
.Query <User>()
.Where(x => x.Name == "oren")
.ToListAsync(); // create an index to backup
await session
.Query <Order>()
.Where(x => x.Freight > 20)
.ToListAsync(); // create an index to backup
session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup
await session.SaveChangesAsync();
}
var googleCloudSettings = GetGoogleCloudSettings();
var config = Backup.CreateBackupConfiguration(backupType: BackupType.Snapshot, googleCloudSettings: googleCloudSettings);
var backupTaskId = Backup.UpdateConfigAndRunBackup(Server, config, store);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
var status = Backup.RunBackupAndReturnStatus(Server, backupTaskId, store, isFullBackup: false, expectedEtag: lastEtag);
// restore the database with a different name
string databaseName = $"restored_database_snapshot-{Guid.NewGuid()}";
var subfolderGoogleCloudSettings = GetGoogleCloudSettings(status.FolderName);
var restoreFromGoogleCloudConfiguration = new RestoreFromGoogleCloudConfiguration
{
DatabaseName = databaseName,
Settings = subfolderGoogleCloudSettings
};
using (Backup.RestoreDatabaseFromCloud(store, restoreFromGoogleCloudConfiguration, TimeSpan.FromSeconds(300)))
{
using (var session = store.OpenAsyncSession(databaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.NotNull(users["users/1"]);
Assert.NotNull(users["users/2"]);
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(2, stats.CountOfIndexes);
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Contains($"A:8-{originalDatabase.DbBase64Id}", databaseChangeVector);
Assert.Contains($"A:10-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
public bool ExecuteReplicationOnce(TcpConnectionOptions tcpConnectionOptions, OutgoingReplicationStatsScope stats, ref long next)
{
EnsureValidStats(stats);
var wasInterrupted = false;
var delay = GetDelayReplication();
var currentNext = next;
using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
try
{
// we scan through the documents to send to the other side, we need to be careful about
// filtering a lot of documents, because we need to let the other side know about this, and
// at the same time, we need to send a heartbeat to keep the tcp connection alive
_lastEtag = _parent._lastSentDocumentEtag;
_parent.CancellationToken.ThrowIfCancellationRequested();
var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo();
long prevLastEtag = _lastEtag;
var replicationState = new ReplicationState
{
BatchSize = _parent._database.Configuration.Replication.MaxItemsCount,
MaxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend,
CurrentNext = currentNext,
Delay = delay,
Context = documentsContext,
LastTransactionMarker = -1,
NumberOfItemsSent = 0,
Size = 0L,
MissingTxMarkers = new HashSet <short>()
};
using (_stats.Storage.Start())
{
foreach (var item in GetReplicationItems(_parent._database, documentsContext, _lastEtag, _stats, _parent.SupportedFeatures.Replication.CaseInsensitiveCounters))
{
_parent.CancellationToken.ThrowIfCancellationRequested();
if (replicationState.LastTransactionMarker != item.TransactionMarker)
{
replicationState.Item = item;
if (CanContinueBatch(replicationState, ref next) == false)
{
wasInterrupted = true;
break;
}
replicationState.LastTransactionMarker = item.TransactionMarker;
}
_stats.Storage.RecordInputAttempt();
// here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundary
if (MissingAttachmentsInLastBatch &&
item.Type == ReplicationBatchItem.ReplicationItemType.Document &&
item is DocumentReplicationItem docItem &&
docItem.Flags.Contain(DocumentFlags.HasAttachments))
{
var type = (docItem.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision: AttachmentType.Document;
foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, docItem.Id, docItem.ChangeVector))
{
// we need to filter attachments that are been sent in the same batch as the document
if (attachment.Etag >= prevLastEtag)
{
if (attachment.TransactionMarker != item.TransactionMarker)
{
replicationState.MissingTxMarkers.Add(attachment.TransactionMarker);
}
continue;
}
var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash);
attachment.Stream = stream;
var attachmentItem = AttachmentReplicationItem.From(documentsContext, attachment);
AddReplicationItemToBatch(attachmentItem, _stats.Storage, skippedReplicationItemsInfo);
replicationState.Size += attachmentItem.Size;
}
}
_lastEtag = item.Etag;
if (AddReplicationItemToBatch(item, _stats.Storage, skippedReplicationItemsInfo) == false)
{
// this item won't be needed anymore
item.Dispose();
continue;
}
replicationState.Size += item.Size;
replicationState.NumberOfItemsSent++;
}
}
if (_log.IsInfoEnabled)
{
if (skippedReplicationItemsInfo.SkippedItems > 0)
{
var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector);
_log.Info(message);
}
var msg = $"Found {_orderedReplicaItems.Count:#,#;;0} documents " +
$"and {_replicaAttachmentStreams.Count} attachment's streams " +
$"to replicate to {_parent.Node.FromString()}, ";
var encryptionSize = documentsContext.Transaction.InnerTransaction.LowLevelTransaction.AdditionalMemoryUsageSize.GetValue(SizeUnit.Bytes);
if (encryptionSize > 0)
{
msg += $"encryption buffer overhead size is {new Size(encryptionSize, SizeUnit.Bytes)}, ";
}
msg += $"total size: {new Size(replicationState.Size + encryptionSize, SizeUnit.Bytes)}";
_log.Info(msg);
}
if (_orderedReplicaItems.Count == 0)
{
var hasModification = _lastEtag != _parent._lastSentDocumentEtag;
// ensure that the other server is aware that we skipped
// on (potentially a lot of) documents to send, and we update
// the last etag they have from us on the other side
_parent._lastSentDocumentEtag = _lastEtag;
_parent._lastDocumentSentTime = DateTime.UtcNow;
var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext);
_parent.SendHeartbeat(changeVector);
return(hasModification);
}
_parent.CancellationToken.ThrowIfCancellationRequested();
try
{
using (_stats.Network.Start())
{
SendDocumentsBatch(documentsContext, _stats.Network);
tcpConnectionOptions._lastEtagSent = _lastEtag;
tcpConnectionOptions.RegisterBytesSent(replicationState.Size);
if (MissingAttachmentsInLastBatch)
{
return(false);
}
}
}
catch (OperationCanceledException)
{
if (_log.IsInfoEnabled)
{
_log.Info("Received cancellation notification while sending document replication batch.");
}
throw;
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Failed to send document replication batch", e);
}
throw;
}
MissingAttachmentsInLastBatch = false;
return(true);
}
finally
{
foreach (var item in _orderedReplicaItems)
{
item.Value.Dispose();
}
_orderedReplicaItems.Clear();
_replicaAttachmentStreams.Clear();
}
}
}
public bool ExecuteReplicationOnce(OutgoingReplicationStatsScope stats, ref DateTime next)
{
EnsureValidStats(stats);
var wasInterrupted = false;
var delay = GetDelayReplication();
using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
try
{
// we scan through the documents to send to the other side, we need to be careful about
// filtering a lot of documents, because we need to let the other side know about this, and
// at the same time, we need to send a heartbeat to keep the tcp connection alive
_lastEtag = _parent._lastSentDocumentEtag;
_parent.CancellationToken.ThrowIfCancellationRequested();
var batchSize = _parent._database.Configuration.Replication.MaxItemsCount;
var maxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend;
long size = 0;
var numberOfItemsSent = 0;
var skippedReplicationItemsInfo = new SkippedReplicationItemsInfo();
short lastTransactionMarker = -1;
long prevLastEtag = _lastEtag;
using (_stats.Storage.Start())
{
foreach (var item in GetReplicationItems(documentsContext, _lastEtag, _stats))
{
if (lastTransactionMarker != item.TransactionMarker)
{
if (delay.Ticks > 0)
{
var nextReplication = item.LastModifiedTicks + delay.Ticks;
if (_parent._database.Time.GetUtcNow().Ticks < nextReplication)
{
next = new DateTime(nextReplication);
wasInterrupted = true;
break;
}
}
lastTransactionMarker = item.TransactionMarker;
if (_parent.SupportedFeatures.Replication.Counters == false)
{
AssertNotCounterForLegacyReplication(item);
}
if (_parent.SupportedFeatures.Replication.ClusterTransaction == false)
{
AssertNotClusterTransactionDocumentForLegacyReplication(item);
}
// Include the attachment's document which is right after its latest attachment.
if ((item.Type == ReplicationBatchItem.ReplicationItemType.Document ||
item.Type == ReplicationBatchItem.ReplicationItemType.DocumentTombstone) &&
// We want to limit batch sizes to reasonable limits.
((maxSizeToSend.HasValue && size > maxSizeToSend.Value.GetValue(SizeUnit.Bytes)) ||
(batchSize.HasValue && numberOfItemsSent > batchSize.Value)))
{
wasInterrupted = true;
break;
}
if (_stats.Storage.CurrentStats.InputCount % 16384 == 0)
{
// ReSharper disable once PossibleLossOfFraction
if ((_parent._parent.MinimalHeartbeatInterval / 2) < _stats.Storage.Duration.TotalMilliseconds)
{
wasInterrupted = true;
break;
}
}
}
_stats.Storage.RecordInputAttempt();
//Here we add missing attachments in the same batch as the document that contains them without modifying the last etag or transaction boundry
if (MissingAttachmentsInLastBatch &&
item.Type == ReplicationBatchItem.ReplicationItemType.Document &&
(item.Flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
var type = (item.Flags & DocumentFlags.Revision) == DocumentFlags.Revision ? AttachmentType.Revision: AttachmentType.Document;
foreach (var attachment in _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentsForDocument(documentsContext, type, item.Id))
{
//We need to filter attachments that are been sent in the same batch as the document
if (attachment.Etag >= prevLastEtag)
{
continue;
}
var stream = _parent._database.DocumentsStorage.AttachmentsStorage.GetAttachmentStream(documentsContext, attachment.Base64Hash);
attachment.Stream = stream;
AddReplicationItemToBatch(ReplicationBatchItem.From(attachment), _stats.Storage, skippedReplicationItemsInfo);
size += attachment.Stream.Length;
}
}
_lastEtag = item.Etag;
if (item.Data != null)
{
size += item.Data.Size;
}
else if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment)
{
size += item.Stream.Length;
}
if (AddReplicationItemToBatch(item, _stats.Storage, skippedReplicationItemsInfo) == false)
{
continue;
}
numberOfItemsSent++;
}
}
if (_log.IsInfoEnabled)
{
if (skippedReplicationItemsInfo.SkippedItems > 0)
{
var message = skippedReplicationItemsInfo.GetInfoForDebug(_parent.LastAcceptedChangeVector);
_log.Info(message);
}
_log.Info($"Found {_orderedReplicaItems.Count:#,#;;0} documents and {_replicaAttachmentStreams.Count} attachment's streams to replicate to {_parent.Node.FromString()}.");
}
if (_orderedReplicaItems.Count == 0)
{
var hasModification = _lastEtag != _parent._lastSentDocumentEtag;
// ensure that the other server is aware that we skipped
// on (potentially a lot of) documents to send, and we update
// the last etag they have from us on the other side
_parent._lastSentDocumentEtag = _lastEtag;
_parent._lastDocumentSentTime = DateTime.UtcNow;
var changeVector = wasInterrupted ? null : DocumentsStorage.GetDatabaseChangeVector(documentsContext);
_parent.SendHeartbeat(changeVector);
return(hasModification);
}
_parent.CancellationToken.ThrowIfCancellationRequested();
try
{
using (_stats.Network.Start())
{
SendDocumentsBatch(documentsContext, _stats.Network);
if (MissingAttachmentsInLastBatch)
{
return(false);
}
}
}
catch (OperationCanceledException)
{
if (_log.IsInfoEnabled)
{
_log.Info("Received cancellation notification while sending document replication batch.");
}
throw;
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Failed to send document replication batch", e);
}
throw;
}
MissingAttachmentsInLastBatch = false;
return(true);
}
finally
{
foreach (var item in _orderedReplicaItems)
{
var value = item.Value;
if (value.Type == ReplicationBatchItem.ReplicationItemType.Attachment)
{
value.Stream.Dispose();
}
else
{
value.Data?.Dispose(); //item.Value.Data is null if tombstone
}
}
_orderedReplicaItems.Clear();
_replicaAttachmentStreams.Clear();
}
}
}
public async Task AddGlobalChangeVectorToNewDocument(bool useSsl)
{
var clusterSize = 3;
var databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(clusterSize, true, 0, useSsl : useSsl);
X509Certificate2 clientCertificate = null;
X509Certificate2 adminCertificate = null;
if (useSsl)
{
var certificates = GenerateAndSaveSelfSignedCertificate();
adminCertificate = RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate1.Value, new Dictionary <string, DatabaseAccess>(), SecurityClearance.ClusterAdmin, server: leader);
clientCertificate = RegisterClientCertificate(certificates.ServerCertificate.Value, certificates.ClientCertificate2.Value, new Dictionary <string, DatabaseAccess>
{
[databaseName] = DatabaseAccess.Admin
}, server: leader);
}
var doc = new DatabaseRecord(databaseName);
using (var store = new DocumentStore()
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Certificate = adminCertificate,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
var topology = databaseResult.Topology;
Assert.Equal(clusterSize, topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Karmel" }, "users/1");
await session.SaveChangesAsync();
}
// we need to wait for database change vector to be updated
// which means that we need to wait for replication to do a full mesh propagation
Assert.True(await WaitForValueOnGroupAsync(topology, serverStore =>
{
var database = serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName).Result;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var cv = DocumentsStorage.GetDatabaseChangeVector(context);
return(cv != null && cv.Contains("A:1-") && cv.Contains("B:1-") && cv.Contains("C:1-"));
}
}, expected: true, timeout: 60000));
public async Task can_create_azure_snapshot_and_restore_using_restore_point()
{
using (var holder = new Azure.AzureClientHolder(AzureFactAttribute.AzureSettings))
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "oren"
}, "users/1");
session.CountersFor("users/1").Increment("likes", 100);
session.SaveChanges();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Snapshot,
AzureSettings = holder.Settings,
IncrementalBackupFrequency = "0 0 1 1 *"
};
var backupTaskId = (store.Maintenance.Send(new UpdatePeriodicBackupOperation(config))).TaskId;
store.Maintenance.Send(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus status = null;
var value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, 4);
Assert.True(4 == value, $"4 == value, Got status: {status != null}, exception: {status?.Error?.Exception}");
Assert.True(status.LastOperationId != null, $"status.LastOperationId != null, Got status: {status != null}, exception: {status?.Error?.Exception}");
var client = store.GetRequestExecutor().HttpClient;
var data = new StringContent(JsonConvert.SerializeObject(holder.Settings), Encoding.UTF8, "application/json");
var response = await client.PostAsync(store.Urls.First() + "/admin/restore/points?type=Azure ", data);
string result = response.Content.ReadAsStringAsync().Result;
var restorePoints = JsonConvert.DeserializeObject <RestorePoints>(result);
Assert.Equal(1, restorePoints.List.Count);
var point = restorePoints.List.First();
var databaseName = $"restored_database-{Guid.NewGuid()}";
holder.Settings.RemoteFolderName = holder.Settings.RemoteFolderName + "/" + status.FolderName;
var restoreOperation = await store.Maintenance.Server.SendAsync(new RestoreBackupOperation(new RestoreFromAzureConfiguration()
{
DatabaseName = databaseName,
Settings = holder.Settings,
DisableOngoingTasks = true,
LastFileNameToRestore = point.FileName,
}));
await restoreOperation.WaitForCompletionAsync(TimeSpan.FromSeconds(60));
using (var store2 = GetDocumentStore(new Options()
{
CreateDatabase = false, ModifyDatabaseName = s => databaseName
}))
{
using (var session = store2.OpenSession(databaseName))
{
var users = session.Load <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
var val = session.CountersFor("users/1").Get("likes");
Assert.Equal(100, val);
}
var originalDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database).Result;
var restoredDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName).Result;
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:4-{originalDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
}
public async Task can_backup_and_restore()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
session.CountersFor("users/1").Increment("likes", 100);
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
GoogleCloudSettings = GoogleCloudFact.GoogleCloudSettings,
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, 4);
Assert.Equal(4, value);
var backupStatus = store.Maintenance.Send(operation);
var backupOperationId = backupStatus.Status.LastOperationId;
var backupOperation = store.Maintenance.Send(new GetOperationStateOperation(backupOperationId.Value));
var backupResult = backupOperation.Result as BackupResult;
Assert.True(backupResult.Counters.Processed);
Assert.Equal(1, backupResult.Counters.ReadCount);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
// restore the database with a different name
var databaseName = $"restored_database-{Guid.NewGuid()}";
GoogleCloudFact.GoogleCloudSettings.RemoteFolderName = $"{backupStatus.Status.FolderName}";
var restoreFromGoogleCloudConfiguration = new RestoreFromGoogleCloudConfiguration()
{
DatabaseName = databaseName,
Settings = GoogleCloudFact.GoogleCloudSettings
};
var googleCloudOperation = new RestoreBackupOperation(restoreFromGoogleCloudConfiguration);
var restoreOperation = store.Maintenance.Server.Send(googleCloudOperation);
restoreOperation.WaitForCompletion(TimeSpan.FromSeconds(30));
{
using (var session = store.OpenAsyncSession(databaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:7-{originalDatabase.DbBase64Id}, A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
public void AcceptIncomingConnection(TcpConnectionOptions tcpConnectionOptions)
{
ReplicationLatestEtagRequest getLatestEtagMessage;
using (tcpConnectionOptions.ContextPool.AllocateOperationContext(out JsonOperationContext context))
using (var readerObject = context.ParseToMemory(
tcpConnectionOptions.Stream,
"IncomingReplication/get-last-etag-message read",
BlittableJsonDocumentBuilder.UsageMode.None,
tcpConnectionOptions.PinnedBuffer))
{
getLatestEtagMessage = JsonDeserializationServer.ReplicationLatestEtagRequest(readerObject);
if (_log.IsInfoEnabled)
{
_log.Info(
$"GetLastEtag: {getLatestEtagMessage.SourceTag}({getLatestEtagMessage.SourceMachineName}) / {getLatestEtagMessage.SourceDatabaseName} ({getLatestEtagMessage.SourceDatabaseId}) - {getLatestEtagMessage.SourceUrl}");
}
}
var connectionInfo = IncomingConnectionInfo.FromGetLatestEtag(getLatestEtagMessage);
try
{
AssertValidConnection(connectionInfo);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Connection from [{connectionInfo}] is rejected.", e);
}
var incomingConnectionRejectionInfos = _incomingRejectionStats.GetOrAdd(connectionInfo,
_ => new ConcurrentQueue <IncomingConnectionRejectionInfo>());
incomingConnectionRejectionInfos.Enqueue(new IncomingConnectionRejectionInfo {
Reason = e.ToString()
});
try
{
tcpConnectionOptions.Dispose();
}
catch
{
// do nothing
}
throw;
}
try
{
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsOperationContext))
using (Database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext))
using (var writer = new BlittableJsonTextWriter(documentsOperationContext, tcpConnectionOptions.Stream))
using (documentsOperationContext.OpenReadTransaction())
using (configurationContext.OpenReadTransaction())
{
var changeVector = DocumentsStorage.GetDatabaseChangeVector(documentsOperationContext);
var lastEtagFromSrc = Database.DocumentsStorage.GetLastReplicateEtagFrom(
documentsOperationContext, getLatestEtagMessage.SourceDatabaseId);
if (_log.IsInfoEnabled)
{
_log.Info($"GetLastEtag response, last etag: {lastEtagFromSrc}");
}
var response = new DynamicJsonValue
{
[nameof(ReplicationMessageReply.Type)] = "Ok",
[nameof(ReplicationMessageReply.MessageType)] = ReplicationMessageType.Heartbeat,
[nameof(ReplicationMessageReply.LastEtagAccepted)] = lastEtagFromSrc,
[nameof(ReplicationMessageReply.NodeTag)] = _server.NodeTag,
[nameof(ReplicationMessageReply.DatabaseChangeVector)] = changeVector
};
documentsOperationContext.Write(writer, response);
writer.Flush();
}
}
catch (Exception)
{
try
{
tcpConnectionOptions.Dispose();
}
catch (Exception)
{
// do nothing
}
throw;
}
var newIncoming = new IncomingReplicationHandler(
tcpConnectionOptions,
getLatestEtagMessage,
this);
newIncoming.Failed += OnIncomingReceiveFailed;
newIncoming.DocumentsReceived += OnIncomingReceiveSucceeded;
if (_log.IsInfoEnabled)
{
_log.Info(
$"Initialized document replication connection from {connectionInfo.SourceDatabaseName} located at {connectionInfo.SourceUrl}");
}
// need to safeguard against two concurrent connection attempts
var newConnection = _incoming.GetOrAdd(newIncoming.ConnectionInfo.SourceDatabaseId, newIncoming);
if (newConnection == newIncoming)
{
newIncoming.Start();
IncomingReplicationAdded?.Invoke(newIncoming);
ForceTryReconnectAll();
}
else
{
newIncoming.Dispose();
}
}
protected async Task can_backup_and_restore_snapshot_internal()
{
var defaultS3Settings = GetS3Settings();
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
await session
.Query <User>()
.Where(x => x.Name == "oren")
.ToListAsync(); // create an index to backup
await session
.Query <Order>()
.Where(x => x.Freight > 20)
.ToListAsync(); // create an index to backup
session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Snapshot,
S3Settings = defaultS3Settings,
IncrementalBackupFrequency = "0 0 1 1 *"
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus backupStatus = null;
var value = WaitForValue(() =>
{
backupStatus = store.Maintenance.Send(operation).Status;
return(backupStatus?.LastEtag);
}, expectedVal: 4, timeout: 30_000);
Assert.True(4 == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," +
$" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}");
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() =>
{
backupStatus = store.Maintenance.Send(operation).Status;
return(backupStatus?.LastEtag);
}, expectedVal: lastEtag, timeout: 30_000);
Assert.True(lastEtag == value, $"gotStatus? {backupStatus != null}, Status Error: {backupStatus?.Error?.Exception}," +
$" S3 Error: {backupStatus?.UploadToS3?.Exception}, LocalBackup Exception: {backupStatus?.LocalBackup?.Exception}");
// restore the database with a different name
string restoredDatabaseName = $"restored_database_snapshot-{Guid.NewGuid()}";
var subfolderS3Settings = GetS3Settings(backupStatus.FolderName);
using (RestoreDatabaseFromCloud(store,
new RestoreFromS3Configuration
{
DatabaseName = restoredDatabaseName,
Settings = subfolderS3Settings
},
TimeSpan.FromSeconds(60)))
{
using (var session = store.OpenAsyncSession(restoredDatabaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.NotNull(users["users/1"]);
Assert.NotNull(users["users/2"]);
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(2, stats.CountOfIndexes);
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(restoredDatabaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:8-{originalDatabase.DbBase64Id}, A:10-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
public async Task DatabaseChangeVectorIsUpdatedCorrectly()
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new CreateSampleDataOperation());
var documentDatabase = (await Databases.GetDocumentDatabaseInstanceFor(store));
var documentsStorage = documentDatabase.DocumentsStorage;
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var lastAttachmentChangeVector = string.Empty;
var lastRevisionChangeVector = string.Empty;
var attachmentStorage = documentDatabase.DocumentsStorage.AttachmentsStorage;
foreach (var document in documentsStorage.GetDocumentsFrom(context, etag: 0))
{
VerifyAttachments();
VerifyRevisions();
void VerifyAttachments()
{
if (document.Data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) == false)
{
return;
}
if (metadata.TryGet(Constants.Documents.Metadata.Attachments, out BlittableJsonReaderArray attachments) == false)
{
return;
}
foreach (BlittableJsonReaderObject attachmentFromDocument in attachments)
{
attachmentFromDocument.TryGet(nameof(AttachmentName.Name), out string attachmentName);
var attachment = attachmentStorage.GetAttachment(context, document.Id, attachmentName, AttachmentType.Document, null);
Assert.True(document.Etag > attachment.Etag);
var conflictStatus = ChangeVectorUtils.GetConflictStatus(document.ChangeVector, attachment.ChangeVector);
Assert.Equal(ConflictStatus.Update, conflictStatus);
var attachmentConflictStatus = ChangeVectorUtils.GetConflictStatus(attachment.ChangeVector, lastAttachmentChangeVector);
if (attachmentConflictStatus == ConflictStatus.Update)
{
lastAttachmentChangeVector = attachment.ChangeVector;
}
}
}
void VerifyRevisions()
{
var revisions = documentsStorage.RevisionsStorage.GetRevisions(context, document.Id, 0, int.MaxValue);
foreach (var revision in revisions.Revisions)
{
var conflictStatus = ChangeVectorUtils.GetConflictStatus(revision.ChangeVector, lastRevisionChangeVector);
if (conflictStatus == ConflictStatus.Update)
{
lastRevisionChangeVector = revision.ChangeVector;
}
}
}
}
var lastChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
var lastCvAttachmentConflictStatus = ChangeVectorUtils.GetConflictStatus(lastAttachmentChangeVector, lastChangeVector);
Assert.Equal(ConflictStatus.AlreadyMerged, lastCvAttachmentConflictStatus);
var lastCvRevisionConflictStatus = ChangeVectorUtils.GetConflictStatus(lastRevisionChangeVector, lastChangeVector);
Assert.Equal(ConflictStatus.AlreadyMerged, lastCvRevisionConflictStatus);
}
}
}
public bool ExecuteReplicationOnce(OutgoingReplicationStatsScope stats)
{
EnsureValidStats(stats);
using (_parent._database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
try
{
// we scan through the documents to send to the other side, we need to be careful about
// filtering a lot of documents, because we need to let the other side know about this, and
// at the same time, we need to send a heartbeat to keep the tcp connection alive
_lastEtag = _parent._lastSentDocumentEtag;
_parent.CancellationToken.ThrowIfCancellationRequested();
var batchSize = _parent._database.Configuration.Replication.MaxItemsCount;
var maxSizeToSend = _parent._database.Configuration.Replication.MaxSizeToSend;
long size = 0;
int numberOfItemsSent = 0;
short lastTransactionMarker = -1;
using (_stats.Storage.Start())
{
foreach (var item in GetDocsConflictsTombstonesRevisionsAndAttachmentsAfter(documentsContext, _lastEtag, _stats))
{
if (lastTransactionMarker != item.TransactionMarker)
{
lastTransactionMarker = item.TransactionMarker;
// Include the attachment's document which is right after its latest attachment.
if ((item.Type == ReplicationBatchItem.ReplicationItemType.Document ||
item.Type == ReplicationBatchItem.ReplicationItemType.DocumentTombstone) &&
// We want to limit batch sizes to reasonable limits.
((maxSizeToSend.HasValue && size > maxSizeToSend.Value.GetValue(SizeUnit.Bytes)) ||
(batchSize.HasValue && numberOfItemsSent > batchSize.Value)))
{
break;
}
}
_stats.Storage.RecordInputAttempt();
_lastEtag = item.Etag;
if (item.Data != null)
{
size += item.Data.Size;
}
else if (item.Type == ReplicationBatchItem.ReplicationItemType.Attachment)
{
size += item.Stream.Length;
}
if (AddReplicationItemToBatch(item, _stats.Storage))
{
numberOfItemsSent++;
}
}
}
if (_log.IsInfoEnabled)
{
_log.Info($"Found {_orderedReplicaItems.Count:#,#;;0} documents and {_replicaAttachmentStreams.Count} attachment's streams to replicate to {_parent.Node.FromString()}.");
}
if (_orderedReplicaItems.Count == 0)
{
var hasModification = _lastEtag != _parent._lastSentDocumentEtag;
// ensure that the other server is aware that we skipped
// on (potentially a lot of) documents to send, and we update
// the last etag they have from us on the other side
_parent._lastSentDocumentEtag = _lastEtag;
_parent._lastDocumentSentTime = DateTime.UtcNow;
_parent.SendHeartbeat(DocumentsStorage.GetDatabaseChangeVector(documentsContext));
return(hasModification);
}
_parent.CancellationToken.ThrowIfCancellationRequested();
try
{
using (_stats.Network.Start())
{
SendDocumentsBatch(documentsContext, _stats.Network);
}
}
catch (OperationCanceledException)
{
if (_log.IsInfoEnabled)
{
_log.Info("Received cancellation notification while sending document replication batch.");
}
throw;
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Failed to send document replication batch", e);
}
throw;
}
return(true);
}
finally
{
foreach (var item in _orderedReplicaItems)
{
var value = item.Value;
if (value.Type == ReplicationBatchItem.ReplicationItemType.Attachment)
{
// TODO: Why are we disposing here?
// Shouldn't the all context be disposed here?
// If not, should we dispose all strings here?
value.Stream.Dispose();
}
else
{
value.Data?.Dispose(); //item.Value.Data is null if tombstone
}
}
_orderedReplicaItems.Clear();
_replicaAttachmentStreams.Clear();
}
}
}
public async Task can_backup_and_restore_snapshot()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var store = GetDocumentStore())
{
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "oren" }, "users/1");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
await session
.Query <User>()
.Where(x => x.Name == "oren")
.ToListAsync(); // create an index to backup
await session
.Query <User>()
.Where(x => x.Age > 20)
.ToListAsync(); // create an index to backup
session.CountersFor("users/1").Increment("likes", 100); //create a counter to backup
await session.SaveChangesAsync();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Snapshot,
LocalSettings = new LocalSettings
{
FolderPath = backupPath
},
IncrementalBackupFrequency = "* * * * *" //every minute
};
var backupTaskId = (await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(config))).TaskId;
await store.Maintenance.SendAsync(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
var value = WaitForValue(() =>
{
var status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, 3);
Assert.Equal(3, value);
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "ayende" }, "users/2");
await session.StoreAsync(new User { Name = "ayende" }, "users/3");
session.CountersFor("users/2").Increment("downloads", 200);
await session.SaveChangesAsync();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
await store.Maintenance.SendAsync(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
// restore the database with a different name
string restoredDatabaseName = $"restored_database_snapshot-{Guid.NewGuid()}";
using (RestoreDatabase(store, new RestoreBackupConfiguration
{
BackupLocation = Directory.GetDirectories(backupPath).First(),
DatabaseName = restoredDatabaseName
}))
{
using (var session = store.OpenAsyncSession(restoredDatabaseName))
{
var users = await session.LoadAsync <User>(new[] { "users/1", "users/2" });
Assert.NotNull(users["users/1"]);
Assert.NotNull(users["users/2"]);
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = await session.CountersFor("users/1").GetAsync("likes");
Assert.Equal(100, val);
val = await session.CountersFor("users/2").GetAsync("downloads");
Assert.Equal(200, val);
}
var stats = await store.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(2, stats.CountOfIndexes);
var originalDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var restoredDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(restoredDatabaseName);
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:3-{originalDatabase.DbBase64Id}, A:9-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
private void FillDatabaseStatistics(DatabaseStatistics stats, DocumentsOperationContext context)
{
using (context.OpenReadTransaction())
{
var indexes = Database.IndexStore.GetIndexes().ToList();
var size = Database.GetSizeOnDisk();
stats.LastDocEtag = DocumentsStorage.ReadLastDocumentEtag(context.Transaction.InnerTransaction);
stats.CountOfDocuments = Database.DocumentsStorage.GetNumberOfDocuments(context);
stats.CountOfRevisionDocuments = Database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context);
stats.CountOfDocumentsConflicts = Database.DocumentsStorage.ConflictsStorage.GetNumberOfDocumentsConflicts(context);
stats.CountOfTombstones = Database.DocumentsStorage.GetNumberOfTombstones(context);
stats.CountOfConflicts = Database.DocumentsStorage.ConflictsStorage.ConflictsCount;
stats.SizeOnDisk = size.Data;
stats.NumberOfTransactionMergerQueueOperations = Database.TxMerger.NumberOfQueuedOperations;
stats.TempBuffersSizeOnDisk = size.TempBuffers;
stats.CountOfCounterEntries = Database.DocumentsStorage.CountersStorage.GetNumberOfCounterEntries(context);
var attachments = Database.DocumentsStorage.AttachmentsStorage.GetNumberOfAttachments(context);
stats.CountOfAttachments = attachments.AttachmentCount;
stats.CountOfUniqueAttachments = attachments.StreamsCount;
stats.CountOfIndexes = indexes.Count;
var statsDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
stats.DatabaseChangeVector = statsDatabaseChangeVector;
stats.DatabaseId = Database.DocumentsStorage.Environment.Base64Id;
stats.Is64Bit = !Database.DocumentsStorage.Environment.Options.ForceUsing32BitsPager && IntPtr.Size == sizeof(long);
stats.Pager = Database.DocumentsStorage.Environment.Options.DataPager.GetType().ToString();
stats.Indexes = new IndexInformation[indexes.Count];
for (var i = 0; i < indexes.Count; i++)
{
var index = indexes[i];
bool isStale;
try
{
isStale = index.IsStale(context);
}
catch (OperationCanceledException)
{
// if the index has just been removed, let us consider it stale
// until it can be safely removed from the list of indexes in the
// database
isStale = true;
}
stats.Indexes[i] = new IndexInformation
{
State = index.State,
IsStale = isStale,
Name = index.Name,
LockMode = index.Definition.LockMode,
Priority = index.Definition.Priority,
Type = index.Type,
LastIndexingTime = index.LastIndexingTime
};
if (stats.LastIndexingTime.HasValue)
{
stats.LastIndexingTime = stats.LastIndexingTime >= index.LastIndexingTime ? stats.LastIndexingTime : index.LastIndexingTime;
}
else
{
stats.LastIndexingTime = index.LastIndexingTime;
}
}
}
}
protected override async Task DoWork()
{
await WaitOrThrowOperationCanceled(_notificationCenter.Options.DatabaseStatsThrottle);
Stats current;
DateTime?lastIndexingErrorTime = null;
using (_database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var indexes = _database.IndexStore.GetIndexes().ToList();
var staleIndexes = 0;
var countOfIndexingErrors = 0L;
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var index in indexes)
{
if (index.IsStale(context))
{
staleIndexes++;
}
var errorCount = index.GetErrorCount();
if (errorCount > 0)
{
var lastError = index.GetLastIndexingErrorTime();
if (lastError != null)
{
if (lastIndexingErrorTime == null || lastError > lastIndexingErrorTime)
{
lastIndexingErrorTime = lastError;
}
}
}
countOfIndexingErrors += errorCount;
}
current = new Stats
{
CountOfConflicts = _database.DocumentsStorage.ConflictsStorage.GetCountOfDocumentsConflicts(context),
CountOfDocuments = _database.DocumentsStorage.GetNumberOfDocuments(context),
CountOfIndexes = indexes.Count,
CountOfStaleIndexes = staleIndexes,
CountOfIndexingErrors = countOfIndexingErrors,
LastEtag = DocumentsStorage.ReadLastEtag(context.Transaction.InnerTransaction),
GlobalChangeVector = DocumentsStorage.GetDatabaseChangeVector(context)
};
current.Collections = _database.DocumentsStorage.GetCollections(context)
.ToDictionary(x => x.Name, x => new DatabaseStatsChanged.ModifiedCollection(x.Name, x.Count, _database.DocumentsStorage.GetLastDocumentChangeVector(context, x.Name)));
}
if (_latest != null && _latest.Equals(current))
{
return;
}
var modifiedCollections = _latest == null?current.Collections.Values.ToList() : ExtractModifiedCollections(current);
_notificationCenter.Add(DatabaseStatsChanged.Create(current.CountOfConflicts, current.CountOfDocuments, current.CountOfIndexes,
current.CountOfStaleIndexes, current.GlobalChangeVector, current.LastEtag, current.CountOfIndexingErrors, lastIndexingErrorTime, modifiedCollections));
_latest = current;
}
public async Task PreviewCollection()
{
var start = GetStart();
var pageSize = GetPageSize();
var collection = GetStringQueryString("collection", required: false);
var bindings = GetStringValuesQueryString("binding", required: false);
var fullBindings = GetStringValuesQueryString("fullBinding", required: false);
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
Document[] documents;
HashSet <LazyStringValue> availableColumns;
HashSet <string> propertiesPreviewToSend;
HashSet <string> fullPropertiesToSend = new HashSet <string>(fullBindings);
long totalResults;
string changeVector;
string etag = null;
if (string.IsNullOrEmpty(collection))
{
changeVector = DocumentsStorage.GetDatabaseChangeVector(context);
totalResults = Database.DocumentsStorage.GetNumberOfDocuments(context);
etag = $"{changeVector}/{totalResults}";
}
else
{
changeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, collection);
totalResults = Database.DocumentsStorage.GetCollection(collection, context).Count;
if (changeVector != null)
{
etag = $"{changeVector}/{totalResults}";
}
}
if (etag != null && GetStringFromHeaders("If-None-Match") == etag)
{
HttpContext.Response.StatusCode = (int)HttpStatusCode.NotModified;
return;
}
HttpContext.Response.Headers["ETag"] = "\"" + etag + "\"";
if (string.IsNullOrEmpty(collection))
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, start, pageSize).ToArray();
availableColumns = ExtractColumnNames(documents, context);
propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : new HashSet <string>();
}
else
{
documents = Database.DocumentsStorage.GetDocumentsInReverseEtagOrder(context, collection, start, pageSize).ToArray();
availableColumns = ExtractColumnNames(documents, context);
propertiesPreviewToSend = bindings.Count > 0 ? new HashSet <string>(bindings) : availableColumns.Take(ColumnsSamplingLimit).Select(x => x.ToString()).ToHashSet();
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteStartObject();
writer.WritePropertyName("Results");
writer.WriteStartArray();
var first = true;
foreach (var document in documents)
{
if (first == false)
{
writer.WriteComma();
}
first = false;
using (document.Data)
{
WriteDocument(writer, context, document, propertiesPreviewToSend, fullPropertiesToSend);
}
}
writer.WriteEndArray();
writer.WriteComma();
writer.WritePropertyName("TotalResults");
writer.WriteInteger(totalResults);
writer.WriteComma();
writer.WriteArray("AvailableColumns", availableColumns);
writer.WriteEndObject();
}
}
}
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx)
{
foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx))
{
if (_token.IsCancellationRequested)
{
yield break;
}
var report = new DatabaseStatusReport
{
Name = dbName,
NodeName = _server.NodeTag
};
if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false)
{
var recorod = _server.Cluster.ReadDatabase(ctx, dbName);
if (recorod == null || recorod.Topology.RelevantFor(_server.NodeTag) == false)
{
continue; // Database does not exists in this server
}
report.Status = DatabaseStatus.Unloaded;
yield return(dbName, report);
continue;
}
if (dbTask.IsFaulted)
{
var extractSingleInnerException = dbTask.Exception.ExtractSingleInnerException();
if (Equals(extractSingleInnerException.Data[DatabasesLandlord.DoNotRemove], true))
{
report.Status = DatabaseStatus.Unloaded;
yield return(dbName, report);
continue;
}
}
if (dbTask.IsCanceled || dbTask.IsFaulted)
{
report.Status = DatabaseStatus.Faulted;
report.Error = dbTask.Exception.ToString();
yield return(dbName, report);
continue;
}
if (dbTask.IsCompleted == false)
{
report.Status = DatabaseStatus.Loading;
yield return(dbName, report);
continue;
}
var dbInstance = dbTask.Result;
var documentsStorage = dbInstance.DocumentsStorage;
var indexStorage = dbInstance.IndexStore;
if (dbInstance.DatabaseShutdown.IsCancellationRequested)
{
report.Status = DatabaseStatus.Shutdown;
yield return(dbName, report);
continue;
}
report.Status = DatabaseStatus.Loaded;
try
{
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var tx = context.OpenReadTransaction())
{
report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction);
report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount;
report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context);
report.DatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
foreach (var outgoing in dbInstance.ReplicationLoader.OutgoingHandlers)
{
var node = outgoing.GetNode();
if (node != null)
{
report.LastSentEtag.Add(node, outgoing._lastSentDocumentEtag);
}
}
if (indexStorage != null)
{
foreach (var index in indexStorage.GetIndexes())
{
var stats = index.GetIndexStats(context);
//We might have old version of this index with the same name
report.LastIndexStats[index.Name] = new DatabaseStatusReport.ObservedIndexStatus
{
LastIndexedEtag = stats.LastProcessedEtag,
IsSideBySide = index.Name.StartsWith(Constants.Documents.Indexing.SideBySideIndexNamePrefix, StringComparison.OrdinalIgnoreCase),
IsStale = stats.IsStale,
State = index.State
};
}
}
}
}
catch (Exception e)
{
report.Error = e.ToString();
}
yield return(dbName, report);
}
}
public void can_backup_and_restore()
{
var azureSettings = GenerateAzureSettings();
InitContainer(azureSettings);
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "oren"
}, "users/1");
session.CountersFor("users/1").Increment("likes", 100);
session.SaveChanges();
}
var config = new PeriodicBackupConfiguration
{
BackupType = BackupType.Backup,
AzureSettings = azureSettings,
IncrementalBackupFrequency = "0 0 1 1 *"
};
var backupTaskId = (store.Maintenance.Send(new UpdatePeriodicBackupOperation(config))).TaskId;
store.Maintenance.Send(new StartBackupOperation(true, backupTaskId));
var operation = new GetPeriodicBackupStatusOperation(backupTaskId);
PeriodicBackupStatus status = null;
var value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
return(status?.LastEtag);
}, 4);
Assert.True(4 == value, $"4 == value, Got status: {status != null}, exception: {status?.Error?.Exception}");
Assert.True(status.LastOperationId != null, $"status.LastOperationId != null, Got status: {status != null}, exception: {status?.Error?.Exception}");
var backupOperation = store.Maintenance.Send(new GetOperationStateOperation(status.LastOperationId.Value));
var backupResult = backupOperation.Result as BackupResult;
Assert.True(backupResult != null && backupResult.Counters.Processed, "backupResult != null && backupResult.Counters.Processed");
Assert.True(1 == backupResult.Counters.ReadCount, "1 == backupResult.Counters.ReadCount");
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "ayende"
}, "users/2");
session.CountersFor("users/2").Increment("downloads", 200);
session.SaveChanges();
}
var lastEtag = store.Maintenance.Send(new GetStatisticsOperation()).LastDocEtag;
store.Maintenance.Send(new StartBackupOperation(false, backupTaskId));
value = WaitForValue(() => store.Maintenance.Send(operation).Status.LastEtag, lastEtag);
Assert.Equal(lastEtag, value);
// restore the database with a different name
var databaseName = $"restored_database-{Guid.NewGuid()}";
azureSettings.RemoteFolderName = status.FolderName;
var restoreFromGoogleCloudConfiguration = new RestoreFromAzureConfiguration()
{
DatabaseName = databaseName,
Settings = azureSettings,
DisableOngoingTasks = true
};
var googleCloudOperation = new RestoreBackupOperation(restoreFromGoogleCloudConfiguration);
var restoreOperation = store.Maintenance.Server.Send(googleCloudOperation);
restoreOperation.WaitForCompletion(TimeSpan.FromSeconds(30));
using (var store2 = GetDocumentStore(new Options()
{
CreateDatabase = false,
ModifyDatabaseName = s => databaseName
}))
{
using (var session = store2.OpenSession(databaseName))
{
var users = session.Load <User>(new[] { "users/1", "users/2" });
Assert.True(users.Any(x => x.Value.Name == "oren"));
Assert.True(users.Any(x => x.Value.Name == "ayende"));
var val = session.CountersFor("users/1").Get("likes");
Assert.Equal(100, val);
val = session.CountersFor("users/2").Get("downloads");
Assert.Equal(200, val);
}
var originalDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database).Result;
var restoredDatabase = Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName).Result;
using (restoredDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx))
using (ctx.OpenReadTransaction())
{
var databaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(ctx);
Assert.Equal($"A:7-{originalDatabase.DbBase64Id}, A:8-{restoredDatabase.DbBase64Id}", databaseChangeVector);
}
}
}
}
private IEnumerable <(string name, DatabaseStatusReport report)> CollectDatabaseInformation(TransactionOperationContext ctx)
{
foreach (var dbName in _server.Cluster.GetDatabaseNames(ctx))
{
if (_token.IsCancellationRequested)
{
yield break;
}
if (_server.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var dbTask) == false)
{
continue; // Database does not exists in this server
}
var report = new DatabaseStatusReport
{
Name = dbName,
NodeName = _server.NodeTag
};
if (dbTask.IsCanceled || dbTask.IsFaulted)
{
report.Status = DatabaseStatus.Faulted;
report.Error = dbTask.Exception.ToString();
yield return(dbName, report);
continue;
}
if (dbTask.IsCompleted == false)
{
report.Status = DatabaseStatus.Loading;
yield return(dbName, report);
continue;
}
var dbInstance = dbTask.Result;
var documentsStorage = dbInstance.DocumentsStorage;
var indexStorage = dbInstance.IndexStore;
if (dbInstance.DatabaseShutdown.IsCancellationRequested)
{
report.Status = DatabaseStatus.Shutdown;
yield return(dbName, report);
continue;
}
report.Status = DatabaseStatus.Loaded;
try
{
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var tx = context.OpenReadTransaction())
{
report.LastEtag = DocumentsStorage.ReadLastEtag(tx.InnerTransaction);
report.LastTombstoneEtag = DocumentsStorage.ReadLastTombstoneEtag(tx.InnerTransaction);
report.NumberOfConflicts = documentsStorage.ConflictsStorage.ConflictsCount;
report.NumberOfDocuments = documentsStorage.GetNumberOfDocuments(context);
report.LastChangeVector = DocumentsStorage.GetDatabaseChangeVector(context);
if (indexStorage != null)
{
foreach (var index in indexStorage.GetIndexes())
{
var stats = index.GetIndexStats(context);
//We might have old version of this index with the same name
report.LastIndexStats.Add(index.Name, new DatabaseStatusReport.ObservedIndexStatus
{
LastIndexedEtag = stats.LastProcessedEtag,
IsSideBySide = false, // TODO: fix this so it get whatever this has side by side or not
IsStale = stats.IsStale
});
}
}
}
}
catch (Exception e)
{
report.Error = e.ToString();
}
yield return(dbName, report);
}
}
