public async Task MissingRevisions4()
{
var(nodes, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore
{
Database = database,
Urls = new[] { leader.WebUrl }
}.Initialize())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
PurgeOnDelete = false,
MinimumRevisionsToKeep = 30
}
};
long index;
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertCommandToBlittable(configuration, context);
(index, _) = await leader.ServerStore.ModifyDatabaseRevisions(context, database, configurationJson, Guid.NewGuid().ToString());
}
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(15));
using (var session = store.OpenSession(new SessionOptions {
TransactionMode = TransactionMode.ClusterWide
}))
{
session.Store(new User()
{
Name = "userT1"
}, "users/1");
session.Delete("users/2");
session.SaveChanges();
await WaitForDocumentInClusterAsync <User>(nodes, database, "users/1", null, TimeSpan.FromSeconds(15));
}
var revisionCountList = new List <long>();
foreach (var server in Servers)
{
var documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
using (documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
revisionCountList.Add(documentDatabase.DocumentsStorage.RevisionsStorage.GetRevisionsCount(context, "users/1"));
}
}
Assert.Equal(revisionCountList[0], revisionCountList[1]);
Assert.Equal(revisionCountList[0], revisionCountList[2]);
}
}
public static async Task <long> SetupRevisions(Raven.Server.ServerWide.ServerStore serverStore, string database, bool purgeOnDelete = true, long minimumRevisionsToKeep = 123)
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 5,
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Disabled = false,
PurgeOnDelete = purgeOnDelete,
MinimumRevisionsToKeep = minimumRevisionsToKeep
},
["Comments"] = new RevisionsCollectionConfiguration
{
Disabled = true
},
["Products"] = new RevisionsCollectionConfiguration
{
Disabled = true
},
}
};
return(await SetupRevisions(serverStore, database, configuration));
}
private async Task SetupRevisions(RavenServer server, string defaultDatabase)
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 10,
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 10
}
}
};
var documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(defaultDatabase);
var res = await documentDatabase.ServerStore.ModifyDatabaseRevisions(context,
defaultDatabase,
DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(configuration, context), Guid.NewGuid().ToString());
foreach (var s in Servers)// need to wait for it on all servers
{
documentDatabase = await s.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(defaultDatabase);
await documentDatabase.RachisLogIndexNotifications.WaitForIndexNotification(res.Item1, s.ServerStore.Engine.OperationTimeout);
}
}
}
public CasinoTest(string orchestratorUrl) : base(orchestratorUrl, "CorruptedCasinoTest", "Karmel")
{
try
{
new BetsIndex().Execute(DocumentStore);
}
catch
{
// ignore
}
// ConfigureExpiration
var configureExpirationOperationResult = DocumentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(new ExpirationConfiguration
{
Disabled = false,
DeleteFrequencyInSec = 600
})).Result;
// ConfigureRevisions
var config = new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
PurgeOnDelete = true,
MinimumRevisionAgeToKeep = TimeSpan.FromSeconds(180)
},
["Bets"] = new RevisionsCollectionConfiguration()
}
};
var configureRevisionsOperationResult = DocumentStore.Maintenance.SendAsync(new ConfigureRevisionsOperation(config)).Result;
}
private static async Task SetupRevisionsForTest(RavenServer server, DocumentStore store, string database)
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration {
Disabled = false, MinimumRevisionsToKeep = int.MaxValue
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration {
Disabled = false, MinimumRevisionsToKeep = int.MaxValue
}
}
};
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertCommandToBlittable(configuration, context);
var(index, _) = await server.ServerStore.ModifyDatabaseRevisions(context, store.Database, configurationJson, Guid.NewGuid().ToString());
var documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await documentDatabase.RachisLogIndexNotifications.WaitForIndexNotification(index, server.ServerStore.Engine.OperationTimeout);
}
}
public void Should_not_allow_having_two_revision_collection_configurations_with_collection_name_that_differs_only_in_casing()
{
using (var store = GetDocumentStore())
{
var configuration = new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
{
"Users", new RevisionsCollectionConfiguration
{
PurgeOnDelete = true
}
},
{
"users", new RevisionsCollectionConfiguration
{
MinimumRevisionsToKeep = 5
}
}
}
};
var ex = Assert.Throws <RavenException>(() =>
store.Maintenance.Send(new ConfigureRevisionsOperation(configuration)));
Assert.Contains("Cannot have two different revision configurations on the same collection", ex.Message);
}
}
public async Task DeleteDocumentAndRevisions()
{
var user = new User {
Name = "Raven"
};
var id = "user/1";
using (var store = GetDocumentStore())
{
await RevisionsHelper.SetupRevisions(Server.ServerStore, store.Database);
using (var session = store.OpenSession())
{
session.Store(user, id);
session.SaveChanges();
}
using (var session = store.OpenAsyncSession())
{
user = await session.LoadAsync <User>(id);
user.Age = 10;
await session.StoreAsync(user);
await session.SaveChangesAsync();
var metadata = session.Advanced.GetMetadataFor(user);
Assert.Equal(DocumentFlags.HasRevisions.ToString(), metadata.GetString(Constants.Documents.Metadata.Flags));
var revisions = await session.Advanced.Revisions.GetForAsync <User>(id);
Assert.Equal(2, revisions.Count);
var configuration = new RevisionsConfiguration()
{
Default = null
};
await RevisionsHelper.SetupRevisions(Server.ServerStore, store.Database, configuration);
session.Delete(id);
await session.SaveChangesAsync();
}
using (var session = store.OpenSession())
{
session.Store(user, id);
session.SaveChanges();
await RevisionsHelper.SetupRevisions(Server.ServerStore, store.Database);
}
using (var session = store.OpenAsyncSession())
{
user = await session.LoadAsync <User>(id);
var revisions = await session.Advanced.Revisions.GetForAsync <User>(id);
Assert.Equal(0, revisions.Count);
}
}
}
private void WriteRevisions(RevisionsConfiguration revisions)
{
if (revisions == null)
{
_writer.WriteNull();
return;
}
_context.Write(_writer, revisions.ToJson());
}
public async Task EnforceEmptyConfigurationWillDeleteAllRevisions()
{
using (var store = GetDocumentStore())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration {
Disabled = false, MinimumRevisionsToKeep = 10
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Dummy"] = new RevisionsCollectionConfiguration {
Disabled = false, MinimumRevisionsToKeep = 10
}
}
};
await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new Product(), "foo/bar");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
var product = await session.LoadAsync <Product>("foo/bar");
var metadata = session.Advanced.GetMetadataFor(product);
Assert.Equal((DocumentFlags.HasRevisions).ToString(), metadata[Constants.Documents.Metadata.Flags]);
var bar = await session.Advanced.Revisions.GetMetadataForAsync("foo/bar");
Assert.Equal(1, bar.Count);
}
configuration.Default = null;
await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration));
var db = await GetDocumentDatabaseInstanceFor(store);
using (var token = new OperationCancelToken(db.Configuration.Databases.OperationTimeout.AsTimeSpan, db.DatabaseShutdown))
await db.DocumentsStorage.RevisionsStorage.EnforceConfiguration(_ => { }, token);
using (var session = store.OpenAsyncSession())
{
var product = await session.LoadAsync <Product>("foo/bar");
var metadata = session.Advanced.GetMetadataFor(product);
Assert.False(metadata.Keys.Contains(Constants.Documents.Metadata.Flags));
var foo = await session.Advanced.Revisions.GetMetadataForAsync("foo/bar");
Assert.Equal(0, foo.Count);
}
}
}
public void InitializeFromDatabaseRecord(DatabaseRecord dbRecord)
{
try
{
var revisions = dbRecord.Revisions;
if (revisions == null ||
(revisions.Default == null && revisions.Collections.Count == 0))
{
Configuration = null;
return;
}
if (revisions.Equals(Configuration))
{
return;
}
Configuration = revisions;
using (var tx = _database.DocumentsStorage.Environment.WriteTransaction())
{
foreach (var collection in Configuration.Collections)
{
if (collection.Value.Disabled)
{
continue;
}
EnsureRevisionTableCreated(tx, new CollectionName(collection.Key));
}
CreateTrees(tx);
tx.Commit();
}
if (_logger.IsInfoEnabled)
{
_logger.Info("Revisions configuration changed");
}
}
catch (Exception e)
{
var msg = "Cannot enable revisions for documents as the revisions configuration " +
"in the database record is missing or not valid.";
_database.NotificationCenter.Add(AlertRaised.Create(
_database.Name,
$"Revisions error in {_database.Name}", msg,
AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, _database.Name));
if (_logger.IsOperationsEnabled)
{
_logger.Operations(msg, e);
}
}
}
public async Task ZeroMinimumRevisionsToKeepShouldWork()
{
using (var store = GetDocumentStore())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 10
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 0
}
}
};
await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User(), "foo");
await session.StoreAsync(new Product(), "bar");
await session.SaveChangesAsync();
}
using (var session = store.OpenAsyncSession())
{
var user = await session.LoadAsync <User>("foo");
var metadata = session.Advanced.GetMetadataFor(user);
Assert.False(metadata.Keys.Contains(Constants.Documents.Metadata.Flags));
var foo = await session.Advanced.Revisions.GetMetadataForAsync("foo");
Assert.Equal(0, foo.Count);
var product = await session.LoadAsync <Product>("bar");
metadata = session.Advanced.GetMetadataFor(product);
Assert.Equal((DocumentFlags.HasRevisions).ToString(), metadata[Constants.Documents.Metadata.Flags]);
var bar = await session.Advanced.Revisions.GetMetadataForAsync("bar");
Assert.Equal(1, bar.Count);
}
}
}
public RevisionsStorage(DocumentDatabase database, Transaction tx)
{
_database = database;
_documentsStorage = _database.DocumentsStorage;
_logger = LoggingSource.Instance.GetLogger <RevisionsStorage>(database.Name);
Operations = new RevisionsOperations(_database);
ConflictConfiguration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
MinimumRevisionAgeToKeep = TimeSpan.FromDays(45),
Disabled = false
}
};
CreateTrees(tx);
}
private static async Task ConfigureRevisions()
{
var config = new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
PurgeOnDelete = true,
MinimumRevisionAgeToKeep = TimeSpan.FromSeconds(180)
},
["Bets"] = new RevisionsCollectionConfiguration()
}
};
await Store.Maintenance.SendAsync(new ConfigureRevisionsOperation(config)).ConfigureAwait(false);
}
private void WriteRevisions(RevisionsConfiguration revisions)
{
if (revisions == null)
{
_writer.WriteNull();
return;
}
_writer.WriteStartObject();
_writer.WritePropertyName(nameof(revisions.Default));
WriteRevisionsCollectionConfiguration(revisions.Default);
_writer.WriteComma();
_writer.WritePropertyName(nameof(revisions.Collections));
if (revisions.Collections == null)
{
_writer.WriteNull();
}
else
{
_writer.WriteStartObject();
var first = true;
foreach (var collection in revisions.Collections)
{
if (first == false)
{
_writer.WriteComma();
}
first = false;
_writer.WritePropertyName(collection.Key);
WriteRevisionsCollectionConfiguration(collection.Value);
}
_writer.WriteEndObject();
}
_writer.WriteEndObject();
}
public IActionResult Run()
{
#region Demo
#region Step_1
RevisionsConfiguration myRevisionsConfiguration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false
}
};
ConfigureRevisionsOperation revisionsConfigurationOperation = new ConfigureRevisionsOperation(myRevisionsConfiguration);
DocumentStoreHolder.Store.Maintenance.Send(revisionsConfigurationOperation);
#endregion
List <Company> revisions;
using (IDocumentSession session = DocumentStoreHolder.Store.OpenSession())
{
#region Step_2
Company company = session.Load <Company>("companies/7-A");
company.Name = "Name 1";
session.CountersFor("companies/7-A").Increment("MyCounter", 100);
session.SaveChanges();
company.Name = "Name 2";
company.Phone = "052-1234-567";
session.SaveChanges();
#endregion
#region Step_3
revisions = session
.Advanced
.Revisions
.GetFor <Company>("companies/7-A", start: 0, pageSize: 25);
#endregion
}
#endregion
return(Ok(revisions));
}
public static async Task <long> SetupRevisions(Raven.Server.ServerWide.ServerStore serverStore, string database, Action <RevisionsConfiguration> modifyConfiguration = null)
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 5
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Disabled = false,
PurgeOnDelete = true,
MinimumRevisionsToKeep = 123
},
["People"] = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 10
},
["Comments"] = new RevisionsCollectionConfiguration
{
Disabled = true
},
["Products"] = new RevisionsCollectionConfiguration
{
Disabled = true
}
}
};
modifyConfiguration?.Invoke(configuration);
var index = await SetupRevisions(serverStore, database, configuration);
var documentDatabase = await serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
await documentDatabase.RachisLogIndexNotifications.WaitForIndexNotification(index, serverStore.Engine.OperationTimeout);
return(index);
}
public async Task DeleteDocumentWithoutResolvedFlagAfterEnableRevisions()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User {
Name = "Toli"
}, "users/1");
session.SaveChanges();
}
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 5
}
};
var database = await GetDatabase(store.Database);
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(configuration, context);
var(index, _) = await database.ServerStore.ModifyDatabaseRevisions(context, store.Database, configurationJson, Guid.NewGuid().ToString());
await database.RachisLogIndexNotifications.WaitForIndexNotification(index, database.ServerStore.Engine.OperationTimeout);
}
using (var session = store.OpenSession())
{
session.Delete("users/1");
session.SaveChanges();
}
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
context.OpenReadTransaction();
var count = database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context);
Assert.Equal(2, count);
}
}
}
public async Task DeleteDocumentWithResolvedFlagAfterEnableRevisions()
{
using (var store = GetDocumentStore())
{
using (var stream = GetDump("RavenDB_15898.1.ravendbdump"))
{
var operation = await store.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions(), stream);
await operation.WaitForCompletionAsync(TimeSpan.FromMinutes(1));
}
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 5
}
};
var database = await GetDatabase(store.Database);
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(configuration, context);
var(index, _) = await database.ServerStore.ModifyDatabaseRevisions(context, store.Database, configurationJson, Guid.NewGuid().ToString());
await database.RachisLogIndexNotifications.WaitForIndexNotification(index, database.ServerStore.Engine.OperationTimeout);
}
using (var session = store.OpenSession())
{
session.Delete("users/1");
session.SaveChanges();
}
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
context.OpenReadTransaction();
var count = database.DocumentsStorage.RevisionsStorage.GetNumberOfRevisionDocuments(context);
Assert.Equal(2, count);
}
}
}
private async Task SetupRevisions(RavenServer server, string defaultDatabase)
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Active = true,
MinimumRevisionsToKeep = 10,
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Active = true,
MinimumRevisionsToKeep = 10
}
}
};
var documentDatabase = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(defaultDatabase);
var res = await documentDatabase.ServerStore.ModifyDatabaseRevisions(context,
defaultDatabase,
EntityToBlittable.ConvertEntityToBlittable(configuration,
new DocumentConventions(),
context));
foreach (var s in Servers)// need to wait for it on all servers
{
documentDatabase = await s.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(defaultDatabase);
await documentDatabase.RachisLogIndexNotifications.WaitForIndexNotification(res.Item1);
}
}
}
public static async Task SetupRevisions(IDocumentStore store, Raven.Server.ServerWide.ServerStore serverStore, RevisionsConfiguration configuration)
{
if (store == null)
{
throw new ArgumentNullException(nameof(store));
}
if (serverStore == null)
{
throw new ArgumentNullException(nameof(serverStore));
}
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
var result = await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration));
var documentDatabase = await serverStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
await documentDatabase.RachisLogIndexNotifications.WaitForIndexNotification(result.RaftCommandIndex.Value, serverStore.Engine.OperationTimeout);
}
public async Task MissingRevisions7()
{
var(nodes, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
var dbCreation = await CreateDatabaseInClusterInner(new DatabaseRecord(database), 2, leader.WebUrl, null);
using (var storeB = new DocumentStore
{
Database = database,
Urls = new[] { dbCreation.Servers[1].WebUrl },
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
using (var storeA = new DocumentStore
{
Database = database,
Urls = new[] { dbCreation.Servers[0].WebUrl },
Conventions = new DocumentConventions
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
PurgeOnDelete = false,
MinimumRevisionsToKeep = 30
}
};
long index;
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertCommandToBlittable(configuration, context);
(index, _) = await leader.ServerStore.ModifyDatabaseRevisions(context, database, configurationJson, Guid.NewGuid().ToString());
}
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(15));
await StoreInRegularMode(storeA, 3);
await WaitAndAssertForValueAsync(async() =>
{
var documentDatabase = await dbCreation.Servers[1].ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
using (documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
return(documentDatabase.DocumentsStorage.RevisionsStorage.GetRevisionsCount(context, "users/1"));
}, 3);
var result = await leader.ServerStore.SendToLeaderAsync(new DeleteDatabaseCommand(database, Guid.NewGuid().ToString())
{
HardDelete = true,
FromNodes = new[] { dbCreation.Servers[0].ServerStore.NodeTag },
});
await WaitForRaftIndexToBeAppliedInCluster(result.Index, TimeSpan.FromSeconds(10));
await WaitAndAssertForValueAsync(() =>
{
var record = storeB.Maintenance.Server.Send(new GetDatabaseRecordOperation(database));
return(Task.FromResult(record.DeletionInProgress.Count));
}, 0);
var breakRepl = await BreakReplication(dbCreation.Servers[1].ServerStore, database);
// make member directly, so it can perform cluster tx
var record = storeB.Maintenance.Server.Send(new GetDatabaseRecordOperation(database));
record.Topology.Members.Add(dbCreation.Servers[0].ServerStore.NodeTag);
await storeB.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, record.Etag));
// have a doc written by cluster tx before we get any replication
await StoreInTransactionMode(storeB, 1);
breakRepl.Mend();
var val = await WaitForValueAsync(async() => await GetMembersCount(storeB, database), 2, 20000);
Assert.Equal(2, val);
await WaitAndAssertForValueAsync(() =>
{
var record = storeB.Maintenance.Server.Send(new GetDatabaseRecordOperation(database));
return(Task.FromResult(record.DeletionInProgress.Count));
}, 0);
await WaitAndAssertForValueAsync(async() =>
{
var documentDatabase = await dbCreation.Servers[0].ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
using (documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
return(documentDatabase.DocumentsStorage.RevisionsStorage.GetRevisionsCount(context, "users/1"));
}, 4);
}
}
public async Task MissingRevisions()
{
var(nodes, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore
{
Database = database,
Urls = new[] { leader.WebUrl }
}.Initialize())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
PurgeOnDelete = false,
MinimumRevisionsToKeep = 30
}
};
long index;
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertCommandToBlittable(configuration, context);
(index, _) = await leader.ServerStore.ModifyDatabaseRevisions(context, database, configurationJson, Guid.NewGuid().ToString());
}
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(15));
await StoreInTransactionMode(store, 1);
await StoreInRegularMode(store, 10);
await DeleteInTransactionMode(store, 1);
RavenServer testServer = Servers.FirstOrDefault(server => server.ServerStore.IsLeader() == false);
var result = await DisposeServerAndWaitForFinishOfDisposalAsync(testServer);
var val = await WaitForValueAsync(async() => await GetMembersCount(store, database), 2, 20000);
Assert.Equal(2, val);
testServer = GetNewServer(new ServerCreationOptions
{
DeletePrevious = true,
RunInMemory = false,
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = result.Url
}
});
using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)))
{
await testServer.ServerStore.Engine.WaitForState(RachisState.Follower, cts.Token);
}
await WaitAndAssertForValueAsync(async() => await GetMembersCount(store, database), 3, 20000);
using (var session = store.OpenSession(new SessionOptions {
}))
{
session.Store(new User()
{
Name = "userT"
}, "users/1");
session.SaveChanges();
await WaitForDocumentInClusterAsync <User>(nodes, database, "users/1", null, TimeSpan.FromSeconds(15));
}
var documentDatabase = await testServer.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
{
var res = WaitForValue(() =>
{
using (documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
return(documentDatabase.DocumentsStorage.RevisionsStorage.GetRevisionsCount(context, "users/1"));
}, 13, 15000);
Assert.Equal(13, res);
}
}
}
private static async Task <long> SetupRevisions(Raven.Server.ServerWide.ServerStore serverStore, string database, RevisionsConfiguration configuration)
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertCommandToBlittable(configuration, context);
var(index, _) = await serverStore.ModifyDatabaseRevisions(context, database, configurationJson);
return(index);
}
}
public async Task MissingRevisions3()
{
var(nodes, leader) = await CreateRaftCluster(3, watcherCluster : true);
var database = GetDatabaseName();
await CreateDatabaseInClusterInner(new DatabaseRecord(database), 3, leader.WebUrl, null);
using (var store = new DocumentStore
{
Database = database,
Urls = new[] { leader.WebUrl }
}.Initialize())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
PurgeOnDelete = false,
MinimumRevisionsToKeep = 30
}
};
long index;
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertCommandToBlittable(configuration, context);
(index, _) = await leader.ServerStore.ModifyDatabaseRevisions(context, database, configurationJson, Guid.NewGuid().ToString());
}
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(15));
var testServer = Servers.First(server => server.ServerStore.IsLeader() == false);
await StoreInTransactionMode(store, 1);
await StoreInRegularMode(store, 3);
await DeleteInTransactionMode(store, 1);
var result = await leader.ServerStore.SendToLeaderAsync(new DeleteDatabaseCommand(database, Guid.NewGuid().ToString())
{
HardDelete = true,
FromNodes = new[] { testServer.ServerStore.NodeTag },
});
await WaitForRaftIndexToBeAppliedInCluster(result.Index, TimeSpan.FromSeconds(10));
var val = await WaitForValueAsync(async() => await GetMembersCount(store, database), 2, 20000);
Assert.Equal(2, val);
var delCount = await WaitForValueAsync(() =>
{
var record = store.Maintenance.Server.Send(new GetDatabaseRecordOperation(database));
return(record.DeletionInProgress.Count);
}, 0);
Assert.Equal(0, delCount);
await store.Maintenance.Server.SendAsync(new AddDatabaseNodeOperation(database, testServer.ServerStore.NodeTag));
var documentDatabase = await testServer.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(database);
await WaitAndAssertForValueAsync(() =>
{
using (documentDatabase.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
return(Task.FromResult(documentDatabase.DocumentsStorage.RevisionsStorage.GetRevisionsCount(context, "users/1")));
}, 5);
}
}
public async Task ForceRevisionCreationWhenRevisionConfigurationIsSet()
{
using (var store = GetDocumentStore())
{
// Define revisions settings
var configuration = new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
{
"Companies", new RevisionsCollectionConfiguration
{
PurgeOnDelete = true,
MinimumRevisionsToKeep = 5
}
}
}
};
var result = await store.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration));
var companyId = "";
using (var session = store.OpenSession())
{
var company = new Company {
Name = "HR"
};
session.Store(company);
companyId = company.Id;
session.SaveChanges();
var revisionsCount = session.Advanced.Revisions.GetFor <Company>(company.Id).Count;
Assert.Equal(1, revisionsCount); // one revision because configuration is set
session.Advanced.Revisions.ForceRevisionCreationFor <Company>(company);
session.SaveChanges();
revisionsCount = session.Advanced.Revisions.GetFor <Company>(company.Id).Count;
Assert.Equal(1, revisionsCount); // no new revision created - already exists due to configuration settings
session.Advanced.Revisions.ForceRevisionCreationFor <Company>(company);
session.SaveChanges();
company.Name = "HR V2";
session.SaveChanges();
List <Company> revisions = session.Advanced.Revisions.GetFor <Company>(companyId);
revisionsCount = revisions.Count;
Assert.Equal(2, revisionsCount);
Assert.Equal("HR V2", revisions[0].Name);
}
using (var session = store.OpenSession())
{
session.Advanced.Revisions.ForceRevisionCreationFor(companyId);
session.SaveChanges();
List <Company> revisions = session.Advanced.Revisions.GetFor <Company>(companyId);
var revisionsCount = revisions.Count;
Assert.Equal(2, revisionsCount);
Assert.Equal("HR V2", revisions[0].Name);
}
}
}
public async Task PlainRevisionsSubscriptionsCompareDocs()
{
using (var store = GetDocumentStore())
{
var subscriptionId = await store.Subscriptions.CreateAsync <Revision <User> >();
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Active = true,
MinimumRevisionsToKeep = 5,
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Active = true
},
["Dons"] = new RevisionsCollectionConfiguration
{
Active = true,
}
}
};
await Server.ServerStore.ModifyDatabaseRevisions(context,
store.Database,
EntityToBlittable.ConvertEntityToBlittable(configuration,
new DocumentConventions(),
context));
}
for (var j = 0; j < 10; j++)
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Name = $"users1 ver {j}",
Age = j
}, "users/1");
session.Store(new Company()
{
Name = $"dons1 ver {j}"
}, "dons/1");
session.SaveChanges();
}
}
using (var sub = store.Subscriptions.Open <Revision <User> >(new SubscriptionConnectionOptions(subscriptionId)))
{
var mre = new AsyncManualResetEvent();
var names = new HashSet <string>();
var maxAge = -1;
GC.KeepAlive(sub.Run(a =>
{
foreach (var item in a.Items)
{
var x = item.Result;
if (x.Current.Age > maxAge && x.Current.Age > (x.Previous?.Age ?? -1))
{
names.Add(x.Current?.Name + x.Previous?.Name);
maxAge = x.Current.Age;
}
names.Add(x.Current?.Name + x.Previous?.Name);
if (names.Count == 10)
{
mre.Set();
}
}
}));
Assert.True(await mre.WaitAsync(_reasonableWaitTime));
}
}
}
public async Task RevisionsSubscriptionsWithCustomScriptCompareDocs()
{
using (var store = GetDocumentStore())
{
var subscriptionId = await store.Subscriptions.CreateAsync(new SubscriptionCreationOptions
{
Query = @"
declare function match(d){
return d.Current.Age > d.Previous.Age;
}
from Users (Revisions = true) as d
where match(d)
select { Id: id(d.Current), Age: d.Current.Age }
"
});
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Active = true,
MinimumRevisionsToKeep = 5,
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Active = true
},
["Dons"] = new RevisionsCollectionConfiguration
{
Active = true,
}
}
};
await Server.ServerStore.ModifyDatabaseRevisions(context,
store.Database,
EntityToBlittable.ConvertEntityToBlittable(configuration,
new DocumentConventions(),
context));
}
for (int i = 0; i < 10; i++)
{
for (var j = 0; j < 10; j++)
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Name = $"users{i} ver {j}",
Age = j
}, "users/" + i);
session.Store(new Company()
{
Name = $"dons{i} ver {j}"
}, "companies/" + i);
session.SaveChanges();
}
}
}
using (var sub = store.Subscriptions.Open <Result>(new SubscriptionConnectionOptions(subscriptionId)))
{
var mre = new AsyncManualResetEvent();
var names = new HashSet <string>();
var maxAge = -1;
GC.KeepAlive(sub.Run(x =>
{
foreach (var item in x.Items)
{
if (item.Result.Age > maxAge)
{
names.Add(item.Result.Id + item.Result.Age);
maxAge = item.Result.Age;
}
if (names.Count == 9)
{
mre.Set();
}
}
}));
Assert.True(await mre.WaitAsync(_reasonableWaitTime));
}
}
}
public async Task RDBCL_801()
{
DoNotReuseServer();
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(new User
{
Name = "Karmel"
}, "users/1");
session.SaveChanges();
}
var subscriptionId = await store.Subscriptions.CreateAsync <Revision <User> >();
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configuration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
MinimumRevisionsToKeep = 5,
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Users"] = new RevisionsCollectionConfiguration
{
Disabled = false
},
["Dons"] = new RevisionsCollectionConfiguration
{
Disabled = false
}
}
};
await Server.ServerStore.ModifyDatabaseRevisions(context,
store.Database,
DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(configuration,
context), Guid.NewGuid().ToString());
}
using (var sub = store.Subscriptions.GetSubscriptionWorker <Revision <User> >(new SubscriptionWorkerOptions(subscriptionId)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(5)
}))
{
GC.KeepAlive(sub.Run(x =>
{
}));
var acks = 0;
await Task.Delay(5000);
using (Server.ServerStore.Engine.ContextPool.AllocateOperationContext(out ClusterOperationContext context))
using (context.OpenReadTransaction())
{
foreach (var entry in Server.ServerStore.Engine.LogHistory.GetHistoryLogs(context))
{
var type = entry[nameof(RachisLogHistory.LogHistoryColumn.Type)].ToString();
if (type == nameof(AcknowledgeSubscriptionBatchCommand))
{
acks++;
}
}
}
Assert.True(acks < 50, $"{acks}");
}
}
}
protected static async Task CanExportWithPulsatingReadTransaction_ActualTest(int numberOfUsers, int numberOfCountersPerUser, int numberOfRevisionsPerDocument,
int numberOfOrders, int deleteUserFactor, DocumentStore storeToExport, string file, DocumentStore storeToImport, string fileAfterDeletions,
DocumentStore storeToAfterDeletions)
{
if (numberOfRevisionsPerDocument > 0)
{
var configuration = new RevisionsConfiguration {
Default = new RevisionsCollectionConfiguration {
Disabled = false, MinimumRevisionsToKeep = 10
}
};
await storeToExport.Maintenance.SendAsync(new ConfigureRevisionsOperation(configuration));
}
using (var bulk = storeToExport.BulkInsert())
{
for (int i = 0; i < Math.Max(numberOfUsers, numberOfOrders); i++)
{
if (i < numberOfUsers)
{
bulk.Store(new User(), "users/" + i);
}
if (i < numberOfOrders)
{
bulk.Store(new Order(), "orders/" + i);
}
}
}
if (numberOfRevisionsPerDocument > 2)
{
for (int j = 0; j < numberOfRevisionsPerDocument; j++)
{
using (var bulk = storeToExport.BulkInsert())
{
for (int i = 0; i < Math.Max(numberOfUsers, numberOfOrders); i++)
{
if (i < numberOfUsers)
{
bulk.Store(new User()
{
Name = i + " " + j
}, "users/" + i);
}
if (i < numberOfOrders)
{
bulk.Store(new Order()
{
Company = i + " " + j
}, "orders/" + i);
}
}
}
}
}
using (var session = storeToExport.OpenSession())
{
for (int i = 0; i < numberOfUsers; i++)
{
for (int j = 0; j < numberOfCountersPerUser; j++)
{
session.CountersFor("users/" + i).Increment("counter/" + j, 100);
}
}
session.SaveChanges();
}
var originalStats = await storeToExport.Maintenance.SendAsync(new GetStatisticsOperation());
var options = new DatabaseSmugglerExportOptions();
var operation = await storeToExport.Smuggler.ExportAsync(options, file);
var result = await operation.WaitForCompletionAsync(TimeSpan.FromMinutes(2));
SmugglerResult.SmugglerProgress progress = ((SmugglerResult)result).Progress as SmugglerResult.SmugglerProgress;
Assert.Equal(originalStats.CountOfDocuments, progress.Documents.ReadCount);
Assert.Equal(originalStats.CountOfCounterEntries, progress.Counters.ReadCount);
Assert.Equal(originalStats.CountOfRevisionDocuments, progress.RevisionDocuments.ReadCount);
operation = await storeToImport.Smuggler.ImportAsync(new DatabaseSmugglerImportOptions(), file);
await operation.WaitForCompletionAsync(TimeSpan.FromMinutes(2));
var stats = await storeToImport.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(numberOfUsers + numberOfOrders, stats.CountOfDocuments);
Assert.Equal(numberOfUsers, stats.CountOfCounterEntries);
var expectedNumberOfRevisions = (numberOfUsers + numberOfOrders) * numberOfRevisionsPerDocument;
if (numberOfCountersPerUser > 0)
{
// if we added counters then additional revisions were created
expectedNumberOfRevisions += numberOfUsers;
}
Assert.Equal(expectedNumberOfRevisions, stats.CountOfRevisionDocuments);
// deleting some docs
var deletedUsers = 0;
using (var session = storeToExport.OpenSession())
{
for (int i = 0; i < numberOfUsers; i++)
{
if (i % deleteUserFactor != 0)
{
continue;
}
session.Delete("users/" + i);
deletedUsers++;
}
session.SaveChanges();
}
// import to new db
var originalStatsAfterDeletions = await storeToExport.Maintenance.SendAsync(new GetStatisticsOperation());
options.OperateOnTypes |= DatabaseItemType.Tombstones;
operation = await storeToExport.Smuggler.ExportAsync(options, fileAfterDeletions);
result = await operation.WaitForCompletionAsync(TimeSpan.FromMinutes(2));
progress = ((SmugglerResult)result).Progress as SmugglerResult.SmugglerProgress;
Assert.Equal(originalStatsAfterDeletions.CountOfDocuments, progress.Documents.ReadCount);
Assert.Equal(originalStatsAfterDeletions.CountOfCounterEntries, progress.Counters.ReadCount);
Assert.Equal(originalStatsAfterDeletions.CountOfRevisionDocuments, progress.RevisionDocuments.ReadCount);
Assert.Equal(originalStatsAfterDeletions.CountOfTombstones, progress.Tombstones.ReadCount);
var importOptions = new DatabaseSmugglerImportOptions();
importOptions.OperateOnTypes |= DatabaseItemType.Tombstones;
operation = await storeToAfterDeletions.Smuggler.ImportAsync(importOptions, fileAfterDeletions);
await operation.WaitForCompletionAsync(TimeSpan.FromMinutes(2));
var statsAfterDeletions = await storeToAfterDeletions.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(numberOfUsers - deletedUsers + numberOfOrders, statsAfterDeletions.CountOfDocuments);
Assert.Equal(numberOfUsers - deletedUsers, statsAfterDeletions.CountOfCounterEntries);
Assert.Equal(expectedNumberOfRevisions, statsAfterDeletions.CountOfRevisionDocuments);
Assert.Equal(deletedUsers, statsAfterDeletions.CountOfTombstones);
}
private static async Task <long> SetupRevisions(Raven.Server.ServerWide.ServerStore serverStore, string database, RevisionsConfiguration configuration)
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var configurationJson = EntityToBlittable.ConvertEntityToBlittable(configuration, DocumentConventions.Default, context);
(long etag, _) = await serverStore.ModifyDatabaseRevisions(context, database, configurationJson);
return(etag);
}
}
