public override void Execute(TransactionOperationContext context, Table items, long index, DatabaseRecord record, RachisState state, out object result)
{
var resultDict = new Dictionary <string, long>();
var identities = context.Transaction.InnerTransaction.ReadTree(ClusterStateMachine.Identities);
foreach (var kvp in Identities)
{
var itemKey = GetStorageKey(DatabaseName, kvp.Key);
using (Slice.From(context.Allocator, itemKey, out var key))
{
bool isSet;
if (Force == false)
{
isSet = identities.AddMax(key, kvp.Value);
}
else
{
identities.Add(key, kvp.Value);
isSet = true;
}
long newVal;
if (isSet)
{
newVal = kvp.Value;
}
else
{
var rc = identities.ReadLong(key);
newVal = rc ?? -1; // '-1' should not happen
}
var keyString = key.ToString().ToLowerInvariant();
resultDict.TryGetValue(keyString, out var oldVal);
resultDict[keyString] = Math.Max(oldVal, newVal);
}
}
result = resultDict;
}
public override string UpdateDatabaseRecord(DatabaseRecord record, long etag)
{
Definition.Etag = etag;
record.AddIndex(Definition);
return(null);
}
protected DocumentStore GetDocumentStore(Options options = null, [CallerMemberName] string caller = null)
{
try
{
lock (_getDocumentStoreSync)
{
options = options ?? Options.Default;
var serverToUse = options.Server ?? Server;
var name = GetDatabaseName(caller);
if (options.ModifyDatabaseName != null)
{
name = options.ModifyDatabaseName(name) ?? name;
}
var hardDelete = true;
var runInMemory = true;
var pathToUse = options.Path;
if (pathToUse == null)
{
pathToUse = NewDataPath(name);
}
else
{
hardDelete = false;
runInMemory = false;
}
var doc = new DatabaseRecord(name)
{
Settings =
{
[RavenConfiguration.GetKey(x => x.Replication.ReplicationMinimalHeartbeat)] = "1",
[RavenConfiguration.GetKey(x => x.Replication.RetryReplicateAfter)] = "1",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = runInMemory.ToString(),
[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = pathToUse,
[RavenConfiguration.GetKey(x => x.Core.ThrowIfAnyIndexCannotBeOpened)] = "true",
[RavenConfiguration.GetKey(x => x.Indexing.MinNumberOfMapAttemptsAfterWhichBatchWillBeCanceledIfRunningLowOnMemory)] = int.MaxValue.ToString()
}
};
options.ModifyDatabaseRecord?.Invoke(doc);
var store = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.ClientCertificate
};
options.ModifyDocumentStore?.Invoke(store);
//This gives too much error details in most cases, we don't need this now
store.RequestExecutorCreated += (sender, executor) =>
{
executor.AdditionalErrorInformation += sb => sb.AppendLine().Append(GetLastStatesFromAllServersOrderedByTime());
};
store.Initialize();
if (options.CreateDatabase)
{
foreach (var server in Servers)
{
using (server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
context.OpenReadTransaction();
if (server.ServerStore.Cluster.Read(context, Constants.Documents.Prefix + name) != null)
{
throw new InvalidOperationException($"Database '{name}' already exists");
}
}
}
DatabasePutResult result;
if (options.AdminCertificate != null)
{
using (var adminStore = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.AdminCertificate
}.Initialize())
{
result = adminStore.Maintenance.Server.Send(new CreateDatabaseOperation(doc, options.ReplicationFactor));
}
}
else
{
result = store.Maintenance.Server.Send(new CreateDatabaseOperation(doc, options.ReplicationFactor));
}
Assert.True(result.RaftCommandIndex > 0); //sanity check
var timeout = TimeSpan.FromMinutes(Debugger.IsAttached ? 5 : 1);
AsyncHelpers.RunSync(async() =>
{
await WaitForRaftIndexToBeAppliedInCluster(result.RaftCommandIndex, timeout);
});
}
store.BeforeDispose += (sender, args) =>
{
if (CreatedStores.TryRemove(store) == false)
{
return; // can happen if we are wrapping the store inside sharded one
}
foreach (var server in Servers)
{
if (server.Disposed)
{
continue;
}
var serverUrl = UseFiddler(server.WebUrl);
if (store.Urls.Any(url => serverUrl.Contains(url)) == false)
{
continue;
}
try
{
var databaseTask = server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(name, options.IgnoreDisabledDatabase);
if (databaseTask != null && databaseTask.IsCompleted == false)
{
// if we are disposing store before database had chance to load then we need to wait
databaseTask.Wait();
}
}
catch (DatabaseDisabledException)
{
// ignoring
}
catch (DatabaseNotRelevantException)
{
continue;
}
if (options.DeleteDatabaseOnDispose)
{
DeleteDatabaseResult result;
try
{
if (options.AdminCertificate != null)
{
using (var adminStore = new DocumentStore
{
Urls = UseFiddler(serverToUse.WebUrl),
Database = name,
Certificate = options.AdminCertificate
}.Initialize())
{
result = adminStore.Maintenance.Server.Send(new DeleteDatabasesOperation(name, hardDelete));
}
}
else
{
result = store.Maintenance.Server.Send(new DeleteDatabasesOperation(name, hardDelete));
}
}
catch (DatabaseDoesNotExistException)
{
continue;
}
catch (NoLeaderException)
{
continue;
}
}
}
};
CreatedStores.Add(store);
return(store);
}
}
catch (TimeoutException te)
{
throw new TimeoutException($"{te.Message} {Environment.NewLine} {te.StackTrace}{Environment.NewLine}Servers states:{Environment.NewLine}{GetLastStatesFromAllServersOrderedByTime()}");
}
}
public void Initialize(DatabaseRecord record)
{
LoadProcesses(record, record.RavenEtls, record.SqlEtls, toRemove: null);
}
public async Task CanPassNodeTagToRestoreBackupOperation()
{
var myBackupsList = new List <MyBackup>();
var backupPath = NewDataPath(suffix: "BackupFolder");
var clusterSize = 3;
var databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(clusterSize, false, useSsl : false);
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
await store.Maintenance.SendAsync(new CreateSampleDataOperation());
var myNodesList = databaseResult.Topology.AllNodes.ToList();
Assert.True(clusterSize == myNodesList.Count, $"clusterSize({clusterSize}) == myNodesList.Count({myNodesList.Count})");
foreach (var node in myNodesList)
{
var myGuid = Guid.NewGuid();
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = Path.Combine(backupPath, myGuid.ToString())
},
FullBackupFrequency = "0 0 1 1 *", // once a year on 1st january at 00:00
BackupType = BackupType.Backup,
Name = $"Task_{node}_{myGuid}",
MentorNode = node
};
var result = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
await WaitForRaftIndexToBeAppliedInCluster(result.RaftCommandIndex, TimeSpan.FromSeconds(15));
var res = await store.Maintenance.SendAsync(new GetOngoingTaskInfoOperation(result.TaskId, OngoingTaskType.Backup));
Assert.NotNull(res);
Assert.True(node == res.MentorNode, $"node({node}) == res.MentorNode({res.MentorNode})");
Assert.True(node == res.ResponsibleNode.NodeTag, $"node({node}) == res.ResponsibleNode.NodeTag({res.ResponsibleNode.NodeTag})");
myBackupsList.Add(new MyBackup
{
BackupTaskId = result.TaskId,
Guid = myGuid,
NodeTag = res.ResponsibleNode.NodeTag
});
}
foreach (var myBackup in myBackupsList)
{
var res = await store.Maintenance.SendAsync(new StartBackupOperation(isFullBackup : true, myBackup.BackupTaskId));
Assert.True(myBackup.NodeTag == res.Result.ResponsibleNode, $"myBackup.NodeTag({myBackup.NodeTag}) == res.ResponsibleNode({res.Result.ResponsibleNode})");
var operation = new GetPeriodicBackupStatusOperation(myBackup.BackupTaskId);
PeriodicBackupStatus status = null;
var value = WaitForValue(() =>
{
status = store.Maintenance.Send(operation).Status;
if (status?.LastEtag == null)
{
return(false);
}
return(true);
}, true);
Assert.True(value, $"Got status: {status != null}, exception: {status?.Error?.Exception}, LocalBackup.Exception: {status?.LocalBackup?.Exception}");
Assert.True(status != null, $"status != null, exception: {status?.Error?.Exception}, LocalBackup.Exception: {status?.LocalBackup?.Exception}");
Assert.True(myBackup.NodeTag == status.NodeTag, $"myBackup.NodeTag({myBackup.NodeTag}) == status.NodeTag({status.NodeTag})");
var prePath = Path.Combine(backupPath, myBackup.Guid.ToString());
myBackup.BackupPath = Path.Combine(prePath, status.FolderName);
}
var dbs = new List <string>();
foreach (var myBackup in myBackupsList)
{
var name = $"restored_DB1_{myBackup.NodeTag}";
var restoreConfig = new RestoreBackupConfiguration
{
DatabaseName = name,
BackupLocation = myBackup.BackupPath
};
dbs.Add(name);
var restoreBackupTask = store.Maintenance.Server.Send(new RestoreBackupOperation(restoreConfig, myBackup.NodeTag));
restoreBackupTask.WaitForCompletion(TimeSpan.FromSeconds(30));
}
var dbNames = await store.Maintenance.Server.SendAsync(new GetDatabaseNamesOperation(0, int.MaxValue));
Assert.Equal(clusterSize + 1, dbNames.Length);
dbs.ForEach(db => Assert.True(dbNames.Contains(db), $"numOfDbs.Contains(db), db = {db}"));
}
}
public async Task CanToggleTaskState()
{
var clusterSize = 3;
var databaseName = "TestDB";
var leader = await CreateRaftClusterAndGetLeader(clusterSize);
ModifyOngoingTaskResult addWatcherRes;
UpdatePeriodicBackupOperationResult updateBackupResult;
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
var watcher = new ExternalReplication("Watcher1", "Connection");
addWatcherRes = await AddWatcherToReplicationTopology((DocumentStore)store, watcher, new[] { "http://127.0.0.1:9090" });
var backupConfig = new PeriodicBackupConfiguration
{
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
updateBackupResult = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
}
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var taskId = addWatcherRes.TaskId;
var op = new ToggleOngoingTaskStateOperation(taskId, OngoingTaskType.Replication, true);
await store.Maintenance.SendAsync(op);
var result = await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Replication);
Assert.Equal(OngoingTaskState.Disabled, result.TaskState);
taskId = updateBackupResult.TaskId;
op = new ToggleOngoingTaskStateOperation(taskId, OngoingTaskType.Backup, false);
await store.Maintenance.SendAsync(op);
result = await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Backup);
Assert.Equal(OngoingTaskState.Enabled, result.TaskState);
}
}
public static ExpiredDocumentsCleaner LoadConfigurations(DocumentDatabase database, DatabaseRecord dbRecord, ExpiredDocumentsCleaner expiredDocumentsCleaner)
{
try
{
if (dbRecord.Expiration == null)
{
expiredDocumentsCleaner?.Dispose();
return(null);
}
if (dbRecord.Expiration.Equals(expiredDocumentsCleaner?.Configuration))
{
return(expiredDocumentsCleaner);
}
expiredDocumentsCleaner?.Dispose();
if (dbRecord.Expiration.Active == false)
{
return(null);
}
var cleaner = new ExpiredDocumentsCleaner(database, dbRecord.Expiration);
cleaner.Start();
return(cleaner);
}
catch (Exception e)
{
const string msg = "Cannot enable expired documents cleaner as the configuration record is not valid.";
database.NotificationCenter.Add(AlertRaised.Create($"Expiration error in {database.Name}", msg,
AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, database.Name));
var logger = LoggingSource.Instance.GetLogger <ExpiredDocumentsCleaner>(database.Name);
if (logger.IsOperationsEnabled)
{
logger.Operations(msg, e);
}
return(null);
}
}
public override string UpdateDatabaseRecord(DatabaseRecord record, long etag)
{
record.Expiration = Configuration;
return(null);
}
public void HandleDatabaseRecordChange(DatabaseRecord record)
{
if (record == null)
{
return;
}
var myRavenEtl = new List <RavenEtlConfiguration>();
var mySqlEtl = new List <SqlEtlConfiguration>();
foreach (var config in record.RavenEtls)
{
if (IsMyEtlTask <RavenEtlConfiguration, RavenConnectionString>(record, config))
{
myRavenEtl.Add(config);
}
}
foreach (var config in record.SqlEtls)
{
if (IsMyEtlTask <SqlEtlConfiguration, SqlConnectionString>(record, config))
{
mySqlEtl.Add(config);
}
}
var toRemove = _processes.GroupBy(x => x.ConfigurationName).ToDictionary(x => x.Key, x => x.ToList());
foreach (var processesPerConfig in _processes.GroupBy(x => x.ConfigurationName))
{
var process = processesPerConfig.First();
Debug.Assert(processesPerConfig.All(x => x.GetType() == process.GetType()));
if (process is RavenEtl ravenEtl)
{
RavenEtlConfiguration existing = null;
foreach (var config in myRavenEtl)
{
if (ravenEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(processesPerConfig.Key);
myRavenEtl.Remove(existing);
}
}
else if (process is SqlEtl sqlEtl)
{
SqlEtlConfiguration existing = null;
foreach (var config in mySqlEtl)
{
if (sqlEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(processesPerConfig.Key);
mySqlEtl.Remove(existing);
}
}
else
{
throw new InvalidOperationException($"Unknown ETL process type: {process.GetType()}");
}
}
Parallel.ForEach(toRemove, x =>
{
foreach (var process in x.Value)
{
try
{
process.Stop();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to stop ETL process {process.Name} on the database record change", e);
}
}
}
});
LoadProcesses(record, myRavenEtl, mySqlEtl, toRemove.SelectMany(x => x.Value).ToList());
// unsubscribe old etls _after_ we start new processes to ensure the tombstone cleaner
// constantly keeps track of tombstones processed by ETLs so it won't delete them during etl processes reloading
foreach (var processesPerConfig in toRemove)
{
foreach (var process in processesPerConfig.Value)
{
_database.TombstoneCleaner.Unsubscribe(process);
}
}
Parallel.ForEach(toRemove, x =>
{
foreach (var process in x.Value)
{
try
{
process.Dispose();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to dispose ETL process {process.Name} on the database record change", e);
}
}
}
});
}
public RavenConfiguration CreateDatabaseConfiguration(StringSegment databaseName, bool ignoreDisabledDatabase, bool ignoreBeenDeleted, bool ignoreNotRelevant, DatabaseRecord databaseRecord)
{
if (databaseRecord.DatabaseState == DatabaseStateStatus.RestoreInProgress)
{
throw new DatabaseRestoringException(databaseName + " is currently being restored");
}
if (databaseRecord.Disabled && ignoreDisabledDatabase == false)
{
throw new DatabaseDisabledException(databaseName + " has been disabled");
}
var databaseIsBeenDeleted = databaseRecord.DeletionInProgress != null &&
databaseRecord.DeletionInProgress.TryGetValue(_serverStore.NodeTag, out DeletionInProgressStatus deletionInProgress) &&
deletionInProgress != DeletionInProgressStatus.No;
if (ignoreBeenDeleted == false && databaseIsBeenDeleted)
{
throw new DatabaseDisabledException(databaseName + " is currently being deleted on " + _serverStore.NodeTag);
}
if (ignoreNotRelevant == false && databaseRecord.Topology.RelevantFor(_serverStore.NodeTag) == false &&
databaseIsBeenDeleted == false)
{
throw new DatabaseNotRelevantException(databaseName + " is not relevant for " + _serverStore.NodeTag);
}
return(CreateConfiguration(databaseRecord));
}
public void UpdateDatabaseRecord(DatabaseRecord databaseRecord)
{
databaseRecord.Expiration = Configuration;
}
public void DeleteDatabase(string dbName, DeletionInProgressStatus deletionInProgress, DatabaseRecord record)
{
IDisposable removeLockAndReturn = null;
string databaseId;
try
{
try
{
removeLockAndReturn = DatabasesCache.RemoveLockAndReturn(dbName, CompleteDatabaseUnloading, out var database);
databaseId = database?.DbBase64Id;
}
catch (AggregateException ae) when(nameof(DeleteDatabase).Equals(ae.InnerException.Data["Source"]))
{
// this is already in the process of being deleted, we can just exit and let another thread handle it
return;
}
catch (DatabaseConcurrentLoadTimeoutException e)
{
if (e.Data.Contains(nameof(DeleteDatabase)))
{
// This case is when a deletion request was issued during the loading of the database.
// The DB will be deleted after actually finishing the loading process
return;
}
throw;
}
if (deletionInProgress == DeletionInProgressStatus.HardDelete)
{
RavenConfiguration configuration;
try
{
configuration = CreateDatabaseConfiguration(dbName, ignoreDisabledDatabase: true, ignoreBeenDeleted: true, ignoreNotRelevant: true,
databaseRecord: record);
}
catch (Exception ex)
{
configuration = null;
if (_logger.IsInfoEnabled)
{
_logger.Info("Could not create database configuration", ex);
}
}
// this can happen if the database record was already deleted
if (configuration != null)
{
DatabaseHelper.DeleteDatabaseFiles(configuration);
}
}
// At this point the db record still exists but the db was effectively deleted
// from this node so we can also remove its secret key from this node.
if (record.Encrypted)
{
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
using (var tx = context.OpenWriteTransaction())
{
_serverStore.DeleteSecretKey(context, dbName);
tx.Commit();
}
}
// delete the cache info
DeleteDatabaseCachedInfo(dbName, _serverStore);
}
finally
{
removeLockAndReturn?.Dispose();
}
NotifyLeaderAboutRemoval(dbName, databaseId);
}
public static DynamicJsonValue ToJson(ServerStore serverStore, TransactionOperationContext context, DatabaseRecord record, long databaseIndex)
{
var mapping = SnmpDatabase.GetIndexMapping(context, serverStore, record.DatabaseName);
var djv = new DynamicJsonValue();
if (mapping.Count == 0)
{
return(djv);
}
foreach (var indexName in record.Indexes.Keys)
{
if (mapping.TryGetValue(indexName, out var index) == false)
{
continue;
}
var array = new DynamicJsonArray();
foreach (var field in typeof(Indexes).GetFields())
{
var fieldValue = GetFieldValue(field);
var databaseOid = string.Format(fieldValue.Oid, databaseIndex);
var indexOid = string.Format(databaseOid, index);
array.Add(CreateJsonItem(Root + indexOid, fieldValue.Description));
}
djv[indexName] = array;
}
return(djv);
}
public void Initialize(DatabaseRecord record)
{
LoadProcesses(record);
}
private void SmugglerRestore(
string backupDirectory,
DocumentDatabase database,
DocumentsOperationContext context,
DatabaseRecord databaseRecord,
Action <IOperationProgress> onProgress,
RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
_filesToRestore = _filesToRestore
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToList();
if (_filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
OperateOnTypes = ~(DatabaseItemType.CompareExchange | DatabaseItemType.Identities)
};
options.OperateOnTypes |= DatabaseItemType.LegacyDocumentDeletions;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachments;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachmentDeletions;
var oldOperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
for (var i = 0; i < _filesToRestore.Count - 1; i++)
{
result.AddInfo($"Restoring file {(i + 1):#,#;;0}/{_filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
var filePath = Path.Combine(backupDirectory, _filesToRestore[i]);
ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options,
onDatabaseRecordAction: smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = Path.Combine(backupDirectory, _filesToRestore.Last());
result.AddInfo($"Restoring file {_filesToRestore.Count:#,#;;0}/{_filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options,
onIndexAction: indexAndType =>
{
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
},
onDatabaseRecordAction: smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
databaseRecord.Revisions = smugglerDatabaseRecord.Revisions;
databaseRecord.Expiration = smugglerDatabaseRecord.Expiration;
databaseRecord.RavenConnectionStrings = smugglerDatabaseRecord.RavenConnectionStrings;
databaseRecord.SqlConnectionStrings = smugglerDatabaseRecord.SqlConnectionStrings;
databaseRecord.Client = smugglerDatabaseRecord.Client;
});
}
public override string UpdateDatabaseRecord(DatabaseRecord record, long etag)
{
record.RevisionsForConflicts = Configuration;
return(null);
}
public async Task MoveToPassiveWhenRefusedConnectionFromAllNodes()
{
//DebuggerAttachedTimeout.DisableLongTimespan = true;
var clusterSize = 3;
var databaseName = "MoveToPassiveWhenRefusedConnectionFromAllNodes";
var leader = await CreateRaftClusterAndGetLeader(clusterSize, false, 0, customSettings : new Dictionary <string, string>()
{
[RavenConfiguration.GetKey(x => x.Cluster.ElectionTimeout)] = "600"
});
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.Members.Count);
await WaitForRaftIndexToBeAppliedInCluster(databaseResult.RaftCommandIndex, TimeSpan.FromSeconds(10));
using (var session = store.OpenAsyncSession())
{
await session.StoreAsync(new User());
await session.SaveChangesAsync();
}
var dataDir = Servers[1].Configuration.Core.DataDirectory.FullPath.Split('/').Last();
var urls = new[] { Servers[1].WebUrl };
var nodeTag = Servers[1].ServerStore.NodeTag;
// kill the process and remove the node from topology
DisposeServerAndWaitForFinishOfDisposal(Servers[1]);
await ActionWithLeader((l) => l.ServerStore.RemoveFromClusterAsync(nodeTag));
using (leader.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
var val = await WaitForValueAsync(() =>
{
using (context.OpenReadTransaction())
{
return(Servers[2].ServerStore.GetClusterTopology(context).AllNodes.Count);
}
}, clusterSize - 1);
Assert.Equal(clusterSize - 1, val);
val = await WaitForValueAsync(() =>
{
using (context.OpenReadTransaction())
{
return(Servers[0].ServerStore.GetClusterTopology(context).AllNodes.Count);
}
}, clusterSize - 1);
Assert.Equal(clusterSize - 1, val);
}
// bring the node back to live and ensure that he moves to passive state
Servers[1] = GetNewServer(new Dictionary <string, string>
{
{ RavenConfiguration.GetKey(x => x.Core.PublicServerUrl), urls[0] },
{ RavenConfiguration.GetKey(x => x.Core.ServerUrls), urls[0] },
{ RavenConfiguration.GetKey(x => x.Cluster.ElectionTimeout), "600" }
}, runInMemory: false, deletePrevious: false, partialPath: dataDir);
Assert.True(await Servers[1].ServerStore.WaitForState(RachisState.Passive, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(30)), "1st assert");
// rejoin the node to the cluster
await ActionWithLeader((l) => l.ServerStore.AddNodeToClusterAsync(urls[0], nodeTag));
Assert.True(await Servers[1].ServerStore.WaitForState(RachisState.Follower, CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(30)), "2nd assert");
}
}
public static void CreateDatabase(IDocumentStore defaultStore, string dbName)
{
var dbRecord = new DatabaseRecord(dbName);
defaultStore.Maintenance.Server.Send(new CreateDatabaseOperation(dbRecord));
}
public async Task CanGetTaskInfo()
{
var clusterSize = 3;
var databaseName = "TestDB";
var leader = await CreateRaftClusterAndGetLeader(clusterSize);
ModifyOngoingTaskResult addWatcherRes;
UpdatePeriodicBackupOperationResult updateBackupResult;
AddEtlOperationResult addRavenEtlResult;
AddEtlOperationResult addSqlEtlResult;
RavenEtlConfiguration etlConfiguration;
SqlEtlConfiguration sqlConfiguration;
ExternalReplication watcher;
SqlConnectionString sqlConnectionString;
var sqlScript = @"
var orderData = {
Id: __document_id,
OrderLinesCount: this.OrderLines.length,
TotalCost: 0
};
loadToOrders(orderData);
";
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers)
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
watcher = new ExternalReplication("Watcher1", "Connection")
{
Name = "MyExternalReplication"
};
addWatcherRes = await AddWatcherToReplicationTopology((DocumentStore)store, watcher, new [] { leader.WebUrl });
var backupConfig = new PeriodicBackupConfiguration
{
Name = "backup1",
LocalSettings = new LocalSettings
{
FolderPath = NewDataPath(suffix: "BackupFolder")
},
AzureSettings = new AzureSettings
{
StorageContainer = "abc"
},
FullBackupFrequency = "* */1 * * *",
IncrementalBackupFrequency = "* */2 * * *",
Disabled = true
};
updateBackupResult = await store.Maintenance.SendAsync(new UpdatePeriodicBackupOperation(backupConfig));
store.Maintenance.Send(new PutConnectionStringOperation <RavenConnectionString>(new RavenConnectionString
{
Name = "cs",
TopologyDiscoveryUrls = new [] { "http://127.0.0.1:8080" },
Database = "Northwind",
}));
etlConfiguration = new RavenEtlConfiguration()
{
Name = "tesst",
ConnectionStringName = "cs",
Transforms =
{
new Transformation()
{
Name = "loadAll",
Collections ={ "Users" },
Script = "loadToUsers(this)"
}
}
};
addRavenEtlResult = store.Maintenance.Send(new AddEtlOperation <RavenConnectionString>(etlConfiguration));
sqlConnectionString = new SqlConnectionString
{
Name = "abc",
ConnectionString = @"Data Source=localhost\sqlexpress;Integrated Security=SSPI;Connection Timeout=3" + $";Initial Catalog=SqlReplication-{store.Database};",
FactoryName = "System.Data.SqlClient"
};
store.Maintenance.Send(new PutConnectionStringOperation <SqlConnectionString>(sqlConnectionString));
sqlConfiguration = new SqlEtlConfiguration()
{
Name = "abc",
ConnectionStringName = "abc",
SqlTables =
{
new SqlEtlTable {
TableName = "Orders", DocumentIdColumn = "Id", InsertOnlyMode = false
},
new SqlEtlTable {
TableName = "OrderLines", DocumentIdColumn = "OrderId", InsertOnlyMode = false
},
},
Transforms =
{
new Transformation()
{
Name = "OrdersAndLines",
Collections = new List <string>{
"Orders"
},
Script = sqlScript
}
}
};
addSqlEtlResult = store.Maintenance.Send(new AddEtlOperation <SqlConnectionString>(sqlConfiguration));
}
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions =
{
DisableTopologyUpdates = true
}
}.Initialize())
{
var taskId = addWatcherRes.TaskId;
var replicationResult = (OngoingTaskReplication) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Replication);
Assert.Equal(watcher.Database, replicationResult.DestinationDatabase);
Assert.Equal(watcher.Url, replicationResult.DestinationUrl);
Assert.Equal(watcher.Name, replicationResult.TaskName);
taskId = updateBackupResult.TaskId;
var backupResult = (OngoingTaskBackup) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.Backup);
Assert.Equal("Local", backupResult.BackupDestinations[0]);
Assert.Equal("Azure", backupResult.BackupDestinations[1]);
Assert.Equal("backup1", backupResult.TaskName);
Assert.Equal(OngoingTaskState.Disabled, backupResult.TaskState);
taskId = addRavenEtlResult.TaskId;
var etlResult = (OngoingTaskRavenEtlDetails) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.RavenEtl);
Assert.Equal("cs", etlResult.Configuration.ConnectionStringName);
Assert.Equal("tesst", etlResult.Configuration.Name);
Assert.Equal("loadAll", etlResult.Configuration.Transforms[0].Name);
Assert.Equal("loadToUsers(this)", etlResult.Configuration.Transforms[0].Script);
Assert.Equal("Users", etlResult.Configuration.Transforms[0].Collections[0]);
Assert.Equal(etlConfiguration.Name, etlResult?.TaskName);
taskId = addSqlEtlResult.TaskId;
var sqlResult = (OngoingTaskSqlEtlDetails) await GetTaskInfo((DocumentStore)store, taskId, OngoingTaskType.SqlEtl);
Assert.Equal("abc", sqlResult.Configuration.ConnectionStringName);
Assert.Equal("abc", sqlResult.Configuration.Name);
Assert.Equal("OrdersAndLines", sqlResult.Configuration.Transforms[0].Name);
Assert.Equal(sqlScript, sqlResult.Configuration.Transforms[0].Script);
Assert.Equal("Orders", sqlResult.Configuration.Transforms[0].Collections[0]);
Assert.NotNull(sqlResult.Configuration.SqlTables);
Assert.Equal(sqlConfiguration.Name, sqlResult?.TaskName);
}
}
public async Task CanIdleDatabaseInCluster()
{
const int clusterSize = 3;
var databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(3, customSettings : new Dictionary <string, string>()
{
[RavenConfiguration.GetKey(x => x.Cluster.MoveToRehabGraceTime)] = "1",
[RavenConfiguration.GetKey(x => x.Cluster.AddReplicaTimeout)] = "1",
[RavenConfiguration.GetKey(x => x.Cluster.ElectionTimeout)] = "300",
[RavenConfiguration.GetKey(x => x.Cluster.StabilizationTime)] = "1",
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Replication.RetryMaxTimeout)] = "1",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
});
DatabasePutResult databaseResult;
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
}
Assert.Equal(clusterSize, databaseResult.Topology.AllNodes.Count());
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(databaseResult.RaftCommandIndex);
}
foreach (var server in Servers.Where(s => databaseResult.NodesAddedTo.Any(n => n == s.WebUrl)))
{
await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(databaseName);
}
var now = DateTime.Now;
var nextNow = now + TimeSpan.FromSeconds(300);
while (now < nextNow && GetIdleCount() < clusterSize)
{
await Task.Delay(3000);
now = DateTime.Now;
}
foreach (var server in Servers)
{
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
Assert.True(server.ServerStore.IdleDatabases.TryGetValue(databaseName, out var dictionary));
// new incoming replications not saved in IdleDatabases
Assert.Equal(0, dictionary.Count);
}
var rnd = new Random();
var index = rnd.Next(0, Servers.Count - 1);
using (var store = new DocumentStore
{
Urls = new[] { Servers[index].WebUrl },
Database = databaseName
}.Initialize())
{
await store.Maintenance.SendAsync(new GetStatisticsOperation());
}
Assert.Equal(2, GetIdleCount());
using (var store = new DocumentStore
{
Urls = new[] { Servers[index].WebUrl },
Database = databaseName
}.Initialize())
{
using (var s = store.OpenAsyncSession())
{
await s.StoreAsync(new User()
{
Name = "Egor"
}, "foo/bar");
await s.SaveChangesAsync();
}
}
nextNow = DateTime.Now + TimeSpan.FromSeconds(300);
while (now < nextNow && GetIdleCount() > 0)
{
await Task.Delay(3000);
now = DateTime.Now;
}
Assert.Equal(0, GetIdleCount());
foreach (var server in Servers)
{
using (var store = new DocumentStore
{
Urls = new[] { server.WebUrl },
Database = databaseName
}.Initialize())
{
var docs = (await store.Maintenance.SendAsync(new GetStatisticsOperation())).CountOfDocuments;
Assert.Equal(1, docs);
}
}
index = rnd.Next(0, Servers.Count - 1);
nextNow = DateTime.Now + TimeSpan.FromSeconds(300);
using (var store = new DocumentStore
{
Urls = new[] { Servers[index].WebUrl },
Database = databaseName
}.Initialize())
{
while (now < nextNow && GetIdleCount() < 2)
{
await Task.Delay(3000);
await store.Maintenance.SendAsync(new GetStatisticsOperation());
now = DateTime.Now;
}
}
Assert.Equal(2, GetIdleCount());
}
public void HandleDatabaseRecordChange(DatabaseRecord record)
{
if (record == null)
{
return;
}
var myRavenEtl = new List <RavenEtlConfiguration>();
var mySqlEtl = new List <SqlEtlConfiguration>();
foreach (var config in record.RavenEtls)
{
if (IsMyEtlTask <RavenEtlConfiguration, RavenConnectionString>(record, config))
{
myRavenEtl.Add(config);
}
}
foreach (var config in record.SqlEtls)
{
if (IsMyEtlTask <SqlEtlConfiguration, SqlConnectionString>(record, config))
{
mySqlEtl.Add(config);
}
}
var toRemove = new List <EtlProcess>(_processes);
foreach (var etlProcess in _processes)
{
if (etlProcess is RavenEtl ravenEtl)
{
RavenEtlConfiguration existing = null;
foreach (var config in myRavenEtl)
{
if (ravenEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(etlProcess);
myRavenEtl.Remove(existing);
}
}
if (etlProcess is SqlEtl sqlEtl)
{
SqlEtlConfiguration existing = null;
foreach (var config in mySqlEtl)
{
if (sqlEtl.Configuration.IsEqual(config))
{
existing = config;
break;
}
}
if (existing != null)
{
toRemove.Remove(etlProcess);
mySqlEtl.Remove(existing);
}
}
}
Parallel.ForEach(toRemove, x =>
{
try
{
x.Stop();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to stop ETL process {x.Name} on the database record change", e);
}
}
});
LoadProcesses(record, myRavenEtl, mySqlEtl, toRemove);
// unsubscribe old etls _after_ we start new processes to ensure the tombstone cleaner
// constantly keeps track of tombstones processed by ETLs so it won't delete them during etl processes reloading
foreach (var process in toRemove)
{
_database.DocumentTombstoneCleaner.Unsubscribe(process);
}
Parallel.ForEach(toRemove, x =>
{
try
{
x.Dispose();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to dispose ETL process {x.Name} on the database record change", e);
}
}
});
}
public DatabaseRecord GetDatabaseRecord()
{
var databaseRecord = new DatabaseRecord();
ReadObject(reader =>
{
if (reader.TryGet(nameof(databaseRecord.Revisions), out BlittableJsonReaderObject revisions) &&
revisions != null)
{
try
{
databaseRecord.Revisions = JsonDeserializationCluster.RevisionsConfiguration(revisions);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Wasn't able to import the reivions configuration from smuggler file. Skiping.", e);
}
}
}
if (reader.TryGet(nameof(databaseRecord.Expiration), out BlittableJsonReaderObject expiration) &&
expiration != null)
{
try
{
databaseRecord.Expiration = JsonDeserializationCluster.ExpirationConfiguration(expiration);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Wasn't able to import the expiration configuration from smuggler file. Skiping.", e);
}
}
}
if (reader.TryGet(nameof(databaseRecord.RavenConnectionStrings), out BlittableJsonReaderObject ravenConnectionStrings) &&
ravenConnectionStrings != null)
{
try
{
foreach (var connectionName in ravenConnectionStrings.GetPropertyNames())
{
if (ravenConnectionStrings.TryGet(connectionName, out BlittableJsonReaderObject connection) == false)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Wasn't able to import the RavenDB connection string {connectionName} from smuggler file. Skiping.");
}
continue;
}
var connectionString = JsonDeserializationCluster.RavenConnectionString(connection);
databaseRecord.RavenConnectionStrings[connectionString.Name] = connectionString;
}
}
catch (Exception e)
{
databaseRecord.RavenConnectionStrings.Clear();
if (_log.IsInfoEnabled)
{
_log.Info("Wasn't able to import the RavenDB connection strings from smuggler file. Skiping.", e);
}
}
}
if (reader.TryGet(nameof(databaseRecord.SqlConnectionStrings), out BlittableJsonReaderObject sqlConnectionStrings) &&
sqlConnectionStrings != null)
{
try
{
foreach (var connectionName in sqlConnectionStrings.GetPropertyNames())
{
if (ravenConnectionStrings.TryGet(connectionName, out BlittableJsonReaderObject connection) == false)
{
if (_log.IsInfoEnabled)
{
_log.Info($"Wasn't able to import the SQL connection string {connectionName} from smuggler file. Skiping.");
}
continue;
}
var connectionString = JsonDeserializationCluster.SqlConnectionString(connection);
databaseRecord.SqlConnectionStrings[connectionString.Name] = connectionString;
}
}
catch (Exception e)
{
databaseRecord.SqlConnectionStrings.Clear();
if (_log.IsInfoEnabled)
{
_log.Info("Wasn't able to import the SQL connection strings from smuggler file. Skiping.", e);
}
}
}
if (reader.TryGet(nameof(databaseRecord.Client), out BlittableJsonReaderObject client) &&
client != null)
{
try
{
databaseRecord.Client = JsonDeserializationCluster.ClientConfiguration(expiration);
}
catch (Exception e)
{
if (_log.IsInfoEnabled)
{
_log.Info("Wasn't able to import the client configuration from smuggler file. Skiping.", e);
}
}
}
});
return(databaseRecord);
}
public async Task CanPassNodeTagToRestorePatchOperation()
{
var clusterSize = 3;
var databaseName = GetDatabaseName();
var leader = await CreateRaftClusterAndGetLeader(clusterSize, false, useSsl : false);
var myNodesList = new List <string>();
using (var store = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
}.Initialize())
{
var doc = new DatabaseRecord(databaseName);
var databaseResult = await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(doc, clusterSize));
myNodesList.AddRange(databaseResult.Topology.AllNodes);
store.Maintenance.Send(new PutIndexesOperation(new[] { new IndexDefinition
{
Maps = { "from doc in docs.Items select new { doc.Name }" },
Name = "MyIndex"
} }));
using (var commands = store.Commands())
{
await commands.PutAsync("items/1", null, new { Name = "testname" }, new Dictionary <string, object>
{
{ Constants.Documents.Metadata.Collection, "Items" }
});
WaitForIndexing(store);
var operation = store.Operations.Send(new PatchByQueryOperation(new IndexQuery
{
Query = "FROM INDEX 'MyIndex' UPDATE { this.NewName = 'NewValue'; } "
}));
var opStatus = store.Maintenance.Send(new GetOperationStateOperation(operation.Id));
Assert.NotNull(opStatus);
foreach (var node in myNodesList)
{
var op = store.Maintenance.Send(new GetOperationStateOperation(operation.Id, node));
if (node == operation.NodeTag)
{
Assert.NotNull(op);
}
else
{
Assert.Null(op);
}
}
operation.WaitForCompletion(TimeSpan.FromSeconds(15));
dynamic document = await commands.GetAsync("items/1");
Assert.Equal("NewValue", document.NewName.ToString());
}
}
}
private void OpenIndexesFromRecord(PathSetting path, DatabaseRecord record)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Starting to load indexes from record");
}
List <Exception> exceptions = null;
if (_documentDatabase.Configuration.Core.ThrowIfAnyIndexCannotBeOpened)
{
exceptions = new List <Exception>();
}
// delete all unrecognized index directories
//foreach (var indexDirectory in new DirectoryInfo(path.FullPath).GetDirectories().Concat(indexesCustomPaths.Values.SelectMany(x => new DirectoryInfo(x).GetDirectories())))
//{
// if (record.Indexes.ContainsKey(indexDirectory.Name) == false)
// {
// IOExtensions.DeleteDirectory(indexDirectory.FullName);
// continue;
// }
// // delete all redundant index instances
// var indexInstances = indexDirectory.GetDirectories();
// if (indexInstances.Length > 2)
// {
// var orderedIndexes = indexInstances.OrderByDescending(x =>
// int.Parse(x.Name.Substring(x.Name.LastIndexOf("\\") +1)));
// foreach (var indexToRemove in orderedIndexes.Skip(2))
// {
// Directory.Delete(indexToRemove.FullName);
// }
// }
//}
foreach (var kvp in record.Indexes)
{
if (_documentDatabase.DatabaseShutdown.IsCancellationRequested)
{
return;
}
var name = kvp.Key;
var definition = kvp.Value;
var safeName = IndexDefinitionBase.GetIndexNameSafeForFileSystem(definition.Name);
var indexPath = path.Combine(safeName).FullPath;
if (Directory.Exists(indexPath))
{
OpenIndex(path, indexPath, exceptions, name);
}
}
foreach (var kvp in record.AutoIndexes)
{
if (_documentDatabase.DatabaseShutdown.IsCancellationRequested)
{
return;
}
var name = kvp.Key;
var definition = kvp.Value;
var safeName = IndexDefinitionBase.GetIndexNameSafeForFileSystem(definition.Name);
var indexPath = path.Combine(safeName).FullPath;
if (Directory.Exists(indexPath))
{
OpenIndex(path, indexPath, exceptions, name);
}
}
if (exceptions != null && exceptions.Count > 0)
{
throw new AggregateException("Could not load some of the indexes", exceptions);
}
}
public override void UpdateDatabaseRecord(DatabaseRecord record, long etag)
{
Configuration?.InitializeRollupAndRetention();
record.TimeSeries = Configuration;
}
private void NotifyFeaturesAboutValueChange(DatabaseRecord record)
{
SubscriptionStorage?.HandleDatabaseValueChange(record);
}
public override void UpdateDatabaseRecord(DatabaseRecord record, long etag)
{
record.SqlConnectionStrings[ConnectionString.Name] = ConnectionString;
}
private void OnDatabaseRecordChanged(DatabaseRecord record)
{
DatabaseRecordChanged?.Invoke(record);
}
public CreateDatabaseOperationWithoutNameValidation(DatabaseRecord databaseRecord, int replicationFactor = 1)
{
_databaseRecord = databaseRecord;
_replicationFactor = replicationFactor;
}
protected override BlittableJsonReaderObject GetUpdatedValue(long index, DatabaseRecord record, JsonOperationContext context, BlittableJsonReaderObject existingValue, RachisState state)
{
throw new NotSupportedException();
}
