public ImportCommand(JsonOperationContext context, DatabaseSmugglerImportOptions options, Stream stream, long operationId, TaskCompletionSource <object> tcs, DatabaseSmuggler parent)
{
_stream = stream ?? throw new ArgumentNullException(nameof(stream));
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_options = DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(options, context);
_operationId = operationId;
_tcs = tcs ?? throw new ArgumentNullException(nameof(tcs));
_parent = parent ?? throw new ArgumentNullException(nameof(parent));
}
public async Task <Operation> ExportAsync(DatabaseSmugglerOptions options, DatabaseSmuggler toDatabase, CancellationToken token = default(CancellationToken))
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (toDatabase == null)
{
throw new ArgumentNullException(nameof(toDatabase));
}
var result = await ExportAsync(options, async stream =>
{
await toDatabase.ImportAsync(options, stream, token).ConfigureAwait(false);
}, token).ConfigureAwait(false);
return(result);
}
public ImportCommand(DocumentConventions conventions, JsonOperationContext context, DatabaseSmugglerImportOptions options, Stream stream, long operationId, TaskCompletionSource <object> tcs, DatabaseSmuggler parent, string nodeTag)
{
_stream = stream ?? throw new ArgumentNullException(nameof(stream));
if (conventions == null)
{
throw new ArgumentNullException(nameof(conventions));
}
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_options = EntityToBlittable.ConvertCommandToBlittable(options, context);
_operationId = operationId;
_tcs = tcs ?? throw new ArgumentNullException(nameof(tcs));
_parent = parent ?? throw new ArgumentNullException(nameof(parent));
SelectedNodeTag = nodeTag;
}
public StreamContentWithConfirmation(Stream content, TaskCompletionSource <object> tcs, DatabaseSmuggler parent) : base(content)
{
_tcs = tcs ?? throw new ArgumentNullException(nameof(tcs));
_parent = parent;
}
public async Task <Operation> ExportAsync(DatabaseSmugglerExportOptions options, DatabaseSmuggler toDatabase, CancellationToken token = default)
{
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (toDatabase == null)
{
throw new ArgumentNullException(nameof(toDatabase));
}
Operation operation = null;
var importOptions = new DatabaseSmugglerImportOptions(options);
var tcs = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
await ExportAsync(options, async stream =>
{
try
{
operation = await toDatabase.ImportAsync(importOptions, stream, token).ConfigureAwait(false);
tcs.TrySetResult(null);
}
catch (Exception e)
{
tcs.TrySetException(e);
throw;
}
}, token).ConfigureAwait(false);
await tcs.Task.ConfigureAwait(false);
return(operation);
}
protected async Task SmugglerRestore(DocumentDatabase database, List <string> filesToRestore, DocumentsOperationContext context,
DatabaseRecord databaseRecord, Action <IOperationProgress> onProgress, RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
if (filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup, we'll take the indexes from the last file
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
SkipRevisionCreation = true
};
options.OperateOnTypes |= DatabaseItemType.LegacyDocumentDeletions;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachments;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachmentDeletions;
#pragma warning disable 618
options.OperateOnTypes |= DatabaseItemType.Counters;
#pragma warning restore 618
var oldOperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
for (var i = 0; i < filesToRestore.Count - 1; i++)
{
result.AddInfo($"Restoring file {(i + 1):#,#;;0}/{filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
var filePath = GetBackupPath(filesToRestore[i]);
await ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options, isLastFile : false,
onDatabaseRecordAction : smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = GetBackupPath(filesToRestore.Last());
result.AddInfo($"Restoring file {filesToRestore.Count:#,#;;0}/{filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
await ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options, isLastFile : true,
onIndexAction : indexAndType =>
{
if (this.RestoreFromConfiguration.SkipIndexes)
{
return;
}
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
},
onDatabaseRecordAction : smugglerDatabaseRecord =>
{
databaseRecord.ConflictSolverConfig = smugglerDatabaseRecord.ConflictSolverConfig;
foreach (var setting in smugglerDatabaseRecord.Settings)
{
databaseRecord.Settings[setting.Key] = setting.Value;
}
databaseRecord.SqlEtls = smugglerDatabaseRecord.SqlEtls;
databaseRecord.RavenEtls = smugglerDatabaseRecord.RavenEtls;
databaseRecord.PeriodicBackups = smugglerDatabaseRecord.PeriodicBackups;
databaseRecord.ExternalReplications = smugglerDatabaseRecord.ExternalReplications;
databaseRecord.Sorters = smugglerDatabaseRecord.Sorters;
databaseRecord.SinkPullReplications = smugglerDatabaseRecord.SinkPullReplications;
databaseRecord.HubPullReplications = smugglerDatabaseRecord.HubPullReplications;
databaseRecord.Revisions = smugglerDatabaseRecord.Revisions;
databaseRecord.Expiration = smugglerDatabaseRecord.Expiration;
databaseRecord.RavenConnectionStrings = smugglerDatabaseRecord.RavenConnectionStrings;
databaseRecord.SqlConnectionStrings = smugglerDatabaseRecord.SqlConnectionStrings;
databaseRecord.Client = smugglerDatabaseRecord.Client;
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
private void SmugglerRestore(
string backupDirectory,
DocumentDatabase database,
DocumentsOperationContext context,
DatabaseRecord databaseRecord,
Action <IOperationProgress> onProgress,
RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
_filesToRestore = _filesToRestore
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToList();
if (_filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
OperateOnTypes = ~(DatabaseItemType.CompareExchange | DatabaseItemType.Identities)
};
options.OperateOnTypes |= DatabaseItemType.LegacyDocumentDeletions;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachments;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachmentDeletions;
var oldOperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
for (var i = 0; i < _filesToRestore.Count - 1; i++)
{
var filePath = Path.Combine(backupDirectory, _filesToRestore[i]);
ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options,
onDatabaseRecordAction: smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = Path.Combine(backupDirectory, _filesToRestore.Last());
ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options,
onIndexAction: indexAndType =>
{
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
},
onDatabaseRecordAction: smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
databaseRecord.Revisions = smugglerDatabaseRecord.Revisions;
databaseRecord.Expiration = smugglerDatabaseRecord.Expiration;
databaseRecord.RavenConnectionStrings = smugglerDatabaseRecord.RavenConnectionStrings;
databaseRecord.SqlConnectionStrings = smugglerDatabaseRecord.SqlConnectionStrings;
databaseRecord.Client = smugglerDatabaseRecord.Client;
});
}
private void SmugglerRestore(
string backupDirectory,
DocumentDatabase database,
DocumentsOperationContext context,
DatabaseRecord databaseRecord,
Action <IOperationProgress> onProgress,
RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
_filesToRestore = _filesToRestore
.Where(file =>
{
var extension = Path.GetExtension(file);
return
(Constants.Documents.PeriodicBackup.IncrementalBackupExtension.Equals(extension, StringComparison.OrdinalIgnoreCase) ||
Constants.Documents.PeriodicBackup.FullBackupExtension.Equals(extension, StringComparison.OrdinalIgnoreCase));
})
.OrderBy(x => x)
.ToList();
if (_filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = ~(DatabaseItemType.CmpXchg | DatabaseItemType.Identities)
};
var oldOperateOnTypes = options.OperateOnTypes;
options.OperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
for (var i = 0; i < _filesToRestore.Count - 1; i++)
{
var filePath = Path.Combine(backupDirectory, _filesToRestore[i]);
ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options);
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = Path.Combine(backupDirectory, _filesToRestore.Last());
ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options,
onIndexAction: indexAndType =>
{
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
});
}
