private void ImportSingleBackupFile(DocumentDatabase database,
Action <IOperationProgress> onProgress, RestoreResult restoreResult,
string filePath, DocumentsOperationContext context,
DatabaseDestination destination, DatabaseSmugglerOptionsServerSide options,
Action <IndexDefinitionAndType> onIndexAction = null)
{
using (var fileStream = File.Open(filePath, FileMode.Open))
using (var stream = new GZipStream(new BufferedStream(fileStream, 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var source = new StreamSource(stream, context, database))
{
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, options, result: restoreResult, onProgress: onProgress, token: _operationCancelToken.Token)
{
OnIndexAction = onIndexAction,
};
smuggler.Execute();
}
}
private void RestoreFromLastFile(Action <IOperationProgress> onProgress, DocumentDatabase database, string lastFile, DocumentsOperationContext context, RestoreResult result)
{
var destination = new DatabaseDestination(database);
var smugglerOptions = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
OperateOnTypes = DatabaseItemType.CompareExchange | DatabaseItemType.Identities,
SkipRevisionCreation = true
};
var lastPath = Path.Combine(_restoreConfiguration.BackupLocation, lastFile);
if (Path.GetExtension(lastPath) == Constants.Documents.PeriodicBackup.SnapshotExtension)
{
using (var zip = ZipFile.Open(lastPath, ZipArchiveMode.Read, System.Text.Encoding.UTF8))
{
foreach (var entry in zip.Entries)
{
if (entry.Name == RestoreSettings.SmugglerValuesFileName)
{
using (var input = entry.Open())
using (var uncompressed = new GZipStream(input, CompressionMode.Decompress))
{
var source = new StreamSource(uncompressed, context, database);
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, smugglerOptions, onProgress: onProgress, token: _operationCancelToken.Token);
smuggler.Execute();
}
break;
}
}
}
}
else
{
ImportSingleBackupFile(database, onProgress, null, lastPath, context, destination, smugglerOptions);
}
result.Identities.Processed = true;
result.CompareExchange.Processed = true;
onProgress.Invoke(result.Progress);
}
public async Task SearchForPageShouldNotSkipLastPage()
{
using (var inputStream = GetDump("RavenDB-13937.ravendbdump"))
using (var stream = new GZipStream(inputStream, CompressionMode.Decompress))
{
using (var database = CreateDocumentDatabase())
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context, database))
{
var editRevisions = new EditRevisionsConfigurationCommand(new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Orders"] = new RevisionsCollectionConfiguration
{
Disabled = false
}
}
}, database.Name, RaftIdGenerator.NewId());
var(index, _) = await database.ServerStore.SendToLeaderAsync(editRevisions);
await database.RachisLogIndexNotifications.WaitForIndexNotification(index, database.ServerStore.Engine.OperationTimeout);
var destination = new DatabaseDestination(database);
var smuggler = await(new DatabaseSmuggler(database, source, destination, database.Time, new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Documents | DatabaseItemType.RevisionDocuments | DatabaseItemType.Attachments |
DatabaseItemType.Indexes,
SkipRevisionCreation = true
}).ExecuteAsync());
using (context.OpenReadTransaction())
{
var(revisions, count) = database.DocumentsStorage.RevisionsStorage.GetRevisions(context, "Orders/825-A", 0, int.MaxValue);
Assert.Equal(count, revisions.Length);
}
}
}
}
private async Task <SmugglerResult> MigrateDatabase(string json, bool readLegacyEtag)
{
var response = await RunWithAuthRetry(async() =>
{
var url = $"{ServerUrl}/databases/{DatabaseName}/studio-tasks/exportDatabase";
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = content
};
var responseMessage = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancelToken.Token);
return(responseMessage);
});
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export database from server: {ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
using (var responseStream = await response.Content.ReadAsStreamAsync())
using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress))
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide
{
#pragma warning disable 618
ReadLegacyEtag = readLegacyEtag,
#pragma warning restore 618
RemoveAnalyzers = RemoveAnalyzers
};
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, Result, OnProgress, CancelToken.Token);
return(smuggler.Execute());
}
}
public async Task PostImport()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var options = DatabaseSmugglerOptionsServerSide.Create(HttpContext, context);
using (var stream = new GZipStream(new BufferedStream(await GetImportStream(), 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var token = CreateOperationToken())
using (var source = new StreamSource(stream, context))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time, options, token: token.Token);
var result = smuggler.Execute();
WriteImportResult(context, result, ResponseBodyStream());
}
}
}
private async Task ImportSingleBackupFile(DocumentDatabase database,
Action <IOperationProgress> onProgress, RestoreResult restoreResult,
string filePath, DocumentsOperationContext context,
DatabaseDestination destination, DatabaseSmugglerOptionsServerSide options, bool isLastFile,
Action <IndexDefinitionAndType> onIndexAction = null,
Action <DatabaseRecord> onDatabaseRecordAction = null)
{
using (var fileStream = await GetStream(filePath))
using (var inputStream = GetInputStream(fileStream, database.MasterKey))
using (var gzipStream = new GZipStream(inputStream, CompressionMode.Decompress))
using (var source = new StreamSource(gzipStream, context, database))
{
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, options, result: restoreResult, onProgress: onProgress, token: _operationCancelToken.Token)
{
OnIndexAction = onIndexAction,
OnDatabaseRecordAction = onDatabaseRecordAction
};
smuggler.Execute(ensureStepsProcessed: false, isLastFile);
}
}
/// <summary>
/// Creates report destinations. Can be overriden to add headers or to initialize the destination in some way.
/// </summary>
/// <param name="opts"></param>
public virtual void AddDestinations(IsIdentifiableAbstractOptions opts)
{
IReportDestination destination;
// Default is to write out CSV results
if (!string.IsNullOrWhiteSpace(opts.DestinationCsvFolder))
{
destination = new CsvDestination(opts, _reportName);
}
else if (!string.IsNullOrWhiteSpace(opts.DestinationConnectionString))
{
destination = new DatabaseDestination(opts, _reportName);
}
else
{
opts.DestinationCsvFolder = Environment.CurrentDirectory;
destination = new CsvDestination(opts, _reportName);
}
Destinations.Add(destination);
}
private async Task MigrateDocuments(string lastEtag)
{
var response = await RunWithAuthRetry(async() =>
{
var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/streams/docs?etag={lastEtag}";
var request = new HttpRequestMessage(HttpMethod.Get, url);
var responseMessage = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token);
return(responseMessage);
});
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export documents from server: {Options.ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
using (var responseStream = await response.Content.ReadAsStreamAsync())
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(responseStream, context, Parameters.Database))
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
#pragma warning disable 618
ReadLegacyEtag = true,
#pragma warning restore 618
TransformScript = Options.TransformScript,
OperateOnTypes = Options.OperateOnTypes
};
var smuggler = new DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token);
// since we will be migrating indexes as separate task don't ensureStepsProcessed at this point
smuggler.Execute(ensureStepsProcessed: false);
}
}
protected async Task SmugglerRestore(DocumentDatabase database, List <string> filesToRestore, DocumentsOperationContext context,
DatabaseRecord databaseRecord, Action <IOperationProgress> onProgress, RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
if (filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup, we'll take the indexes from the last file
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
SkipRevisionCreation = true
};
options.OperateOnTypes |= DatabaseItemType.LegacyDocumentDeletions;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachments;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachmentDeletions;
#pragma warning disable 618
options.OperateOnTypes |= DatabaseItemType.Counters;
#pragma warning restore 618
var oldOperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
for (var i = 0; i < filesToRestore.Count - 1; i++)
{
result.AddInfo($"Restoring file {(i + 1):#,#;;0}/{filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
var filePath = GetBackupPath(filesToRestore[i]);
await ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options, isLastFile : false,
onDatabaseRecordAction : smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = GetBackupPath(filesToRestore.Last());
result.AddInfo($"Restoring file {filesToRestore.Count:#,#;;0}/{filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
await ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options, isLastFile : true,
onIndexAction : indexAndType =>
{
if (this.RestoreFromConfiguration.SkipIndexes)
{
return;
}
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
},
onDatabaseRecordAction : smugglerDatabaseRecord =>
{
databaseRecord.ConflictSolverConfig = smugglerDatabaseRecord.ConflictSolverConfig;
foreach (var setting in smugglerDatabaseRecord.Settings)
{
databaseRecord.Settings[setting.Key] = setting.Value;
}
databaseRecord.SqlEtls = smugglerDatabaseRecord.SqlEtls;
databaseRecord.RavenEtls = smugglerDatabaseRecord.RavenEtls;
databaseRecord.PeriodicBackups = smugglerDatabaseRecord.PeriodicBackups;
databaseRecord.ExternalReplications = smugglerDatabaseRecord.ExternalReplications;
databaseRecord.Sorters = smugglerDatabaseRecord.Sorters;
databaseRecord.SinkPullReplications = smugglerDatabaseRecord.SinkPullReplications;
databaseRecord.HubPullReplications = smugglerDatabaseRecord.HubPullReplications;
databaseRecord.Revisions = smugglerDatabaseRecord.Revisions;
databaseRecord.Expiration = smugglerDatabaseRecord.Expiration;
databaseRecord.RavenConnectionStrings = smugglerDatabaseRecord.RavenConnectionStrings;
databaseRecord.SqlConnectionStrings = smugglerDatabaseRecord.SqlConnectionStrings;
databaseRecord.Client = smugglerDatabaseRecord.Client;
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
private void SmugglerRestore(
string backupDirectory,
DocumentDatabase database,
DocumentsOperationContext context,
DatabaseRecord databaseRecord,
Action <IOperationProgress> onProgress,
RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
_filesToRestore = _filesToRestore
.Where(BackupUtils.IsBackupFile)
.OrderBackups()
.ToList();
if (_filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
OperateOnTypes = ~(DatabaseItemType.CompareExchange | DatabaseItemType.Identities)
};
options.OperateOnTypes |= DatabaseItemType.LegacyDocumentDeletions;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachments;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachmentDeletions;
var oldOperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
for (var i = 0; i < _filesToRestore.Count - 1; i++)
{
var filePath = Path.Combine(backupDirectory, _filesToRestore[i]);
ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options,
onDatabaseRecordAction: smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = Path.Combine(backupDirectory, _filesToRestore.Last());
ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options,
onIndexAction: indexAndType =>
{
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
},
onDatabaseRecordAction: smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
databaseRecord.Revisions = smugglerDatabaseRecord.Revisions;
databaseRecord.Expiration = smugglerDatabaseRecord.Expiration;
databaseRecord.RavenConnectionStrings = smugglerDatabaseRecord.RavenConnectionStrings;
databaseRecord.SqlConnectionStrings = smugglerDatabaseRecord.SqlConnectionStrings;
databaseRecord.Client = smugglerDatabaseRecord.Client;
});
}
private async Task BulkImport(BlockingCollection <Func <Task <Stream> > > files, string directory)
{
var results = new ConcurrentQueue <SmugglerResult>();
var tasks = new Task[Math.Max(1, ProcessorInfo.ProcessorCount / 2)];
var finalResult = new SmugglerResult();
for (int i = 0; i < tasks.Length; i++)
{
tasks[i] = Task.Run(async() =>
{
while (files.IsCompleted == false)
{
Func <Task <Stream> > getFile;
try
{
getFile = files.Take();
}
catch (Exception)
{
//TODO : add logging, _silently_ skipping is a bad idea
continue;
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var file = await getFile())
using (var stream = new GZipStream(new BufferedStream(file, 128 * Voron.Global.Constants.Size.Kilobyte), CompressionMode.Decompress))
using (var source = new StreamSource(stream, context))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time);
var result = smuggler.Execute();
results.Enqueue(result);
}
}
});
}
await Task.WhenAll(tasks);
while (results.TryDequeue(out SmugglerResult importResult))
{
finalResult.Documents.SkippedCount += importResult.Documents.SkippedCount;
finalResult.Documents.ReadCount += importResult.Documents.ReadCount;
finalResult.Documents.ErroredCount += importResult.Documents.ErroredCount;
finalResult.Documents.LastEtag = Math.Max(finalResult.Documents.LastEtag, importResult.Documents.LastEtag);
finalResult.Documents.Attachments = importResult.Documents.Attachments;
finalResult.RevisionDocuments.ReadCount += importResult.RevisionDocuments.ReadCount;
finalResult.RevisionDocuments.ErroredCount += importResult.RevisionDocuments.ErroredCount;
finalResult.RevisionDocuments.LastEtag = Math.Max(finalResult.RevisionDocuments.LastEtag, importResult.RevisionDocuments.LastEtag);
finalResult.RevisionDocuments.Attachments = importResult.RevisionDocuments.Attachments;
finalResult.Identities.ReadCount += importResult.Identities.ReadCount;
finalResult.Identities.ErroredCount += importResult.Identities.ErroredCount;
finalResult.Indexes.ReadCount += importResult.Indexes.ReadCount;
finalResult.Indexes.ErroredCount += importResult.Indexes.ErroredCount;
foreach (var message in importResult.Messages)
{
finalResult.AddMessage(message);
}
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext finalContext))
{
var memoryStream = new MemoryStream();
WriteImportResult(finalContext, finalResult, memoryStream);
memoryStream.Position = 0;
try
{
using (var output = File.Create(Path.Combine(directory, "smuggler.results.txt")))
{
memoryStream.CopyTo(output);
}
}
catch (Exception)
{
// ignore any failure here
}
memoryStream.Position = 0;
memoryStream.CopyTo(ResponseBodyStream());
}
}
private void ImportSingleBackupFile(DocumentDatabase database,
Action <IOperationProgress> onProgress, RestoreResult restoreResult,
string filePath, DocumentsOperationContext context,
DatabaseDestination destination, DatabaseSmugglerOptionsServerSide options,
Action <IndexDefinitionAndType> onIndexAction = null,
Action <(string Prefix, long Value)> onIdentityAction = null)
private void SmugglerRestore(
string backupDirectory,
DocumentDatabase database,
DatabaseRecord databaseRecord,
RestoreSettings restoreSettings,
Action <IOperationProgress> onProgress,
RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
_filesToRestore = _filesToRestore
.Where(file =>
{
var extension = Path.GetExtension(file);
return
(Constants.Documents.PeriodicBackup.IncrementalBackupExtension.Equals(extension, StringComparison.OrdinalIgnoreCase) ||
Constants.Documents.PeriodicBackup.FullBackupExtension.Equals(extension, StringComparison.OrdinalIgnoreCase));
})
.OrderBy(x => x)
.ToList();
if (_filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
restoreSettings.Identities = new Dictionary <string, long>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide();
var oldOperateOnTypes = DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
for (var i = 0; i < _filesToRestore.Count - 1; i++)
{
var filePath = Path.Combine(backupDirectory, _filesToRestore[i]);
ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options);
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = Path.Combine(backupDirectory, _filesToRestore.Last());
ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options,
onIndexAction: indexAndType =>
{
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBase)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
},
onIdentityAction: identity => restoreSettings.Identities[identity.Prefix] = identity.Value);
}
}
public async Task Attachments()
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.Initialize(options, null, buildVersion: default);
using (var documentActions = destination.Documents())
using (var buffered = new BufferedStream(RequestBodyStream()))
using (var reader = new BsonReader(buffered))
{
var result = LegacyAttachmentUtils.GetObject(reader);
const string idProperty = "@id";
const string etagProperty = "@etag";
const string metadataProperty = "@metadata";
const string dataProperty = "data";
string lastAttachmentEtag = null;
var progress = new SmugglerProgressBase.CountsWithLastEtag();
foreach (var attachmentObject in result.Values)
{
if (!(attachmentObject is Dictionary <string, object> attachmentDictionary))
{
throw new InvalidDataException("attachmentObject isn't a Dictionary<string, object>");
}
if (attachmentDictionary.TryGetValue(idProperty, out var attachmentKeyObject) == false)
{
throw new InvalidDataException($"{idProperty} doesn't exist");
}
if (!(attachmentKeyObject is string attachmentKey))
{
throw new InvalidDataException($"{idProperty} isn't of type string");
}
if (attachmentDictionary.TryGetValue(etagProperty, out var lastAttachmentEtagObject) == false)
{
throw new InvalidDataException($"{etagProperty} doesn't exist");
}
if (!(lastAttachmentEtagObject is byte[] lastAttachmentEtagByteArray))
{
throw new InvalidDataException($"{etagProperty} isn't of type byte[]");
}
lastAttachmentEtag = LegacyAttachmentUtils.ByteArrayToEtagString(lastAttachmentEtagByteArray);
if (attachmentDictionary.TryGetValue(metadataProperty, out object metadataObject) == false)
{
throw new InvalidDataException($"{metadataProperty} doesn't exist");
}
if (!(metadataObject is Dictionary <string, object> metadata))
{
throw new InvalidDataException($"{idProperty} isn't of type string");
}
if (metadata.TryGetValue("Raven-Delete-Marker", out var deletedObject) && deletedObject is bool deletedObjectAsBool && deletedObjectAsBool)
{
var id = StreamSource.GetLegacyAttachmentId(attachmentKey);
documentActions.DeleteDocument(id);
continue;
}
var djv = new DynamicJsonValue();
foreach (var keyValue in metadata)
{
var key = keyValue.Key;
if (key.Equals("Raven-Replication-Source") ||
key.Equals("Raven-Replication-Version") ||
key.Equals("Raven-Replication-History"))
{
continue;
}
djv[key] = keyValue.Value;
}
var contextToUse = documentActions.GetContextForNewDocument();
var metadataBlittable = contextToUse.ReadObject(djv, "metadata");
if (attachmentDictionary.TryGetValue(dataProperty, out object dataObject) == false)
{
throw new InvalidDataException($"{dataProperty} doesn't exist");
}
if (!(dataObject is byte[] data))
{
throw new InvalidDataException($"{dataProperty} isn't of type byte[]");
}
using (var dataStream = new MemoryStream(data))
{
var attachment = new DocumentItem.AttachmentStream
{
Stream = documentActions.GetTempStream()
};
var attachmentDetails = StreamSource.GenerateLegacyAttachmentDetails(contextToUse, dataStream, attachmentKey, metadataBlittable, ref attachment);
var documentItem = new DocumentItem
{
Document = new Document
{
Data = StreamSource.WriteDummyDocumentForAttachment(contextToUse, attachmentDetails),
Id = attachmentDetails.Id,
ChangeVector = string.Empty,
Flags = DocumentFlags.HasAttachments,
LastModified = Database.Time.GetUtcNow()
},
Attachments = new List <DocumentItem.AttachmentStream>
{
attachment
}
};
documentActions.WriteDocument(documentItem, progress);
}
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var replicationSource = GetSourceReplicationInformation(context, GetRemoteServerInstanceId(), out var documentId);
replicationSource.LastAttachmentEtag = lastAttachmentEtag;
replicationSource.Source = GetFromServer();
replicationSource.LastModified = DateTime.UtcNow;
await SaveSourceReplicationInformation(replicationSource, context, documentId);
}
}
}
private async Task MigrateDatabase(long operationId, ImportInfo importInfo)
{
var startDocumentEtag = importInfo?.LastEtag ?? 0;
var startRaftIndex = importInfo?.LastRaftIndex ?? 0;
var url = $"{Options.ServerUrl}/databases/{Options.DatabaseName}/smuggler/export?operationId={operationId}&startEtag={startDocumentEtag}&startRaftIndex={startRaftIndex}";
var databaseSmugglerOptionsServerSide = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = Options.OperateOnTypes,
RemoveAnalyzers = Options.RemoveAnalyzers
};
if (importInfo != null)
{
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.Documents))
{
databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.Tombstones;
Options.OperateOnTypes |= DatabaseItemType.Tombstones;
}
if (Options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange))
{
databaseSmugglerOptionsServerSide.OperateOnTypes |= DatabaseItemType.CompareExchangeTombstones;
Options.OperateOnTypes |= DatabaseItemType.CompareExchangeTombstones;
}
}
var json = JsonConvert.SerializeObject(databaseSmugglerOptionsServerSide);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var request = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = content
};
var response = await Parameters.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, Parameters.CancelToken.Token);
if (response.IsSuccessStatusCode == false)
{
var responseString = await response.Content.ReadAsStringAsync();
throw new InvalidOperationException($"Failed to export database from server: {Options.ServerUrl}, " +
$"status code: {response.StatusCode}, " +
$"error: {responseString}");
}
await using (var responseStream = await response.Content.ReadAsStreamAsync())
await using (var stream = new GZipStream(responseStream, mode: CompressionMode.Decompress))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var source = new StreamSource(stream, context, Parameters.Database))
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
TransformScript = Options.TransformScript,
OperateOnTypes = Options.OperateOnTypes,
OperateOnDatabaseRecordTypes = Options.OperateOnDatabaseRecordTypes
};
var smuggler = new Documents.DatabaseSmuggler(Parameters.Database, source, destination, Parameters.Database.Time, options, Parameters.Result, Parameters.OnProgress, Parameters.CancelToken.Token);
await smuggler.ExecuteAsync();
}
}
private async Task MigrateAttachments(string lastEtag, SmugglerResult parametersResult)
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.Initialize(options, parametersResult, buildVersion: default);
using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var attachmentsArray = await GetAttachmentsList(lastEtag, transactionOperationContext);
if (attachmentsArray.Length == 0)
{
var count = Parameters.Result.Documents.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} legacy attachment{(count > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
}
return;
}
foreach (var attachmentObject in attachmentsArray)
{
var blittable = attachmentObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("attachmentObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("Key", out string key) == false)
{
throw new InvalidDataException("Key doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var dataStream = await GetAttachmentStream(key);
if (dataStream == null)
{
Parameters.Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
documentActions.DeleteDocument(id);
continue;
}
var contextToUse = documentActions.GetContextForNewDocument();
using (var old = metadata)
metadata = metadata.Clone(contextToUse);
WriteDocumentWithAttachment(documentActions, contextToUse, dataStream, key, metadata);
Parameters.Result.Documents.ReadCount++;
if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} legacy attachments.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
sp.Restart();
}
}
var lastAttachment = attachmentsArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastAttachment != null, "lastAttachment != null");
if (lastAttachment.TryGet("Etag", out string etag))
{
lastEtag = Parameters.Result.LegacyLastAttachmentEtag = etag;
}
}
}
}
public async Task PostCreateSampleData()
{
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
using (context.OpenReadTransaction())
{
foreach (var collection in Database.DocumentsStorage.GetCollections(context))
{
if (collection.Count > 0)
{
throw new InvalidOperationException("You cannot create sample data in a database that already contains documents");
}
}
}
var operateOnTypesAsString = GetStringValuesQueryString("operateOnTypes", required: false);
var operateOnTypes = GetOperateOnTypes(operateOnTypesAsString);
if (operateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments))
{
var editRevisions = new EditRevisionsConfigurationCommand(new RevisionsConfiguration
{
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
["Orders"] = new RevisionsCollectionConfiguration
{
Disabled = false
}
}
}, Database.Name, GetRaftRequestIdFromQuery() + "/revisions");
var(index, _) = await ServerStore.SendToLeaderAsync(editRevisions);
await Database.RachisLogIndexNotifications.WaitForIndexNotification(index, Database.ServerStore.Engine.OperationTimeout);
}
if (operateOnTypes.HasFlag(DatabaseItemType.TimeSeries))
{
var tsConfig = new TimeSeriesConfiguration
{
NamedValues = new Dictionary <string, Dictionary <string, string[]> >
{
["Companies"] = new Dictionary <string, string[]>
{
["StockPrices"] = new[] { "Open", "Close", "High", "Low", "Volume" }
},
["Employees"] = new Dictionary <string, string[]>
{
["HeartRates"] = new[] { "BPM" }
}
}
};
var editTimeSeries = new EditTimeSeriesConfigurationCommand(tsConfig, Database.Name, GetRaftRequestIdFromQuery() + "/time-series");
var(index, _) = await ServerStore.SendToLeaderAsync(editTimeSeries);
await Database.RachisLogIndexNotifications.WaitForIndexNotification(index, Database.ServerStore.Engine.OperationTimeout);
}
await using (var sampleData = typeof(SampleDataHandler).Assembly
.GetManifestResourceStream("Raven.Server.Web.Studio.EmbeddedData.Northwind.ravendbdump"))
{
await using (var stream = new GZipStream(sampleData, CompressionMode.Decompress))
using (var source = new StreamSource(stream, context, Database))
{
var destination = new DatabaseDestination(Database);
var smuggler = new DatabaseSmuggler(Database, source, destination, Database.Time,
options: new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = operateOnTypes,
SkipRevisionCreation = true
});
await smuggler.ExecuteAsync();
}
}
await NoContent();
}
private async Task <string> MigrateRavenFs(string lastEtag, SmugglerResult parametersResult)
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.InitializeAsync(options, parametersResult, _buildVersion);
using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
await using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var ravenFsHeadersArray = await GetRavenFsHeadersArray(lastEtag, transactionOperationContext);
if (ravenFsHeadersArray.Length == 0)
{
var count = Parameters.Result.Documents.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} RavenFS file{(count > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
}
return(lastEtag);
}
foreach (var headerObject in ravenFsHeadersArray)
{
var blittable = headerObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("headerObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("FullPath", out string fullPath) == false)
{
throw new InvalidDataException("FullPath doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var key = fullPath.TrimStart('/');
var dataStream = await GetRavenFsStream(key);
if (dataStream == null)
{
Parameters.Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
await documentActions.DeleteDocumentAsync(id);
continue;
}
var contextToUse = documentActions.GetContextForNewDocument();
metadata = GetCleanMetadata(metadata, contextToUse);
await WriteDocumentWithAttachmentAsync(documentActions, contextToUse, dataStream, key, metadata);
Parameters.Result.Documents.ReadCount++;
if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} " +
$"RavenFS file{(Parameters.Result.Documents.ReadCount > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
sp.Restart();
}
}
var lastFile = ravenFsHeadersArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastFile != null, "lastAttachment != null");
if (lastFile.TryGet("Etag", out string etag))
{
lastEtag = etag;
}
}
}
}
private async Task MigrateAttachments(string lastEtag)
{
var destination = new DatabaseDestination(Database);
using (Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var attatchmentsArray = await GetAttachmentsList(lastEtag, transactionOperationContext);
if (attatchmentsArray.Length == 0)
{
var count = Result.Documents.Attachments.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} legacy attachment{(count > 1 ? "s" : string.Empty)}.";
Result.AddInfo(message);
OnProgress.Invoke(Result.Progress);
}
return;
}
foreach (var attachmentObject in attatchmentsArray)
{
var blittable = attachmentObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("attchmentObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("Key", out string key) == false)
{
throw new InvalidDataException("Key doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var dataStream = await GetAttachmentStream(key);
if (dataStream == null)
{
Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
documentActions.DeleteDocument(id);
continue;
}
WriteDocumentWithAttachment(documentActions, context, dataStream, key, metadata);
Result.Documents.ReadCount++;
if (Result.Documents.Attachments.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Result.Documents.Attachments.ReadCount:#,#;;0} legacy attachments.";
Result.AddInfo(message);
OnProgress.Invoke(Result.Progress);
sp.Restart();
}
}
var lastAttachment = attatchmentsArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastAttachment != null, "lastAttachment != null");
if (lastAttachment.TryGet("Etag", out string etag))
{
lastEtag = Result.LegacyLastAttachmentEtag = etag;
}
}
}
}
