private void EnsureSnapshotProcessed(DatabaseSummary databaseSummary, SmugglerResult snapshotSmugglerResult, long indexesCount)
{
_backupResult.SnapshotBackup.Processed = true;
_backupResult.DatabaseRecord.Processed = true;
_backupResult.RevisionDocuments.Attachments.Processed = true;
_backupResult.Tombstones.Processed = true;
_backupResult.Indexes.Processed = true;
_backupResult.Indexes.ReadCount = indexesCount;
_backupResult.Documents.Processed = true;
_backupResult.Documents.ReadCount = databaseSummary.DocumentsCount;
_backupResult.Documents.Attachments.Processed = true;
_backupResult.Documents.Attachments.ReadCount = databaseSummary.AttachmentsCount;
_backupResult.Counters.Processed = true;
_backupResult.Counters.ReadCount = databaseSummary.CounterEntriesCount;
_backupResult.RevisionDocuments.Processed = true;
_backupResult.RevisionDocuments.ReadCount = databaseSummary.RevisionsCount;
_backupResult.Conflicts.Processed = true;
_backupResult.Conflicts.ReadCount = databaseSummary.ConflictsCount;
_backupResult.Identities.Processed = true;
_backupResult.Identities.ReadCount = snapshotSmugglerResult.Identities.ReadCount;
_backupResult.CompareExchange.Processed = true;
_backupResult.CompareExchange.ReadCount = snapshotSmugglerResult.CompareExchange.ReadCount;
_backupResult.CompareExchangeTombstones.Processed = true;
_backupResult.Subscriptions.Processed = true;
_backupResult.Subscriptions.ReadCount = snapshotSmugglerResult.Subscriptions.ReadCount;
_backupResult.TimeSeries.Processed = true;
_backupResult.TimeSeries.ReadCount = databaseSummary.TimeSeriesSegmentsCount;
}
private void EnsureSnapshotProcessed(
DocumentsOperationContext context,
SmugglerResult snapshotSmugglerResult,
long indexesCount)
{
_backupResult.SnapshotBackup.Processed = true;
_backupResult.DatabaseRecord.Processed = true;
_backupResult.RevisionDocuments.Attachments.Processed = true;
_backupResult.Tombstones.Processed = true;
_backupResult.Indexes.Processed = true;
_backupResult.Indexes.ReadCount = indexesCount;
var summary = _database.GetDatabaseSummary(context);
_backupResult.Documents.Processed = true;
_backupResult.Documents.ReadCount = summary.DocumentsCount;
_backupResult.Documents.Attachments.Processed = true;
_backupResult.Documents.Attachments.ReadCount = summary.AttachmentsCount;
_backupResult.RevisionDocuments.Processed = true;
_backupResult.RevisionDocuments.ReadCount = summary.RevisionsCount;
_backupResult.Conflicts.Processed = true;
_backupResult.Conflicts.ReadCount = summary.ConflictsCount;
_backupResult.Identities.Processed = true;
_backupResult.Identities.ReadCount = snapshotSmugglerResult.Identities.ReadCount;
_backupResult.CompareExchange.Processed = true;
_backupResult.CompareExchange.ReadCount = snapshotSmugglerResult.CompareExchange.ReadCount;
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_result = result;
_returnBuffer = _context.GetManagedBuffer(out _buffer);
_state = new JsonParserState();
_parser = new UnmanagedJsonParser(_context, _state, "file");
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json.", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartObject)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start object, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
buildVersion = ReadBuildVersion();
_buildVersionType = BuildVersion.Type(buildVersion);
_readLegacyEtag = options.ReadLegacyEtag;
return(new DisposableAction(() =>
{
_parser.Dispose();
_returnBuffer.Dispose();
_returnWriteBuffer.Dispose();
}));
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Counters))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
}
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
}));
}
private static void WriteImportResult(JsonOperationContext context, SmugglerResult result, Stream stream)
{
using (var writer = new BlittableJsonTextWriter(context, stream))
{
var json = result.ToJson();
context.Write(writer, json);
}
}
public SmugglerCounterBatchCommand(DocumentDatabase database, SmugglerResult result)
{
_database = database;
_result = result;
_counterGroups = new List <CounterGroupDetail>();
_counterUpdates = new Dictionary <string, Document>();
_toDispose = new List <IDisposable>();
_resetContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
}
private SmugglerProgressBase.Counts ProcessIdentities(SmugglerResult result)
{
using (var actions = _destination.Identities())
{
foreach (var kvp in _source.GetIdentities())
{
_token.ThrowIfCancellationRequested();
result.Identities.ReadCount++;
if (kvp.Equals(default))
private void CompletePendingTransactions(SmugglerResult result)
{
// If we export documents from a database,
// we should wait for all the pending transactions to be completed first.
var shouldExecute = _options.ExecutePendingClusterTransactions ||
(_options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) && _source is DatabaseSource);
if (shouldExecute == false)
{
return;
}
_database.ExectueClusterTransactionOnDatabase(WaitForClusterTransactionCompletion);
void WaitForClusterTransactionCompletion(IReadOnlyList <Task> transactionTasks)
{
result.AddInfo($"Has to processing {transactionTasks.Count} cluster transactions before the export can take place.");
_onProgress.Invoke(result.Progress);
for (var index = 0; index < transactionTasks.Count; index++)
{
var task = transactionTasks[index];
_token.ThrowIfCancellationRequested();
while (task.IsCompleted == false)
{
_token.ThrowIfCancellationRequested();
if (task.Wait((int)TimeSpan.FromSeconds(10).TotalMilliseconds, _token) == false)
{
_token.ThrowIfCancellationRequested();
result.AddInfo($"Processing cluster transaction {index}.");
_onProgress.Invoke(result.Progress);
}
}
if (task.IsCompletedSuccessfully)
{
result.AddInfo($"Cluster transaction {index} out of {transactionTasks.Count} is completed.");
_onProgress.Invoke(result.Progress);
}
if (task.IsCanceled)
{
result.AddInfo($"Cluster transaction {index} was canceled.");
_onProgress.Invoke(result.Progress);
}
if (task.IsFaulted)
{
_result.AddError($"Cluster transaction {index} is faulted: {task.Exception}.");
_onProgress.Invoke(result.Progress);
}
}
}
}
public Migrator_V2(
string serverUrl,
string databaseName,
SmugglerResult result,
Action <IOperationProgress> onProgress,
DocumentDatabase database,
HttpClient client,
OperationCancelToken cancelToken)
: base(serverUrl, databaseName, result, onProgress, database, cancelToken)
{
_client = client;
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
buildVersion = 40;
_reader = new StreamReader(_stream);
_csvReader = new CsvReader(_reader);
_csvReader.Configuration.Delimiter = ",";
return(new DisposableAction(() =>
{
_reader.Dispose();
_csvReader.Dispose();
}));
}
public static void EnsureProcessed(SmugglerResult result)
{
EnsureStepProcessed(result.DatabaseRecord);
EnsureStepProcessed(result.Documents);
EnsureStepProcessed(result.Documents.Attachments);
EnsureStepProcessed(result.RevisionDocuments);
EnsureStepProcessed(result.RevisionDocuments.Attachments);
EnsureStepProcessed(result.Tombstones);
EnsureStepProcessed(result.Conflicts);
EnsureStepProcessed(result.Indexes);
EnsureStepProcessed(result.Identities);
EnsureStepProcessed(result.CompareExchange);
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeTransaction.Dispose();
_returnContext.Dispose();
}));
}
protected AbstractMigrator(MigratorOptions options)
{
MigrationStateKey = options.MigrationStateKey;
ServerUrl = options.ServerUrl;
DatabaseName = options.DatabaseName;
HttpClient = options.HttpClient;
OperateOnTypes = options.OperateOnTypes;
RemoveAnalyzers = options.RemoveAnalyzers;
ImportRavenFs = options.ImportRavenFs;
Result = options.Result;
OnProgress = options.OnProgress;
Database = options.Database;
CancelToken = options.CancelToken;
}
protected AbstractMigrator(
string serverUrl,
string databaseName,
SmugglerResult result,
Action <IOperationProgress> onProgress,
DocumentDatabase database,
OperationCancelToken cancelToken)
{
ServerUrl = serverUrl;
DatabaseName = databaseName;
Result = result;
OnProgress = onProgress;
Database = database;
CancelToken = cancelToken;
}
public Task <SmugglerInitializeResult> InitializeAsync(DatabaseSmugglerOptionsServerSide options, SmugglerResult result)
{
_reader = new StreamReader(_stream);
_csvReader = new CsvReader(_reader, _csvHelperConfig);
_result = result;
var disposable = new DisposableAction(() =>
{
_reader.Dispose();
_csvReader.Dispose();
});
return(Task.FromResult(new SmugglerInitializeResult(disposable, ServerVersion.DevBuildNumber)));
}
public IDisposable Initialize(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, out long buildVersion)
{
buildVersion = ServerVersion.DevBuildNumber;
_reader = new StreamReader(_stream);
_csvReader = new CsvReader(_reader, _csvHelperConfig);
_result = result;
return(new DisposableAction(() =>
{
_reader.Dispose();
_csvReader.Dispose();
}));
}
public Migrator_V3(
string serverUrl,
string databaseName,
SmugglerResult result,
Action <IOperationProgress> onProgress,
DocumentDatabase database,
HttpClient client,
string migrationStateKey,
MajorVersion marjorVersion,
OperationCancelToken cancelToken)
: base(serverUrl, databaseName, result, onProgress, database, cancelToken)
{
_client = client;
_migrationStateKey = migrationStateKey;
_majorVersion = marjorVersion;
}
public static void EnsureProcessed(SmugglerResult result, bool skipped = true)
{
EnsureStepProcessed(result.DatabaseRecord, skipped);
EnsureStepProcessed(result.Documents, skipped);
EnsureStepProcessed(result.Documents.Attachments, skipped);
EnsureStepProcessed(result.RevisionDocuments, skipped);
EnsureStepProcessed(result.RevisionDocuments.Attachments, skipped);
EnsureStepProcessed(result.Counters, skipped);
EnsureStepProcessed(result.Tombstones, skipped);
EnsureStepProcessed(result.Conflicts, skipped);
EnsureStepProcessed(result.Indexes, skipped);
EnsureStepProcessed(result.Identities, skipped);
EnsureStepProcessed(result.CompareExchange, skipped);
EnsureStepProcessed(result.CompareExchangeTombstones, skipped);
EnsureStepProcessed(result.Subscriptions, skipped);
EnsureStepProcessed(result.TimeSeries, skipped);
public Importer(
string serverUrl,
string sourceDatabaseName,
SmugglerResult result,
Action <IOperationProgress> onProgress,
DocumentDatabase database,
string migrationStateKey,
OperationCancelToken cancelToken)
: base(serverUrl, sourceDatabaseName, result, onProgress, database, cancelToken)
{
_migrationStateKey = migrationStateKey;
_requestExecutor = RequestExecutor.CreateForSingleNodeWithoutConfigurationUpdates(ServerUrl, DatabaseName, Database.ServerStore.Server.ClusterCertificateHolder.Certificate, DocumentConventions.Default);
_client = _requestExecutor.HttpClient;
}
protected AbstractMigrator(MigratorOptions options)
{
MigrationStateKey = options.MigrationStateKey;
ServerUrl = options.ServerUrl;
DatabaseName = options.DatabaseName;
HttpClient = options.HttpClient;
ApiKey = options.ApiKey;
EnableBasicAuthenticationOverUnsecuredHttp = options.EnableBasicAuthenticationOverUnsecuredHttp;
SkipServerCertificateValidation = options.SkipServerCertificateValidation;
OperateOnTypes = options.OperateOnTypes;
RemoveAnalyzers = options.RemoveAnalyzers;
ImportRavenFs = options.ImportRavenFs;
Result = options.Result;
OnProgress = options.OnProgress;
Database = options.Database;
CancelToken = options.CancelToken;
}
public IDisposable Initialize(DatabaseSmugglerOptions options, SmugglerResult result, long buildVersion)
{
_gzipStream = new GZipStream(_stream, CompressionMode.Compress, leaveOpen: true);
_writer = new BlittableJsonTextWriter(_context, _gzipStream);
_writer.WriteStartObject();
_writer.WritePropertyName("BuildVersion");
_writer.WriteInteger(buildVersion);
return(new DisposableAction(() =>
{
_writer.WriteEndObject();
_writer.Dispose();
_gzipStream.Dispose();
}));
}
public SmugglerProgress(SmugglerResult result)
{
_result = result;
Message = _result?.Message;
DatabaseRecord = _result?.DatabaseRecord;
Documents = _result?.Documents;
RevisionDocuments = _result?.RevisionDocuments;
Tombstones = _result?.Tombstones;
Conflicts = _result?.Conflicts;
Identities = _result?.Identities;
Indexes = _result?.Indexes;
CompareExchange = _result?.CompareExchange;
Counters = _result?.Counters;
CompareExchangeTombstones = _result?.CompareExchangeTombstones;
Subscriptions = _result?.Subscriptions;
TimeSeries = _result?.TimeSeries;
ReplicationHubCertificates = _result?.ReplicationHubCertificates;
}
public DatabaseSmuggler(DocumentDatabase database, ISmugglerSource source, ISmugglerDestination destination, SystemTime time,
DatabaseSmugglerOptionsServerSide options = null, SmugglerResult result = null, Action <IOperationProgress> onProgress = null,
CancellationToken token = default)
{
_database = database;
_source = source;
_destination = destination;
_options = options ?? new DatabaseSmugglerOptionsServerSide();
_result = result;
_token = token;
if (string.IsNullOrWhiteSpace(_options.TransformScript) == false)
{
_patcher = new SmugglerPatcher(_options, database);
}
_time = time;
_onProgress = onProgress ?? (progress => { });
}
public IDisposable Initialize(DatabaseSmugglerOptionsServerSide options, SmugglerResult result, out long buildVersion)
{
_currentTypeIndex = 0;
if (options.OperateOnTypes.HasFlag(DatabaseItemType.Documents) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.RevisionDocuments) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Tombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Conflicts) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CounterGroups) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries))
{
_returnContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
_disposeTransaction = _context.OpenReadTransaction();
LastEtag = DocumentsStorage.ReadLastEtag(_disposeTransaction.InnerTransaction);
LastDatabaseChangeVector = DocumentsStorage.GetDatabaseChangeVector(_disposeTransaction.InnerTransaction);
}
if (options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchange) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Identities) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.CompareExchangeTombstones) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.Subscriptions) ||
options.OperateOnTypes.HasFlag(DatabaseItemType.ReplicationHubCertificates))
{
_returnServerContext = _database.ServerStore.ContextPool.AllocateOperationContext(out _serverContext);
_disposeServerTransaction = _serverContext.OpenReadTransaction();
using (var rawRecord = _database.ServerStore.Cluster.ReadRawDatabaseRecord(_serverContext, _database.Name))
{
LastRaftIndex = rawRecord.EtagForBackup;
}
}
buildVersion = ServerVersion.Build;
return(new DisposableAction(() =>
{
_disposeServerTransaction?.Dispose();
_returnServerContext?.Dispose();
_disposeTransaction?.Dispose();
_returnContext?.Dispose();
}));
}
public void Can_Build_Serializator_For_SmugglerResult()
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var result = new SmugglerResult();
result.AddError("MessageA");
result.AddInfo("MessageB");
result.AddWarning("MessageC");
result.AddMessage("MessageD");
var djv = result.ToJson();
var json = context.ReadObject(djv, "smuggler/result");
var result2 = JsonDeserializationClient.SmugglerResult(json);
Assert.Equal(result.Messages, result2.Messages);
var result3 = DocumentConventions.Default.Serialization.DefaultConverter.FromBlittable <SmugglerResult>(json);
Assert.Equal(result.Messages, result3.Messages);
}
}
public DatabaseSmuggler(DocumentDatabase database, ISmugglerSource source, ISmugglerDestination destination, SystemTime time,
DatabaseSmugglerOptionsServerSide options = null, SmugglerResult result = null, Action <IOperationProgress> onProgress = null,
CancellationToken token = default)
{
_database = database;
_source = source;
_destination = destination;
_options = options ?? new DatabaseSmugglerOptionsServerSide();
_result = result;
_token = token;
if (string.IsNullOrWhiteSpace(_options.TransformScript) == false)
{
_patcher = new SmugglerPatcher(_options, database);
}
Debug.Assert((source is DatabaseSource && destination is DatabaseDestination) == false,
"When both source and destination are database, we might get into a delayed write for the dest while the " +
"source already pulsed its' read transaction, resulting in bad memory read.");
_time = time;
_onProgress = onProgress ?? (progress => { });
}
private void ProcessDocumentsWithDuplicateCollection(SmugglerResult result)
{
var didWork = false;
var count = 0;
using (var actions = _destination.Documents())
{
foreach (var item in actions.GetDocumentsWithDuplicateCollection())
{
if (didWork == false)
{
result.AddInfo("Starting to process documents with duplicate collection.");
didWork = true;
}
actions.WriteDocument(item, result.Documents);
count++;
}
}
if (didWork)
{
result.AddInfo($"Finished processing '{count}' documents with duplicate collection.");
}
}
private async Task <string> MigrateRavenFs(string lastEtag, SmugglerResult parametersResult)
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.InitializeAsync(options, parametersResult, _buildVersion);
using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
await using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var ravenFsHeadersArray = await GetRavenFsHeadersArray(lastEtag, transactionOperationContext);
if (ravenFsHeadersArray.Length == 0)
{
var count = Parameters.Result.Documents.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} RavenFS file{(count > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
}
return(lastEtag);
}
foreach (var headerObject in ravenFsHeadersArray)
{
var blittable = headerObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("headerObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("FullPath", out string fullPath) == false)
{
throw new InvalidDataException("FullPath doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var key = fullPath.TrimStart('/');
var dataStream = await GetRavenFsStream(key);
if (dataStream == null)
{
Parameters.Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
await documentActions.DeleteDocumentAsync(id);
continue;
}
var contextToUse = documentActions.GetContextForNewDocument();
metadata = GetCleanMetadata(metadata, contextToUse);
await WriteDocumentWithAttachmentAsync(documentActions, contextToUse, dataStream, key, metadata);
Parameters.Result.Documents.ReadCount++;
if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} " +
$"RavenFS file{(Parameters.Result.Documents.ReadCount > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
sp.Restart();
}
}
var lastFile = ravenFsHeadersArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastFile != null, "lastAttachment != null");
if (lastFile.TryGet("Etag", out string etag))
{
lastEtag = etag;
}
}
}
}
private async Task MigrateAttachments(string lastEtag, SmugglerResult parametersResult)
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.Initialize(options, parametersResult, buildVersion: default);
using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var attachmentsArray = await GetAttachmentsList(lastEtag, transactionOperationContext);
if (attachmentsArray.Length == 0)
{
var count = Parameters.Result.Documents.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} legacy attachment{(count > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
}
return;
}
foreach (var attachmentObject in attachmentsArray)
{
var blittable = attachmentObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("attachmentObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("Key", out string key) == false)
{
throw new InvalidDataException("Key doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var dataStream = await GetAttachmentStream(key);
if (dataStream == null)
{
Parameters.Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
documentActions.DeleteDocument(id);
continue;
}
var contextToUse = documentActions.GetContextForNewDocument();
using (var old = metadata)
metadata = metadata.Clone(contextToUse);
WriteDocumentWithAttachment(documentActions, contextToUse, dataStream, key, metadata);
Parameters.Result.Documents.ReadCount++;
if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} legacy attachments.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
sp.Restart();
}
}
var lastAttachment = attachmentsArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastAttachment != null, "lastAttachment != null");
if (lastAttachment.TryGet("Etag", out string etag))
{
lastEtag = Parameters.Result.LegacyLastAttachmentEtag = etag;
}
}
}
}
private void SkipType(DatabaseItemType type, SmugglerResult result, bool ensureStepProcessed = true)
{
result.AddInfo($"Skipping '{type}' processing.");
_onProgress.Invoke(result.Progress);
SmugglerProgressBase.Counts counts;
switch (type)
{
case DatabaseItemType.DatabaseRecord:
counts = result.DatabaseRecord;
break;
case DatabaseItemType.Documents:
counts = result.Documents;
break;
case DatabaseItemType.RevisionDocuments:
counts = result.RevisionDocuments;
break;
case DatabaseItemType.Tombstones:
counts = result.Tombstones;
break;
case DatabaseItemType.Conflicts:
counts = result.Conflicts;
break;
case DatabaseItemType.Indexes:
counts = result.Indexes;
break;
case DatabaseItemType.Identities:
counts = result.Identities;
break;
case DatabaseItemType.CompareExchange:
counts = result.CompareExchange;
break;
case DatabaseItemType.Counters:
counts = result.Counters;
break;
case DatabaseItemType.LegacyDocumentDeletions:
counts = new SmugglerProgressBase.Counts();
break;
default:
throw new ArgumentOutOfRangeException(nameof(type), type, null);
}
void OnSkipped(long skipped)
{
if (type == DatabaseItemType.Documents)
{
result.Documents.SkippedCount = skipped;
}
if (skipped % 10000 != 0)
{
return;
}
result.AddInfo($"Skipped {skipped:#,#;;0} {type.ToString().ToLowerInvariant()}");
_onProgress.Invoke(result.Progress);
}
var numberOfItemsSkipped = _source.SkipType(type, onSkipped: OnSkipped);
if (ensureStepProcessed == false)
{
return;
}
counts.Skipped = true;
counts.Processed = true;
if (numberOfItemsSkipped > 0)
{
counts.ReadCount = numberOfItemsSkipped;
result.AddInfo($"Skipped '{type}' processing. Skipped {numberOfItemsSkipped:#,#;;0} items.");
}
else
{
result.AddInfo($"Skipped '{type}' processing.");
}
_onProgress.Invoke(result.Progress);
}