private async Task <string> MigrateRavenFs(string lastEtag, SmugglerResult parametersResult)
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.InitializeAsync(options, parametersResult, _buildVersion);
using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
await using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var ravenFsHeadersArray = await GetRavenFsHeadersArray(lastEtag, transactionOperationContext);
if (ravenFsHeadersArray.Length == 0)
{
var count = Parameters.Result.Documents.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} RavenFS file{(count > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
}
return(lastEtag);
}
foreach (var headerObject in ravenFsHeadersArray)
{
var blittable = headerObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("headerObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("FullPath", out string fullPath) == false)
{
throw new InvalidDataException("FullPath doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var key = fullPath.TrimStart('/');
var dataStream = await GetRavenFsStream(key);
if (dataStream == null)
{
Parameters.Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
await documentActions.DeleteDocumentAsync(id);
continue;
}
var contextToUse = documentActions.GetContextForNewDocument();
metadata = GetCleanMetadata(metadata, contextToUse);
await WriteDocumentWithAttachmentAsync(documentActions, contextToUse, dataStream, key, metadata);
Parameters.Result.Documents.ReadCount++;
if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} " +
$"RavenFS file{(Parameters.Result.Documents.ReadCount > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
sp.Restart();
}
}
var lastFile = ravenFsHeadersArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastFile != null, "lastAttachment != null");
if (lastFile.TryGet("Etag", out string etag))
{
lastEtag = etag;
}
}
}
}
private async Task MigrateAttachments(string lastEtag)
{
var destination = new DatabaseDestination(Database);
using (Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var attatchmentsArray = await GetAttachmentsList(lastEtag, transactionOperationContext);
if (attatchmentsArray.Length == 0)
{
var count = Result.Documents.Attachments.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} legacy attachment{(count > 1 ? "s" : string.Empty)}.";
Result.AddInfo(message);
OnProgress.Invoke(Result.Progress);
}
return;
}
foreach (var attachmentObject in attatchmentsArray)
{
var blittable = attachmentObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("attchmentObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("Key", out string key) == false)
{
throw new InvalidDataException("Key doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var dataStream = await GetAttachmentStream(key);
if (dataStream == null)
{
Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
documentActions.DeleteDocument(id);
continue;
}
WriteDocumentWithAttachment(documentActions, context, dataStream, key, metadata);
Result.Documents.ReadCount++;
if (Result.Documents.Attachments.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Result.Documents.Attachments.ReadCount:#,#;;0} legacy attachments.";
Result.AddInfo(message);
OnProgress.Invoke(Result.Progress);
sp.Restart();
}
}
var lastAttachment = attatchmentsArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastAttachment != null, "lastAttachment != null");
if (lastAttachment.TryGet("Etag", out string etag))
{
lastEtag = Result.LegacyLastAttachmentEtag = etag;
}
}
}
}
private async Task MigrateAttachments(string lastEtag, SmugglerResult parametersResult)
{
var destination = new DatabaseDestination(Parameters.Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.Initialize(options, parametersResult, buildVersion: default);
using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (var documentActions = destination.Documents())
{
var sp = Stopwatch.StartNew();
while (true)
{
var attachmentsArray = await GetAttachmentsList(lastEtag, transactionOperationContext);
if (attachmentsArray.Length == 0)
{
var count = Parameters.Result.Documents.ReadCount;
if (count > 0)
{
var message = $"Read {count:#,#;;0} legacy attachment{(count > 1 ? "s" : string.Empty)}.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
}
return;
}
foreach (var attachmentObject in attachmentsArray)
{
var blittable = attachmentObject as BlittableJsonReaderObject;
if (blittable == null)
{
throw new InvalidDataException("attachmentObject isn't a BlittableJsonReaderObject");
}
if (blittable.TryGet("Key", out string key) == false)
{
throw new InvalidDataException("Key doesn't exist");
}
if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new InvalidDataException("Metadata doesn't exist");
}
var dataStream = await GetAttachmentStream(key);
if (dataStream == null)
{
Parameters.Result.Tombstones.ReadCount++;
var id = StreamSource.GetLegacyAttachmentId(key);
documentActions.DeleteDocument(id);
continue;
}
var contextToUse = documentActions.GetContextForNewDocument();
using (var old = metadata)
metadata = metadata.Clone(contextToUse);
WriteDocumentWithAttachment(documentActions, contextToUse, dataStream, key, metadata);
Parameters.Result.Documents.ReadCount++;
if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000)
{
var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} legacy attachments.";
Parameters.Result.AddInfo(message);
Parameters.OnProgress.Invoke(Parameters.Result.Progress);
sp.Restart();
}
}
var lastAttachment = attachmentsArray.Last() as BlittableJsonReaderObject;
Debug.Assert(lastAttachment != null, "lastAttachment != null");
if (lastAttachment.TryGet("Etag", out string etag))
{
lastEtag = Parameters.Result.LegacyLastAttachmentEtag = etag;
}
}
}
}
public async Task Attachments()
{
var destination = new DatabaseDestination(Database);
var options = new DatabaseSmugglerOptionsServerSide
{
OperateOnTypes = DatabaseItemType.Attachments,
SkipRevisionCreation = true
};
destination.Initialize(options, null, buildVersion: default);
using (var documentActions = destination.Documents())
using (var buffered = new BufferedStream(RequestBodyStream()))
using (var reader = new BsonReader(buffered))
{
var result = LegacyAttachmentUtils.GetObject(reader);
const string idProperty = "@id";
const string etagProperty = "@etag";
const string metadataProperty = "@metadata";
const string dataProperty = "data";
string lastAttachmentEtag = null;
var progress = new SmugglerProgressBase.CountsWithLastEtag();
foreach (var attachmentObject in result.Values)
{
if (!(attachmentObject is Dictionary <string, object> attachmentDictionary))
{
throw new InvalidDataException("attachmentObject isn't a Dictionary<string, object>");
}
if (attachmentDictionary.TryGetValue(idProperty, out var attachmentKeyObject) == false)
{
throw new InvalidDataException($"{idProperty} doesn't exist");
}
if (!(attachmentKeyObject is string attachmentKey))
{
throw new InvalidDataException($"{idProperty} isn't of type string");
}
if (attachmentDictionary.TryGetValue(etagProperty, out var lastAttachmentEtagObject) == false)
{
throw new InvalidDataException($"{etagProperty} doesn't exist");
}
if (!(lastAttachmentEtagObject is byte[] lastAttachmentEtagByteArray))
{
throw new InvalidDataException($"{etagProperty} isn't of type byte[]");
}
lastAttachmentEtag = LegacyAttachmentUtils.ByteArrayToEtagString(lastAttachmentEtagByteArray);
if (attachmentDictionary.TryGetValue(metadataProperty, out object metadataObject) == false)
{
throw new InvalidDataException($"{metadataProperty} doesn't exist");
}
if (!(metadataObject is Dictionary <string, object> metadata))
{
throw new InvalidDataException($"{idProperty} isn't of type string");
}
if (metadata.TryGetValue("Raven-Delete-Marker", out var deletedObject) && deletedObject is bool deletedObjectAsBool && deletedObjectAsBool)
{
var id = StreamSource.GetLegacyAttachmentId(attachmentKey);
documentActions.DeleteDocument(id);
continue;
}
var djv = new DynamicJsonValue();
foreach (var keyValue in metadata)
{
var key = keyValue.Key;
if (key.Equals("Raven-Replication-Source") ||
key.Equals("Raven-Replication-Version") ||
key.Equals("Raven-Replication-History"))
{
continue;
}
djv[key] = keyValue.Value;
}
var contextToUse = documentActions.GetContextForNewDocument();
var metadataBlittable = contextToUse.ReadObject(djv, "metadata");
if (attachmentDictionary.TryGetValue(dataProperty, out object dataObject) == false)
{
throw new InvalidDataException($"{dataProperty} doesn't exist");
}
if (!(dataObject is byte[] data))
{
throw new InvalidDataException($"{dataProperty} isn't of type byte[]");
}
using (var dataStream = new MemoryStream(data))
{
var attachment = new DocumentItem.AttachmentStream
{
Stream = documentActions.GetTempStream()
};
var attachmentDetails = StreamSource.GenerateLegacyAttachmentDetails(contextToUse, dataStream, attachmentKey, metadataBlittable, ref attachment);
var documentItem = new DocumentItem
{
Document = new Document
{
Data = StreamSource.WriteDummyDocumentForAttachment(contextToUse, attachmentDetails),
Id = attachmentDetails.Id,
ChangeVector = string.Empty,
Flags = DocumentFlags.HasAttachments,
LastModified = Database.Time.GetUtcNow()
},
Attachments = new List <DocumentItem.AttachmentStream>
{
attachment
}
};
documentActions.WriteDocument(documentItem, progress);
}
}
using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
var replicationSource = GetSourceReplicationInformation(context, GetRemoteServerInstanceId(), out var documentId);
replicationSource.LastAttachmentEtag = lastAttachmentEtag;
replicationSource.Source = GetFromServer();
replicationSource.LastModified = DateTime.UtcNow;
await SaveSourceReplicationInformation(replicationSource, context, documentId);
}
}
}
