private async Task MigrateAttachments(string lastEtag, SmugglerResult parametersResult) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = DatabaseItemType.Attachments, SkipRevisionCreation = true }; destination.Initialize(options, parametersResult, buildVersion: default); using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext)) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) using (var documentActions = destination.Documents()) { var sp = Stopwatch.StartNew(); while (true) { var attachmentsArray = await GetAttachmentsList(lastEtag, transactionOperationContext); if (attachmentsArray.Length == 0) { var count = Parameters.Result.Documents.ReadCount; if (count > 0) { var message = $"Read {count:#,#;;0} legacy attachment{(count > 1 ? "s" : string.Empty)}."; Parameters.Result.AddInfo(message); Parameters.OnProgress.Invoke(Parameters.Result.Progress); } return; } foreach (var attachmentObject in attachmentsArray) { var blittable = attachmentObject as BlittableJsonReaderObject; if (blittable == null) { throw new InvalidDataException("attachmentObject isn't a BlittableJsonReaderObject"); } if (blittable.TryGet("Key", out string key) == false) { throw new InvalidDataException("Key doesn't exist"); } if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false) { throw new InvalidDataException("Metadata doesn't exist"); } var dataStream = await GetAttachmentStream(key); if (dataStream == null) { Parameters.Result.Tombstones.ReadCount++; var id = StreamSource.GetLegacyAttachmentId(key); documentActions.DeleteDocument(id); continue; } var contextToUse = documentActions.GetContextForNewDocument(); using (var old = metadata) metadata = metadata.Clone(contextToUse); WriteDocumentWithAttachment(documentActions, contextToUse, dataStream, key, metadata); Parameters.Result.Documents.ReadCount++; if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000) { var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} legacy attachments."; Parameters.Result.AddInfo(message); Parameters.OnProgress.Invoke(Parameters.Result.Progress); sp.Restart(); } } var lastAttachment = attachmentsArray.Last() as BlittableJsonReaderObject; Debug.Assert(lastAttachment != null, "lastAttachment != null"); if (lastAttachment.TryGet("Etag", out string etag)) { lastEtag = Parameters.Result.LegacyLastAttachmentEtag = etag; } } } }
public async Task Attachments() { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = DatabaseItemType.Attachments, SkipRevisionCreation = true }; destination.Initialize(options, null, buildVersion: default); using (var documentActions = destination.Documents()) using (var buffered = new BufferedStream(RequestBodyStream())) using (var reader = new BsonReader(buffered)) { var result = LegacyAttachmentUtils.GetObject(reader); const string idProperty = "@id"; const string etagProperty = "@etag"; const string metadataProperty = "@metadata"; const string dataProperty = "data"; string lastAttachmentEtag = null; var progress = new SmugglerProgressBase.CountsWithLastEtag(); foreach (var attachmentObject in result.Values) { if (!(attachmentObject is Dictionary <string, object> attachmentDictionary)) { throw new InvalidDataException("attachmentObject isn't a Dictionary<string, object>"); } if (attachmentDictionary.TryGetValue(idProperty, out var attachmentKeyObject) == false) { throw new InvalidDataException($"{idProperty} doesn't exist"); } if (!(attachmentKeyObject is string attachmentKey)) { throw new InvalidDataException($"{idProperty} isn't of type string"); } if (attachmentDictionary.TryGetValue(etagProperty, out var lastAttachmentEtagObject) == false) { throw new InvalidDataException($"{etagProperty} doesn't exist"); } if (!(lastAttachmentEtagObject is byte[] lastAttachmentEtagByteArray)) { throw new InvalidDataException($"{etagProperty} isn't of type byte[]"); } lastAttachmentEtag = LegacyAttachmentUtils.ByteArrayToEtagString(lastAttachmentEtagByteArray); if (attachmentDictionary.TryGetValue(metadataProperty, out object metadataObject) == false) { throw new InvalidDataException($"{metadataProperty} doesn't exist"); } if (!(metadataObject is Dictionary <string, object> metadata)) { throw new InvalidDataException($"{idProperty} isn't of type string"); } if (metadata.TryGetValue("Raven-Delete-Marker", out var deletedObject) && deletedObject is bool deletedObjectAsBool && deletedObjectAsBool) { var id = StreamSource.GetLegacyAttachmentId(attachmentKey); documentActions.DeleteDocument(id); continue; } var djv = new DynamicJsonValue(); foreach (var keyValue in metadata) { var key = keyValue.Key; if (key.Equals("Raven-Replication-Source") || key.Equals("Raven-Replication-Version") || key.Equals("Raven-Replication-History")) { continue; } djv[key] = keyValue.Value; } var contextToUse = documentActions.GetContextForNewDocument(); var metadataBlittable = contextToUse.ReadObject(djv, "metadata"); if (attachmentDictionary.TryGetValue(dataProperty, out object dataObject) == false) { throw new InvalidDataException($"{dataProperty} doesn't exist"); } if (!(dataObject is byte[] data)) { throw new InvalidDataException($"{dataProperty} isn't of type byte[]"); } using (var dataStream = new MemoryStream(data)) { var attachment = new DocumentItem.AttachmentStream { Stream = documentActions.GetTempStream() }; var attachmentDetails = StreamSource.GenerateLegacyAttachmentDetails(contextToUse, dataStream, attachmentKey, metadataBlittable, ref attachment); var documentItem = new DocumentItem { Document = new Document { Data = StreamSource.WriteDummyDocumentForAttachment(contextToUse, attachmentDetails), Id = attachmentDetails.Id, ChangeVector = string.Empty, Flags = DocumentFlags.HasAttachments, LastModified = Database.Time.GetUtcNow() }, Attachments = new List <DocumentItem.AttachmentStream> { attachment } }; documentActions.WriteDocument(documentItem, progress); } } using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var replicationSource = GetSourceReplicationInformation(context, GetRemoteServerInstanceId(), out var documentId); replicationSource.LastAttachmentEtag = lastAttachmentEtag; replicationSource.Source = GetFromServer(); replicationSource.LastModified = DateTime.UtcNow; await SaveSourceReplicationInformation(replicationSource, context, documentId); } } }