private async Task <string> MigrateRavenFs(string lastEtag, SmugglerResult parametersResult) { var destination = new DatabaseDestination(Parameters.Database); var options = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = DatabaseItemType.Attachments, SkipRevisionCreation = true }; destination.InitializeAsync(options, parametersResult, _buildVersion); using (Parameters.Database.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext)) using (Parameters.Database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) await using (var documentActions = destination.Documents()) { var sp = Stopwatch.StartNew(); while (true) { var ravenFsHeadersArray = await GetRavenFsHeadersArray(lastEtag, transactionOperationContext); if (ravenFsHeadersArray.Length == 0) { var count = Parameters.Result.Documents.ReadCount; if (count > 0) { var message = $"Read {count:#,#;;0} RavenFS file{(count > 1 ? "s" : string.Empty)}."; Parameters.Result.AddInfo(message); Parameters.OnProgress.Invoke(Parameters.Result.Progress); } return(lastEtag); } foreach (var headerObject in ravenFsHeadersArray) { var blittable = headerObject as BlittableJsonReaderObject; if (blittable == null) { throw new InvalidDataException("headerObject isn't a BlittableJsonReaderObject"); } if (blittable.TryGet("FullPath", out string fullPath) == false) { throw new InvalidDataException("FullPath doesn't exist"); } if (blittable.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false) { throw new InvalidDataException("Metadata doesn't exist"); } var key = fullPath.TrimStart('/'); var dataStream = await GetRavenFsStream(key); if (dataStream == null) { Parameters.Result.Tombstones.ReadCount++; var id = StreamSource.GetLegacyAttachmentId(key); await documentActions.DeleteDocumentAsync(id); continue; } var contextToUse = documentActions.GetContextForNewDocument(); metadata = GetCleanMetadata(metadata, contextToUse); await WriteDocumentWithAttachmentAsync(documentActions, contextToUse, dataStream, key, metadata); Parameters.Result.Documents.ReadCount++; if (Parameters.Result.Documents.ReadCount % 50 == 0 || sp.ElapsedMilliseconds > 3000) { var message = $"Read {Parameters.Result.Documents.ReadCount:#,#;;0} " + $"RavenFS file{(Parameters.Result.Documents.ReadCount > 1 ? "s" : string.Empty)}."; Parameters.Result.AddInfo(message); Parameters.OnProgress.Invoke(Parameters.Result.Progress); sp.Restart(); } } var lastFile = ravenFsHeadersArray.Last() as BlittableJsonReaderObject; Debug.Assert(lastFile != null, "lastAttachment != null"); if (lastFile.TryGet("Etag", out string etag)) { lastEtag = etag; } } } }
public async Task Attachments() { var destination = new DatabaseDestination(Database); var options = new DatabaseSmugglerOptionsServerSide { OperateOnTypes = DatabaseItemType.Attachments, SkipRevisionCreation = true }; await using (destination.InitializeAsync(options, null, buildVersion: default)) await using (var documentActions = destination.Documents()) await using (var buffered = new BufferedStream(RequestBodyStream())) #pragma warning disable CS0618 // Type or member is obsolete using (var reader = new BsonReader(buffered)) #pragma warning restore CS0618 // Type or member is obsolete { var result = LegacyAttachmentUtils.GetObject(reader); const string idProperty = "@id"; const string etagProperty = "@etag"; const string metadataProperty = "@metadata"; const string dataProperty = "data"; string lastAttachmentEtag = null; var progress = new SmugglerProgressBase.CountsWithLastEtagAndAttachments(); foreach (var attachmentObject in result.Values) { if (!(attachmentObject is Dictionary <string, object> attachmentDictionary)) { throw new InvalidDataException("attachmentObject isn't a Dictionary<string, object>"); } if (attachmentDictionary.TryGetValue(idProperty, out var attachmentKeyObject) == false) { throw new InvalidDataException($"{idProperty} doesn't exist"); } if (!(attachmentKeyObject is string attachmentKey)) { throw new InvalidDataException($"{idProperty} isn't of type string"); } if (attachmentDictionary.TryGetValue(etagProperty, out var lastAttachmentEtagObject) == false) { throw new InvalidDataException($"{etagProperty} doesn't exist"); } if (!(lastAttachmentEtagObject is byte[] lastAttachmentEtagByteArray)) { throw new InvalidDataException($"{etagProperty} isn't of type byte[]"); } lastAttachmentEtag = LegacyAttachmentUtils.ByteArrayToEtagString(lastAttachmentEtagByteArray); if (attachmentDictionary.TryGetValue(metadataProperty, out object metadataObject) == false) { throw new InvalidDataException($"{metadataProperty} doesn't exist"); } if (!(metadataObject is Dictionary <string, object> metadata)) { throw new InvalidDataException($"{idProperty} isn't of type string"); } if (metadata.TryGetValue("Raven-Delete-Marker", out var deletedObject) && deletedObject is bool deletedObjectAsBool && deletedObjectAsBool) { var id = StreamSource.GetLegacyAttachmentId(attachmentKey); await documentActions.DeleteDocumentAsync(id); continue; } var djv = new DynamicJsonValue(); foreach (var keyValue in metadata) { var key = keyValue.Key; if (key.Equals("Raven-Replication-Source") || key.Equals("Raven-Replication-Version") || key.Equals("Raven-Replication-History")) { continue; } djv[key] = keyValue.Value; } var contextToUse = documentActions.GetContextForNewDocument(); var metadataBlittable = contextToUse.ReadObject(djv, "metadata"); if (attachmentDictionary.TryGetValue(dataProperty, out object dataObject) == false) { throw new InvalidDataException($"{dataProperty} doesn't exist"); } if (!(dataObject is byte[] data)) { throw new InvalidDataException($"{dataProperty} isn't of type byte[]"); } await using (var dataStream = new MemoryStream(data)) { var attachment = new DocumentItem.AttachmentStream { Stream = documentActions.GetTempStream() }; var attachmentDetails = StreamSource.GenerateLegacyAttachmentDetails(contextToUse, dataStream, attachmentKey, metadataBlittable, ref attachment); var documentItem = new DocumentItem { Document = new Document { Data = StreamSource.WriteDummyDocumentForAttachment(contextToUse, attachmentDetails), Id = attachmentDetails.Id, ChangeVector = string.Empty, Flags = DocumentFlags.HasAttachments, LastModified = Database.Time.GetUtcNow() }, Attachments = new List <DocumentItem.AttachmentStream> { attachment } }; await documentActions.WriteDocumentAsync(documentItem, progress); } } using (ContextPool.AllocateOperationContext(out DocumentsOperationContext context)) { var replicationSource = GetSourceReplicationInformation(context, GetRemoteServerInstanceId(), out var documentId); replicationSource.LastAttachmentEtag = lastAttachmentEtag; replicationSource.Source = GetFromServer(); replicationSource.LastModified = DateTime.UtcNow; await SaveSourceReplicationInformation(replicationSource, context, documentId); } } }