public static LegacyAttachmentDetails GenerateLegacyAttachmentDetails( DocumentsOperationContext context, Stream decodedStream, string key, BlittableJsonReaderObject metadata, ref DocumentItem.AttachmentStream attachment) { var stream = attachment.Stream; var hash = AsyncHelpers.RunSync(() => AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, decodedStream, stream, CancellationToken.None)); attachment.Stream.Flush(); var lazyHash = context.GetLazyString(hash); attachment.Base64HashDispose = Slice.External(context.Allocator, lazyHash, out attachment.Base64Hash); var tag = $"{DummyDocumentPrefix}{key}{RecordSeparator}d{RecordSeparator}{key}{RecordSeparator}{hash}{RecordSeparator}"; var lazyTag = context.GetLazyString(tag); attachment.TagDispose = Slice.External(context.Allocator, lazyTag, out attachment.Tag); var id = GetLegacyAttachmentId(key); var lazyId = context.GetLazyString(id); attachment.Data = context.ReadObject(metadata, id); return(new LegacyAttachmentDetails { Id = lazyId, Hash = hash, Key = key, Size = attachment.Stream.Length, Tag = tag, Metadata = attachment.Data }); }
internal unsafe LegacyAttachmentDetails ProcessLegacyAttachment( DocumentsOperationContext context, BlittableJsonReaderObject data, ref DocumentItem.AttachmentStream attachment) { if (data.TryGet("Key", out string key) == false) { throw new ArgumentException("The key of legacy attachment is missing its key property."); } if (data.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false) { throw new ArgumentException($"Metadata of legacy attachment with key={key} is missing"); } if (data.TryGet("Data", out string base64data) == false) { throw new ArgumentException($"Data of legacy attachment with key={key} is missing"); } if (_readLegacyEtag && data.TryGet("Etag", out string etag)) { _result.LegacyLastAttachmentEtag = etag; } var memoryStream = new MemoryStream(); fixed(char *pdata = base64data) { memoryStream.SetLength(Base64.FromBase64_ComputeResultLength(pdata, base64data.Length)); fixed(byte *buffer = memoryStream.GetBuffer()) Base64.FromBase64_Decode(pdata, base64data.Length, buffer, (int)memoryStream.Length); } memoryStream.Position = 0; return(GenerateLegacyAttachmentDetails(context, memoryStream, key, metadata, ref attachment)); }
private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var legacyImport = _buildVersionType == BuildVersionType.V3; var modifier = new BlittableMetadataModifier(context) { ReadFirstEtagOfLegacyRevision = legacyImport, ReadLegacyEtag = _readLegacyEtag, OperateOnTypes = _operateOnTypes }; var builder = CreateBuilder(context, modifier); try { List <DocumentItem.AttachmentStream> attachments = null; while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type)) { if (type != DocumentItem.ExportDocumentType.Attachment) { var msg = $"Ignoring an item of type `{type}`. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.AddWarning(msg); continue; } if (attachments == null) { attachments = new List <DocumentItem.AttachmentStream>(); } var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; ProcessAttachmentStream(context, data, ref attachment); attachments.Add(attachment); continue; } if (legacyImport) { if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix)) { data.Modifications = new DynamicJsonValue { [Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection } }; } } if (data.Modifications != null) { data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } _result.LegacyLastDocumentEtag = modifier.LegacyEtag; yield return(new DocumentItem { Document = new Document { Data = data, Id = modifier.Id, ChangeVector = modifier.ChangeVector, Flags = modifier.Flags, NonPersistentFlags = modifier.NonPersistentFlags, LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(), }, Attachments = attachments }); attachments = null; } } finally { builder.Dispose(); } }
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var modifier = new BlittableMetadataModifier(context); var builder = CreateBuilder(context, modifier); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment); if (ShouldSkip(attachmentInfo)) { continue; } var dummyDoc = new DocumentItem { Document = new Document { Data = WriteDummyDocumentForAttachment(context, attachmentInfo), Id = attachmentInfo.Id, ChangeVector = string.Empty, Flags = DocumentFlags.HasAttachments, NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler, LastModified = _database.Time.GetUtcNow(), }, Attachments = new List <DocumentItem.AttachmentStream> { attachment } }; yield return(dummyDoc); } } finally { builder.Dispose(); } }
public unsafe void ProcessAttachmentStream(DocumentsOperationContext context, BlittableJsonReaderObject data, ref DocumentItem.AttachmentStream attachment) { if (data.TryGet(nameof(AttachmentName.Hash), out LazyStringValue hash) == false || data.TryGet(nameof(AttachmentName.Size), out long size) == false || data.TryGet(nameof(DocumentItem.AttachmentStream.Tag), out LazyStringValue tag) == false) { throw new ArgumentException($"Data of attachment stream is not valid: {data}"); } if (_writeBuffer == null) { _returnWriteBuffer = _context.GetManagedBuffer(out _writeBuffer); } attachment.Data = data; attachment.Base64HashDispose = Slice.External(context.Allocator, hash, out attachment.Base64Hash); attachment.TagDispose = Slice.External(context.Allocator, tag, out attachment.Tag); while (size > 0) { var sizeToRead = (int)Math.Min(_writeBuffer.Length, size); var read = _parser.Copy(_writeBuffer.Pointer, sizeToRead); attachment.Stream.Write(_writeBuffer.Buffer.Array, _writeBuffer.Buffer.Offset, read.BytesRead); if (read.Done == false) { var read2 = _peepingTomStream.Read(_buffer.Buffer.Array, _buffer.Buffer.Offset, _buffer.Length); if (read2 == 0) { throw new EndOfStreamException("Stream ended without reaching end of stream content"); } _parser.SetBuffer(_buffer, 0, read2); } size -= read.BytesRead; } attachment.Stream.Flush(); }