public static LegacyAttachmentDetails GenerateLegacyAttachmentDetails(
DocumentsOperationContext context,
Stream decodedStream,
string key,
BlittableJsonReaderObject metadata,
ref DocumentItem.AttachmentStream attachment)
{
var stream = attachment.Stream;
var hash = AsyncHelpers.RunSync(() => AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, decodedStream, stream, CancellationToken.None));
attachment.Stream.Flush();
var lazyHash = context.GetLazyString(hash);
attachment.Base64HashDispose = Slice.External(context.Allocator, lazyHash, out attachment.Base64Hash);
var tag = $"{DummyDocumentPrefix}{key}{RecordSeparator}d{RecordSeparator}{key}{RecordSeparator}{hash}{RecordSeparator}";
var lazyTag = context.GetLazyString(tag);
attachment.TagDispose = Slice.External(context.Allocator, lazyTag, out attachment.Tag);
var id = GetLegacyAttachmentId(key);
var lazyId = context.GetLazyString(id);
attachment.Data = context.ReadObject(metadata, id);
return(new LegacyAttachmentDetails
{
Id = lazyId,
Hash = hash,
Key = key,
Size = attachment.Stream.Length,
Tag = tag,
Metadata = attachment.Data
});
}
internal unsafe LegacyAttachmentDetails ProcessLegacyAttachment(
DocumentsOperationContext context,
BlittableJsonReaderObject data,
ref DocumentItem.AttachmentStream attachment)
{
if (data.TryGet("Key", out string key) == false)
{
throw new ArgumentException("The key of legacy attachment is missing its key property.");
}
if (data.TryGet("Metadata", out BlittableJsonReaderObject metadata) == false)
{
throw new ArgumentException($"Metadata of legacy attachment with key={key} is missing");
}
if (data.TryGet("Data", out string base64data) == false)
{
throw new ArgumentException($"Data of legacy attachment with key={key} is missing");
}
if (_readLegacyEtag && data.TryGet("Etag", out string etag))
{
_result.LegacyLastAttachmentEtag = etag;
}
var memoryStream = new MemoryStream();
fixed(char *pdata = base64data)
{
memoryStream.SetLength(Base64.FromBase64_ComputeResultLength(pdata, base64data.Length));
fixed(byte *buffer = memoryStream.GetBuffer())
Base64.FromBase64_Decode(pdata, base64data.Length, buffer, (int)memoryStream.Length);
}
memoryStream.Position = 0;
return(GenerateLegacyAttachmentDetails(context, memoryStream, key, metadata, ref attachment));
}
private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var legacyImport = _buildVersionType == BuildVersionType.V3;
var modifier = new BlittableMetadataModifier(context)
{
ReadFirstEtagOfLegacyRevision = legacyImport,
ReadLegacyEtag = _readLegacyEtag,
OperateOnTypes = _operateOnTypes
};
var builder = CreateBuilder(context, modifier);
try
{
List <DocumentItem.AttachmentStream> attachments = null;
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
{
if (type != DocumentItem.ExportDocumentType.Attachment)
{
var msg = $"Ignoring an item of type `{type}`. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.AddWarning(msg);
continue;
}
if (attachments == null)
{
attachments = new List <DocumentItem.AttachmentStream>();
}
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
ProcessAttachmentStream(context, data, ref attachment);
attachments.Add(attachment);
continue;
}
if (legacyImport)
{
if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
{
data.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
}
};
}
}
if (data.Modifications != null)
{
data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
_result.LegacyLastDocumentEtag = modifier.LegacyEtag;
yield return(new DocumentItem
{
Document = new Document
{
Data = data,
Id = modifier.Id,
ChangeVector = modifier.ChangeVector,
Flags = modifier.Flags,
NonPersistentFlags = modifier.NonPersistentFlags,
LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
},
Attachments = attachments
});
attachments = null;
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var modifier = new BlittableMetadataModifier(context);
var builder = CreateBuilder(context, modifier);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment);
if (ShouldSkip(attachmentInfo))
{
continue;
}
var dummyDoc = new DocumentItem
{
Document = new Document
{
Data = WriteDummyDocumentForAttachment(context, attachmentInfo),
Id = attachmentInfo.Id,
ChangeVector = string.Empty,
Flags = DocumentFlags.HasAttachments,
NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
LastModified = _database.Time.GetUtcNow(),
},
Attachments = new List <DocumentItem.AttachmentStream>
{
attachment
}
};
yield return(dummyDoc);
}
}
finally
{
builder.Dispose();
}
}
public unsafe void ProcessAttachmentStream(DocumentsOperationContext context, BlittableJsonReaderObject data, ref DocumentItem.AttachmentStream attachment)
{
if (data.TryGet(nameof(AttachmentName.Hash), out LazyStringValue hash) == false ||
data.TryGet(nameof(AttachmentName.Size), out long size) == false ||
data.TryGet(nameof(DocumentItem.AttachmentStream.Tag), out LazyStringValue tag) == false)
{
throw new ArgumentException($"Data of attachment stream is not valid: {data}");
}
if (_writeBuffer == null)
{
_returnWriteBuffer = _context.GetManagedBuffer(out _writeBuffer);
}
attachment.Data = data;
attachment.Base64HashDispose = Slice.External(context.Allocator, hash, out attachment.Base64Hash);
attachment.TagDispose = Slice.External(context.Allocator, tag, out attachment.Tag);
while (size > 0)
{
var sizeToRead = (int)Math.Min(_writeBuffer.Length, size);
var read = _parser.Copy(_writeBuffer.Pointer, sizeToRead);
attachment.Stream.Write(_writeBuffer.Buffer.Array, _writeBuffer.Buffer.Offset, read.BytesRead);
if (read.Done == false)
{
var read2 = _peepingTomStream.Read(_buffer.Buffer.Array, _buffer.Buffer.Offset, _buffer.Length);
if (read2 == 0)
{
throw new EndOfStreamException("Stream ended without reaching end of stream content");
}
_parser.SetBuffer(_buffer, 0, read2);
}
size -= read.BytesRead;
}
attachment.Stream.Flush();
}