public IEnumerable <DocumentItem> GetRevisionDocuments(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var revisionsStorage = _database.DocumentsStorage.RevisionsStorage;
if (revisionsStorage.Configuration == null)
{
yield break;
}
var enumerator = new PulsedTransactionEnumerator <Document, DocumentsIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetRevisionsFromCollections(_context, state));
}
return(revisionsStorage.GetRevisionsFrom(_context, state.StartEtag, int.MaxValue));
},
new DocumentsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) // initial state
{
StartEtag = _startDocumentEtag,
StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
});
while (enumerator.MoveNext())
{
yield return(new DocumentItem
{
Document = enumerator.Current
});
}
}
public IEnumerable <DocumentItem> GetRevisionDocuments(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var revisionsStorage = _database.DocumentsStorage.RevisionsStorage;
if (revisionsStorage.Configuration == null)
{
yield break;
}
var documents = revisionsStorage.GetRevisionsFrom(_context, _startDocumentEtag, int.MaxValue);
foreach (var document in documents)
{
yield return(new DocumentItem
{
Document = document
});
}
}
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions)
{
return(ReadConflicts(actions));
}
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var modifier = new BlittableMetadataModifier(context);
var builder = CreateBuilder(context, modifier);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment);
if (ShouldSkip(attachmentInfo))
{
continue;
}
var dummyDoc = new DocumentItem
{
Document = new Document
{
Data = WriteDummyDocumentForAttachment(context, attachmentInfo),
Id = attachmentInfo.Id,
ChangeVector = string.Empty,
Flags = DocumentFlags.HasAttachments,
NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
LastModified = _database.Time.GetUtcNow(),
},
Attachments = new List <DocumentItem.AttachmentStream>
{
attachment
}
};
yield return(dummyDoc);
}
}
finally
{
builder.Dispose();
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public async IAsyncEnumerable <DocumentConflict> GetConflictsAsync(List <string> collectionsToExport, INewDocumentActions actions)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <DocumentConflict, DocumentConflictsIterationState>(_context,
state =>
{
if (collectionsToExport.Count != 0)
{
return(GetConflictsFromCollections(_context, collectionsToExport.ToHashSet(), state));
}
return(_database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, state.StartEtag));
},
new DocumentConflictsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag,
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public IEnumerable <DocumentItem> GetLegacyAttachments(INewDocumentActions actions)
{
return(ReadLegacyAttachments(actions));
}
public async IAsyncEnumerable <DocumentItem> GetDocumentsAsync(List <string> collectionsToExport, INewDocumentActions actions)
{
var line = 0;
while (await _csvReader.ReadAsync())
{
line++;
if (ProcessFieldsIfNeeded())
{
continue;
}
var context = actions.GetContextForNewDocument();
DocumentItem item;
try
{
item = ConvertRecordToDocumentItem(context, _csvReader.Context.Parser.Record, _csvReaderFieldHeaders, _collection);
}
catch (Exception e)
{
_result.AddError($"Fail to parse CSV line {line}, Error:{e}");
_result.Documents.ErroredCount++;
continue;
}
yield return(item);
}
}
public IAsyncEnumerable <DocumentItem> GetLegacyAttachmentsAsync(INewDocumentActions actions)
{
return(AsyncEnumerable.Empty <DocumentItem>());
}
public IEnumerable <DocumentTombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions)
{
return(Enumerable.Empty <DocumentTombstone>());
}
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions)
{
yield break;
}
public IEnumerable <DocumentItem> GetDocuments(List <string> collectionsToExport, INewDocumentActions actions)
{
while (_csvReader.Read())
{
if (ProcessFieldsIfNeeded())
{
continue;
}
var context = actions.GetContextForNewDocument();
yield return(ConvertRecordToDocumentItem(context, _csvReader.Context.Record, _csvReaderFieldHeaders, _collection));
}
}
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <DocumentConflict, DocumentConflictsIterationState>(_context,
state =>
{
if (collectionsToExport.Count != 0)
{
return(GetConflictsFromCollections(_context, collectionsToExport.ToHashSet(), state));
}
return(_database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, state.StartEtag));
},
new DocumentConflictsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag,
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public IEnumerable <Tombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <Tombstone, TombstonesIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetTombstonesFromCollections(_context, state));
}
return(_database.DocumentsStorage.GetTombstonesFrom(_context, state.StartEtag, 0, int.MaxValue));
},
new TombstonesIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag, StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public IEnumerable <Tombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var tombstones = collectionsToExport.Count > 0
? _database.DocumentsStorage.GetTombstonesFrom(_context, collectionsToExport, _startDocumentEtag, int.MaxValue)
: _database.DocumentsStorage.GetTombstonesFrom(_context, _startDocumentEtag, 0, int.MaxValue);
foreach (var tombstone in tombstones)
{
yield return(tombstone);
}
}
public IAsyncEnumerable <DocumentItem> GetRevisionDocumentsAsync(List <string> collectionsToExport, INewDocumentActions actions)
{
return(AsyncEnumerable.Empty <DocumentItem>());
}
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var conflicts = _database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, _startDocumentEtag);
if (collectionsToExport.Count > 0)
{
foreach (var conflict in conflicts)
{
if (collectionsToExport.Contains(conflict.Collection) == false)
{
continue;
}
yield return(conflict);
}
yield break;
}
foreach (var conflict in conflicts)
{
yield return(conflict);
}
}
public IAsyncEnumerable <Tombstone> GetTombstonesAsync(List <string> collectionsToExport, INewDocumentActions actions)
{
return(AsyncEnumerable.Empty <Tombstone>());
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
public async IAsyncEnumerable <Tombstone> GetTombstonesAsync(List <string> collectionsToExport, INewDocumentActions actions)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
Debug.Assert(_context != null);
var enumerator = new PulsedTransactionEnumerator <Tombstone, TombstonesIterationState>(_context,
state =>
{
if (state.StartEtagByCollection.Count != 0)
{
return(GetTombstonesFromCollections(_context, state));
}
return(_database.DocumentsStorage.GetTombstonesFrom(_context, state.StartEtag, 0, long.MaxValue));
},
new TombstonesIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit)
{
StartEtag = _startDocumentEtag,
StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag)
});
while (enumerator.MoveNext())
{
yield return(enumerator.Current);
}
}
public IAsyncEnumerable <DocumentConflict> GetConflictsAsync(List <string> collectionsToExport, INewDocumentActions actions)
{
return(AsyncEnumerable.Empty <DocumentConflict>());
}
public IEnumerable <DocumentItem> GetRevisionDocuments(List <string> collectionsToExport, INewDocumentActions actions)
{
return(ReadDocuments(actions));
}
private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var tombstone = new Tombstone();
if (data.TryGet("Key", out tombstone.LowerId) &&
data.TryGet(nameof(Tombstone.Type), out string type) &&
data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) &&
data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified))
{
if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false)
{
var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. ";
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
continue;
}
tombstone.Type = tombstoneType;
yield return(tombstone);
}
else
{
var msg = "Ignoring an invalied tombstone which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Tombstones.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
public IEnumerable <Tombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions)
{
return(ReadTombstones(actions));
}
private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var builder = CreateBuilder(context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
var conflict = new DocumentConflict();
if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) &&
data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) &&
data.TryGet(nameof(DocumentConflict.Flags), out string flags) &&
data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) &&
data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) &&
data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) &&
data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc))
{
conflict.Flags = Enum.Parse <DocumentFlags>(flags);
if (conflict.Doc != null) // This is null for conflict that was generated from tombstone
{
conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
yield return(conflict);
}
else
{
var msg = "Ignoring an invalied conflict which you try to import. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.Conflicts.ErroredCount++;
_result.AddWarning(msg);
}
}
}
finally
{
builder.Dispose();
}
}
private IEnumerable <BlittableJsonReaderObject> ReadArray(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var builder = CreateBuilder(_context, null);
try
{
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading array", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = _context;
var context = actions.GetContextForNewDocument();
if (_context != oldContext)
{
builder.Dispose();
builder = CreateBuilder(context, null);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type) &&
type == DocumentItem.ExportDocumentType.Attachment)
{
// skip document attachments, documents with attachments are handled separately
SkipAttachmentStream(data);
continue;
}
yield return(data);
}
}
finally
{
builder.Dispose();
}
}
public IEnumerable <DocumentItem> GetDocuments(List <string> collectionsToExport, INewDocumentActions actions)
{
Debug.Assert(_context != null);
var documents = collectionsToExport.Count != 0
? _database.DocumentsStorage.GetDocumentsFrom(_context, collectionsToExport, _startDocumentEtag, int.MaxValue)
: _database.DocumentsStorage.GetDocumentsFrom(_context, _startDocumentEtag, 0, int.MaxValue);
foreach (var document in documents)
{
yield return(new DocumentItem
{
Document = document
});
}
}
private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType != JsonParserToken.StartArray)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser);
}
var context = _context;
var legacyImport = _buildVersionType == BuildVersionType.V3;
var modifier = new BlittableMetadataModifier(context)
{
ReadFirstEtagOfLegacyRevision = legacyImport,
ReadLegacyEtag = _readLegacyEtag
};
var builder = CreateBuilder(context, modifier);
try
{
List <DocumentItem.AttachmentStream> attachments = null;
while (true)
{
if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false)
{
UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser);
}
if (_state.CurrentTokenType == JsonParserToken.EndArray)
{
break;
}
if (actions != null)
{
var oldContext = context;
context = actions.GetContextForNewDocument();
if (oldContext != context)
{
builder.Dispose();
builder = CreateBuilder(context, modifier);
}
}
builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_context.CachedProperties.NewDocument();
ReadObject(builder);
var data = builder.CreateReader();
builder.Reset();
if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type))
{
if (type != DocumentItem.ExportDocumentType.Attachment)
{
var msg = $"Ignoring an item of type `{type}`. " + data;
if (_log.IsOperationsEnabled)
{
_log.Operations(msg);
}
_result.AddWarning(msg);
continue;
}
if (attachments == null)
{
attachments = new List <DocumentItem.AttachmentStream>();
}
var attachment = new DocumentItem.AttachmentStream
{
Stream = actions.GetTempStream()
};
ProcessAttachmentStream(context, data, ref attachment);
attachments.Add(attachment);
continue;
}
if (legacyImport)
{
if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix))
{
data.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
}
};
}
}
if (data.Modifications != null)
{
data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
_result.LegacyLastDocumentEtag = modifier.LegacyEtag;
yield return(new DocumentItem
{
Document = new Document
{
Data = data,
Id = modifier.Id,
ChangeVector = modifier.ChangeVector,
Flags = modifier.Flags,
NonPersistentFlags = modifier.NonPersistentFlags,
LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(),
},
Attachments = attachments
});
attachments = null;
}
}
finally
{
builder.Dispose();
}
}
public IEnumerable <DocumentItem> GetDocuments(List <string> collectionsToExport, INewDocumentActions actions)
{
while (_csvReader.Read())
{
ProcessFieldsIfNeeded();
yield return(ConvertRecordToDocumentItem(_csvReader.CurrentRecord, _csvReader.FieldHeaders, _collection));
}
}
