public IEnumerable <DocumentItem> GetRevisionDocuments(List <string> collectionsToExport, INewDocumentActions actions) { Debug.Assert(_context != null); var revisionsStorage = _database.DocumentsStorage.RevisionsStorage; if (revisionsStorage.Configuration == null) { yield break; } var enumerator = new PulsedTransactionEnumerator <Document, DocumentsIterationState>(_context, state => { if (state.StartEtagByCollection.Count != 0) { return(GetRevisionsFromCollections(_context, state)); } return(revisionsStorage.GetRevisionsFrom(_context, state.StartEtag, int.MaxValue)); }, new DocumentsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) // initial state { StartEtag = _startDocumentEtag, StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag) }); while (enumerator.MoveNext()) { yield return(new DocumentItem { Document = enumerator.Current }); } }
public IEnumerable <DocumentItem> GetRevisionDocuments(List <string> collectionsToExport, INewDocumentActions actions) { Debug.Assert(_context != null); var revisionsStorage = _database.DocumentsStorage.RevisionsStorage; if (revisionsStorage.Configuration == null) { yield break; } var documents = revisionsStorage.GetRevisionsFrom(_context, _startDocumentEtag, int.MaxValue); foreach (var document in documents) { yield return(new DocumentItem { Document = document }); } }
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions) { return(ReadConflicts(actions)); }
private IEnumerable <DocumentItem> ReadLegacyAttachments(INewDocumentActions actions) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var modifier = new BlittableMetadataModifier(context); var builder = CreateBuilder(context, modifier); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading legacy attachments", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; var attachmentInfo = ProcessLegacyAttachment(context, data, ref attachment); if (ShouldSkip(attachmentInfo)) { continue; } var dummyDoc = new DocumentItem { Document = new Document { Data = WriteDummyDocumentForAttachment(context, attachmentInfo), Id = attachmentInfo.Id, ChangeVector = string.Empty, Flags = DocumentFlags.HasAttachments, NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler, LastModified = _database.Time.GetUtcNow(), }, Attachments = new List <DocumentItem.AttachmentStream> { attachment } }; yield return(dummyDoc); } } finally { builder.Dispose(); } }
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously public async IAsyncEnumerable <DocumentConflict> GetConflictsAsync(List <string> collectionsToExport, INewDocumentActions actions) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { Debug.Assert(_context != null); var enumerator = new PulsedTransactionEnumerator <DocumentConflict, DocumentConflictsIterationState>(_context, state => { if (collectionsToExport.Count != 0) { return(GetConflictsFromCollections(_context, collectionsToExport.ToHashSet(), state)); } return(_database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, state.StartEtag)); }, new DocumentConflictsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) { StartEtag = _startDocumentEtag, }); while (enumerator.MoveNext()) { yield return(enumerator.Current); } }
public IEnumerable <DocumentItem> GetLegacyAttachments(INewDocumentActions actions) { return(ReadLegacyAttachments(actions)); }
public async IAsyncEnumerable <DocumentItem> GetDocumentsAsync(List <string> collectionsToExport, INewDocumentActions actions) { var line = 0; while (await _csvReader.ReadAsync()) { line++; if (ProcessFieldsIfNeeded()) { continue; } var context = actions.GetContextForNewDocument(); DocumentItem item; try { item = ConvertRecordToDocumentItem(context, _csvReader.Context.Parser.Record, _csvReaderFieldHeaders, _collection); } catch (Exception e) { _result.AddError($"Fail to parse CSV line {line}, Error:{e}"); _result.Documents.ErroredCount++; continue; } yield return(item); } }
public IAsyncEnumerable <DocumentItem> GetLegacyAttachmentsAsync(INewDocumentActions actions) { return(AsyncEnumerable.Empty <DocumentItem>()); }
public IEnumerable <DocumentTombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions) { return(Enumerable.Empty <DocumentTombstone>()); }
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions) { yield break; }
public IEnumerable <DocumentItem> GetDocuments(List <string> collectionsToExport, INewDocumentActions actions) { while (_csvReader.Read()) { if (ProcessFieldsIfNeeded()) { continue; } var context = actions.GetContextForNewDocument(); yield return(ConvertRecordToDocumentItem(context, _csvReader.Context.Record, _csvReaderFieldHeaders, _collection)); } }
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions) { Debug.Assert(_context != null); var enumerator = new PulsedTransactionEnumerator <DocumentConflict, DocumentConflictsIterationState>(_context, state => { if (collectionsToExport.Count != 0) { return(GetConflictsFromCollections(_context, collectionsToExport.ToHashSet(), state)); } return(_database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, state.StartEtag)); }, new DocumentConflictsIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) { StartEtag = _startDocumentEtag, }); while (enumerator.MoveNext()) { yield return(enumerator.Current); } }
public IEnumerable <Tombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions) { Debug.Assert(_context != null); var enumerator = new PulsedTransactionEnumerator <Tombstone, TombstonesIterationState>(_context, state => { if (state.StartEtagByCollection.Count != 0) { return(GetTombstonesFromCollections(_context, state)); } return(_database.DocumentsStorage.GetTombstonesFrom(_context, state.StartEtag, 0, int.MaxValue)); }, new TombstonesIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) { StartEtag = _startDocumentEtag, StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag) }); while (enumerator.MoveNext()) { yield return(enumerator.Current); } }
public IEnumerable <Tombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions) { Debug.Assert(_context != null); var tombstones = collectionsToExport.Count > 0 ? _database.DocumentsStorage.GetTombstonesFrom(_context, collectionsToExport, _startDocumentEtag, int.MaxValue) : _database.DocumentsStorage.GetTombstonesFrom(_context, _startDocumentEtag, 0, int.MaxValue); foreach (var tombstone in tombstones) { yield return(tombstone); } }
public IAsyncEnumerable <DocumentItem> GetRevisionDocumentsAsync(List <string> collectionsToExport, INewDocumentActions actions) { return(AsyncEnumerable.Empty <DocumentItem>()); }
public IEnumerable <DocumentConflict> GetConflicts(List <string> collectionsToExport, INewDocumentActions actions) { Debug.Assert(_context != null); var conflicts = _database.DocumentsStorage.ConflictsStorage.GetConflictsFrom(_context, _startDocumentEtag); if (collectionsToExport.Count > 0) { foreach (var conflict in conflicts) { if (collectionsToExport.Contains(conflict.Collection) == false) { continue; } yield return(conflict); } yield break; } foreach (var conflict in conflicts) { yield return(conflict); } }
public IAsyncEnumerable <Tombstone> GetTombstonesAsync(List <string> collectionsToExport, INewDocumentActions actions) { return(AsyncEnumerable.Empty <Tombstone>()); }
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously public async IAsyncEnumerable <Tombstone> GetTombstonesAsync(List <string> collectionsToExport, INewDocumentActions actions) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { Debug.Assert(_context != null); var enumerator = new PulsedTransactionEnumerator <Tombstone, TombstonesIterationState>(_context, state => { if (state.StartEtagByCollection.Count != 0) { return(GetTombstonesFromCollections(_context, state)); } return(_database.DocumentsStorage.GetTombstonesFrom(_context, state.StartEtag, 0, long.MaxValue)); }, new TombstonesIterationState(_context, _database.Configuration.Databases.PulseReadTransactionLimit) { StartEtag = _startDocumentEtag, StartEtagByCollection = collectionsToExport.ToDictionary(x => x, x => _startDocumentEtag) }); while (enumerator.MoveNext()) { yield return(enumerator.Current); } }
public IAsyncEnumerable <DocumentConflict> GetConflictsAsync(List <string> collectionsToExport, INewDocumentActions actions) { return(AsyncEnumerable.Empty <DocumentConflict>()); }
public IEnumerable <DocumentItem> GetRevisionDocuments(List <string> collectionsToExport, INewDocumentActions actions) { return(ReadDocuments(actions)); }
private IEnumerable <Tombstone> ReadTombstones(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var builder = CreateBuilder(context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var tombstone = new Tombstone(); if (data.TryGet("Key", out tombstone.LowerId) && data.TryGet(nameof(Tombstone.Type), out string type) && data.TryGet(nameof(Tombstone.Collection), out tombstone.Collection) && data.TryGet(nameof(Tombstone.LastModified), out tombstone.LastModified)) { if (Enum.TryParse <Tombstone.TombstoneType>(type, out var tombstoneType) == false) { var msg = $"Ignoring a tombstone of type `{type}` which is not supported in 4.0. "; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Tombstones.ErroredCount++; _result.AddWarning(msg); continue; } tombstone.Type = tombstoneType; yield return(tombstone); } else { var msg = "Ignoring an invalied tombstone which you try to import. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Tombstones.ErroredCount++; _result.AddWarning(msg); } } } finally { builder.Dispose(); } }
public IEnumerable <Tombstone> GetTombstones(List <string> collectionsToExport, INewDocumentActions actions) { return(ReadTombstones(actions)); }
private IEnumerable <DocumentConflict> ReadConflicts(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var builder = CreateBuilder(context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); var conflict = new DocumentConflict(); if (data.TryGet(nameof(DocumentConflict.Id), out conflict.Id) && data.TryGet(nameof(DocumentConflict.Collection), out conflict.Collection) && data.TryGet(nameof(DocumentConflict.Flags), out string flags) && data.TryGet(nameof(DocumentConflict.ChangeVector), out conflict.ChangeVector) && data.TryGet(nameof(DocumentConflict.Etag), out conflict.Etag) && data.TryGet(nameof(DocumentConflict.LastModified), out conflict.LastModified) && data.TryGet(nameof(DocumentConflict.Doc), out conflict.Doc)) { conflict.Flags = Enum.Parse <DocumentFlags>(flags); if (conflict.Doc != null) // This is null for conflict that was generated from tombstone { conflict.Doc = context.ReadObject(conflict.Doc, conflict.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } yield return(conflict); } else { var msg = "Ignoring an invalied conflict which you try to import. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.Conflicts.ErroredCount++; _result.AddWarning(msg); } } } finally { builder.Dispose(); } }
private IEnumerable <BlittableJsonReaderObject> ReadArray(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var builder = CreateBuilder(_context, null); try { while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading array", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = _context; var context = actions.GetContextForNewDocument(); if (_context != oldContext) { builder.Dispose(); builder = CreateBuilder(context, null); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type) && type == DocumentItem.ExportDocumentType.Attachment) { // skip document attachments, documents with attachments are handled separately SkipAttachmentStream(data); continue; } yield return(data); } } finally { builder.Dispose(); } }
public IEnumerable <DocumentItem> GetDocuments(List <string> collectionsToExport, INewDocumentActions actions) { Debug.Assert(_context != null); var documents = collectionsToExport.Count != 0 ? _database.DocumentsStorage.GetDocumentsFrom(_context, collectionsToExport, _startDocumentEtag, int.MaxValue) : _database.DocumentsStorage.GetDocumentsFrom(_context, _startDocumentEtag, 0, int.MaxValue); foreach (var document in documents) { yield return(new DocumentItem { Document = document }); } }
private IEnumerable <DocumentItem> ReadDocuments(INewDocumentActions actions = null) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json", _peepingTomStream, _parser); } if (_state.CurrentTokenType != JsonParserToken.StartArray) { UnmanagedJsonParserHelper.ThrowInvalidJson("Expected start array, but got " + _state.CurrentTokenType, _peepingTomStream, _parser); } var context = _context; var legacyImport = _buildVersionType == BuildVersionType.V3; var modifier = new BlittableMetadataModifier(context) { ReadFirstEtagOfLegacyRevision = legacyImport, ReadLegacyEtag = _readLegacyEtag }; var builder = CreateBuilder(context, modifier); try { List <DocumentItem.AttachmentStream> attachments = null; while (true) { if (UnmanagedJsonParserHelper.Read(_peepingTomStream, _parser, _state, _buffer) == false) { UnmanagedJsonParserHelper.ThrowInvalidJson("Unexpected end of json while reading docs", _peepingTomStream, _parser); } if (_state.CurrentTokenType == JsonParserToken.EndArray) { break; } if (actions != null) { var oldContext = context; context = actions.GetContextForNewDocument(); if (oldContext != context) { builder.Dispose(); builder = CreateBuilder(context, modifier); } } builder.Renew("import/object", BlittableJsonDocumentBuilder.UsageMode.ToDisk); _context.CachedProperties.NewDocument(); ReadObject(builder); var data = builder.CreateReader(); builder.Reset(); if (data.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) && metadata.TryGet(DocumentItem.ExportDocumentType.Key, out string type)) { if (type != DocumentItem.ExportDocumentType.Attachment) { var msg = $"Ignoring an item of type `{type}`. " + data; if (_log.IsOperationsEnabled) { _log.Operations(msg); } _result.AddWarning(msg); continue; } if (attachments == null) { attachments = new List <DocumentItem.AttachmentStream>(); } var attachment = new DocumentItem.AttachmentStream { Stream = actions.GetTempStream() }; ProcessAttachmentStream(context, data, ref attachment); attachments.Add(attachment); continue; } if (legacyImport) { if (modifier.Id.Contains(HiLoHandler.RavenHiloIdPrefix)) { data.Modifications = new DynamicJsonValue { [Constants.Documents.Metadata.Key] = new DynamicJsonValue { [Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection } }; } } if (data.Modifications != null) { data = context.ReadObject(data, modifier.Id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); } _result.LegacyLastDocumentEtag = modifier.LegacyEtag; yield return(new DocumentItem { Document = new Document { Data = data, Id = modifier.Id, ChangeVector = modifier.ChangeVector, Flags = modifier.Flags, NonPersistentFlags = modifier.NonPersistentFlags, LastModified = modifier.LastModified ?? _database.Time.GetUtcNow(), }, Attachments = attachments }); attachments = null; } } finally { builder.Dispose(); } }
public IEnumerable <DocumentItem> GetDocuments(List <string> collectionsToExport, INewDocumentActions actions) { while (_csvReader.Read()) { ProcessFieldsIfNeeded(); yield return(ConvertRecordToDocumentItem(_csvReader.CurrentRecord, _csvReader.FieldHeaders, _collection)); } }