public void Set(string name, string key, RavenJObject data, UuidType uuidType) { Api.JetSetCurrentIndex(session, Lists, "by_name_and_key"); Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey); Api.MakeKey(session, Lists, key, Encoding.Unicode, MakeKeyGrbit.None); var exists = Api.TrySeek(session, Lists, SeekGrbit.SeekEQ); using (var update = new Update(session, Lists, exists ? JET_prep.Replace : JET_prep.Insert)) { Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["name"], name, Encoding.Unicode); Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["key"], key, Encoding.Unicode); Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["etag"], uuidGenerator.CreateSequentialUuid(uuidType).TransformToValueForEsentSorting()); Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["created_at"], SystemTime.UtcNow); using (var columnStream = new ColumnStream(session, Lists, tableColumnsCache.ListsColumns["data"])) { if (exists) columnStream.SetLength(0); using (Stream stream = new BufferedStream(columnStream)) { data.WriteTo(stream); stream.Flush(); } } update.Save(); } }
public Guid AddAttachment(string key, Guid? etag, Stream data, RavenJObject headers) { AssertValidEtag(key, etag, "PUT"); if(data == null) { var attachment = GetAttachment(key); if(attachment == null) throw new InvalidOperationException("When adding new attachment, the attachment data must be specified"); attachment.Metadata = headers; return AddAttachment(key, etag, attachment.Data(), headers); } var ms = new MemoryStream(); headers.WriteTo(ms); data.CopyTo(ms); var newEtag = generator.CreateSequentialUuid(); var result = storage.Attachments.Put(new RavenJObject { {"key", key}, {"etag", newEtag.ToByteArray()} }, ms.ToArray()); if (result == false) throw new ConcurrencyException("PUT attempted on attachment '" + key + "' while it was locked by another transaction"); logger.Debug("Adding attachment {0}", key); return newEtag; }
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data) { Etag etag = uuidGenerator.CreateSequentialUuid(UuidType.MappedResults); using (var update = new Update(session, MappedResults, JET_prep.Insert)) { Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], view, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"], docId, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"], reduceKey, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["hashed_reduce_key"], HashReduceKey(reduceKey)); var mapBucket = IndexingUtil.MapBucket(docId); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"], mapBucket); using (Stream stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]))) { using (var dataStream = documentCodecs.Aggregate(stream, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(dataStream); dataStream.Flush(); } } Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"], etag.TransformToValueForEsentSorting()); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"], SystemTime.UtcNow.ToBinary()); update.Save(); } }
public void PutReducedResult(string view, string reduceKey, int level, int sourceBucket, int bucket, RavenJObject data) { Guid etag = uuidGenerator.CreateSequentialUuid(); using (var update = new Update(session, ReducedResults, JET_prep.Insert)) { Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["view"], view, Encoding.Unicode); Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["level"], level); Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["reduce_key"], reduceKey, Encoding.Unicode); Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["hashed_reduce_key"], HashReduceKey(reduceKey)); Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["bucket"], bucket); Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["source_bucket"], sourceBucket); using (Stream stream = new BufferedStream(new ColumnStream(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["data"]))) { using (var dataStream = documentCodecs.Aggregate(stream, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(dataStream); dataStream.Flush(); } } Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["etag"], etag.TransformToValueForEsentSorting()); Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["timestamp"], SystemTime.UtcNow); update.Save(); } }
public void Set(string name, string key, RavenJObject data, UuidType type) { var memoryStream = new MemoryStream(); data.WriteTo(memoryStream); storage.Lists.Put(new RavenJObject { {"name", name}, {"key", key}, {"etag", generator.CreateSequentialUuid(type).ToByteArray()} }, memoryStream.ToArray()); }
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data, byte[] viewAndReduceKeyHashed) { var ms = new MemoryStream(); data.WriteTo(ms); var byteArray = generator.CreateSequentialUuid().ToByteArray(); var key = new RavenJObject { {"view", view}, {"reduceKey", reduceKey}, {"docId", docId}, {"etag", byteArray}, {"timestamp", DateTime.Now} }; storage.MappedResults.Put(key, ms.ToArray()); }
public Task<Etag> ExportAttachmentsDeletion(JsonTextWriter jsonWriter, Etag startAttachmentsDeletionEtag, Etag maxAttachmentEtag) { var lastEtag = startAttachmentsDeletionEtag; database.TransactionalStorage.Batch(accessor => { foreach (var listItem in accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, startAttachmentsDeletionEtag, maxAttachmentEtag, int.MaxValue)) { var o = new RavenJObject { {"Key", listItem.Key} }; o.WriteTo(jsonWriter); lastEtag = listItem.Etag; } }); return new CompletedTask<Etag>(lastEtag); }
public Etag AddDocumentInTransaction(string key, Etag etag, RavenJObject data, RavenJObject metadata, TransactionInformation transactionInformation) { var readResult = storage.Documents.Read(new RavenJObject { { "key", key } }); if (readResult != null) // update { StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation); AssertValidEtag(key, readResult, storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } }), etag, "DELETE"); var ravenJObject = ((RavenJObject)readResult.Key.CloneToken()); ravenJObject["txId"] = transactionInformation.Id.ToByteArray(); if (storage.Documents.UpdateKey(ravenJObject) == false) throw new ConcurrencyException("PUT attempted on document '" + key + "' that is currently being modified by another transaction"); } else { readResult = storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } }); StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation); } storage.Transactions.UpdateKey(new RavenJObject { {"txId", transactionInformation.Id.ToByteArray()}, {"timeout", SystemTime.UtcNow.Add(transactionInformation.Timeout)} }); var ms = new MemoryStream(); metadata.WriteTo(ms); using (var stream = documentCodecs.Aggregate<Lazy<AbstractDocumentCodec>, Stream>(ms, (memoryStream, codec) => codec.Value.Encode(key, data, metadata, memoryStream))) { data.WriteTo(stream); stream.Flush(); } var newEtag = generator.CreateSequentialUuid(UuidType.DocumentTransactions); storage.DocumentsModifiedByTransactions.Put(new RavenJObject { {"key", key}, {"etag", newEtag.ToByteArray()}, {"modified", SystemTime.UtcNow}, {"txId", transactionInformation.Id.ToByteArray()} }, ms.ToArray()); return newEtag; }
public Guid AddAttachment(string key, Guid? etag, byte[] data, RavenJObject headers) { AssertValidEtag(key, etag, "PUT"); var ms = new MemoryStream(); headers.WriteTo(ms); ms.Write(data,0,data.Length); var newEtag = generator.CreateSequentialUuid(); var result = storage.Attachments.Put(new RavenJObject { {"key", key}, {"etag", newEtag.ToByteArray()} }, ms.ToArray()); if (result == false) throw new ConcurrencyException("PUT attempted on attachment '" + key + "' while it was locked by another transaction"); logger.Debug("Adding attachment {0}", key); return newEtag; }
public Guid Add(string key, string tag, Guid? etag, RavenJObject metadata, RavenJObject document) { EnsureEtagMatches(key, etag); var stream = new PooledMemoryStream(); metadata.WriteTo(new BsonWriter(stream)); document.WriteTo(new BsonWriter(stream)); var newGuid = Guid.NewGuid(); Storage.Documents.Put(new RavenJObject { {"key", key}, {"tag", tag}, {"etag", newGuid.ToByteArray()} }, stream); return newGuid; }
public Guid AddDocumentInTransaction(string key, Guid? etag, RavenJObject data, RavenJObject metadata, TransactionInformation transactionInformation) { var readResult = storage.Documents.Read(new RavenJObject {{"key", key}}); if (readResult != null) // update { StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation); AssertValidEtag(key, readResult, storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } }), etag); ((RavenJObject)readResult.Key)["txId"] = transactionInformation.Id.ToByteArray(); if (storage.Documents.UpdateKey(readResult.Key) == false) throw new ConcurrencyException("PUT attempted on document '" + key + "' that is currently being modified by another transaction"); } else { readResult = storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } }); StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation); } storage.Transactions.UpdateKey(new RavenJObject { {"txId", transactionInformation.Id.ToByteArray()}, {"timeout", SystemTime.UtcNow.Add(transactionInformation.Timeout)} }); var ms = new MemoryStream(); metadata.WriteTo(ms); var dataBytes = documentCodecs.Aggregate(data.ToBytes(), (bytes, codec) => codec.Encode(key, data, metadata, bytes)); ms.Write(dataBytes, 0, dataBytes.Length); var newEtag = generator.CreateSequentialUuid(); storage.DocumentsModifiedByTransactions.Put(new RavenJObject { {"key", key}, {"etag", newEtag.ToByteArray()}, {"modified", SystemTime.UtcNow}, {"txId", transactionInformation.Id.ToByteArray()} }, ms.ToArray()); return newEtag; }
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data) { var ms = new MemoryStream(); using (var stream = documentCodecs.Aggregate((Stream)ms, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(stream); } var byteArray = generator.CreateSequentialUuid(UuidType.MappedResults).ToByteArray(); var key = new RavenJObject { {"view", view}, {"reduceKey", reduceKey}, {"docId", docId}, {"etag", byteArray}, {"bucket", IndexingUtil.MapBucket(docId)}, {"timestamp", SystemTime.UtcNow} }; storage.MappedResults.Put(key, ms.ToArray()); }
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data, byte[] viewAndReduceKeyHashed) { Guid etag = uuidGenerator.CreateSequentialUuid(); using (var update = new Update(session, MappedResults, JET_prep.Insert)) { Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], view, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"], docId, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"], reduceKey, Encoding.Unicode); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key_and_view_hashed"], viewAndReduceKeyHashed); using (var stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"]))) { data.WriteTo(stream); stream.Flush(); } Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"], etag.TransformToValueForEsentSorting()); Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"], SystemTime.Now); update.Save(); } }
public void Write(RavenJObject ravenJObject, params JsonConverter[] converters) { ravenJObject.WriteTo(GetCurrentJsonTextWriter(), converters); SpinWriterIfReachedMaxSize(); }
protected Task<Etag> ExportDocumentsDeletion(SmugglerOptions options, JsonTextWriter jsonWriter, Etag startDocsEtag, Etag maxEtag) { var lastEtag = startDocsEtag; database.TransactionalStorage.Batch(accessor => { foreach (var listItem in accessor.Lists.Read(Constants.RavenPeriodicBackupsDocsTombstones, startDocsEtag, maxEtag, int.MaxValue)) { var o = new RavenJObject { {"Key", listItem.Key} }; o.WriteTo(jsonWriter); lastEtag = listItem.Etag; } }); return new CompletedTask<Etag>(lastEtag); }
public Guid AddDocumentInTransaction(string key, Guid?etag, RavenJObject data, RavenJObject metadata, TransactionInformation transactionInformation) { Api.JetSetCurrentIndex(session, Documents, "by_key"); Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey); var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ); if (isUpdate) { EnsureNotLockedByTransaction(key, transactionInformation.Id); EnsureDocumentEtagMatchInTransaction(key, etag); using (var update = new Update(session, Documents, JET_prep.Replace)) { Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["locked_by_transaction"], transactionInformation.Id.ToByteArray()); update.Save(); } } else { EnsureDocumentIsNotCreatedInAnotherTransaction(key, transactionInformation.Id); } EnsureTransactionExists(transactionInformation); Guid newEtag = uuidGenerator.CreateSequentialUuid(UuidType.DocumentTransactions); Api.JetSetCurrentIndex(session, DocumentsModifiedByTransactions, "by_key"); Api.MakeKey(session, DocumentsModifiedByTransactions, key, Encoding.Unicode, MakeKeyGrbit.NewKey); var isUpdateInTransaction = Api.TrySeek(session, DocumentsModifiedByTransactions, SeekGrbit.SeekEQ); using (var update = new Update(session, DocumentsModifiedByTransactions, isUpdateInTransaction ? JET_prep.Replace : JET_prep.Insert)) { Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["key"], key, Encoding.Unicode); using (var columnStream = new ColumnStream(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["data"])) { if (isUpdate) { columnStream.SetLength(0); } using (Stream stream = new BufferedStream(columnStream)) using (var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current))) { data.WriteTo(finalStream); finalStream.Flush(); } } Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["etag"], newEtag.TransformToValueForEsentSorting()); using (var columnStream = new ColumnStream(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["metadata"])) { if (isUpdate) { columnStream.SetLength(0); } using (Stream stream = new BufferedStream(columnStream)) { metadata.WriteTo(stream); stream.Flush(); } } Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["last_modified"], SystemTime.UtcNow.ToBinary()); Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["delete_document"], false); Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["locked_by_transaction"], transactionInformation.Id.ToByteArray()); update.Save(); } logger.Debug("Inserted a new document with key '{0}', update: {1}, in transaction: {2}", key, isUpdate, transactionInformation.Id); return(newEtag); }
private void ReplicateAttachment(IStorageActionsAccessor actions, string id, RavenJObject metadata, byte[] data, Guid lastEtag, string src) { var existingAttachment = actions.Attachments.GetAttachment(id); if (existingAttachment == null) { log.Debug("New attachment {0} replicated successfully from {1}", id, src); actions.Attachments.AddAttachment(id, Guid.Empty, new MemoryStream(data), metadata); return; } // we just got the same version from the same source - request playback again? // at any rate, not an error, moving on if (existingAttachment.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) == metadata.Value <string>(ReplicationConstants.RavenReplicationSource) && existingAttachment.Metadata.Value <int>(ReplicationConstants.RavenReplicationVersion) == metadata.Value <int>(ReplicationConstants.RavenReplicationVersion)) { return; } var existingDocumentIsInConflict = existingAttachment.Metadata[ReplicationConstants.RavenReplicationConflict] != null; if (existingDocumentIsInConflict == false && // if the current document is not in conflict, we can continue without having to keep conflict semantics (IsDirectChildOfCurrentAttachment(existingAttachment, metadata))) // this update is direct child of the existing doc, so we are fine with overwriting this { log.Debug("Existing document {0} replicated successfully from {1}", id, src); actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata); return; } if (ReplicationConflictResolvers.Any(replicationConflictResolver => replicationConflictResolver.TryResolve(id, metadata, data, existingAttachment))) { actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata); return; } var newDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(metadata, lastEtag); metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true)); actions.Attachments.AddAttachment(newDocumentConflictId, null, new MemoryStream(data), metadata); if (existingDocumentIsInConflict) // the existing document is in conflict { log.Debug("Conflicted document {0} has a new version from {1}, adding to conflicted documents", id, src); // just update the current doc with the new conflict document var conflictArray = existingAttachment.Metadata.Value <RavenJArray>("Conflicts"); if (conflictArray == null) { existingAttachment.Metadata["Conflicts"] = conflictArray = new RavenJArray(); } conflictArray.Add(RavenJToken.FromObject(newDocumentConflictId)); actions.Attachments.AddAttachment(id, existingAttachment.Etag, existingAttachment.Data(), existingAttachment.Metadata); return; } log.Debug("Existing document {0} is in conflict with replicated version from {1}, marking document as conflicted", id, src); // we have a new conflict // move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingAttachment.Etag); existingAttachment.Metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true)); actions.Attachments.AddAttachment(existingDocumentConflictId, null, existingAttachment.Data(), existingAttachment.Metadata); var conflictAttachment = new RavenJObject { { "Conflicts", new RavenJArray(existingDocumentConflictId, newDocumentConflictId) } }; var memoryStream = new MemoryStream(); conflictAttachment.WriteTo(memoryStream); memoryStream.Position = 0; actions.Attachments.AddAttachment(id, null, memoryStream, new RavenJObject { { ReplicationConstants.RavenReplicationConflict, true }, { "@Http-Status-Code", 409 }, { "@Http-Status-Description", "Conflict" } }); }
public virtual void WriteTo(JsonWriter writer) { inner.WriteTo(writer); }
public void SetConfig(string name, RavenJObject metadata) { var builder = new StringBuilder(); using (var writer = new JsonTextWriter(new StringWriter(builder))) metadata.WriteTo(writer); string metadataString = builder.ToString(); Api.JetSetCurrentIndex(session, Config, "by_name"); Api.MakeKey(session, Config, name, Encoding.Unicode, MakeKeyGrbit.NewKey); var prep = Api.TrySeek(session, Config, SeekGrbit.SeekEQ) ? JET_prep.Replace : JET_prep.Insert; using (var update = new Update(session, Config, prep)) { Api.SetColumn(session, Config, tableColumnsCache.ConfigColumns["name"], name, Encoding.Unicode); Api.SetColumn(session, Config, tableColumnsCache.ConfigColumns["metadata"], metadataString, Encoding.Unicode); update.Save(); } }
public void PutReducedResult(int view, string reduceKey, int level, int sourceBucket, int bucket, RavenJObject data) { var reduceResultsByViewAndReduceKeyAndLevelAndSourceBucket = tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByViewAndReduceKeyAndLevelAndSourceBucket); var reduceResultsByViewAndReduceKeyAndLevelAndBucket = tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByViewAndReduceKeyAndLevelAndBucket); var reduceResultsByViewAndReduceKeyAndLevel = tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByViewAndReduceKeyAndLevel); var reduceResultsByView = tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByView); var reduceResultsData = tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.Data); var ms = CreateStream(); using ( var stream = documentCodecs.Aggregate((Stream) new UndisposableStream(ms), (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(stream); stream.Flush(); } var id = generator.CreateSequentialUuid(UuidType.MappedResults); var idAsSlice = (Slice)id.ToString(); var reduceResult = new Structure<ReduceResultFields>(tableStorage.ReduceResults.Schema) .Set(ReduceResultFields.IndexId, view) .Set(ReduceResultFields.Etag, id.ToByteArray()) .Set(ReduceResultFields.ReduceKey, reduceKey) .Set(ReduceResultFields.Level, level) .Set(ReduceResultFields.SourceBucket, sourceBucket) .Set(ReduceResultFields.Bucket, bucket) .Set(ReduceResultFields.Timestamp, SystemTime.UtcNow.ToBinary()); tableStorage.ReduceResults.AddStruct(writeBatch.Value, idAsSlice, reduceResult, 0); ms.Position = 0; reduceResultsData.Add(writeBatch.Value, idAsSlice, ms, 0); var viewKey = CreateViewKey(view); var viewAndReduceKeyAndLevel = CreateReduceResultsKey(view,reduceKey, level); var viewAndReduceKeyAndLevelAndSourceBucket = CreateReduceResultsWithBucketKey(view, reduceKey, level, sourceBucket); var viewAndReduceKeyAndLevelAndBucket = CreateReduceResultsWithBucketKey(view, reduceKey, level, bucket); reduceResultsByViewAndReduceKeyAndLevelAndSourceBucket.MultiAdd(writeBatch.Value, viewAndReduceKeyAndLevelAndSourceBucket, idAsSlice); reduceResultsByViewAndReduceKeyAndLevel.MultiAdd(writeBatch.Value, viewAndReduceKeyAndLevel, idAsSlice); reduceResultsByViewAndReduceKeyAndLevelAndBucket.MultiAdd(writeBatch.Value, viewAndReduceKeyAndLevelAndBucket, idAsSlice); reduceResultsByView.MultiAdd(writeBatch.Value, viewKey, idAsSlice); }
public Task WriteDocumentAsync(RavenJObject document, CancellationToken cancellationToken) { document.WriteTo(Writer); return(new CompletedTask()); }
public async Task AppendAsync(RavenJObject data) { if (disposed) throw new ObjectDisposedException("EventStream"); var nextEtag = _options.Status.NextEtag(); data["@etag"] = nextEtag.ToString(); data["@date"] = DateTime.UtcNow.ToString("o"); using (var stream = new BufferPoolMemoryStream(_options.BufferPool)) { var bsonWriter = new BsonWriter(stream); data.WriteTo(bsonWriter); bsonWriter.Flush(); stream.Position = 0; var mine = new PendingWrite(stream, nextEtag); _pending.Enqueue(mine); while (mine.Done() == false && _pending.Peek() != mine) { await _writeCompletedEvent.WaitAsync(); } if (mine.Done()) return; await AppendInternalAsync(mine); } }
public void Write(RavenJObject result) { result.WriteTo(writer, Default.Converters); writer.WriteRaw(Environment.NewLine); }
public void PutMappedResult(int view, string docId, string reduceKey, RavenJObject data) { var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId); var mappedResultsByView = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByView); var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey); var mappedResultsByViewAndReduceKeyAndSourceBucket = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKeyAndSourceBucket); var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data); var ms = CreateStream(); using (var stream = documentCodecs.Aggregate((Stream) new UndisposableStream(ms), (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(stream); stream.Flush(); } var id = generator.CreateSequentialUuid(UuidType.MappedResults); var idAsString = id.ToString(); var bucket = IndexingUtil.MapBucket(docId); var reduceKeyHash = HashKey(reduceKey); tableStorage.MappedResults.Add( writeBatch.Value, idAsString, new RavenJObject { { "view", view }, { "reduceKey", reduceKey }, { "docId", docId }, { "etag", id.ToByteArray() }, { "bucket", bucket }, { "timestamp", SystemTime.UtcNow } }, 0); ms.Position = 0; mappedResultsData.Add(writeBatch.Value, idAsString, ms, 0); mappedResultsByViewAndDocumentId.MultiAdd(writeBatch.Value, CreateKey(view, docId), idAsString); mappedResultsByView.MultiAdd(writeBatch.Value, CreateKey(view), idAsString); mappedResultsByViewAndReduceKey.MultiAdd(writeBatch.Value, CreateKey(view, reduceKey, reduceKeyHash), idAsString); mappedResultsByViewAndReduceKeyAndSourceBucket.MultiAdd(writeBatch.Value, CreateKey(view, reduceKey, reduceKeyHash, bucket), idAsString); }
private void WriteAttachments(JsonTextWriter jsonWriter) { long totalAttachmentsCount = 0; storage.Batch(accsesor => totalAttachmentsCount = accsesor.Attachments.GetAttachmentsCount()); if (totalAttachmentsCount == 0) return; var lastEtag = Etag.Empty; long currentAttachmentsCount = 0; do { var previousAttachmentCount = currentAttachmentsCount; try { storage.Batch(accsesor => { var attachments = accsesor.Attachments.GetAttachmentsAfter(lastEtag, batchSize, long.MaxValue); foreach (var attachmentInformation in attachments) { var attachment = accsesor.Attachments.GetAttachment(attachmentInformation.Key); if (attachment == null) { ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Couldn't find attachment '{0}'", attachmentInformation.Key); continue; } var data = attachment.Data; attachment.Data = () => { var memoryStream = new MemoryStream(); storage.Batch(accessor => data().CopyTo(memoryStream)); memoryStream.Position = 0; return memoryStream; }; var attachmentData = attachment.Data().ReadData(); if (attachmentData == null) { ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "No data was found for attachment '{0}'", attachment.Key); continue; } var ravenJsonObj = new RavenJObject { { "Data", attachmentData }, { "Metadata", attachmentInformation.Metadata }, { "Key", attachmentInformation.Key }, { "Etag", new RavenJValue(attachmentInformation.Etag.ToString()) } }; ravenJsonObj.WriteTo(jsonWriter); lastEtag = attachmentInformation.Etag; currentAttachmentsCount++; if (currentAttachmentsCount % batchSize == 0) ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount); } }); } catch (Exception e) { lastEtag = lastEtag.IncrementBy(1); currentAttachmentsCount++; ReportCorrupted("attachment", currentAttachmentsCount, e.Message); } finally { if (currentAttachmentsCount > previousAttachmentCount) ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount); } } while (currentAttachmentsCount < totalAttachmentsCount); }
public void Write(RavenJObject result) { result.WriteTo(writer, Default.Converters); }
private void ReplicateAttachment(IStorageActionsAccessor actions, string id, RavenJObject metadata, byte[] data, Guid lastEtag ,string src) { var existingAttachment = actions.Attachments.GetAttachment(id); if (existingAttachment == null) { log.Debug("New attachment {0} replicated successfully from {1}", id, src); actions.Attachments.AddAttachment(id, Guid.Empty, new MemoryStream(data), metadata); return; } // we just got the same version from the same source - request playback again? // at any rate, not an error, moving on if(existingAttachment.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) == metadata.Value<string>(ReplicationConstants.RavenReplicationSource) && existingAttachment.Metadata.Value<int>(ReplicationConstants.RavenReplicationVersion) == metadata.Value<int>(ReplicationConstants.RavenReplicationVersion)) { return; } var existingDocumentIsInConflict = existingAttachment.Metadata[ReplicationConstants.RavenReplicationConflict] != null; if (existingDocumentIsInConflict == false && // if the current document is not in conflict, we can continue without having to keep conflict semantics (IsDirectChildOfCurrentAttachment(existingAttachment, metadata))) // this update is direct child of the existing doc, so we are fine with overwriting this { log.Debug("Existing document {0} replicated successfully from {1}", id, src); actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata); return; } if (ReplicationConflictResolvers.Any(replicationConflictResolver => replicationConflictResolver.TryResolve(id, metadata, data, existingAttachment))) { actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata); return; } var newDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(metadata, lastEtag); metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true)); actions.Attachments.AddAttachment(newDocumentConflictId, null, new MemoryStream(data), metadata); if (existingDocumentIsInConflict) // the existing document is in conflict { log.Debug("Conflicted document {0} has a new version from {1}, adding to conflicted documents", id, src); // just update the current doc with the new conflict document var conflictArray = existingAttachment.Metadata.Value<RavenJArray>("Conflicts"); if (conflictArray == null) existingAttachment.Metadata["Conflicts"] = conflictArray = new RavenJArray(); conflictArray.Add(RavenJToken.FromObject(newDocumentConflictId)); actions.Attachments.AddAttachment(id, existingAttachment.Etag, existingAttachment.Data(), existingAttachment.Metadata); return; } log.Debug("Existing document {0} is in conflict with replicated version from {1}, marking document as conflicted", id, src); // we have a new conflict // move the existing doc to a conflict and create a conflict document var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingAttachment.Etag); existingAttachment.Metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true)); actions.Attachments.AddAttachment(existingDocumentConflictId, null, existingAttachment.Data(), existingAttachment.Metadata); var conflictAttachment = new RavenJObject { {"Conflicts", new RavenJArray(existingDocumentConflictId, newDocumentConflictId)} }; var memoryStream = new MemoryStream(); conflictAttachment.WriteTo(memoryStream); memoryStream.Position = 0; actions.Attachments.AddAttachment(id, null, memoryStream, new RavenJObject { {ReplicationConstants.RavenReplicationConflict, true}, {"@Http-Status-Code", 409}, {"@Http-Status-Description", "Conflict"} }); }
public AddDocumentResult AddDocument(string key, Guid?etag, RavenJObject data, RavenJObject metadata) { if (key != null && Encoding.Unicode.GetByteCount(key) >= 2048) { throw new ArgumentException(string.Format("The key must be a maximum of 2,048 bytes in Unicode, 1,024 characters, key is: '{0}'", key), "key"); } Api.JetSetCurrentIndex(session, Documents, "by_key"); Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey); var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ); if (isUpdate) { EnsureNotLockedByTransaction(key, null); EnsureDocumentEtagMatch(key, etag, "PUT"); } else { if (etag != null && etag != Guid.Empty) // expected something to be there. { throw new ConcurrencyException("PUT attempted on document '" + key + "' using a non current etag (document deleted)") { ExpectedETag = etag.Value } } ; EnsureDocumentIsNotCreatedInAnotherTransaction(key, Guid.NewGuid()); if (Api.TryMoveFirst(session, Details)) { Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["document_count"], 1); } } Guid newEtag = uuidGenerator.CreateSequentialUuid(); DateTime savedAt; using (var update = new Update(session, Documents, isUpdate ? JET_prep.Replace : JET_prep.Insert)) { Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode); using (Stream stream = new BufferedStream(new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"]))) using (var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current))) { data.WriteTo(finalStream); finalStream.Flush(); } Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"], newEtag.TransformToValueForEsentSorting()); savedAt = SystemTime.UtcNow; Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt); using (Stream stream = new BufferedStream(new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"]))) { metadata.WriteTo(stream); stream.Flush(); } update.Save(); } logger.Debug("Inserted a new document with key '{0}', update: {1}, ", key, isUpdate); cacher.RemoveCachedDocument(key, newEtag); return(new AddDocumentResult { Etag = newEtag, SavedAt = savedAt, Updated = isUpdate }); }
private void WriteAttachmentMetadata(Slice key, Etag etag, RavenJObject headers) { var memoryStream = CreateStream(); memoryStream.Write(etag); headers.WriteTo(memoryStream); memoryStream.Position = 0; metadataIndex.Add(writeBatch.Value, key, memoryStream); }
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout) { timeout.Delay(); item.WriteTo(writer); }
public void PutMappedResult(int view, string docId, string reduceKey, RavenJObject data) { var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId); var mappedResultsByView = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByView); var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey); var mappedResultsByViewAndReduceKeyAndSourceBucket = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKeyAndSourceBucket); var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data); var ms = CreateStream(); using (var stream = documentCodecs.Aggregate((Stream) new UndisposableStream(ms), (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(stream); stream.Flush(); } var id = generator.CreateSequentialUuid(UuidType.MappedResults); var idSlice = (Slice)id.ToString(); var bucket = IndexingUtil.MapBucket(docId); var reduceKeyHash = HashKey(reduceKey); var mappedResult = new Structure<MappedResultFields>(tableStorage.MappedResults.Schema) .Set(MappedResultFields.IndexId, view) .Set(MappedResultFields.Bucket, bucket) .Set(MappedResultFields.Timestamp, SystemTime.UtcNow.ToBinary()) .Set(MappedResultFields.ReduceKey, reduceKey) .Set(MappedResultFields.DocId, docId) .Set(MappedResultFields.Etag, id.ToByteArray()); tableStorage.MappedResults.AddStruct( writeBatch.Value, idSlice, mappedResult, 0); ms.Position = 0; mappedResultsData.Add(writeBatch.Value, idSlice, ms, 0); string viewKey = CreateKey(view); string viewReduceKey = AppendToKey(viewKey, ReduceKeySizeLimited(reduceKey)); string viewReduceHashKey = AppendToKey(viewReduceKey, reduceKeyHash); mappedResultsByViewAndDocumentId.MultiAdd(writeBatch.Value, (Slice)AppendToKey(viewKey, docId), idSlice); mappedResultsByView.MultiAdd(writeBatch.Value, (Slice)viewKey, idSlice); mappedResultsByViewAndReduceKey.MultiAdd(writeBatch.Value, (Slice)viewReduceHashKey, idSlice); mappedResultsByViewAndReduceKeyAndSourceBucket.MultiAdd(writeBatch.Value, (Slice)AppendToKey(viewReduceHashKey, bucket), idSlice); }
public AddDocumentResult AddDocument(string key, Etag etag, RavenJObject data, RavenJObject metadata) { if (key == null) { throw new ArgumentNullException("key"); } var byteCount = Encoding.Unicode.GetByteCount(key); if (byteCount >= 2048) { throw new ArgumentException(string.Format("The key must be a maximum of 2,048 bytes in Unicode, 1,024 characters, key is: '{0}'", key), "key"); } try { Api.JetSetCurrentIndex(session, Documents, "by_key"); Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey); var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ); Etag existingEtag = null; if (isUpdate) { existingEtag = EnsureDocumentEtagMatch(key, etag, "PUT"); } else { if (etag != null && etag != Etag.Empty) // expected something to be there. { throw new ConcurrencyException("PUT attempted on document '" + key + "' using a non current etag (document deleted)") { ExpectedETag = etag } } ; if (Api.TryMoveFirst(session, Details)) { Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["document_count"], 1); } } Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents); DateTime savedAt; try { using (var update = new Update(session, Documents, isUpdate ? JET_prep.Replace : JET_prep.Insert)) { Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode); using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"])) { if (isUpdate) { columnStream.SetLength(0); // empty the existing value, since we are going to overwrite the entire thing } using (Stream stream = new BufferedStream(columnStream)) using ( var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current)) ) { data.WriteTo(finalStream); finalStream.Flush(); } } Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"], newEtag.TransformToValueForEsentSorting()); savedAt = SystemTime.UtcNow; Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary()); using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"])) { if (isUpdate) { columnStream.SetLength(0); } using (Stream stream = new BufferedStream(columnStream)) { metadata.WriteTo(stream); stream.Flush(); } } update.Save(); } } catch (EsentErrorException e) { if (e.Error == JET_err.KeyDuplicate || e.Error == JET_err.WriteConflict) { throw new ConcurrencyException("PUT attempted on document '" + key + "' concurrently", e); } throw; } logger.Debug("Inserted a new document with key '{0}', update: {1}, ", key, isUpdate); cacher.RemoveCachedDocument(key, newEtag); return(new AddDocumentResult { Etag = newEtag, PrevEtag = existingEtag, SavedAt = savedAt, Updated = isUpdate }); } catch (EsentKeyDuplicateException e) { throw new ConcurrencyException("Illegal duplicate key " + key, e); } }
public AddDocumentResult InsertDocument(string key, RavenJObject data, RavenJObject metadata, bool overwriteExisting) { var prep = JET_prep.Insert; bool isUpdate = false; Etag existingETag = null; if (overwriteExisting) { Api.JetSetCurrentIndex(session, Documents, "by_key"); Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey); isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ); if (isUpdate) { existingETag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"])); prep = JET_prep.Replace; } } try { using (var update = new Update(session, Documents, prep)) { Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode); using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"])) { if (isUpdate) { columnStream.SetLength(0); } using (Stream stream = new BufferedStream(columnStream)) using (var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current))) { data.WriteTo(finalStream); finalStream.Flush(); } } Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents); Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"], newEtag.TransformToValueForEsentSorting()); DateTime savedAt = SystemTime.UtcNow; Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary()); using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"])) { if (isUpdate) { columnStream.SetLength(0); } using (Stream stream = new BufferedStream(columnStream)) { metadata.WriteTo(stream); stream.Flush(); } } update.Save(); return(new AddDocumentResult { Etag = newEtag, PrevEtag = existingETag, SavedAt = savedAt, Updated = isUpdate }); } } catch (EsentKeyDuplicateException e) { throw new ConcurrencyException("Illegal duplicate key " + key, e); } }
private bool WriteDocumentData(string key, string normalizedKey, Etag etag, RavenJObject data, RavenJObject metadata, out Etag newEtag, out Etag existingEtag, out DateTime savedAt) { var normalizedKeySlice = (Slice)normalizedKey; var keyByEtagDocumentIndex = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag); ushort? existingVersion; var isUpdate = tableStorage.Documents.Contains(Snapshot, normalizedKeySlice, writeBatch.Value, out existingVersion); existingEtag = null; if (isUpdate) { existingEtag = EnsureDocumentEtagMatch(normalizedKey, etag, "PUT"); keyByEtagDocumentIndex.Delete(writeBatch.Value, existingEtag); } else if (etag != null && etag != Etag.Empty) { throw new ConcurrencyException("PUT attempted on document '" + key + "' using a non current etag (document deleted)") { ExpectedETag = etag }; } var dataStream = CreateStream(); using (var finalDataStream = documentCodecs.Aggregate((Stream)new UndisposableStream(dataStream), (current, codec) => codec.Encode(normalizedKey, data, metadata, current))) { data.WriteTo(finalDataStream); finalDataStream.Flush(); } dataStream.Position = 0; tableStorage.Documents.Add(writeBatch.Value, normalizedKeySlice, dataStream, existingVersion ?? 0); newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents); savedAt = SystemTime.UtcNow; var isUpdated = PutDocumentMetadataInternal(key, normalizedKeySlice, metadata, newEtag, savedAt); keyByEtagDocumentIndex.Add(writeBatch.Value, newEtag, normalizedKey); return isUpdated; }
private void WriteAttachments(JsonTextWriter jsonWriter) { long totalAttachmentsCount = 0; storage.Batch(accsesor => totalAttachmentsCount = accsesor.Attachments.GetAttachmentsCount()); if (totalAttachmentsCount == 0) { return; } var lastEtag = Etag.Empty; long currentAttachmentsCount = 0; do { var previousAttachmentCount = currentAttachmentsCount; try { storage.Batch(accsesor => { var attachments = accsesor.Attachments.GetAttachmentsAfter(lastEtag, batchSize, long.MaxValue); foreach (var attachmentInformation in attachments) { var attachment = accsesor.Attachments.GetAttachment(attachmentInformation.Key); if (attachment == null) { ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Couldn't find attachment '{0}'", attachmentInformation.Key); continue; } var data = attachment.Data; attachment.Data = () => { var memoryStream = new MemoryStream(); storage.Batch(accessor => data().CopyTo(memoryStream)); memoryStream.Position = 0; return(memoryStream); }; var attachmentData = attachment.Data().ReadData(); if (attachmentData == null) { ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "No data was found for attachment '{0}'", attachment.Key); continue; } var ravenJsonObj = new RavenJObject { { "Data", attachmentData }, { "Metadata", attachmentInformation.Metadata }, { "Key", attachmentInformation.Key }, { "Etag", new RavenJValue(attachmentInformation.Etag.ToString()) } }; ravenJsonObj.WriteTo(jsonWriter); lastEtag = attachmentInformation.Etag; currentAttachmentsCount++; if (currentAttachmentsCount % batchSize == 0) { ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount); } } }); } catch (Exception e) { lastEtag = lastEtag.IncrementBy(1); currentAttachmentsCount++; ReportCorrupted("attachment", currentAttachmentsCount, e.Message); } finally { if (currentAttachmentsCount > previousAttachmentCount) { ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount); } } } while (currentAttachmentsCount < totalAttachmentsCount); }
public void PutReducedResult(string name, string reduceKey, int level, int sourceBucket, int bucket, RavenJObject data) { var ms = new MemoryStream(); using (var stream = documentCodecs.Aggregate((Stream)ms, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds))) { data.WriteTo(stream); } var etag = generator.CreateSequentialUuid().ToByteArray(); storage.ReduceResults.Put(new RavenJObject { {"view", name}, {"etag", etag}, {"reduceKey", reduceKey}, {"level", level}, {"sourceBucket", sourceBucket}, {"bucket", bucket}, {"timestamp", SystemTime.UtcNow} }, ms.ToArray()); }
public static void WaitForIndexing(IDocumentStore store, string database = null, TimeSpan?timeout = null) { var databaseCommands = store.DatabaseCommands; if (database != null) { databaseCommands = databaseCommands.ForDatabase(database); } timeout = timeout ?? (Debugger.IsAttached ? TimeSpan.FromMinutes(15) : TimeSpan.FromMinutes(1)); var sp = Stopwatch.StartNew(); while (sp.Elapsed < timeout.Value) { var databaseStatistics = databaseCommands.GetStatistics(); if (databaseStatistics.Indexes.All(x => x.IsStale == false)) { return; } if (databaseStatistics.Indexes.Any(x => x.State == IndexState.Error)) { break; } Thread.Sleep(32); } var request = databaseCommands.CreateRequest("/indexes/performance", HttpMethod.Get); var perf = request.ReadResponseJson(); request = databaseCommands.CreateRequest("/indexes/errors", HttpMethod.Get); var errors = request.ReadResponseJson(); var total = new RavenJObject { ["Errors"] = errors, ["Performance"] = perf }; var file = Path.GetTempFileName() + ".json"; using (var writer = File.CreateText(file)) { var jsonTextWriter = new JsonTextWriter(writer); total.WriteTo(jsonTextWriter); jsonTextWriter.Flush(); } var stats = databaseCommands.GetStatistics(); var corrupted = stats.Indexes.Where(x => x.State == IndexState.Error).ToList(); if (corrupted.Count > 0) { throw new InvalidOperationException( $"The following indexes are with error state: {string.Join(",", corrupted.Select(x => x.Name))} - details at " + file); } throw new TimeoutException("The indexes stayed stale for more than " + timeout.Value + ", stats at " + file); }