public void Set(string name, string key, RavenJObject data, UuidType uuidType)
{
Api.JetSetCurrentIndex(session, Lists, "by_name_and_key");
Api.MakeKey(session, Lists, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
Api.MakeKey(session, Lists, key, Encoding.Unicode, MakeKeyGrbit.None);
var exists = Api.TrySeek(session, Lists, SeekGrbit.SeekEQ);
using (var update = new Update(session, Lists, exists ? JET_prep.Replace : JET_prep.Insert))
{
Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["name"], name, Encoding.Unicode);
Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["key"], key, Encoding.Unicode);
Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["etag"], uuidGenerator.CreateSequentialUuid(uuidType).TransformToValueForEsentSorting());
Api.SetColumn(session, Lists, tableColumnsCache.ListsColumns["created_at"], SystemTime.UtcNow);
using (var columnStream = new ColumnStream(session, Lists, tableColumnsCache.ListsColumns["data"]))
{
if (exists)
columnStream.SetLength(0);
using (Stream stream = new BufferedStream(columnStream))
{
data.WriteTo(stream);
stream.Flush();
}
}
update.Save();
}
}
public Guid AddAttachment(string key, Guid? etag, Stream data, RavenJObject headers)
{
AssertValidEtag(key, etag, "PUT");
if(data == null)
{
var attachment = GetAttachment(key);
if(attachment == null)
throw new InvalidOperationException("When adding new attachment, the attachment data must be specified");
attachment.Metadata = headers;
return AddAttachment(key, etag, attachment.Data(), headers);
}
var ms = new MemoryStream();
headers.WriteTo(ms);
data.CopyTo(ms);
var newEtag = generator.CreateSequentialUuid();
var result = storage.Attachments.Put(new RavenJObject
{
{"key", key},
{"etag", newEtag.ToByteArray()}
}, ms.ToArray());
if (result == false)
throw new ConcurrencyException("PUT attempted on attachment '" + key + "' while it was locked by another transaction");
logger.Debug("Adding attachment {0}", key);
return newEtag;
}
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data)
{
Etag etag = uuidGenerator.CreateSequentialUuid(UuidType.MappedResults);
using (var update = new Update(session, MappedResults, JET_prep.Insert))
{
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], view, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"], docId, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"], reduceKey, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["hashed_reduce_key"], HashReduceKey(reduceKey));
var mapBucket = IndexingUtil.MapBucket(docId);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"], mapBucket);
using (Stream stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"])))
{
using (var dataStream = documentCodecs.Aggregate(stream, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(dataStream);
dataStream.Flush();
}
}
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"], etag.TransformToValueForEsentSorting());
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"], SystemTime.UtcNow.ToBinary());
update.Save();
}
}
public void PutReducedResult(string view, string reduceKey, int level, int sourceBucket, int bucket, RavenJObject data)
{
Guid etag = uuidGenerator.CreateSequentialUuid();
using (var update = new Update(session, ReducedResults, JET_prep.Insert))
{
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["view"], view, Encoding.Unicode);
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["level"], level);
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["reduce_key"], reduceKey, Encoding.Unicode);
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["hashed_reduce_key"], HashReduceKey(reduceKey));
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["bucket"], bucket);
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["source_bucket"], sourceBucket);
using (Stream stream = new BufferedStream(new ColumnStream(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["data"])))
{
using (var dataStream = documentCodecs.Aggregate(stream, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(dataStream);
dataStream.Flush();
}
}
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["etag"], etag.TransformToValueForEsentSorting());
Api.SetColumn(session, ReducedResults, tableColumnsCache.ReduceResultsColumns["timestamp"], SystemTime.UtcNow);
update.Save();
}
}
public void Set(string name, string key, RavenJObject data, UuidType type)
{
var memoryStream = new MemoryStream();
data.WriteTo(memoryStream);
storage.Lists.Put(new RavenJObject
{
{"name", name},
{"key", key},
{"etag", generator.CreateSequentialUuid(type).ToByteArray()}
}, memoryStream.ToArray());
}
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data, byte[] viewAndReduceKeyHashed)
{
var ms = new MemoryStream();
data.WriteTo(ms);
var byteArray = generator.CreateSequentialUuid().ToByteArray();
var key = new RavenJObject
{
{"view", view},
{"reduceKey", reduceKey},
{"docId", docId},
{"etag", byteArray},
{"timestamp", DateTime.Now}
};
storage.MappedResults.Put(key, ms.ToArray());
}
public Task<Etag> ExportAttachmentsDeletion(JsonTextWriter jsonWriter, Etag startAttachmentsDeletionEtag, Etag maxAttachmentEtag)
{
var lastEtag = startAttachmentsDeletionEtag;
database.TransactionalStorage.Batch(accessor =>
{
foreach (var listItem in accessor.Lists.Read(Constants.RavenPeriodicExportsAttachmentsTombstones, startAttachmentsDeletionEtag, maxAttachmentEtag, int.MaxValue))
{
var o = new RavenJObject
{
{"Key", listItem.Key}
};
o.WriteTo(jsonWriter);
lastEtag = listItem.Etag;
}
});
return new CompletedTask<Etag>(lastEtag);
}
public Etag AddDocumentInTransaction(string key, Etag etag, RavenJObject data, RavenJObject metadata, TransactionInformation transactionInformation)
{
var readResult = storage.Documents.Read(new RavenJObject { { "key", key } });
if (readResult != null) // update
{
StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation);
AssertValidEtag(key, readResult, storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } }), etag, "DELETE");
var ravenJObject = ((RavenJObject)readResult.Key.CloneToken());
ravenJObject["txId"] = transactionInformation.Id.ToByteArray();
if (storage.Documents.UpdateKey(ravenJObject) == false)
throw new ConcurrencyException("PUT attempted on document '" + key +
"' that is currently being modified by another transaction");
}
else
{
readResult = storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } });
StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation);
}
storage.Transactions.UpdateKey(new RavenJObject
{
{"txId", transactionInformation.Id.ToByteArray()},
{"timeout", SystemTime.UtcNow.Add(transactionInformation.Timeout)}
});
var ms = new MemoryStream();
metadata.WriteTo(ms);
using (var stream = documentCodecs.Aggregate<Lazy<AbstractDocumentCodec>, Stream>(ms, (memoryStream, codec) => codec.Value.Encode(key, data, metadata, memoryStream)))
{
data.WriteTo(stream);
stream.Flush();
}
var newEtag = generator.CreateSequentialUuid(UuidType.DocumentTransactions);
storage.DocumentsModifiedByTransactions.Put(new RavenJObject
{
{"key", key},
{"etag", newEtag.ToByteArray()},
{"modified", SystemTime.UtcNow},
{"txId", transactionInformation.Id.ToByteArray()}
}, ms.ToArray());
return newEtag;
}
public Guid AddAttachment(string key, Guid? etag, byte[] data, RavenJObject headers)
{
AssertValidEtag(key, etag, "PUT");
var ms = new MemoryStream();
headers.WriteTo(ms);
ms.Write(data,0,data.Length);
var newEtag = generator.CreateSequentialUuid();
var result = storage.Attachments.Put(new RavenJObject
{
{"key", key},
{"etag", newEtag.ToByteArray()}
}, ms.ToArray());
if (result == false)
throw new ConcurrencyException("PUT attempted on attachment '" + key + "' while it was locked by another transaction");
logger.Debug("Adding attachment {0}", key);
return newEtag;
}
public Guid Add(string key, string tag, Guid? etag, RavenJObject metadata, RavenJObject document)
{
EnsureEtagMatches(key, etag);
var stream = new PooledMemoryStream();
metadata.WriteTo(new BsonWriter(stream));
document.WriteTo(new BsonWriter(stream));
var newGuid = Guid.NewGuid();
Storage.Documents.Put(new RavenJObject
{
{"key", key},
{"tag", tag},
{"etag", newGuid.ToByteArray()}
}, stream);
return newGuid;
}
public Guid AddDocumentInTransaction(string key, Guid? etag, RavenJObject data, RavenJObject metadata, TransactionInformation transactionInformation)
{
var readResult = storage.Documents.Read(new RavenJObject {{"key", key}});
if (readResult != null) // update
{
StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation);
AssertValidEtag(key, readResult, storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } }), etag);
((RavenJObject)readResult.Key)["txId"] = transactionInformation.Id.ToByteArray();
if (storage.Documents.UpdateKey(readResult.Key) == false)
throw new ConcurrencyException("PUT attempted on document '" + key +
"' that is currently being modified by another transaction");
}
else
{
readResult = storage.DocumentsModifiedByTransactions.Read(new RavenJObject { { "key", key } });
StorageHelper.AssertNotModifiedByAnotherTransaction(storage, this, key, readResult, transactionInformation);
}
storage.Transactions.UpdateKey(new RavenJObject
{
{"txId", transactionInformation.Id.ToByteArray()},
{"timeout", SystemTime.UtcNow.Add(transactionInformation.Timeout)}
});
var ms = new MemoryStream();
metadata.WriteTo(ms);
var dataBytes = documentCodecs.Aggregate(data.ToBytes(), (bytes, codec) => codec.Encode(key, data, metadata, bytes));
ms.Write(dataBytes, 0, dataBytes.Length);
var newEtag = generator.CreateSequentialUuid();
storage.DocumentsModifiedByTransactions.Put(new RavenJObject
{
{"key", key},
{"etag", newEtag.ToByteArray()},
{"modified", SystemTime.UtcNow},
{"txId", transactionInformation.Id.ToByteArray()}
}, ms.ToArray());
return newEtag;
}
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data)
{
var ms = new MemoryStream();
using (var stream = documentCodecs.Aggregate((Stream)ms, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(stream);
}
var byteArray = generator.CreateSequentialUuid(UuidType.MappedResults).ToByteArray();
var key = new RavenJObject
{
{"view", view},
{"reduceKey", reduceKey},
{"docId", docId},
{"etag", byteArray},
{"bucket", IndexingUtil.MapBucket(docId)},
{"timestamp", SystemTime.UtcNow}
};
storage.MappedResults.Put(key, ms.ToArray());
}
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data, byte[] viewAndReduceKeyHashed)
{
Guid etag = uuidGenerator.CreateSequentialUuid();
using (var update = new Update(session, MappedResults, JET_prep.Insert))
{
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], view, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"], docId, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"], reduceKey, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key_and_view_hashed"], viewAndReduceKeyHashed);
using (var stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"])))
{
data.WriteTo(stream);
stream.Flush();
}
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"], etag.TransformToValueForEsentSorting());
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"], SystemTime.Now);
update.Save();
}
}
public void Write(RavenJObject ravenJObject, params JsonConverter[] converters)
{
ravenJObject.WriteTo(GetCurrentJsonTextWriter(), converters);
SpinWriterIfReachedMaxSize();
}
protected Task<Etag> ExportDocumentsDeletion(SmugglerOptions options, JsonTextWriter jsonWriter, Etag startDocsEtag, Etag maxEtag)
{
var lastEtag = startDocsEtag;
database.TransactionalStorage.Batch(accessor =>
{
foreach (var listItem in accessor.Lists.Read(Constants.RavenPeriodicBackupsDocsTombstones, startDocsEtag, maxEtag, int.MaxValue))
{
var o = new RavenJObject
{
{"Key", listItem.Key}
};
o.WriteTo(jsonWriter);
lastEtag = listItem.Etag;
}
});
return new CompletedTask<Etag>(lastEtag);
}
public Guid AddDocumentInTransaction(string key, Guid?etag, RavenJObject data, RavenJObject metadata, TransactionInformation transactionInformation)
{
Api.JetSetCurrentIndex(session, Documents, "by_key");
Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ);
if (isUpdate)
{
EnsureNotLockedByTransaction(key, transactionInformation.Id);
EnsureDocumentEtagMatchInTransaction(key, etag);
using (var update = new Update(session, Documents, JET_prep.Replace))
{
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["locked_by_transaction"], transactionInformation.Id.ToByteArray());
update.Save();
}
}
else
{
EnsureDocumentIsNotCreatedInAnotherTransaction(key, transactionInformation.Id);
}
EnsureTransactionExists(transactionInformation);
Guid newEtag = uuidGenerator.CreateSequentialUuid(UuidType.DocumentTransactions);
Api.JetSetCurrentIndex(session, DocumentsModifiedByTransactions, "by_key");
Api.MakeKey(session, DocumentsModifiedByTransactions, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
var isUpdateInTransaction = Api.TrySeek(session, DocumentsModifiedByTransactions, SeekGrbit.SeekEQ);
using (var update = new Update(session, DocumentsModifiedByTransactions, isUpdateInTransaction ? JET_prep.Replace : JET_prep.Insert))
{
Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["key"], key, Encoding.Unicode);
using (var columnStream = new ColumnStream(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["data"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
using (var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current)))
{
data.WriteTo(finalStream);
finalStream.Flush();
}
}
Api.SetColumn(session, DocumentsModifiedByTransactions,
tableColumnsCache.DocumentsModifiedByTransactionsColumns["etag"],
newEtag.TransformToValueForEsentSorting());
using (var columnStream = new ColumnStream(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["metadata"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
{
metadata.WriteTo(stream);
stream.Flush();
}
}
Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["last_modified"], SystemTime.UtcNow.ToBinary());
Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["delete_document"], false);
Api.SetColumn(session, DocumentsModifiedByTransactions, tableColumnsCache.DocumentsModifiedByTransactionsColumns["locked_by_transaction"], transactionInformation.Id.ToByteArray());
update.Save();
}
logger.Debug("Inserted a new document with key '{0}', update: {1}, in transaction: {2}",
key, isUpdate, transactionInformation.Id);
return(newEtag);
}
private void ReplicateAttachment(IStorageActionsAccessor actions, string id, RavenJObject metadata, byte[] data, Guid lastEtag, string src)
{
var existingAttachment = actions.Attachments.GetAttachment(id);
if (existingAttachment == null)
{
log.Debug("New attachment {0} replicated successfully from {1}", id, src);
actions.Attachments.AddAttachment(id, Guid.Empty, new MemoryStream(data), metadata);
return;
}
// we just got the same version from the same source - request playback again?
// at any rate, not an error, moving on
if (existingAttachment.Metadata.Value <string>(ReplicationConstants.RavenReplicationSource) == metadata.Value <string>(ReplicationConstants.RavenReplicationSource) &&
existingAttachment.Metadata.Value <int>(ReplicationConstants.RavenReplicationVersion) == metadata.Value <int>(ReplicationConstants.RavenReplicationVersion))
{
return;
}
var existingDocumentIsInConflict = existingAttachment.Metadata[ReplicationConstants.RavenReplicationConflict] != null;
if (existingDocumentIsInConflict == false && // if the current document is not in conflict, we can continue without having to keep conflict semantics
(IsDirectChildOfCurrentAttachment(existingAttachment, metadata))) // this update is direct child of the existing doc, so we are fine with overwriting this
{
log.Debug("Existing document {0} replicated successfully from {1}", id, src);
actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata);
return;
}
if (ReplicationConflictResolvers.Any(replicationConflictResolver => replicationConflictResolver.TryResolve(id, metadata, data, existingAttachment)))
{
actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata);
return;
}
var newDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(metadata, lastEtag);
metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true));
actions.Attachments.AddAttachment(newDocumentConflictId, null, new MemoryStream(data), metadata);
if (existingDocumentIsInConflict) // the existing document is in conflict
{
log.Debug("Conflicted document {0} has a new version from {1}, adding to conflicted documents", id, src);
// just update the current doc with the new conflict document
var conflictArray = existingAttachment.Metadata.Value <RavenJArray>("Conflicts");
if (conflictArray == null)
{
existingAttachment.Metadata["Conflicts"] = conflictArray = new RavenJArray();
}
conflictArray.Add(RavenJToken.FromObject(newDocumentConflictId));
actions.Attachments.AddAttachment(id, existingAttachment.Etag, existingAttachment.Data(), existingAttachment.Metadata);
return;
}
log.Debug("Existing document {0} is in conflict with replicated version from {1}, marking document as conflicted", id, src);
// we have a new conflict
// move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingAttachment.Etag);
existingAttachment.Metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true));
actions.Attachments.AddAttachment(existingDocumentConflictId, null, existingAttachment.Data(), existingAttachment.Metadata);
var conflictAttachment = new RavenJObject
{
{ "Conflicts", new RavenJArray(existingDocumentConflictId, newDocumentConflictId) }
};
var memoryStream = new MemoryStream();
conflictAttachment.WriteTo(memoryStream);
memoryStream.Position = 0;
actions.Attachments.AddAttachment(id, null,
memoryStream,
new RavenJObject
{
{ ReplicationConstants.RavenReplicationConflict, true },
{ "@Http-Status-Code", 409 },
{ "@Http-Status-Description", "Conflict" }
});
}
public virtual void WriteTo(JsonWriter writer)
{
inner.WriteTo(writer);
}
public void SetConfig(string name, RavenJObject metadata)
{
var builder = new StringBuilder();
using (var writer = new JsonTextWriter(new StringWriter(builder)))
metadata.WriteTo(writer);
string metadataString = builder.ToString();
Api.JetSetCurrentIndex(session, Config, "by_name");
Api.MakeKey(session, Config, name, Encoding.Unicode, MakeKeyGrbit.NewKey);
var prep = Api.TrySeek(session, Config, SeekGrbit.SeekEQ) ? JET_prep.Replace : JET_prep.Insert;
using (var update = new Update(session, Config, prep))
{
Api.SetColumn(session, Config, tableColumnsCache.ConfigColumns["name"], name, Encoding.Unicode);
Api.SetColumn(session, Config, tableColumnsCache.ConfigColumns["metadata"], metadataString, Encoding.Unicode);
update.Save();
}
}
public void PutReducedResult(int view, string reduceKey, int level, int sourceBucket, int bucket, RavenJObject data)
{
var reduceResultsByViewAndReduceKeyAndLevelAndSourceBucket =
tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByViewAndReduceKeyAndLevelAndSourceBucket);
var reduceResultsByViewAndReduceKeyAndLevelAndBucket =
tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByViewAndReduceKeyAndLevelAndBucket);
var reduceResultsByViewAndReduceKeyAndLevel =
tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByViewAndReduceKeyAndLevel);
var reduceResultsByView =
tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.ByView);
var reduceResultsData = tableStorage.ReduceResults.GetIndex(Tables.ReduceResults.Indices.Data);
var ms = CreateStream();
using (
var stream = documentCodecs.Aggregate((Stream) new UndisposableStream(ms),
(ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(stream);
stream.Flush();
}
var id = generator.CreateSequentialUuid(UuidType.MappedResults);
var idAsSlice = (Slice)id.ToString();
var reduceResult = new Structure<ReduceResultFields>(tableStorage.ReduceResults.Schema)
.Set(ReduceResultFields.IndexId, view)
.Set(ReduceResultFields.Etag, id.ToByteArray())
.Set(ReduceResultFields.ReduceKey, reduceKey)
.Set(ReduceResultFields.Level, level)
.Set(ReduceResultFields.SourceBucket, sourceBucket)
.Set(ReduceResultFields.Bucket, bucket)
.Set(ReduceResultFields.Timestamp, SystemTime.UtcNow.ToBinary());
tableStorage.ReduceResults.AddStruct(writeBatch.Value, idAsSlice, reduceResult, 0);
ms.Position = 0;
reduceResultsData.Add(writeBatch.Value, idAsSlice, ms, 0);
var viewKey = CreateViewKey(view);
var viewAndReduceKeyAndLevel = CreateReduceResultsKey(view,reduceKey, level);
var viewAndReduceKeyAndLevelAndSourceBucket = CreateReduceResultsWithBucketKey(view, reduceKey, level, sourceBucket);
var viewAndReduceKeyAndLevelAndBucket = CreateReduceResultsWithBucketKey(view, reduceKey, level, bucket);
reduceResultsByViewAndReduceKeyAndLevelAndSourceBucket.MultiAdd(writeBatch.Value, viewAndReduceKeyAndLevelAndSourceBucket, idAsSlice);
reduceResultsByViewAndReduceKeyAndLevel.MultiAdd(writeBatch.Value, viewAndReduceKeyAndLevel, idAsSlice);
reduceResultsByViewAndReduceKeyAndLevelAndBucket.MultiAdd(writeBatch.Value, viewAndReduceKeyAndLevelAndBucket, idAsSlice);
reduceResultsByView.MultiAdd(writeBatch.Value, viewKey, idAsSlice);
}
public Task WriteDocumentAsync(RavenJObject document, CancellationToken cancellationToken)
{
document.WriteTo(Writer);
return(new CompletedTask());
}
public async Task AppendAsync(RavenJObject data)
{
if (disposed)
throw new ObjectDisposedException("EventStream");
var nextEtag = _options.Status.NextEtag();
data["@etag"] = nextEtag.ToString();
data["@date"] = DateTime.UtcNow.ToString("o");
using (var stream = new BufferPoolMemoryStream(_options.BufferPool))
{
var bsonWriter = new BsonWriter(stream);
data.WriteTo(bsonWriter);
bsonWriter.Flush();
stream.Position = 0;
var mine = new PendingWrite(stream, nextEtag);
_pending.Enqueue(mine);
while (mine.Done() == false && _pending.Peek() != mine)
{
await _writeCompletedEvent.WaitAsync();
}
if (mine.Done())
return;
await AppendInternalAsync(mine);
}
}
public void Write(RavenJObject result)
{
result.WriteTo(writer, Default.Converters);
writer.WriteRaw(Environment.NewLine);
}
public void PutMappedResult(int view, string docId, string reduceKey, RavenJObject data)
{
var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
var mappedResultsByView = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByView);
var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey);
var mappedResultsByViewAndReduceKeyAndSourceBucket = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKeyAndSourceBucket);
var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data);
var ms = CreateStream();
using (var stream = documentCodecs.Aggregate((Stream) new UndisposableStream(ms), (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(stream);
stream.Flush();
}
var id = generator.CreateSequentialUuid(UuidType.MappedResults);
var idAsString = id.ToString();
var bucket = IndexingUtil.MapBucket(docId);
var reduceKeyHash = HashKey(reduceKey);
tableStorage.MappedResults.Add(
writeBatch.Value,
idAsString,
new RavenJObject
{
{ "view", view },
{ "reduceKey", reduceKey },
{ "docId", docId },
{ "etag", id.ToByteArray() },
{ "bucket", bucket },
{ "timestamp", SystemTime.UtcNow }
}, 0);
ms.Position = 0;
mappedResultsData.Add(writeBatch.Value, idAsString, ms, 0);
mappedResultsByViewAndDocumentId.MultiAdd(writeBatch.Value, CreateKey(view, docId), idAsString);
mappedResultsByView.MultiAdd(writeBatch.Value, CreateKey(view), idAsString);
mappedResultsByViewAndReduceKey.MultiAdd(writeBatch.Value, CreateKey(view, reduceKey, reduceKeyHash), idAsString);
mappedResultsByViewAndReduceKeyAndSourceBucket.MultiAdd(writeBatch.Value, CreateKey(view, reduceKey, reduceKeyHash, bucket), idAsString);
}
private void WriteAttachments(JsonTextWriter jsonWriter)
{
long totalAttachmentsCount = 0;
storage.Batch(accsesor => totalAttachmentsCount = accsesor.Attachments.GetAttachmentsCount());
if (totalAttachmentsCount == 0)
return;
var lastEtag = Etag.Empty;
long currentAttachmentsCount = 0;
do
{
var previousAttachmentCount = currentAttachmentsCount;
try
{
storage.Batch(accsesor =>
{
var attachments = accsesor.Attachments.GetAttachmentsAfter(lastEtag, batchSize, long.MaxValue);
foreach (var attachmentInformation in attachments)
{
var attachment = accsesor.Attachments.GetAttachment(attachmentInformation.Key);
if (attachment == null)
{
ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Couldn't find attachment '{0}'", attachmentInformation.Key);
continue;
}
var data = attachment.Data;
attachment.Data = () =>
{
var memoryStream = new MemoryStream();
storage.Batch(accessor => data().CopyTo(memoryStream));
memoryStream.Position = 0;
return memoryStream;
};
var attachmentData = attachment.Data().ReadData();
if (attachmentData == null)
{
ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "No data was found for attachment '{0}'", attachment.Key);
continue;
}
var ravenJsonObj = new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", attachmentInformation.Metadata },
{ "Key", attachmentInformation.Key },
{ "Etag", new RavenJValue(attachmentInformation.Etag.ToString()) }
};
ravenJsonObj.WriteTo(jsonWriter);
lastEtag = attachmentInformation.Etag;
currentAttachmentsCount++;
if (currentAttachmentsCount % batchSize == 0)
ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount);
}
});
}
catch (Exception e)
{
lastEtag = lastEtag.IncrementBy(1);
currentAttachmentsCount++;
ReportCorrupted("attachment", currentAttachmentsCount, e.Message);
}
finally
{
if (currentAttachmentsCount > previousAttachmentCount)
ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount);
}
} while (currentAttachmentsCount < totalAttachmentsCount);
}
public void Write(RavenJObject result)
{
result.WriteTo(writer, Default.Converters);
}
private void ReplicateAttachment(IStorageActionsAccessor actions, string id, RavenJObject metadata, byte[] data, Guid lastEtag ,string src)
{
var existingAttachment = actions.Attachments.GetAttachment(id);
if (existingAttachment == null)
{
log.Debug("New attachment {0} replicated successfully from {1}", id, src);
actions.Attachments.AddAttachment(id, Guid.Empty, new MemoryStream(data), metadata);
return;
}
// we just got the same version from the same source - request playback again?
// at any rate, not an error, moving on
if(existingAttachment.Metadata.Value<string>(ReplicationConstants.RavenReplicationSource) == metadata.Value<string>(ReplicationConstants.RavenReplicationSource)
&& existingAttachment.Metadata.Value<int>(ReplicationConstants.RavenReplicationVersion) == metadata.Value<int>(ReplicationConstants.RavenReplicationVersion))
{
return;
}
var existingDocumentIsInConflict = existingAttachment.Metadata[ReplicationConstants.RavenReplicationConflict] != null;
if (existingDocumentIsInConflict == false && // if the current document is not in conflict, we can continue without having to keep conflict semantics
(IsDirectChildOfCurrentAttachment(existingAttachment, metadata))) // this update is direct child of the existing doc, so we are fine with overwriting this
{
log.Debug("Existing document {0} replicated successfully from {1}", id, src);
actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata);
return;
}
if (ReplicationConflictResolvers.Any(replicationConflictResolver => replicationConflictResolver.TryResolve(id, metadata, data, existingAttachment)))
{
actions.Attachments.AddAttachment(id, null, new MemoryStream(data), metadata);
return;
}
var newDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(metadata, lastEtag);
metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true));
actions.Attachments.AddAttachment(newDocumentConflictId, null, new MemoryStream(data), metadata);
if (existingDocumentIsInConflict) // the existing document is in conflict
{
log.Debug("Conflicted document {0} has a new version from {1}, adding to conflicted documents", id, src);
// just update the current doc with the new conflict document
var conflictArray = existingAttachment.Metadata.Value<RavenJArray>("Conflicts");
if (conflictArray == null)
existingAttachment.Metadata["Conflicts"] = conflictArray = new RavenJArray();
conflictArray.Add(RavenJToken.FromObject(newDocumentConflictId));
actions.Attachments.AddAttachment(id, existingAttachment.Etag, existingAttachment.Data(), existingAttachment.Metadata);
return;
}
log.Debug("Existing document {0} is in conflict with replicated version from {1}, marking document as conflicted", id, src);
// we have a new conflict
// move the existing doc to a conflict and create a conflict document
var existingDocumentConflictId = id + "/conflicts/" + HashReplicationIdentifier(existingAttachment.Etag);
existingAttachment.Metadata.Add(ReplicationConstants.RavenReplicationConflict, RavenJToken.FromObject(true));
actions.Attachments.AddAttachment(existingDocumentConflictId, null, existingAttachment.Data(), existingAttachment.Metadata);
var conflictAttachment = new RavenJObject
{
{"Conflicts", new RavenJArray(existingDocumentConflictId, newDocumentConflictId)}
};
var memoryStream = new MemoryStream();
conflictAttachment.WriteTo(memoryStream);
memoryStream.Position = 0;
actions.Attachments.AddAttachment(id, null,
memoryStream,
new RavenJObject
{
{ReplicationConstants.RavenReplicationConflict, true},
{"@Http-Status-Code", 409},
{"@Http-Status-Description", "Conflict"}
});
}
public AddDocumentResult AddDocument(string key, Guid?etag, RavenJObject data, RavenJObject metadata)
{
if (key != null && Encoding.Unicode.GetByteCount(key) >= 2048)
{
throw new ArgumentException(string.Format("The key must be a maximum of 2,048 bytes in Unicode, 1,024 characters, key is: '{0}'", key), "key");
}
Api.JetSetCurrentIndex(session, Documents, "by_key");
Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ);
if (isUpdate)
{
EnsureNotLockedByTransaction(key, null);
EnsureDocumentEtagMatch(key, etag, "PUT");
}
else
{
if (etag != null && etag != Guid.Empty) // expected something to be there.
{
throw new ConcurrencyException("PUT attempted on document '" + key +
"' using a non current etag (document deleted)")
{
ExpectedETag = etag.Value
}
}
;
EnsureDocumentIsNotCreatedInAnotherTransaction(key, Guid.NewGuid());
if (Api.TryMoveFirst(session, Details))
{
Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["document_count"], 1);
}
}
Guid newEtag = uuidGenerator.CreateSequentialUuid();
DateTime savedAt;
using (var update = new Update(session, Documents, isUpdate ? JET_prep.Replace : JET_prep.Insert))
{
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode);
using (Stream stream = new BufferedStream(new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"])))
using (var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current)))
{
data.WriteTo(finalStream);
finalStream.Flush();
}
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"], newEtag.TransformToValueForEsentSorting());
savedAt = SystemTime.UtcNow;
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt);
using (Stream stream = new BufferedStream(new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"])))
{
metadata.WriteTo(stream);
stream.Flush();
}
update.Save();
}
logger.Debug("Inserted a new document with key '{0}', update: {1}, ",
key, isUpdate);
cacher.RemoveCachedDocument(key, newEtag);
return(new AddDocumentResult
{
Etag = newEtag,
SavedAt = savedAt,
Updated = isUpdate
});
}
private void WriteAttachmentMetadata(Slice key, Etag etag, RavenJObject headers)
{
var memoryStream = CreateStream();
memoryStream.Write(etag);
headers.WriteTo(memoryStream);
memoryStream.Position = 0;
metadataIndex.Add(writeBatch.Value, key, memoryStream);
}
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout)
{
timeout.Delay();
item.WriteTo(writer);
}
private static void WriteToStream(JsonWriter writer, RavenJObject item, CancellationTimeout timeout)
{
timeout.Delay();
item.WriteTo(writer);
}
public void Write(RavenJObject result)
{
result.WriteTo(writer, Default.Converters);
writer.WriteRaw(Environment.NewLine);
}
public void PutMappedResult(int view, string docId, string reduceKey, RavenJObject data)
{
var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
var mappedResultsByView = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByView);
var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey);
var mappedResultsByViewAndReduceKeyAndSourceBucket = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKeyAndSourceBucket);
var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data);
var ms = CreateStream();
using (var stream = documentCodecs.Aggregate((Stream) new UndisposableStream(ms), (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(stream);
stream.Flush();
}
var id = generator.CreateSequentialUuid(UuidType.MappedResults);
var idSlice = (Slice)id.ToString();
var bucket = IndexingUtil.MapBucket(docId);
var reduceKeyHash = HashKey(reduceKey);
var mappedResult = new Structure<MappedResultFields>(tableStorage.MappedResults.Schema)
.Set(MappedResultFields.IndexId, view)
.Set(MappedResultFields.Bucket, bucket)
.Set(MappedResultFields.Timestamp, SystemTime.UtcNow.ToBinary())
.Set(MappedResultFields.ReduceKey, reduceKey)
.Set(MappedResultFields.DocId, docId)
.Set(MappedResultFields.Etag, id.ToByteArray());
tableStorage.MappedResults.AddStruct(
writeBatch.Value,
idSlice,
mappedResult, 0);
ms.Position = 0;
mappedResultsData.Add(writeBatch.Value, idSlice, ms, 0);
string viewKey = CreateKey(view);
string viewReduceKey = AppendToKey(viewKey, ReduceKeySizeLimited(reduceKey));
string viewReduceHashKey = AppendToKey(viewReduceKey, reduceKeyHash);
mappedResultsByViewAndDocumentId.MultiAdd(writeBatch.Value, (Slice)AppendToKey(viewKey, docId), idSlice);
mappedResultsByView.MultiAdd(writeBatch.Value, (Slice)viewKey, idSlice);
mappedResultsByViewAndReduceKey.MultiAdd(writeBatch.Value, (Slice)viewReduceHashKey, idSlice);
mappedResultsByViewAndReduceKeyAndSourceBucket.MultiAdd(writeBatch.Value, (Slice)AppendToKey(viewReduceHashKey, bucket), idSlice);
}
public AddDocumentResult AddDocument(string key, Etag etag, RavenJObject data, RavenJObject metadata)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
var byteCount = Encoding.Unicode.GetByteCount(key);
if (byteCount >= 2048)
{
throw new ArgumentException(string.Format("The key must be a maximum of 2,048 bytes in Unicode, 1,024 characters, key is: '{0}'", key), "key");
}
try
{
Api.JetSetCurrentIndex(session, Documents, "by_key");
Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
var isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ);
Etag existingEtag = null;
if (isUpdate)
{
existingEtag = EnsureDocumentEtagMatch(key, etag, "PUT");
}
else
{
if (etag != null && etag != Etag.Empty) // expected something to be there.
{
throw new ConcurrencyException("PUT attempted on document '" + key +
"' using a non current etag (document deleted)")
{
ExpectedETag = etag
}
}
;
if (Api.TryMoveFirst(session, Details))
{
Api.EscrowUpdate(session, Details, tableColumnsCache.DetailsColumns["document_count"], 1);
}
}
Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
DateTime savedAt;
try
{
using (var update = new Update(session, Documents, isUpdate ? JET_prep.Replace : JET_prep.Insert))
{
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode);
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"]))
{
if (isUpdate)
{
columnStream.SetLength(0); // empty the existing value, since we are going to overwrite the entire thing
}
using (Stream stream = new BufferedStream(columnStream))
using (
var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current))
)
{
data.WriteTo(finalStream);
finalStream.Flush();
}
}
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"],
newEtag.TransformToValueForEsentSorting());
savedAt = SystemTime.UtcNow;
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary());
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
{
metadata.WriteTo(stream);
stream.Flush();
}
}
update.Save();
}
}
catch (EsentErrorException e)
{
if (e.Error == JET_err.KeyDuplicate || e.Error == JET_err.WriteConflict)
{
throw new ConcurrencyException("PUT attempted on document '" + key + "' concurrently", e);
}
throw;
}
logger.Debug("Inserted a new document with key '{0}', update: {1}, ",
key, isUpdate);
cacher.RemoveCachedDocument(key, newEtag);
return(new AddDocumentResult
{
Etag = newEtag,
PrevEtag = existingEtag,
SavedAt = savedAt,
Updated = isUpdate
});
}
catch (EsentKeyDuplicateException e)
{
throw new ConcurrencyException("Illegal duplicate key " + key, e);
}
}
public void Write(RavenJObject result)
{
result.WriteTo(writer, Default.Converters);
}
public AddDocumentResult InsertDocument(string key, RavenJObject data, RavenJObject metadata, bool overwriteExisting)
{
var prep = JET_prep.Insert;
bool isUpdate = false;
Etag existingETag = null;
if (overwriteExisting)
{
Api.JetSetCurrentIndex(session, Documents, "by_key");
Api.MakeKey(session, Documents, key, Encoding.Unicode, MakeKeyGrbit.NewKey);
isUpdate = Api.TrySeek(session, Documents, SeekGrbit.SeekEQ);
if (isUpdate)
{
existingETag = Etag.Parse(Api.RetrieveColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"]));
prep = JET_prep.Replace;
}
}
try
{
using (var update = new Update(session, Documents, prep))
{
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["key"], key, Encoding.Unicode);
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["data"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
using (var finalStream = documentCodecs.Aggregate(stream, (current, codec) => codec.Encode(key, data, metadata, current)))
{
data.WriteTo(finalStream);
finalStream.Flush();
}
}
Etag newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["etag"], newEtag.TransformToValueForEsentSorting());
DateTime savedAt = SystemTime.UtcNow;
Api.SetColumn(session, Documents, tableColumnsCache.DocumentsColumns["last_modified"], savedAt.ToBinary());
using (var columnStream = new ColumnStream(session, Documents, tableColumnsCache.DocumentsColumns["metadata"]))
{
if (isUpdate)
{
columnStream.SetLength(0);
}
using (Stream stream = new BufferedStream(columnStream))
{
metadata.WriteTo(stream);
stream.Flush();
}
}
update.Save();
return(new AddDocumentResult
{
Etag = newEtag,
PrevEtag = existingETag,
SavedAt = savedAt,
Updated = isUpdate
});
}
}
catch (EsentKeyDuplicateException e)
{
throw new ConcurrencyException("Illegal duplicate key " + key, e);
}
}
private bool WriteDocumentData(string key, string normalizedKey, Etag etag, RavenJObject data, RavenJObject metadata, out Etag newEtag, out Etag existingEtag, out DateTime savedAt)
{
var normalizedKeySlice = (Slice)normalizedKey;
var keyByEtagDocumentIndex = tableStorage.Documents.GetIndex(Tables.Documents.Indices.KeyByEtag);
ushort? existingVersion;
var isUpdate = tableStorage.Documents.Contains(Snapshot, normalizedKeySlice, writeBatch.Value, out existingVersion);
existingEtag = null;
if (isUpdate)
{
existingEtag = EnsureDocumentEtagMatch(normalizedKey, etag, "PUT");
keyByEtagDocumentIndex.Delete(writeBatch.Value, existingEtag);
}
else if (etag != null && etag != Etag.Empty)
{
throw new ConcurrencyException("PUT attempted on document '" + key + "' using a non current etag (document deleted)")
{
ExpectedETag = etag
};
}
var dataStream = CreateStream();
using (var finalDataStream = documentCodecs.Aggregate((Stream)new UndisposableStream(dataStream),
(current, codec) => codec.Encode(normalizedKey, data, metadata, current)))
{
data.WriteTo(finalDataStream);
finalDataStream.Flush();
}
dataStream.Position = 0;
tableStorage.Documents.Add(writeBatch.Value, normalizedKeySlice, dataStream, existingVersion ?? 0);
newEtag = uuidGenerator.CreateSequentialUuid(UuidType.Documents);
savedAt = SystemTime.UtcNow;
var isUpdated = PutDocumentMetadataInternal(key, normalizedKeySlice, metadata, newEtag, savedAt);
keyByEtagDocumentIndex.Add(writeBatch.Value, newEtag, normalizedKey);
return isUpdated;
}
private void WriteAttachments(JsonTextWriter jsonWriter)
{
long totalAttachmentsCount = 0;
storage.Batch(accsesor => totalAttachmentsCount = accsesor.Attachments.GetAttachmentsCount());
if (totalAttachmentsCount == 0)
{
return;
}
var lastEtag = Etag.Empty;
long currentAttachmentsCount = 0;
do
{
var previousAttachmentCount = currentAttachmentsCount;
try
{
storage.Batch(accsesor =>
{
var attachments = accsesor.Attachments.GetAttachmentsAfter(lastEtag, batchSize, long.MaxValue);
foreach (var attachmentInformation in attachments)
{
var attachment = accsesor.Attachments.GetAttachment(attachmentInformation.Key);
if (attachment == null)
{
ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "Couldn't find attachment '{0}'", attachmentInformation.Key);
continue;
}
var data = attachment.Data;
attachment.Data = () =>
{
var memoryStream = new MemoryStream();
storage.Batch(accessor => data().CopyTo(memoryStream));
memoryStream.Position = 0;
return(memoryStream);
};
var attachmentData = attachment.Data().ReadData();
if (attachmentData == null)
{
ConsoleUtils.ConsoleWriteLineWithColor(ConsoleColor.Red, "No data was found for attachment '{0}'", attachment.Key);
continue;
}
var ravenJsonObj = new RavenJObject
{
{ "Data", attachmentData },
{ "Metadata", attachmentInformation.Metadata },
{ "Key", attachmentInformation.Key },
{ "Etag", new RavenJValue(attachmentInformation.Etag.ToString()) }
};
ravenJsonObj.WriteTo(jsonWriter);
lastEtag = attachmentInformation.Etag;
currentAttachmentsCount++;
if (currentAttachmentsCount % batchSize == 0)
{
ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount);
}
}
});
}
catch (Exception e)
{
lastEtag = lastEtag.IncrementBy(1);
currentAttachmentsCount++;
ReportCorrupted("attachment", currentAttachmentsCount, e.Message);
}
finally
{
if (currentAttachmentsCount > previousAttachmentCount)
{
ReportProgress("attachments", currentAttachmentsCount, totalAttachmentsCount);
}
}
} while (currentAttachmentsCount < totalAttachmentsCount);
}
public void PutReducedResult(string name, string reduceKey, int level, int sourceBucket, int bucket, RavenJObject data)
{
var ms = new MemoryStream();
using (var stream = documentCodecs.Aggregate((Stream)ms, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(stream);
}
var etag = generator.CreateSequentialUuid().ToByteArray();
storage.ReduceResults.Put(new RavenJObject
{
{"view", name},
{"etag", etag},
{"reduceKey", reduceKey},
{"level", level},
{"sourceBucket", sourceBucket},
{"bucket", bucket},
{"timestamp", SystemTime.UtcNow}
}, ms.ToArray());
}
public static void WaitForIndexing(IDocumentStore store, string database = null, TimeSpan?timeout = null)
{
var databaseCommands = store.DatabaseCommands;
if (database != null)
{
databaseCommands = databaseCommands.ForDatabase(database);
}
timeout = timeout ?? (Debugger.IsAttached
? TimeSpan.FromMinutes(15)
: TimeSpan.FromMinutes(1));
var sp = Stopwatch.StartNew();
while (sp.Elapsed < timeout.Value)
{
var databaseStatistics = databaseCommands.GetStatistics();
if (databaseStatistics.Indexes.All(x => x.IsStale == false))
{
return;
}
if (databaseStatistics.Indexes.Any(x => x.State == IndexState.Error))
{
break;
}
Thread.Sleep(32);
}
var request = databaseCommands.CreateRequest("/indexes/performance", HttpMethod.Get);
var perf = request.ReadResponseJson();
request = databaseCommands.CreateRequest("/indexes/errors", HttpMethod.Get);
var errors = request.ReadResponseJson();
var total = new RavenJObject
{
["Errors"] = errors,
["Performance"] = perf
};
var file = Path.GetTempFileName() + ".json";
using (var writer = File.CreateText(file))
{
var jsonTextWriter = new JsonTextWriter(writer);
total.WriteTo(jsonTextWriter);
jsonTextWriter.Flush();
}
var stats = databaseCommands.GetStatistics();
var corrupted = stats.Indexes.Where(x => x.State == IndexState.Error).ToList();
if (corrupted.Count > 0)
{
throw new InvalidOperationException(
$"The following indexes are with error state: {string.Join(",", corrupted.Select(x => x.Name))} - details at " + file);
}
throw new TimeoutException("The indexes stayed stale for more than " + timeout.Value + ", stats at " + file);
}
