internal static BlittableJsonReaderObject EnsureLowerCasedIndexIdProperty(DocumentsOperationContext context, BlittableJsonReaderObject json,
ElasticSearchIndexWithRecords index)
{
if (json.TryGet(index.DocumentIdProperty, out LazyStringValue idProperty))
{
using (var old = json)
{
json.Modifications = new DynamicJsonValue(json)
{
[index.DocumentIdProperty] = LowerCaseDocumentIdProperty(idProperty)
};
json = context.ReadObject(json, "es-etl-load");
}
}
else if (json.Modifications != null)
{
// document id property was not added by user, so we inserted the lowercased id in ElasticSearchDocumentTransformer.LoadToFunction
#if DEBUG
var docIdProperty = json.Modifications.Properties.First(x => x.Name == index.DocumentIdProperty);
Debug.Assert(docIdProperty.Value.ToString() == docIdProperty.Value.ToString().ToLowerInvariant());
#endif
json = context.ReadObject(json, "es-etl-load");
}
return(json);
}
protected override long ExecuteCmd(DocumentsOperationContext context)
{
var hiLoDocumentId = RavenHiloIdPrefix + Key;
var prefix = Key + Separator;
var newDoc = new DynamicJsonValue();
BlittableJsonReaderObject hiloDocReader = null;
try
{
try
{
hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentId)?.Data;
}
catch (DocumentConflictException e)
{
throw new InvalidDataException("Failed to fetch HiLo document due to a conflict on the document. " +
"This shouldn't happen, since it this conflict should've been resolved during replication. " +
"This exception should not happen and is likely a bug.", e);
}
if (hiloDocReader == null)
{
OldMax = LastRangeMax;
newDoc["Max"] = OldMax + Capacity;
newDoc[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = CollectionName.HiLoCollection
};
using (var freshHilo = context.ReadObject(newDoc, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo);
}
else
{
hiloDocReader.TryGet("Max", out long oldMax);
OldMax = Math.Max(oldMax, LastRangeMax);
hiloDocReader.Modifications = new DynamicJsonValue(hiloDocReader)
{
["Max"] = OldMax + Capacity
};
using (var freshHilo = context.ReadObject(hiloDocReader, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo);
}
Prefix = prefix;
}
finally
{
hiloDocReader?.Dispose();
}
return(1);
}
protected override long ExecuteCmd(DocumentsOperationContext context)
{
var hiLoDocumentId = RavenHiloIdPrefix + Key;
var document = Database.DocumentsStorage.Get(context, hiLoDocumentId);
if (document == null)
{
return(1);
}
document.Data.TryGet("Max", out long oldMax);
if (oldMax != End || Last > oldMax)
{
return(1);
}
document.Data.Modifications = new DynamicJsonValue
{
["Max"] = Last
};
using (var hiloReader = context.ReadObject(document.Data, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
Database.DocumentsStorage.Put(context, hiLoDocumentId, null, hiloReader);
}
return(1);
}
public static LegacyAttachmentDetails GenerateLegacyAttachmentDetails(
DocumentsOperationContext context,
Stream decodedStream,
string key,
BlittableJsonReaderObject metadata,
ref DocumentItem.AttachmentStream attachment)
{
var stream = attachment.Stream;
var hash = AsyncHelpers.RunSync(() => AttachmentsStorageHelper.CopyStreamToFileAndCalculateHash(context, decodedStream, stream, CancellationToken.None));
attachment.Stream.Flush();
var lazyHash = context.GetLazyString(hash);
attachment.Base64HashDispose = Slice.External(context.Allocator, lazyHash, out attachment.Base64Hash);
var tag = $"{DummyDocumentPrefix}{key}{RecordSeparator}d{RecordSeparator}{key}{RecordSeparator}{hash}{RecordSeparator}";
var lazyTag = context.GetLazyString(tag);
attachment.TagDispose = Slice.External(context.Allocator, lazyTag, out attachment.Tag);
var id = GetLegacyAttachmentId(key);
var lazyId = context.GetLazyString(id);
attachment.Data = context.ReadObject(metadata, id);
return(new LegacyAttachmentDetails
{
Id = lazyId,
Hash = hash,
Key = key,
Size = attachment.Stream.Length,
Tag = tag,
Metadata = attachment.Data
});
}
public static BlittableJsonReaderObject WriteDummyDocumentForAttachment(DocumentsOperationContext context, LegacyAttachmentDetails details)
{
var attachment = new DynamicJsonValue
{
["Name"] = details.Key,
["Hash"] = details.Hash,
["ContentType"] = string.Empty,
["Size"] = details.Size,
};
var attachments = new DynamicJsonArray();
attachments.Add(attachment);
var metadata = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = "@files",
[Constants.Documents.Metadata.Attachments] = attachments,
[Constants.Documents.Metadata.LegacyAttachmentsMetadata] = details.Metadata
};
var djv = new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = metadata,
};
return(context.ReadObject(djv, details.Id));
}
private static void HandleResultsWithoutSelect <TResult>(
DocumentsOperationContext documentsContext,
List <Match> matchResults, TResult final) where TResult : QueryResultServerSide <Document>
{
foreach (var match in matchResults)
{
if (match.Empty)
{
continue;
}
if (match.Count == 1) //if we don't have multiple results in each row, we can "flatten" the row
{
final.AddResult(match.GetFirstResult());
continue;
}
var resultAsJson = new DynamicJsonValue();
match.PopulateVertices(resultAsJson);
var result = new Document
{
Data = documentsContext.ReadObject(resultAsJson, "graph/result"),
};
final.AddResult(result);
}
}
private void WriteAllEnvs(BlittableJsonTextWriter writer, DocumentsOperationContext context)
{
var envs = Database.GetAllStoragesEnvironment();
bool first = true;
foreach (var env in envs)
{
if (env == null)
{
continue;
}
if (!first)
{
writer.WriteComma();
}
first = false;
writer.WriteStartObject();
writer.WritePropertyName("Environment");
writer.WriteString(env.Name);
writer.WriteComma();
writer.WritePropertyName("Type");
writer.WriteString(env.Type.ToString());
writer.WriteComma();
var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(GetDetailedReport(env, false));
writer.WritePropertyName("Report");
writer.WriteObject(context.ReadObject(djv, env.Name));
writer.WriteEndObject();
}
}
public override void Execute(DocumentsOperationContext context, RavenTransaction tx)
{
var hiLoDocumentKey = RavenKeyGeneratorsHilo + Key;
var document = Database.DocumentsStorage.Get(context, hiLoDocumentKey);
if (document == null)
{
return;
}
long oldMax;
document.Data.TryGet("Max", out oldMax);
if (oldMax != End || Last > oldMax)
{
return;
}
document.Data.Modifications = new DynamicJsonValue()
{
["Max"] = Last,
};
using (var hiloReader = context.ReadObject(document.Data, hiLoDocumentKey, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
Database.DocumentsStorage.Put(context, hiLoDocumentKey, null, hiloReader);
}
}
public void AddIncludesToResult(TimeSeriesRangeResult rangeResult)
{
if (rangeResult == null || _includes?.Properties.Count > 0 == false)
{
return;
}
rangeResult.Includes = _context.ReadObject(_includes, "TimeSeriesRangeIncludes/" + _docId);
}
public override int Execute(DocumentsOperationContext context)
{
var hiLoDocumentId = RavenIdGeneratorsHilo + Key;
var prefix = Key + Separator;
long oldMax = 0;
var newDoc = new DynamicJsonValue();
BlittableJsonReaderObject hiloDocReader = null;
try
{
try
{
hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentId)?.Data;
}
catch (DocumentConflictException e)
{
throw new InvalidDataException("Failed to fetch HiLo document due to a conflict on the document. " +
"This shouldn't happen, since it this conflict should've been resolved during replication. " +
"This exception should not happen and is likely a bug.", e);
}
if (hiloDocReader != null)
{
var prop = new BlittableJsonReaderObject.PropertyDetails();
foreach (var propertyId in hiloDocReader.GetPropertiesByInsertionOrder())
{
hiloDocReader.GetPropertyByIndex(propertyId, ref prop);
if (prop.Name == "Max")
{
oldMax = (long)prop.Value;
continue;
}
newDoc[prop.Name] = prop.Value;
}
}
oldMax = Math.Max(oldMax, LastRangeMax);
newDoc["Max"] = oldMax + Capacity;
using (var freshHilo = context.ReadObject(newDoc, hiLoDocumentId, BlittableJsonDocumentBuilder.UsageMode.ToDisk))
Database.DocumentsStorage.Put(context, hiLoDocumentId, null, freshHilo);
OldMax = oldMax;
Prefix = prefix;
}
finally
{
hiloDocReader?.Dispose();
}
return(1);
}
private void WriteLastEtag(DocumentsOperationContext context)
{
var key = Constants.SqlReplication.RavenSqlReplicationStatusPrefix + ReplicationUniqueName;
var document = context.ReadObject(new DynamicJsonValue
{
["Name"] = ReplicationUniqueName,
["LastReplicatedEtag"] = Statistics.LastReplicatedEtag,
["LastTombstonesEtag"] = Statistics.LastTombstonesEtag,
}, key, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
_database.DocumentsStorage.Put(context, key, null, document);
}
public void Delete(DocumentsOperationContext context, string id, Slice lowerId, CollectionName collectionName, string changeVector,
long lastModifiedTicks, NonPersistentDocumentFlags nonPersistentFlags, DocumentFlags flags)
{
using (DocumentIdWorker.GetStringPreserveCase(context, id, out Slice idPtr))
{
var deleteRevisionDocument = context.ReadObject(new DynamicJsonValue
{
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = collectionName.Name
}
}, "RevisionsBin");
Delete(context, lowerId, idPtr, id, collectionName, deleteRevisionDocument, changeVector, lastModifiedTicks, nonPersistentFlags, flags);
}
}
private BlittableJsonReaderObject GetCleanMetadata(BlittableJsonReaderObject metadata, DocumentsOperationContext context)
{
metadata.Modifications = new DynamicJsonValue(metadata);
metadata.Modifications.Remove("Origin");
metadata.Modifications.Remove("Raven-Synchronization-Version");
metadata.Modifications.Remove("Raven-Synchronization-Source");
metadata.Modifications.Remove("Creation-Date");
metadata.Modifications.Remove("Raven-Creation-Date");
metadata.Modifications.Remove("Raven-Synchronization-History");
metadata.Modifications.Remove("RavenFS-Size");
metadata.Modifications.Remove("Last-Modified");
metadata.Modifications.Remove("Raven-Last-Modified");
metadata.Modifications.Remove("Content-MD5");
metadata.Modifications.Remove("ETag");
return(context.ReadObject(metadata, MigrationStateKey));
}
public virtual string PutDocument(string key, JsValue document, JsValue metadata, JsValue etagJs, Engine engine)
{
if (document.IsObject() == false)
{
throw new InvalidOperationException(
$"Created document must be a valid object which is not null or empty. Document key: '{key}'.");
}
long?etag = null;
if (etagJs.IsNumber())
{
etag = (long)etagJs.AsNumber();
}
else if (etagJs.IsNull() == false && etagJs.IsUndefined() == false && etagJs.ToString() != "None")
{
throw new InvalidOperationException($"Invalid ETag value for document '{key}'");
}
var data = ToBlittable(document.AsObject());
if (metadata.IsObject())
{
data["@metadata"] = ToBlittable(metadata.AsObject());
}
if (DebugMode)
{
DebugActions.PutDocument.Add(new DynamicJsonValue
{
["Key"] = key,
["Etag"] = etag,
["Data"] = data,
});
}
var dataReader = _context.ReadObject(data, key, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
var put = _database.DocumentsStorage.Put(_context, key, etag, dataReader);
return(put.Key);
}
private void WriteIntermidiateResults(List <GraphQueryRunner.Match> matches)
{
_writer.WritePropertyName("Results");
_writer.WriteStartArray();
var first = true;
foreach (var match in matches)
{
if (first == false)
{
_writer.WriteComma();
}
first = false;
var djv = new DynamicJsonValue();
match.PopulateVertices(djv);
_writer.WriteObject(_ctx.ReadObject(djv, null));
}
_writer.WriteEndArray();
}
public virtual PatchResultData Apply(DocumentsOperationContext context, Document document, PatchRequest patch)
{
if (document == null)
{
return(null);
}
if (string.IsNullOrEmpty(patch.Script))
{
throw new InvalidOperationException("Patch script must be non-null and not empty");
}
var scope = ApplySingleScript(context, document, false, patch);
var modifiedDocument = context.ReadObject(scope.ToBlittable(scope.PatchObject.AsObject()), document.Key); /* TODO: Should not use BlittableJsonDocumentBuilder.UsageMode.ToDisk? */
return(new PatchResultData
{
ModifiedDocument = modifiedDocument ?? document.Data,
DebugInfo = scope.DebugInfo,
});
}
private JsValue GetMetadata(JsValue self, JsValue[] args)
{
if (args.Length != 1 || !(args[0].AsObject() is BlittableObjectInstance boi))
{
throw new InvalidOperationException("getMetadata(doc) must be called with a single entity argument");
}
if (!(boi.Blittable[Constants.Documents.Metadata.Key] is BlittableJsonReaderObject metadata))
{
return(JsValue.Null);
}
metadata.Modifications = new DynamicJsonValue
{
[Constants.Documents.Metadata.ChangeVector] = boi.ChangeVector,
[Constants.Documents.Metadata.Id] = boi.DocumentId,
[Constants.Documents.Metadata.LastModified] = boi.LastModified,
};
metadata = _context.ReadObject(metadata, boi.DocumentId);
return(TranslateToJs(ScriptEngine, _context, metadata));
}
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document,
DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks,
RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null)
{
Debug.Assert(changeVector != null, "Change vector must be set");
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
if (collectionName == null)
{
collectionName = _database.DocumentsStorage.ExtractCollectionName(context, id, document);
}
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var fromSmuggler = (nonPersistentFlags & NonPersistentDocumentFlags.FromSmuggler) == NonPersistentDocumentFlags.FromSmuggler;
var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication;
var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName);
// We want the revision's attachments to have a lower etag than the revision itself
if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments &&
fromSmuggler == false)
{
using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice))
{
if (table.VerifyKeyExists(changeVectorSlice) == false)
{
_documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice);
}
}
}
if (fromReplication)
{
void PutFromRevisionIfChangeVectorIsGreater()
{
bool hasDoc;
TableValueReader tvr;
try
{
hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr);
}
catch (DocumentConflictException)
{
// Do not modify the document.
return;
}
if (hasDoc == false)
{
PutFromRevision();
return;
}
var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr);
if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update)
{
PutFromRevision();
}
void PutFromRevision()
{
_documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector,
flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision);
}
}
PutFromRevisionIfChangeVectorIsGreater();
}
flags |= DocumentFlags.Revision;
var data = context.ReadObject(document, id);
var newEtag = _database.DocumentsStorage.GenerateNextEtag();
var newEtagSwapBytes = Bits.SwapBytes(newEtag);
using (table.Allocate(out TableValueBuilder tvb))
using (Slice.From(context.Allocator, changeVector, out var cv))
{
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(newEtagSwapBytes);
tvb.Add(idPtr);
tvb.Add(data.BasePointer, data.Size);
tvb.Add((int)flags);
tvb.Add(newEtagSwapBytes);
tvb.Add(lastModifiedTicks);
tvb.Add(context.GetTransactionMarker());
var isNew = table.Set(tvb);
if (isNew == false)
{
// It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision.
return;
}
}
if (configuration == null)
{
configuration = GetRevisionsConfiguration(collectionName.Name);
}
DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector);
}
}
private void DeleteCounter(UpdateStep step, LazyStringValue tombstoneKey, DocumentsOperationContext context)
{
var(docId, counterName) = ExtractDocIdAndNameFromCounterTombstone(context, tombstoneKey);
using (docId)
using (counterName)
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, docId, out Slice lowerId, out _))
{
string collection = null;
var docsTable = new Table(DocsSchema, step.ReadTx);
if (docsTable.ReadByKey(lowerId, out var tvr))
{
using (var doc = new BlittableJsonReaderObject(tvr.Read((int)DocumentsTable.Data, out int size), size, context))
{
collection = CollectionName.GetCollectionName(doc);
}
}
var collectionName = new CollectionName(collection);
var table = step.DocumentsStorage.CountersStorage.GetCountersTable(step.WriteTx, collectionName);
if (table.ReadByKey(lowerId, out var existing) == false)
{
return;
}
// (int)CountersTable.Data = 3
var data = new BlittableJsonReaderObject(existing.Read(3, out int oldSize), oldSize, context);
if (data.TryGet(CountersStorage.Values, out BlittableJsonReaderObject counters) == false ||
counters.TryGetMember(counterName, out object counterToDelete) == false ||
counterToDelete is LazyStringValue) // already deleted
{
return;
}
var deleteCv = step.DocumentsStorage.CountersStorage.GenerateDeleteChangeVectorFromRawBlob(data, counterToDelete as BlittableJsonReaderObject.RawBlob);
counters.Modifications = new DynamicJsonValue(counters)
{
[counterName] = deleteCv
};
using (var old = data)
{
data = context.ReadObject(data, null, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
var newEtag = step.DocumentsStorage.GenerateNextEtag();
var newChangeVector = ChangeVectorUtils.NewChangeVector(step.DocumentsStorage.DocumentDatabase.ServerStore.NodeTag, newEtag, _dbId);
using (data)
using (Slice.From(context.Allocator, newChangeVector, out var cv))
using (DocumentIdWorker.GetStringPreserveCase(context, collectionName.Name, out Slice collectionSlice))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(cv);
tvb.Add(data.BasePointer, data.Size);
tvb.Add(collectionSlice);
tvb.Add(context.GetTransactionMarker());
table.Set(tvb);
}
}
}
public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
string expectedChangeVector,
BlittableJsonReaderObject document,
long?lastModifiedTicks = null,
string changeVector = null,
DocumentFlags flags = DocumentFlags.None,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (context.Transaction == null)
{
ThrowRequiresTransaction();
return(default(PutOperationResults)); // never hit
}
#if DEBUG
var documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
#endif
var newEtag = _documentsStorage.GenerateNextEtag();
var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks;
id = BuildDocumentId(id, newEtag, out bool knownNewId);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, document);
var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
var oldValue = default(TableValueReader);
if (knownNewId == false)
{
// delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);
table.ReadByKey(lowerId, out oldValue);
}
BlittableJsonReaderObject oldDoc = null;
if (oldValue.Pointer == null)
{
// expectedChangeVector being null means we don't care, and empty
// means that it must be new
if (string.IsNullOrEmpty(expectedChangeVector) == false)
{
ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
}
}
else
{
// expectedChangeVector has special meaning here
// null - means, don't care, don't check
// "" / empty - means, must be new
// anything else - must match exactly
if (expectedChangeVector != null)
{
var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
{
ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
}
}
oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
if (oldCollectionName != collectionName)
{
ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
}
var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);
if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
flags |= DocumentFlags.HasAttachments;
}
}
}
var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
changeVector = result.ChangeVector;
nonPersistentFlags |= result.NonPersistentFlags;
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
{
flags |= DocumentFlags.Resolved;
}
if (collectionName.IsHiLo == false &&
(flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
{
if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
{
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
#endif
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
}
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
(flags.Contain(DocumentFlags.Resolved) ||
_documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
))
{
var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
ref flags, out RevisionsCollectionConfiguration configuration);
if (shouldVersion)
{
_documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
changeVector, modifiedTicks, configuration, collectionName);
}
}
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(modifiedTicks);
tvb.Add((int)flags);
tvb.Add(context.GetTransactionMarker());
if (oldValue.Pointer == null)
{
table.Insert(tvb);
}
else
{
table.Update(oldValue.Id, tvb);
}
}
if (collectionName.IsHiLo == false)
{
_documentsStorage.ExpirationStorage.Put(context, lowerId, document);
}
context.LastDatabaseChangeVector = changeVector;
_documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
_documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = changeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Put,
});
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
return(new PutOperationResults
{
Etag = newEtag,
Id = id,
Collection = collectionName,
ChangeVector = changeVector,
Flags = flags,
LastModified = new DateTime(modifiedTicks)
});
}
}
internal Document ProjectFromMatch(Match match, JsonOperationContext context, CancellationToken token)
{
var result = new DynamicJsonValue();
result[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Projection] = true
};
var item = new Document();
foreach (var fieldToFetch in FieldsToFetch.Fields.Values)
{
object fieldVal;
string key;
if (fieldToFetch.QueryField?.ExpressionField?.Compound?.Contains("[]") ?? false)
{
key = fieldToFetch.QueryField.ExpressionField.Compound.LastOrDefault().Value;
}
else
{
key = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value;
}
if (fieldToFetch.QueryField.Function != null)
{
var args = new object[fieldToFetch.FunctionArgs.Length + 1];
for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++)
{
var val = match.GetResult(fieldToFetch.FunctionArgs[i].ProjectedName);
switch (val)
{
case Document doc:
doc.EnsureMetadata();
args[i] = doc;
break;
case BlittableJsonReaderObject bjro:
args[i] = bjro;
break;
case List <Match> matchList:
CreateArgsFromMatchList(matchList, args, i);
break;
case MatchCollection matchCollection:
CreateArgsFromMatchList(matchCollection, args, i);
break;
case string s:
args[i] = s;
break;
default:
args[i] = null;
break;
}
}
key = fieldToFetch.ProjectedName ?? (fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value);
fieldVal = GetFunctionValue(fieldToFetch, null, args, token);
var immediateResult = AddProjectionToResult(item, OneScore, FieldsToFetch, result, key, fieldVal);
if (immediateResult.Document != null)
{
return(immediateResult.Document);
}
}
else
{
var val = match.GetResult(fieldToFetch.QueryField.ExpressionField.Compound[0].Value);
switch (val)
{
case Document d:
{
if (TryGetValue(fieldToFetch, d, null, null, null, null, out key, out fieldVal, token) == false)
{
continue;
}
d.EnsureMetadata();
var immediateResult = AddProjectionToResult(d, OneScore, FieldsToFetch, result, key, fieldVal);
if (immediateResult.Document != null)
{
return(immediateResult.Document);
}
break;
}
case BlittableJsonReaderObject bjro:
{
var doc = new Document {
Data = bjro
};
if (TryGetValue(fieldToFetch, doc, null, null, null, null, out key, out fieldVal, token) == false)
{
continue;
}
doc.EnsureMetadata();
var immediateResult = AddProjectionToResult(doc, OneScore, FieldsToFetch, result, key, fieldVal);
if (immediateResult.Document != null)
{
return(immediateResult.Document);
}
break;
}
case MatchCollection matches:
var array = new DynamicJsonArray();
foreach (var m in matches)
{
var djv = new DynamicJsonValue();
m.PopulateVertices(djv);
if (djv.Properties.Count == 0)
{
continue;
}
var matchJson = _context.ReadObject(djv, "graph/arg");
var doc = new Document {
Data = matchJson
};
if (TryGetValue(fieldToFetch, doc, null, null, null, null, out key, out fieldVal, token) == false)
{
continue;
}
doc.EnsureMetadata();
if (ReferenceEquals(doc, fieldVal))
{
fieldVal = doc.Data;
}
array.Add(fieldVal);
}
result[key] = array;
break;
case string s:
result[fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value] = s;
break;
default:
result[fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value] = null;
continue;
}
}
}
return(new Document
{
Data = context.ReadObject(result, "projection result")
});
}
private DocumentItem ConvertRecordToDocumentItem(DocumentsOperationContext context, string[] csvReaderCurrentRecord, string[] csvReaderFieldHeaders, string collection)
{
try
{
var idStr = _hasId ? csvReaderCurrentRecord[_idIndex] : _hasCollection ? $"{csvReaderCurrentRecord[_collectionIndex]}/" : $"{collection}/";
var data = new DynamicJsonValue();
for (int i = 0; i < csvReaderFieldHeaders.Length; i++)
{
//ignoring reserved properties
if (csvReaderFieldHeaders[i][0] == '@')
{
if (_hasCollection && i == _collectionIndex)
{
SetCollectionForDocument(csvReaderCurrentRecord[_collectionIndex], data);
}
continue;
}
if (_nestedPropertyDictionary != null && _nestedPropertyDictionary.TryGetValue(i, out var segments))
{
var nestedData = data;
for (var j = 0; ; j++)
{
//last segment holds the data
if (j == segments.Length - 1)
{
nestedData[segments[j]] = ParseValue(csvReaderCurrentRecord[i]);
break; //we are done
}
//Creating the objects along the path if needed e.g. Foo.Bar.Name will create the 'Bar' oject if needed
if (nestedData[segments[j]] == null)
{
var tmpRef = new DynamicJsonValue();
nestedData[segments[j]] = tmpRef;
nestedData = tmpRef;
}
//We need to advance into the nested object, since it is not the last segment it must be of type 'DynamicJsonValue'
else
{
nestedData = (DynamicJsonValue)nestedData[segments[j]];
}
}
continue;
}
data[csvReaderFieldHeaders[i]] = ParseValue(csvReaderCurrentRecord[i]);
}
if (_hasCollection == false)
{
SetCollectionForDocument(collection, data);
}
return(new DocumentItem
{
Document = new Document
{
Data = context.ReadObject(data, idStr),
Id = context.GetLazyString(idStr),
ChangeVector = string.Empty,
Flags = DocumentFlags.None,
NonPersistentFlags = NonPersistentDocumentFlags.FromSmuggler,
LastModified = _database.Time.GetUtcNow(),
},
Attachments = null
});
}
finally
{
foreach (var disposeMe in _disposibales)
{
disposeMe.Dispose();
}
_disposibales.Clear();
}
}
public unsafe PatchResultData Apply(DocumentsOperationContext context,
string documentKey,
long?etag,
PatchRequest patch,
PatchRequest patchIfMissing,
bool isTestOnly = false,
bool skipPatchIfEtagMismatch = false)
{
var document = _database.DocumentsStorage.Get(context, documentKey);
if (_logger.IsInfoEnabled)
{
_logger.Info(string.Format("Preparing to apply patch on ({0}). Document found?: {1}.", documentKey, document != null));
}
if (etag.HasValue && document != null && document.Etag != etag.Value)
{
System.Diagnostics.Debug.Assert(document.Etag > 0);
if (skipPatchIfEtagMismatch)
{
return(new PatchResultData
{
PatchResult = PatchResult.Skipped
});
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Got concurrent exception while tried to patch the following document: {documentKey}");
}
throw new ConcurrencyException($"Could not patch document '{documentKey}' because non current etag was used")
{
ActualETag = document.Etag,
ExpectedETag = etag.Value,
};
}
var patchRequest = patch;
if (document == null)
{
if (patchIfMissing == null)
{
if (_logger.IsInfoEnabled)
{
_logger.Info("Tried to patch a not exists document and patchIfMissing is null");
}
return(new PatchResultData
{
PatchResult = PatchResult.DocumentDoesNotExists
});
}
patchRequest = patchIfMissing;
}
var scope = ApplySingleScript(context, document, isTestOnly, patchRequest);
var modifiedDocument = context.ReadObject(scope.ToBlittable(scope.PatchObject.AsObject()),
documentKey, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
var result = new PatchResultData
{
PatchResult = PatchResult.NotModified,
OriginalDocument = document?.Data,
DebugInfo = scope.DebugInfo,
};
if (modifiedDocument == null)
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"After applying patch, modifiedDocument is null and document is null? {document == null}");
}
result.PatchResult = PatchResult.Skipped;
return(result);
}
if (isTestOnly)
{
return(new PatchResultData
{
PatchResult = PatchResult.Tested,
OriginalDocument = document?.Data,
ModifiedDocument = modifiedDocument,
DebugActions = scope.DebugActions.GetDebugActions(),
DebugInfo = scope.DebugInfo,
});
}
var putResult = new DocumentsStorage.PutOperationResults();
if (document == null)
{
putResult = _database.DocumentsStorage.Put(context, documentKey, null, modifiedDocument);
}
else
{
var isModified = document.Data.Size != modifiedDocument.Size;
if (isModified == false) // optimization, if size different, no need to compute hash to check
{
var originHash = Hashing.XXHash64.Calculate(document.Data.BasePointer, (ulong)document.Data.Size);
var modifiedHash = Hashing.XXHash64.Calculate(modifiedDocument.BasePointer, (ulong)modifiedDocument.Size);
isModified = originHash != modifiedHash;
}
if (isModified)
{
putResult = _database.DocumentsStorage.Put(context, document.Key, document.Etag,
modifiedDocument);
result.PatchResult = PatchResult.Patched;
}
}
if (putResult.Etag != 0)
{
result.Etag = putResult.Etag;
result.Collection = putResult.Collection;
}
return(result);
}
public void Execute(DocumentsOperationContext context, Transaction indexWriteTransaction)
{
// delete
foreach (string reduceOutputId in _deletedReduceOutputs)
{
var referenceId = _index.OutputReduceToCollection.GetPatternGeneratedIdForReduceOutput(indexWriteTransaction, reduceOutputId);
if (referenceId == null)
{
continue;
}
if (_idsToDeleteByReferenceDocumentId.TryGetValue(referenceId, out var values) == false)
{
values = new HashSet <string>(1, StringComparer.OrdinalIgnoreCase);
_idsToDeleteByReferenceDocumentId.Add(referenceId, values);
}
values.Add(reduceOutputId);
}
foreach (var reduceReferenceIdToReduceOutputIds in _idsToDeleteByReferenceDocumentId)
{
using (var referenceDocument = _database.DocumentsStorage.Get(context, reduceReferenceIdToReduceOutputIds.Key))
{
if (referenceDocument == null)
{
continue;
}
if (referenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray ids) == false)
{
ThrowReduceOutputsPropertyNotFound(referenceDocument.Id);
}
var idsToRemove = reduceReferenceIdToReduceOutputIds.Value;
if (idsToRemove.Count >= ids.Length)
{
Debug.Assert(ids.All(x => idsToRemove.Contains(x.ToString())), $"Found ID in {nameof(idsToRemove)} that aren't in {nameof(ids)}");
foreach (object deletedId in ids)
{
_index.OutputReduceToCollection.DeletePatternGeneratedIdForReduceOutput(indexWriteTransaction, deletedId.ToString());
}
_database.DocumentsStorage.Delete(context, referenceDocument.Id, null);
continue;
}
if (ids.Modifications == null)
{
ids.Modifications = new DynamicJsonArray();
}
var indexesToRemove = new List <int>();
for (int i = ids.Length - 1; i >= 0; i--)
{
var id = ids[i].ToString();
if (idsToRemove.Contains(id))
{
indexesToRemove.Add(i);
}
if (idsToRemove.Count == indexesToRemove.Count)
{
break;
}
}
foreach (int toRemove in indexesToRemove)
{
ids.Modifications.RemoveAt(toRemove);
}
using (var doc = context.ReadObject(referenceDocument.Data, referenceDocument.Id))
{
_database.DocumentsStorage.Put(context, referenceDocument.Id, null, doc);
foreach (var idToRemove in idsToRemove)
{
_index.OutputReduceToCollection.DeletePatternGeneratedIdForReduceOutput(indexWriteTransaction, idToRemove);
}
}
}
}
// put
foreach (var referencesOfReduceOutput in _referencesOfReduceOutputs)
{
using (var existingReferenceDocument = _database.DocumentsStorage.Get(context, referencesOfReduceOutput.Key))
{
var uniqueIds = referencesOfReduceOutput.Value;
if (existingReferenceDocument != null)
{
if (existingReferenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray existingIds) == false)
{
ThrowReduceOutputsPropertyNotFound(existingReferenceDocument.Id);
}
foreach (object id in existingIds)
{
uniqueIds.Add(id.ToString());
}
}
var referenceDoc = new DynamicJsonValue
{
[nameof(OutputReduceToCollectionReference.ReduceOutputs)] = new DynamicJsonArray(uniqueIds),
[Constants.Documents.Metadata.Key] = new DynamicJsonValue
{
[Constants.Documents.Metadata.Collection] = _referencesCollectionName ?? $"{_outputReduceToCollection}/References"
}
};
using (var referenceJson = context.ReadObject(referenceDoc, "reference-of-reduce-output", BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
_database.DocumentsStorage.Delete(context, referencesOfReduceOutput.Key, null);
_database.DocumentsStorage.Put(context, referencesOfReduceOutput.Key, null, referenceJson,
flags: DocumentFlags.Artificial | DocumentFlags.FromIndex);
foreach (var reduceOutputId in referencesOfReduceOutput.Value)
{
_index.OutputReduceToCollection.AddPatternGeneratedIdForReduceOutput(indexWriteTransaction, reduceOutputId,
referencesOfReduceOutput.Key);
}
}
}
}
}
private void CleanupReferenceDocuments(DocumentsOperationContext context, Dictionary <string, HashSet <string> > idsToDeleteByReferenceDocumentId)
{
foreach (var reduceReferenceIdToReduceOutputIds in idsToDeleteByReferenceDocumentId)
{
using (var referenceDocument = _database.DocumentsStorage.Get(context, reduceReferenceIdToReduceOutputIds.Key))
{
if (referenceDocument == null)
{
continue;
}
if (referenceDocument.Data.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray ids) == false)
{
ThrowIdsPropertyNotFound(referenceDocument.Id);
}
var idsToRemove = reduceReferenceIdToReduceOutputIds.Value;
if (ids.Modifications == null)
{
ids.Modifications = new DynamicJsonArray();
}
var indexesToRemove = new List <int>();
for (int i = ids.Length - 1; i >= 0; i--)
{
var id = ids[i].ToString();
if (idsToRemove.Contains(id))
{
indexesToRemove.Add(i);
}
if (idsToRemove.Count == indexesToRemove.Count)
{
break;
}
}
foreach (int toRemove in indexesToRemove)
{
ids.Modifications.RemoveAt(toRemove);
}
using (var doc = context.ReadObject(referenceDocument.Data, referenceDocument.Id))
{
if (doc.TryGet(nameof(OutputReduceToCollectionReference.ReduceOutputs), out BlittableJsonReaderArray updatedIds) == false)
{
ThrowIdsPropertyNotFound(referenceDocument.Id);
}
if (updatedIds.Length == 0)
{
_database.DocumentsStorage.Delete(context, referenceDocument.Id, null);
}
else
{
_database.DocumentsStorage.Put(context, referenceDocument.Id, null, doc);
}
}
}
}
}
private async Task <(List <Match> Matches, GraphQueryPlan QueryPlan, bool NotModified)> GetQueryResults(IndexQueryServerSide query, DocumentsOperationContext documentsContext, long?existingResultEtag, OperationCancelToken token, bool collectIntermediateResults = false)
{
var q = query.Metadata.Query;
var qp = new GraphQueryPlan(query, documentsContext, existingResultEtag, token, Database)
{
CollectIntermediateResults = collectIntermediateResults
};
qp.BuildQueryPlan();
qp.OptimizeQueryPlan(); //TODO: audit optimization
if (query.WaitForNonStaleResults)
{
qp.IsStale = await qp.WaitForNonStaleResults();
}
else
{
await qp.CreateAutoIndexesAndWaitIfNecessary();
}
//for the case where we don't wait for non stale results we will override IsStale in the QueryQueryStep steps
if (documentsContext.Transaction == null || documentsContext.Transaction.Disposed)
{
documentsContext.OpenReadTransaction();
}
qp.ResultEtag = DocumentsStorage.ReadLastEtag(documentsContext.Transaction.InnerTransaction);
if (existingResultEtag.HasValue)
{
if (qp.ResultEtag == existingResultEtag)
{
return(null, null, true);
}
}
await qp.Initialize();
var matchResults = qp.Execute();
if (query.Metadata.OrderBy != null)
{
Sort(matchResults, query.Metadata.OrderBy, Database.Name, query.Query);
}
var filter = q.GraphQuery.Where;
if (filter != null)
{
for (int i = 0; i < matchResults.Count; i++)
{
var resultAsJson = new DynamicJsonValue();
matchResults[i].PopulateVertices(resultAsJson);
using (var result = documentsContext.ReadObject(resultAsJson, "graph/result"))
{
if (filter.IsMatchedBy(result, query.QueryParameters) == false)
{
matchResults[i] = default;
}
}
}
}
if (query.Start > 0)
{
matchResults.RemoveRange(0, Math.Min(query.Start, matchResults.Count));
}
if (query.PageSize < matchResults.Count)
{
matchResults.RemoveRange(query.PageSize, matchResults.Count - query.PageSize);
}
return(matchResults, qp, false);
}
/// <summary>
/// Iterates on a batch in document collection, process it and send documents if found any match
/// </summary>
/// <param name="docsContext"></param>
/// <param name="sendingCurrentBatchStopwatch"></param>
/// <returns>Whether succeeded finding any documents to send</returns>
private async Task <bool> TrySendingBatchToClient(DocumentsOperationContext docsContext, Stopwatch sendingCurrentBatchStopwatch)
{
bool anyDocumentsSentInCurrentIteration = false;
int docsToFlush = 0;
using (var writer = new BlittableJsonTextWriter(docsContext, _buffer))
{
using (docsContext.OpenReadTransaction())
{
foreach (var result in _documentsFetcher.GetDataToSend(docsContext, _startEtag))
{
_startEtag = result.Doc.Etag;
_lastChangeVector = string.IsNullOrEmpty(SubscriptionState.ChangeVectorForNextBatchStartingPoint)
? result.Doc.ChangeVector
: ChangeVectorUtils.MergeVectors(result.Doc.ChangeVector, SubscriptionState.ChangeVectorForNextBatchStartingPoint);
if (result.Doc.Data == null)
{
if (sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
await SendHeartBeat();
sendingCurrentBatchStopwatch.Restart();
}
continue;
}
anyDocumentsSentInCurrentIteration = true;
writer.WriteStartObject();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(TypeSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(DataSegment));
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(DataSegment));
result.Doc.EnsureMetadata();
if (result.Exception != null)
{
var metadata = result.Doc.Data[Client.Constants.Documents.Metadata.Key];
writer.WriteValue(BlittableJsonToken.StartObject,
docsContext.ReadObject(new DynamicJsonValue
{
[Client.Constants.Documents.Metadata.Key] = metadata
}, result.Doc.Id)
);
writer.WriteComma();
writer.WritePropertyName(docsContext.GetLazyStringForFieldWithCaching(ExceptionSegment));
writer.WriteValue(BlittableJsonToken.String, docsContext.GetLazyStringForFieldWithCaching(result.Exception.ToString()));
}
else
{
writer.WriteDocument(docsContext, result.Doc, metadataOnly: false);
}
writer.WriteEndObject();
docsToFlush++;
// perform flush for current batch after 1000ms of running or 1 MB
if (_buffer.Length > Constants.Size.Megabyte ||
sendingCurrentBatchStopwatch.ElapsedMilliseconds > 1000)
{
if (docsToFlush > 0)
{
await FlushDocsToClient(writer, docsToFlush);
docsToFlush = 0;
sendingCurrentBatchStopwatch.Restart();
}
else
{
await SendHeartBeat();
}
}
}
}
if (anyDocumentsSentInCurrentIteration)
{
docsContext.Write(writer, new DynamicJsonValue
{
[nameof(SubscriptionConnectionServerMessage.Type)] = nameof(SubscriptionConnectionServerMessage.MessageType.EndOfBatch)
});
await FlushDocsToClient(writer, docsToFlush, true);
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Finished sending a batch with {docsToFlush} documents for subscription {Options.SubscriptionName}");
}
}
}
return(anyDocumentsSentInCurrentIteration);
}
public void Put(DocumentsOperationContext context, string id, BlittableJsonReaderObject document,
DocumentFlags flags, NonPersistentDocumentFlags nonPersistentFlags, string changeVector, long lastModifiedTicks,
RevisionsCollectionConfiguration configuration = null, CollectionName collectionName = null)
{
Debug.Assert(changeVector != null, "Change vector must be set");
Debug.Assert(lastModifiedTicks != DateTime.MinValue.Ticks, "last modified ticks must be set");
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
if (collectionName == null)
{
collectionName = _database.DocumentsStorage.ExtractCollectionName(context, document);
}
if (configuration == null)
{
configuration = GetRevisionsConfiguration(collectionName.Name);
}
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var fromReplication = (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) == NonPersistentDocumentFlags.FromReplication;
var table = EnsureRevisionTableCreated(context.Transaction.InnerTransaction, collectionName);
// We want the revision's attachments to have a lower etag than the revision itself
if ((flags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
if (flags.Contain(DocumentFlags.Revision) == false)
{
using (Slice.From(context.Allocator, changeVector, out Slice changeVectorSlice))
{
if (table.VerifyKeyExists(changeVectorSlice) == false)
{
_documentsStorage.AttachmentsStorage.RevisionAttachments(context, lowerId, changeVectorSlice);
}
}
}
}
if (document.TryGet(Constants.Documents.Metadata.Key, out BlittableJsonReaderObject metadata) &&
metadata.TryGet(Constants.Documents.Metadata.Counters, out BlittableJsonReaderArray counterNames))
{
var dvj = new DynamicJsonValue();
for (var i = 0; i < counterNames.Length; i++)
{
var counter = counterNames[i].ToString();
var val = _documentsStorage.CountersStorage.GetCounterValue(context, id, counter);
if (val == null)
{
continue;
}
dvj[counter] = val.Value;
}
metadata.Modifications = new DynamicJsonValue(metadata)
{
[Constants.Documents.Metadata.RevisionCounters] = dvj
};
metadata.Modifications.Remove(Constants.Documents.Metadata.Counters);
document.Modifications = new DynamicJsonValue(document)
{
[Constants.Documents.Metadata.Key] = metadata
};
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
}
if (fromReplication)
{
void PutFromRevisionIfChangeVectorIsGreater()
{
bool hasDoc;
TableValueReader tvr;
try
{
hasDoc = _documentsStorage.GetTableValueReaderForDocument(context, lowerId, throwOnConflict: true, tvr: out tvr);
}
catch (DocumentConflictException)
{
// Do not modify the document.
return;
}
if (hasDoc == false)
{
PutFromRevision();
return;
}
var docChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref tvr);
if (ChangeVectorUtils.GetConflictStatus(changeVector, docChangeVector) == ConflictStatus.Update)
{
PutFromRevision();
}
void PutFromRevision()
{
_documentsStorage.Put(context, id, null, document, lastModifiedTicks, changeVector,
flags & ~DocumentFlags.Revision, nonPersistentFlags | NonPersistentDocumentFlags.FromRevision);
}
}
PutFromRevisionIfChangeVectorIsGreater();
}
flags |= DocumentFlags.Revision;
var newEtag = _database.DocumentsStorage.GenerateNextEtag();
var newEtagSwapBytes = Bits.SwapBytes(newEtag);
using (table.Allocate(out TableValueBuilder tvb))
using (Slice.From(context.Allocator, changeVector, out var cv))
{
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(lowerId);
tvb.Add(SpecialChars.RecordSeparator);
tvb.Add(newEtagSwapBytes);
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add((int)flags);
tvb.Add(NotDeletedRevisionMarker);
tvb.Add(lastModifiedTicks);
tvb.Add(context.GetTransactionMarker());
if (flags.Contain(DocumentFlags.Resolved))
{
tvb.Add((int)DocumentFlags.Resolved);
}
else
{
tvb.Add(0);
}
tvb.Add(Bits.SwapBytes(lastModifiedTicks));
var isNew = table.Set(tvb);
if (isNew == false)
{
// It might be just an update from replication as we call this twice, both for the doc delete and for deleteRevision.
return;
}
}
DeleteOldRevisions(context, table, lowerId, collectionName, configuration, nonPersistentFlags, changeVector, lastModifiedTicks);
}
}
public override void Execute(DocumentsOperationContext context, RavenTransaction tx)
{
var hiLoDocumentKey = RavenKeyGeneratorsHilo + Key;
var prefix = Key + Separator;
long oldMax = 0;
var newDoc = new DynamicJsonValue();
BlittableJsonReaderObject hiloDocReader = null, serverPrefixDocReader = null;
try
{
try
{
serverPrefixDocReader = Database.DocumentsStorage.Get(context, RavenKeyServerPrefix)?.Data;
hiloDocReader = Database.DocumentsStorage.Get(context, hiLoDocumentKey)?.Data;
}
catch (DocumentConflictException e)
{
throw new InvalidDataException(@"Failed to fetch HiLo document due to a conflict
on the document. This shouldn't happen, since
it this conflict should've been resolved during replication.
This exception should not happen and is likely a bug.", e);
}
string serverPrefix;
if (serverPrefixDocReader != null &&
serverPrefixDocReader.TryGet("ServerPrefix", out serverPrefix))
{
prefix += serverPrefix;
}
if (hiloDocReader != null)
{
hiloDocReader.TryGet("Max", out oldMax);
var prop = new BlittableJsonReaderObject.PropertyDetails();
for (var i = 0; i < hiloDocReader.Count; i++)
{
hiloDocReader.GetPropertyByIndex(0, ref prop);
if (prop.Name == "Max")
{
continue;
}
newDoc[prop.Name] = prop.Value;
}
}
}
finally
{
serverPrefixDocReader?.Dispose();
hiloDocReader?.Dispose();
}
oldMax = Math.Max(oldMax, LastRangeMax);
newDoc["Max"] = oldMax + Capacity;
using (
var freshHilo = context.ReadObject(newDoc, hiLoDocumentKey,
BlittableJsonDocumentBuilder.UsageMode.ToDisk))
{
Database.DocumentsStorage.Put(context, hiLoDocumentKey, null, freshHilo);
}
OldMax = oldMax;
Prefix = prefix;
}