public unsafe void Invalid_Number_Of_Prop()
{
var blittable = new byte[192]
{
0x0c, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65,
0x20, 0x23, 0x30, 0x00, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x4e,
0x61, 0x6d, 0x65, 0x20, 0x23, 0x30, 0x00, 0x00, 0x05, 0x55,
0x73, 0x65, 0x72, 0x73, 0x00, 0x1d, 0x54, 0x72, 0x79, 0x6f,
0x75, 0x74, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x61,
0x6d, 0x2b, 0x55, 0x73, 0x65, 0x72, 0x2c, 0x20, 0x54, 0x72,
0x79, 0x6f, 0x75, 0x74, 0x73, 0x00, 0x07, 0x75, 0x73, 0x65,
0x72, 0x73, 0x2f, 0x30, 0x00, 0x03, 0x09, 0x06, 0x05, 0x28,
0x05, 0x05, 0x2f, 0x04, 0x05, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F,
0x0a, 0x03, 0x51, 0x55, 0x00, 0x05, 0x47, 0x01, 0x05, 0x3a,
0x02, 0x08, 0x09, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61,
0x6d, 0x65, 0x00, 0x08, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x61,
0x6d, 0x65, 0x00, 0x04, 0x54, 0x61, 0x67, 0x73, 0x00, 0x09,
0x40, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x00,
0x11, 0x52, 0x61, 0x76, 0x65, 0x6e, 0x2d, 0x45, 0x6e, 0x74,
0x69, 0x74, 0x79, 0x2d, 0x4e, 0x61, 0x6d, 0x65, 0x00, 0x0e,
0x52, 0x61, 0x76, 0x65, 0x6e, 0x2d, 0x43, 0x6c, 0x72, 0x2d,
0x54, 0x79, 0x70, 0x65, 0x00, 0x03, 0x40, 0x69, 0x64, 0x00,
0x10, 0x4e, 0x43, 0x39, 0x33, 0x28, 0x15, 0x05, 0x55, 0x01,
0xb4, 0x51
};
var size = 0xc0;
fixed(byte *ptr = &blittable[0])
{
var reader = new BlittableJsonReaderObject(ptr, size, null);
var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation());
Assert.Equal(message.Message, "Number of properties not valid");
}
}
public unsafe ReleaseCacheItem Get(JsonOperationContext context, string url, out string changeVector, out BlittableJsonReaderObject obj)
{
if (_items.TryGetValue(url, out var item))
{
if (item.AddRef())
{
var releaser = new ReleaseCacheItem(item);
changeVector = item.ChangeVector;
obj = item.Ptr != null ? new BlittableJsonReaderObject(item.Ptr, item.Size, context) : null;
#if DEBUG
if (obj != null)
{
obj.BlittableValidation();
}
#endif
if (Logger.IsInfoEnabled)
{
Logger.Info($"Url returned from the cache with etag: {changeVector}. {url}.");
}
return(releaser);
}
}
obj = null;
changeVector = null;
return(new ReleaseCacheItem());
}
internal ReplicationMessageReply HandleServerResponse(BlittableJsonReaderObject replicationBatchReplyMessage, bool allowNotify)
{
replicationBatchReplyMessage.BlittableValidation();
var replicationBatchReply = JsonDeserializationServer.ReplicationMessageReply(replicationBatchReplyMessage);
if (replicationBatchReply.MessageType == "Processing")
{
return(null);
}
if (allowNotify == false && replicationBatchReply.MessageType == "Notify")
{
return(null);
}
DestinationDbId = replicationBatchReply.DatabaseId;
switch (replicationBatchReply.Type)
{
case ReplicationMessageReply.ReplyType.Ok:
UpdateDestinationChangeVector(replicationBatchReply);
OnSuccessfulTwoWaysCommunication();
break;
default:
var msg = $"Received error from remote replication destination. Error received: {replicationBatchReply.Exception}";
if (_log.IsInfoEnabled)
{
_log.Info(msg);
}
break;
}
if (_log.IsInfoEnabled)
{
switch (replicationBatchReply.Type)
{
case ReplicationMessageReply.ReplyType.Ok:
_log.Info(
$"Received reply for replication batch from {Destination.FromString()}. New destination change vector is {LastAcceptedChangeVector}");
break;
case ReplicationMessageReply.ReplyType.Error:
_log.Info(
$"Received reply for replication batch from {Destination.FromString()}. There has been a failure, error string received : {replicationBatchReply.Exception}");
throw new InvalidOperationException(
$"Received failure reply for replication batch. Error string received = {replicationBatchReply.Exception}");
default:
throw new ArgumentOutOfRangeException(nameof(replicationBatchReply),
"Received reply for replication batch with unrecognized type... got " +
replicationBatchReply.Type);
}
}
return(replicationBatchReply);
}
private void FlushDocuments(DocumentsOperationContext context, List <BulkInsertDoc> docsToWrite, ref int totalSize)
{
if (docsToWrite.Count == 0)
{
return;
}
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Writing {docsToWrite.Count:#,#} documents to disk using bulk insert, total {totalSize/1024:#,#} kb to write");
}
Stopwatch sp = Stopwatch.StartNew();
using (var tx = context.OpenWriteTransaction())
{
tx.InnerTransaction.LowLevelTransaction.IsLazyTransaction = true;
foreach (var bulkInsertDoc in docsToWrite)
{
var reader = new BlittableJsonReaderObject(bulkInsertDoc.Pointer, bulkInsertDoc.Used,
context);
reader.BlittableValidation();
string docKey;
BlittableJsonReaderObject metadata;
if (reader.TryGet(Constants.Metadata.Key, out metadata) == false)
{
const string message = "'@metadata' is missing in received document for bulk insert";
throw new InvalidDataException(message);
}
if (metadata.TryGet(Constants.Metadata.Id, out docKey) == false)
{
const string message = "'@id' is missing in received document for bulk insert";
throw new InvalidDataException(message);
}
TcpConnection.DocumentDatabase.DocumentsStorage.Put(context, docKey, null, reader);
}
tx.Commit();
}
foreach (var bulkInsertDoc in docsToWrite)
{
_docsToRelease.Add(bulkInsertDoc);
}
if (_logger.IsInfoEnabled)
{
_logger.Info(
$"Writing {docsToWrite.Count:#,#} documents in bulk insert took {sp.ElapsedMilliseconds:#,#l;0} ms");
}
docsToWrite.Clear();
totalSize = 0;
}
public override unsafe void Read(DocumentsOperationContext context, IncomingReplicationStatsScope stats)
{
// TODO: add stats RavenDB-13470
SetLazyStringValueFromString(context, out Id);
SetLazyStringValueFromString(context, out Collection);
Debug.Assert(Collection != null);
var sizeOfData = *(int *)Reader.ReadExactly(sizeof(int));
var mem = Reader.AllocateMemory(sizeOfData);
Reader.ReadExactly(mem, sizeOfData);
Values = new BlittableJsonReaderObject(mem, sizeOfData, context);
Values.BlittableValidation();
}
public unsafe void Invalid_Null()
{
var blittable = new byte[16]
{
0x01, 0x01, 0x01, 0x00, 0x08, 0x04, 0x74, 0x65, 0x6d, 0x70,
0x00, 0x10, 0x06, 0x01, 0x0b, 0x51
};
fixed(byte *ptr = &blittable[0])
{
var reader = new BlittableJsonReaderObject(ptr, 0x10, null);
var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation());
Assert.Equal(message.Message, "Null not valid");
}
}
public unsafe void Invalid_String()
{
var blittable = new byte[26]
{
0x08, 0x61, 0x62, 0x63, 0x64, 0x0a, 0x61, 0x62, 0x63, 0x01,
0x03, 0x01, 0x0b, 0x00, 0x05, 0x04, 0x74, 0x65, 0x6d, 0x70,
0x00, 0x10, 0x06, 0x0b, 0x15, 0x51
};
fixed(byte *ptr = &blittable[0])
{
var reader = new BlittableJsonReaderObject(ptr, 0x1A, null);
var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation());
Assert.StartsWith("String not valid", message.Message);
}
}
public unsafe void Invalid_Long()
{
var blittable = new byte[26]
{
0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x01, 0x01, 0x0b, 0x00, 0x03, 0x04, 0x74, 0x65, 0x6d, 0x70,
0x00, 0x10, 0x06, 0x0b, 0x15, 0x51
};
fixed(byte *ptr = &blittable[0])
{
var reader = new BlittableJsonReaderObject(ptr, 0x1a, null);
var message = Assert.Throws <FormatException>(() => reader.BlittableValidation());
Assert.Equal(message.Message, "Bad variable size int");
}
}
public unsafe void Invalid_Float()
{
var blittable = new byte[22]
{
0x05, 0x31, 0x2e, 0x33, 0x2e, 0x35, 0x00, 0x01, 0x07, 0x00,
0x04, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x07,
0x11, 0x51
};
fixed(byte *ptr = &blittable[0])
{
var reader = new BlittableJsonReaderObject(ptr, 0x16, null);
var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation());
Assert.Equal(message.Message, "Double not valid (1.3.5)");
}
}
public unsafe void Invalid_Compressed_String()
{
var blittable = new byte[31]
{
0x04, 0x0d, 0x1f, 0x61, 0x01, 0x00, 0xff, 0xff, 0x5f, 0x50,
0x61, 0x61, 0x61, 0x61, 0x61, 0x00, 0x01, 0x10, 0x00, 0x06,
0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x10, 0x1A,
0x51
};
fixed(byte *ptr = &blittable[0])
{
var reader = new BlittableJsonReaderObject(ptr, 0x1F, null);
var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation());
Assert.Equal(message.Message, "Compressed string not valid");
}
}
public unsafe void Invalid_Float()
{
var blittable = new byte[22]
{
0x05, 0x31, 0x2e, 0x33, 0x2e, 0x35, 0x00, 0x01, 0x07, 0x00,
0x04, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x07,
0x11, 0x51
};
fixed(byte *ptr = &blittable[0])
{
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var reader = new BlittableJsonReaderObject(ptr, 0x16, context);
var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation());
Assert.Equal(message.Message, "Number not valid (1.3.5)");
}
}
}
public override unsafe void Read(DocumentsOperationContext context, IncomingReplicationStatsScope stats)
{
IncomingReplicationStatsScope scope;
if (Type == ReplicationItemType.Document)
{
scope = stats.For(ReplicationOperation.Incoming.DocumentRead, start: false);
stats.RecordDocumentRead();
}
else
{
scope = stats.For(ReplicationOperation.Incoming.TombstoneRead, start: false);
stats.RecordDocumentTombstoneRead();
}
using (scope.Start())
{
LastModifiedTicks = *(long *)Reader.ReadExactly(sizeof(long));
Flags = *(DocumentFlags *)Reader.ReadExactly(sizeof(DocumentFlags)) | DocumentFlags.FromReplication;
SetLazyStringValueFromString(context, out Id);
var documentSize = *(int *)Reader.ReadExactly(sizeof(int));
if (documentSize != -1) //if -1, then this is a tombstone
{
using (var read = stats.For(ReplicationOperation.Incoming.DocumentRead).Start())
{
read.RecordDocumentRead();
var mem = Reader.AllocateMemory(documentSize);
Reader.ReadExactly(mem, documentSize);
Data = new BlittableJsonReaderObject(mem, documentSize, context);
Data.BlittableValidation();
}
}
else
{
SetLazyStringValueFromString(context, out Collection);
}
}
}
public unsafe void Set(string url, string changeVector, BlittableJsonReaderObject result)
{
#if DEBUG
result.BlittableValidation();
#endif
var mem = _unmanagedBuffersPool.Allocate(result.Size);
result.CopyTo(mem.Address);
if (Interlocked.Add(ref _totalSize, result.Size) > _maxSize)
{
if (_isFreeSpaceRunning == false)
{
Task.Run(FreeSpace);
}
}
var httpCacheItem = new HttpCacheItem
{
ChangeVector = changeVector,
Ptr = mem.Address,
Size = result.Size,
Allocation = mem,
Cache = this,
Generation = Generation
};
HttpCacheItem old = null;
_items.AddOrUpdate(url, httpCacheItem, (s, oldItem) =>
{
old = oldItem;
return(httpCacheItem);
});
//We need to check if the cache is been disposed after the item was added otherwise we will run into another race condition
//where it started been disposed right after we checked it and before we managed to insert the new cache item.
if (_disposing)
{
//We might have double release here but we have a protection for that.
httpCacheItem.ReleaseRef();
}
old?.ReleaseRef();
}
public RachisEntry ReadRachisEntry(JsonOperationContext context)
{
// we explicitly not disposing this here, because we need to access the entry
BlittableJsonReaderObject json = null;
try
{
using (_disposerLock.EnsureNotDisposed())
{
json = context.Sync.ParseToMemory(_stream, "rachis-entry",
BlittableJsonDocumentBuilder.UsageMode.None, _buffer);
json.BlittableValidation();
ValidateMessage(nameof(RachisEntry), json);
return(JsonDeserializationRachis <RachisEntry> .Deserialize(json));
}
}
catch
{
json?.Dispose();
throw;
}
}
public PutOperationResults PutDocument(DocumentsOperationContext context, string id,
string expectedChangeVector,
BlittableJsonReaderObject document,
long?lastModifiedTicks = null,
string changeVector = null,
DocumentFlags flags = DocumentFlags.None,
NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None)
{
if (context.Transaction == null)
{
ThrowRequiresTransaction();
return(default(PutOperationResults)); // never hit
}
#if DEBUG
var documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
#endif
var newEtag = _documentsStorage.GenerateNextEtag();
var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks;
id = BuildDocumentId(id, newEtag, out bool knownNewId);
using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr))
{
var collectionName = _documentsStorage.ExtractCollectionName(context, document);
var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents));
var oldValue = default(TableValueReader);
if (knownNewId == false)
{
// delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it
DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size);
table.ReadByKey(lowerId, out oldValue);
}
BlittableJsonReaderObject oldDoc = null;
if (oldValue.Pointer == null)
{
// expectedChangeVector being null means we don't care, and empty
// means that it must be new
if (string.IsNullOrEmpty(expectedChangeVector) == false)
{
ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector);
}
}
else
{
// expectedChangeVector has special meaning here
// null - means, don't care, don't check
// "" / empty - means, must be new
// anything else - must match exactly
if (expectedChangeVector != null)
{
var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue);
if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0)
{
ThrowConcurrentException(id, expectedChangeVector, oldChangeVector);
}
}
oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context);
var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc);
if (oldCollectionName != collectionName)
{
ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName);
}
var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue);
if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate &&
(nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication)
{
if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments)
{
flags |= DocumentFlags.HasAttachments;
}
}
}
var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue);
changeVector = result.ChangeVector;
nonPersistentFlags |= result.NonPersistentFlags;
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved))
{
flags |= DocumentFlags.Resolved;
}
if (collectionName.IsHiLo == false &&
(flags & DocumentFlags.Artificial) != DocumentFlags.Artificial)
{
if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags))
{
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
#endif
document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk);
#if DEBUG
documentDebugHash = document.DebugHash;
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
}
if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false &&
(flags.Contain(DocumentFlags.Resolved) ||
_documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null
))
{
var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document,
ref flags, out RevisionsCollectionConfiguration configuration);
if (shouldVersion)
{
_documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags,
changeVector, modifiedTicks, configuration, collectionName);
}
}
}
using (Slice.From(context.Allocator, changeVector, out var cv))
using (table.Allocate(out TableValueBuilder tvb))
{
tvb.Add(lowerId);
tvb.Add(Bits.SwapBytes(newEtag));
tvb.Add(idPtr);
tvb.Add(document.BasePointer, document.Size);
tvb.Add(cv.Content.Ptr, cv.Size);
tvb.Add(modifiedTicks);
tvb.Add((int)flags);
tvb.Add(context.GetTransactionMarker());
if (oldValue.Pointer == null)
{
table.Insert(tvb);
}
else
{
table.Update(oldValue.Id, tvb);
}
}
if (collectionName.IsHiLo == false)
{
_documentsStorage.ExpirationStorage.Put(context, lowerId, document);
}
context.LastDatabaseChangeVector = changeVector;
_documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1);
_documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size);
context.Transaction.AddAfterCommitNotification(new DocumentChange
{
ChangeVector = changeVector,
CollectionName = collectionName.Name,
Id = id,
Type = DocumentChangeTypes.Put,
});
#if DEBUG
if (document.DebugHash != documentDebugHash)
{
throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " +
"this is likely because you are trying to save a document that is already stored and was moved");
}
document.BlittableValidation();
BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true);
AssertMetadataWasFiltered(document);
AttachmentsStorage.AssertAttachments(document, flags);
#endif
return(new PutOperationResults
{
Etag = newEtag,
Id = id,
Collection = collectionName,
ChangeVector = changeVector,
Flags = flags,
LastModified = new DateTime(modifiedTicks)
});
}
}