public unsafe void Invalid_Number_Of_Prop() { var blittable = new byte[192] { 0x0c, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x23, 0x30, 0x00, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x20, 0x23, 0x30, 0x00, 0x00, 0x05, 0x55, 0x73, 0x65, 0x72, 0x73, 0x00, 0x1d, 0x54, 0x72, 0x79, 0x6f, 0x75, 0x74, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x2b, 0x55, 0x73, 0x65, 0x72, 0x2c, 0x20, 0x54, 0x72, 0x79, 0x6f, 0x75, 0x74, 0x73, 0x00, 0x07, 0x75, 0x73, 0x65, 0x72, 0x73, 0x2f, 0x30, 0x00, 0x03, 0x09, 0x06, 0x05, 0x28, 0x05, 0x05, 0x2f, 0x04, 0x05, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F, 0x0a, 0x03, 0x51, 0x55, 0x00, 0x05, 0x47, 0x01, 0x05, 0x3a, 0x02, 0x08, 0x09, 0x46, 0x69, 0x72, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x00, 0x08, 0x4c, 0x61, 0x73, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x00, 0x04, 0x54, 0x61, 0x67, 0x73, 0x00, 0x09, 0x40, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x00, 0x11, 0x52, 0x61, 0x76, 0x65, 0x6e, 0x2d, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2d, 0x4e, 0x61, 0x6d, 0x65, 0x00, 0x0e, 0x52, 0x61, 0x76, 0x65, 0x6e, 0x2d, 0x43, 0x6c, 0x72, 0x2d, 0x54, 0x79, 0x70, 0x65, 0x00, 0x03, 0x40, 0x69, 0x64, 0x00, 0x10, 0x4e, 0x43, 0x39, 0x33, 0x28, 0x15, 0x05, 0x55, 0x01, 0xb4, 0x51 }; var size = 0xc0; fixed(byte *ptr = &blittable[0]) { var reader = new BlittableJsonReaderObject(ptr, size, null); var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation()); Assert.Equal(message.Message, "Number of properties not valid"); } }
public unsafe ReleaseCacheItem Get(JsonOperationContext context, string url, out string changeVector, out BlittableJsonReaderObject obj) { if (_items.TryGetValue(url, out var item)) { if (item.AddRef()) { var releaser = new ReleaseCacheItem(item); changeVector = item.ChangeVector; obj = item.Ptr != null ? new BlittableJsonReaderObject(item.Ptr, item.Size, context) : null; #if DEBUG if (obj != null) { obj.BlittableValidation(); } #endif if (Logger.IsInfoEnabled) { Logger.Info($"Url returned from the cache with etag: {changeVector}. {url}."); } return(releaser); } } obj = null; changeVector = null; return(new ReleaseCacheItem()); }
internal ReplicationMessageReply HandleServerResponse(BlittableJsonReaderObject replicationBatchReplyMessage, bool allowNotify) { replicationBatchReplyMessage.BlittableValidation(); var replicationBatchReply = JsonDeserializationServer.ReplicationMessageReply(replicationBatchReplyMessage); if (replicationBatchReply.MessageType == "Processing") { return(null); } if (allowNotify == false && replicationBatchReply.MessageType == "Notify") { return(null); } DestinationDbId = replicationBatchReply.DatabaseId; switch (replicationBatchReply.Type) { case ReplicationMessageReply.ReplyType.Ok: UpdateDestinationChangeVector(replicationBatchReply); OnSuccessfulTwoWaysCommunication(); break; default: var msg = $"Received error from remote replication destination. Error received: {replicationBatchReply.Exception}"; if (_log.IsInfoEnabled) { _log.Info(msg); } break; } if (_log.IsInfoEnabled) { switch (replicationBatchReply.Type) { case ReplicationMessageReply.ReplyType.Ok: _log.Info( $"Received reply for replication batch from {Destination.FromString()}. New destination change vector is {LastAcceptedChangeVector}"); break; case ReplicationMessageReply.ReplyType.Error: _log.Info( $"Received reply for replication batch from {Destination.FromString()}. There has been a failure, error string received : {replicationBatchReply.Exception}"); throw new InvalidOperationException( $"Received failure reply for replication batch. Error string received = {replicationBatchReply.Exception}"); default: throw new ArgumentOutOfRangeException(nameof(replicationBatchReply), "Received reply for replication batch with unrecognized type... got " + replicationBatchReply.Type); } } return(replicationBatchReply); }
private void FlushDocuments(DocumentsOperationContext context, List <BulkInsertDoc> docsToWrite, ref int totalSize) { if (docsToWrite.Count == 0) { return; } if (_logger.IsInfoEnabled) { _logger.Info( $"Writing {docsToWrite.Count:#,#} documents to disk using bulk insert, total {totalSize/1024:#,#} kb to write"); } Stopwatch sp = Stopwatch.StartNew(); using (var tx = context.OpenWriteTransaction()) { tx.InnerTransaction.LowLevelTransaction.IsLazyTransaction = true; foreach (var bulkInsertDoc in docsToWrite) { var reader = new BlittableJsonReaderObject(bulkInsertDoc.Pointer, bulkInsertDoc.Used, context); reader.BlittableValidation(); string docKey; BlittableJsonReaderObject metadata; if (reader.TryGet(Constants.Metadata.Key, out metadata) == false) { const string message = "'@metadata' is missing in received document for bulk insert"; throw new InvalidDataException(message); } if (metadata.TryGet(Constants.Metadata.Id, out docKey) == false) { const string message = "'@id' is missing in received document for bulk insert"; throw new InvalidDataException(message); } TcpConnection.DocumentDatabase.DocumentsStorage.Put(context, docKey, null, reader); } tx.Commit(); } foreach (var bulkInsertDoc in docsToWrite) { _docsToRelease.Add(bulkInsertDoc); } if (_logger.IsInfoEnabled) { _logger.Info( $"Writing {docsToWrite.Count:#,#} documents in bulk insert took {sp.ElapsedMilliseconds:#,#l;0} ms"); } docsToWrite.Clear(); totalSize = 0; }
public override unsafe void Read(DocumentsOperationContext context, IncomingReplicationStatsScope stats) { // TODO: add stats RavenDB-13470 SetLazyStringValueFromString(context, out Id); SetLazyStringValueFromString(context, out Collection); Debug.Assert(Collection != null); var sizeOfData = *(int *)Reader.ReadExactly(sizeof(int)); var mem = Reader.AllocateMemory(sizeOfData); Reader.ReadExactly(mem, sizeOfData); Values = new BlittableJsonReaderObject(mem, sizeOfData, context); Values.BlittableValidation(); }
public unsafe void Invalid_Null() { var blittable = new byte[16] { 0x01, 0x01, 0x01, 0x00, 0x08, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x01, 0x0b, 0x51 }; fixed(byte *ptr = &blittable[0]) { var reader = new BlittableJsonReaderObject(ptr, 0x10, null); var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation()); Assert.Equal(message.Message, "Null not valid"); } }
public unsafe void Invalid_String() { var blittable = new byte[26] { 0x08, 0x61, 0x62, 0x63, 0x64, 0x0a, 0x61, 0x62, 0x63, 0x01, 0x03, 0x01, 0x0b, 0x00, 0x05, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x0b, 0x15, 0x51 }; fixed(byte *ptr = &blittable[0]) { var reader = new BlittableJsonReaderObject(ptr, 0x1A, null); var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation()); Assert.StartsWith("String not valid", message.Message); } }
public unsafe void Invalid_Long() { var blittable = new byte[26] { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x01, 0x0b, 0x00, 0x03, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x0b, 0x15, 0x51 }; fixed(byte *ptr = &blittable[0]) { var reader = new BlittableJsonReaderObject(ptr, 0x1a, null); var message = Assert.Throws <FormatException>(() => reader.BlittableValidation()); Assert.Equal(message.Message, "Bad variable size int"); } }
public unsafe void Invalid_Float() { var blittable = new byte[22] { 0x05, 0x31, 0x2e, 0x33, 0x2e, 0x35, 0x00, 0x01, 0x07, 0x00, 0x04, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x07, 0x11, 0x51 }; fixed(byte *ptr = &blittable[0]) { var reader = new BlittableJsonReaderObject(ptr, 0x16, null); var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation()); Assert.Equal(message.Message, "Double not valid (1.3.5)"); } }
public unsafe void Invalid_Compressed_String() { var blittable = new byte[31] { 0x04, 0x0d, 0x1f, 0x61, 0x01, 0x00, 0xff, 0xff, 0x5f, 0x50, 0x61, 0x61, 0x61, 0x61, 0x61, 0x00, 0x01, 0x10, 0x00, 0x06, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x10, 0x1A, 0x51 }; fixed(byte *ptr = &blittable[0]) { var reader = new BlittableJsonReaderObject(ptr, 0x1F, null); var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation()); Assert.Equal(message.Message, "Compressed string not valid"); } }
public unsafe void Invalid_Float() { var blittable = new byte[22] { 0x05, 0x31, 0x2e, 0x33, 0x2e, 0x35, 0x00, 0x01, 0x07, 0x00, 0x04, 0x04, 0x74, 0x65, 0x6d, 0x70, 0x00, 0x10, 0x06, 0x07, 0x11, 0x51 }; fixed(byte *ptr = &blittable[0]) { using (var context = JsonOperationContext.ShortTermSingleUse()) { var reader = new BlittableJsonReaderObject(ptr, 0x16, context); var message = Assert.Throws <InvalidDataException>(() => reader.BlittableValidation()); Assert.Equal(message.Message, "Number not valid (1.3.5)"); } } }
public override unsafe void Read(DocumentsOperationContext context, IncomingReplicationStatsScope stats) { IncomingReplicationStatsScope scope; if (Type == ReplicationItemType.Document) { scope = stats.For(ReplicationOperation.Incoming.DocumentRead, start: false); stats.RecordDocumentRead(); } else { scope = stats.For(ReplicationOperation.Incoming.TombstoneRead, start: false); stats.RecordDocumentTombstoneRead(); } using (scope.Start()) { LastModifiedTicks = *(long *)Reader.ReadExactly(sizeof(long)); Flags = *(DocumentFlags *)Reader.ReadExactly(sizeof(DocumentFlags)) | DocumentFlags.FromReplication; SetLazyStringValueFromString(context, out Id); var documentSize = *(int *)Reader.ReadExactly(sizeof(int)); if (documentSize != -1) //if -1, then this is a tombstone { using (var read = stats.For(ReplicationOperation.Incoming.DocumentRead).Start()) { read.RecordDocumentRead(); var mem = Reader.AllocateMemory(documentSize); Reader.ReadExactly(mem, documentSize); Data = new BlittableJsonReaderObject(mem, documentSize, context); Data.BlittableValidation(); } } else { SetLazyStringValueFromString(context, out Collection); } } }
public unsafe void Set(string url, string changeVector, BlittableJsonReaderObject result) { #if DEBUG result.BlittableValidation(); #endif var mem = _unmanagedBuffersPool.Allocate(result.Size); result.CopyTo(mem.Address); if (Interlocked.Add(ref _totalSize, result.Size) > _maxSize) { if (_isFreeSpaceRunning == false) { Task.Run(FreeSpace); } } var httpCacheItem = new HttpCacheItem { ChangeVector = changeVector, Ptr = mem.Address, Size = result.Size, Allocation = mem, Cache = this, Generation = Generation }; HttpCacheItem old = null; _items.AddOrUpdate(url, httpCacheItem, (s, oldItem) => { old = oldItem; return(httpCacheItem); }); //We need to check if the cache is been disposed after the item was added otherwise we will run into another race condition //where it started been disposed right after we checked it and before we managed to insert the new cache item. if (_disposing) { //We might have double release here but we have a protection for that. httpCacheItem.ReleaseRef(); } old?.ReleaseRef(); }
public RachisEntry ReadRachisEntry(JsonOperationContext context) { // we explicitly not disposing this here, because we need to access the entry BlittableJsonReaderObject json = null; try { using (_disposerLock.EnsureNotDisposed()) { json = context.Sync.ParseToMemory(_stream, "rachis-entry", BlittableJsonDocumentBuilder.UsageMode.None, _buffer); json.BlittableValidation(); ValidateMessage(nameof(RachisEntry), json); return(JsonDeserializationRachis <RachisEntry> .Deserialize(json)); } } catch { json?.Dispose(); throw; } }
public PutOperationResults PutDocument(DocumentsOperationContext context, string id, string expectedChangeVector, BlittableJsonReaderObject document, long?lastModifiedTicks = null, string changeVector = null, DocumentFlags flags = DocumentFlags.None, NonPersistentDocumentFlags nonPersistentFlags = NonPersistentDocumentFlags.None) { if (context.Transaction == null) { ThrowRequiresTransaction(); return(default(PutOperationResults)); // never hit } #if DEBUG var documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); #endif var newEtag = _documentsStorage.GenerateNextEtag(); var modifiedTicks = lastModifiedTicks ?? _documentDatabase.Time.GetUtcNow().Ticks; id = BuildDocumentId(id, newEtag, out bool knownNewId); using (DocumentIdWorker.GetLowerIdSliceAndStorageKey(context, id, out Slice lowerId, out Slice idPtr)) { var collectionName = _documentsStorage.ExtractCollectionName(context, document); var table = context.Transaction.InnerTransaction.OpenTable(DocsSchema, collectionName.GetTableName(CollectionTableType.Documents)); var oldValue = default(TableValueReader); if (knownNewId == false) { // delete a tombstone if it exists, if it known that it is a new ID, no need, so we can skip it DeleteTombstoneIfNeeded(context, collectionName, lowerId.Content.Ptr, lowerId.Size); table.ReadByKey(lowerId, out oldValue); } BlittableJsonReaderObject oldDoc = null; if (oldValue.Pointer == null) { // expectedChangeVector being null means we don't care, and empty // means that it must be new if (string.IsNullOrEmpty(expectedChangeVector) == false) { ThrowConcurrentExceptionOnMissingDoc(id, expectedChangeVector); } } else { // expectedChangeVector has special meaning here // null - means, don't care, don't check // "" / empty - means, must be new // anything else - must match exactly if (expectedChangeVector != null) { var oldChangeVector = TableValueToChangeVector(context, (int)DocumentsTable.ChangeVector, ref oldValue); if (string.Compare(expectedChangeVector, oldChangeVector, StringComparison.Ordinal) != 0) { ThrowConcurrentException(id, expectedChangeVector, oldChangeVector); } } oldDoc = new BlittableJsonReaderObject(oldValue.Read((int)DocumentsTable.Data, out int oldSize), oldSize, context); var oldCollectionName = _documentsStorage.ExtractCollectionName(context, oldDoc); if (oldCollectionName != collectionName) { ThrowInvalidCollectionNameChange(id, oldCollectionName, collectionName); } var oldFlags = TableValueToFlags((int)DocumentsTable.Flags, ref oldValue); if ((nonPersistentFlags & NonPersistentDocumentFlags.ByAttachmentUpdate) != NonPersistentDocumentFlags.ByAttachmentUpdate && (nonPersistentFlags & NonPersistentDocumentFlags.FromReplication) != NonPersistentDocumentFlags.FromReplication) { if ((oldFlags & DocumentFlags.HasAttachments) == DocumentFlags.HasAttachments) { flags |= DocumentFlags.HasAttachments; } } } var result = BuildChangeVectorAndResolveConflicts(context, id, lowerId, newEtag, document, changeVector, expectedChangeVector, flags, oldValue); changeVector = result.ChangeVector; nonPersistentFlags |= result.NonPersistentFlags; if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.Resolved)) { flags |= DocumentFlags.Resolved; } if (collectionName.IsHiLo == false && (flags & DocumentFlags.Artificial) != DocumentFlags.Artificial) { if (ShouldRecreateAttachments(context, lowerId, oldDoc, document, ref flags, nonPersistentFlags)) { #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } #endif document = context.ReadObject(document, id, BlittableJsonDocumentBuilder.UsageMode.ToDisk); #if DEBUG documentDebugHash = document.DebugHash; document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif } if (nonPersistentFlags.Contain(NonPersistentDocumentFlags.FromReplication) == false && (flags.Contain(DocumentFlags.Resolved) || _documentDatabase.DocumentsStorage.RevisionsStorage.Configuration != null )) { var shouldVersion = _documentDatabase.DocumentsStorage.RevisionsStorage.ShouldVersionDocument(collectionName, nonPersistentFlags, oldDoc, document, ref flags, out RevisionsCollectionConfiguration configuration); if (shouldVersion) { _documentDatabase.DocumentsStorage.RevisionsStorage.Put(context, id, document, flags, nonPersistentFlags, changeVector, modifiedTicks, configuration, collectionName); } } } using (Slice.From(context.Allocator, changeVector, out var cv)) using (table.Allocate(out TableValueBuilder tvb)) { tvb.Add(lowerId); tvb.Add(Bits.SwapBytes(newEtag)); tvb.Add(idPtr); tvb.Add(document.BasePointer, document.Size); tvb.Add(cv.Content.Ptr, cv.Size); tvb.Add(modifiedTicks); tvb.Add((int)flags); tvb.Add(context.GetTransactionMarker()); if (oldValue.Pointer == null) { table.Insert(tvb); } else { table.Update(oldValue.Id, tvb); } } if (collectionName.IsHiLo == false) { _documentsStorage.ExpirationStorage.Put(context, lowerId, document); } context.LastDatabaseChangeVector = changeVector; _documentDatabase.Metrics.Docs.PutsPerSec.MarkSingleThreaded(1); _documentDatabase.Metrics.Docs.BytesPutsPerSec.MarkSingleThreaded(document.Size); context.Transaction.AddAfterCommitNotification(new DocumentChange { ChangeVector = changeVector, CollectionName = collectionName.Name, Id = id, Type = DocumentChangeTypes.Put, }); #if DEBUG if (document.DebugHash != documentDebugHash) { throw new InvalidDataException("The incoming document " + id + " has changed _during_ the put process, " + "this is likely because you are trying to save a document that is already stored and was moved"); } document.BlittableValidation(); BlittableJsonReaderObject.AssertNoModifications(document, id, assertChildren: true); AssertMetadataWasFiltered(document); AttachmentsStorage.AssertAttachments(document, flags); #endif return(new PutOperationResults { Etag = newEtag, Id = id, Collection = collectionName, ChangeVector = changeVector, Flags = flags, LastModified = new DateTime(modifiedTicks) }); } }