private void AddReplicationItemToBatch(ReplicationBatchIndexItem item) { // destination already has it if (item.ChangeVector.GreaterThan(_parent._destinationLastKnownDocumentChangeVector) == false) { if (_log.IsInfoEnabled) { _log.Info( $"Skipping replication of {item.Name} because destination has a higher change vector. Source: {item.ChangeVector.Format()} < Dest: {_parent._destinationLastKnownDocumentChangeVectorAsString} "); } return; } LastEtag = Math.Max(LastEtag, item.Etag); _orderedReplicaItems.Add(item.Etag, item); }
public void ExecuteReplicationOnce() { _orderedReplicaItems.Clear(); try { var sp = Stopwatch.StartNew(); var timeout = Debugger.IsAttached ? 60 * 1000 : 1000; var configurationContext = _parent._configurationContext; using (configurationContext.OpenReadTransaction()) { while (sp.ElapsedMilliseconds < timeout) { LastEtag = _parent._lastSentIndexOrTransformerEtag; _parent.CancellationToken.ThrowIfCancellationRequested(); var indexAndTransformerMetadata = _parent._database.IndexMetadataPersistence.GetAfter( configurationContext.Transaction.InnerTransaction, configurationContext, LastEtag + 1, 0, 1024); using (var stream = new MemoryStream()) { foreach (var item in indexAndTransformerMetadata) { _parent.CancellationToken.ThrowIfCancellationRequested(); stream.Position = 0; using (var writer = new BlittableJsonTextWriter(configurationContext, stream)) { switch (item.Type) { case IndexEntryType.Index: var index = _parent._database.IndexStore.GetIndex(item.Id); if (index == null) //precaution { throw new InvalidDataException( $"Index with name {item.Name} has metadata, but is not at the index store. This is not supposed to happen and is likely a bug."); } try { IndexProcessor.Export(writer, index, configurationContext, false); } catch (InvalidOperationException e) { if (_log.IsInfoEnabled) { _log.Info( $"Failed to export index definition for replication. Index name = {item.Name}", e); } } break; case IndexEntryType.Transformer: var transformer = _parent._database.TransformerStore.GetTransformer(item.Id); if (transformer == null) //precaution { throw new InvalidDataException( $"Transformer with name {item.Name} has metadata, but is not at the transformer store. This is not supposed to happen and is likely a bug."); } try { TransformerProcessor.Export(writer, transformer, configurationContext); } catch (InvalidOperationException e) { if (_log.IsInfoEnabled) { _log.Info( $"Failed to export transformer definition for replication. Transformer name = {item.Name}", e); } } break; default: throw new ArgumentOutOfRangeException(nameof(item), "Unexpected item type in index/transformer metadata. This is not supposed to happen."); } writer.Flush(); stream.Position = 0; var newItem = new ReplicationBatchIndexItem { Name = item.Name, ChangeVector = item.ChangeVector, Etag = item.Etag, Type = (int)item.Type, Definition = configurationContext.ReadForMemory(stream, "Index/Transformer Replication - Reading definition into memory") }; AddReplicationItemToBatch(newItem); } } } // if we are at the end, we are done if (LastEtag <= _parent._database.IndexMetadataPersistence.ReadLastEtag( configurationContext.Transaction.InnerTransaction)) { break; } } if (_log.IsInfoEnabled) { _log.Info( $"Found {_orderedReplicaItems.Count:#,#;;0} indexes/transformers to replicate to {_parent.Destination.Database} @ {_parent.Destination.Url} in {sp.ElapsedMilliseconds:#,#;;0} ms."); } _parent.CancellationToken.ThrowIfCancellationRequested(); try { using (_parent._documentsContext.OpenReadTransaction()) SendIndexTransformerBatch(); } catch (Exception e) { if (_log.IsInfoEnabled) { _log.Info("Failed to send index/transformer replication batch", e); } throw; } } } finally { //release memory at the end of the operation foreach (var item in _orderedReplicaItems) { item.Value.Definition.Dispose(); } _orderedReplicaItems.Clear(); } }
private unsafe void WriteMetadataToServer(ReplicationBatchIndexItem item) { var changeVectorSize = item.ChangeVector.Length * sizeof(ChangeVectorEntry); var sizeOfNameInBytes = Encoding.UTF8.GetByteCount(item.Name); var requiredSize = changeVectorSize + sizeof(int) + // # of change vector entries sizeof(long) + //size of etag sizeof(int) + // size of type sizeof(int) + //size of name sizeof(int) + //name char count sizeOfNameInBytes + item.Definition.Size + sizeof(int); // size of definition if (requiredSize > _tempBuffer.Length) ThrowTooManyChangeVectorEntries(item.Name, item.ChangeVector, requiredSize); fixed(byte *pTemp = _tempBuffer) { int tempBufferPos = 0; //start writing change vector fixed(ChangeVectorEntry *pChangeVectorEntries = item.ChangeVector) { *(int *)pTemp = item.ChangeVector.Length; //write change vector length tempBufferPos += sizeof(int); //write change vector entries Memory.Copy(pTemp + tempBufferPos, (byte *)pChangeVectorEntries, changeVectorSize); tempBufferPos += changeVectorSize; } //end writing change vector *(long *)(pTemp + tempBufferPos) = item.Etag; //write etag tempBufferPos += sizeof(long); *(int *)(pTemp + tempBufferPos) = item.Type; tempBufferPos += sizeof(int); //start writing index/transformer metadata name *(int *)(pTemp + tempBufferPos) = sizeOfNameInBytes; //write the size of the name string tempBufferPos += sizeof(int); *(int *)(pTemp + tempBufferPos) = item.Name.Length; tempBufferPos += sizeof(int); //start writing the name string characters fixed(char *pName = item.Name) Encoding.UTF8.GetBytes(pName, item.Name.Length, pTemp + tempBufferPos, sizeOfNameInBytes); tempBufferPos += sizeOfNameInBytes; //end writing the name string characters //end writing index/transformer metadata name //start writing index/transformer definition //first write index/transformer definition size *(int *)(pTemp + tempBufferPos) = item.Definition.Size; tempBufferPos += sizeof(int); //start writing definition json data var docReadPos = 0; while (docReadPos < item.Definition.Size) { var sizeToCopy = Math.Min(item.Definition.Size - docReadPos, _tempBuffer.Length - tempBufferPos); if (sizeToCopy == 0) // buffer is full, need to flush it { _stream.Write(_tempBuffer, 0, tempBufferPos); tempBufferPos = 0; continue; } Memory.Copy(pTemp + tempBufferPos, item.Definition.BasePointer + docReadPos, sizeToCopy); tempBufferPos += sizeToCopy; docReadPos += sizeToCopy; } //end writing definition json data //end writing index/transformer definition _stream.Write(_tempBuffer, 0, tempBufferPos); } }