private IEnumerable <KeyData <TKey, TValue> > ReadBlock(int blockSize)
        {
            var blockStartPosition        = checked ((int)this.reader.BaseStream.Position);
            var alignedBlockStartPosition = blockStartPosition - KeyChunkMetadata.Size;

            this.reader.BaseStream.Position = alignedBlockStartPosition + blockSize - sizeof(ulong);
            var expectedChecksum = this.reader.ReadUInt64();

            this.reader.BaseStream.Position = blockStartPosition;

            // Verify checksum.
            var actualChecksum = CRC64.ToCRC64(this.reader.BaseStream.GetBuffer(), alignedBlockStartPosition, blockSize - sizeof(ulong));

            if (actualChecksum != expectedChecksum)
            {
                throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_KeyCheckpoint_FailedToRead_Checksum, actualChecksum, expectedChecksum));
            }

            while (this.reader.BaseStream.Position < (alignedBlockStartPosition + blockSize - sizeof(ulong)))
            {
                var keyData = this.keyCheckpointFile.ReadKey <TKey, TValue>(this.reader, this.keySerializer);
                yield return(keyData);
            }

            Diagnostics.Assert(
                this.reader.BaseStream.Position == (alignedBlockStartPosition + blockSize - sizeof(ulong)),
                this.traceType,
                "Failed to read block of keys due to block alignment mismatch.");
        }
Пример #2
0
        private void WriteValue <TValue>(InMemoryBinaryWriter memoryBuffer, long basePosition, TVersionedItem <TValue> item, byte[] value)
        {
            // Deleted items don't have values.  Only serialize valid items.
            if (item.Kind != RecordKind.DeletedVersion)
            {
                // WriteItemAsync valueSerializer followed by checksum.
                // Serialize the value.
                var valueStartPosition = memoryBuffer.BaseStream.Position;
                memoryBuffer.Write(value);
                var valueEndPosition = memoryBuffer.BaseStream.Position;
                Diagnostics.Assert(
                    valueEndPosition >= valueStartPosition,
                    DifferentialStoreConstants.TraceType,
                    "User's value IStateSerializer moved the stream position backwards unexpectedly!");

                // Write the checksum of just that value's bytes.
                var valueSize = checked ((int)(valueEndPosition - valueStartPosition));
                var checksum  = CRC64.ToCRC64(memoryBuffer.BaseStream.GetBuffer(), checked ((int)valueStartPosition), valueSize);

                // Update the in-memory offset and size for this item.
                item.Offset        = basePosition + valueStartPosition;
                item.ValueSize     = valueSize;
                item.ValueChecksum = checksum;

                // Update checkpoint file in-memory metadata.
                this.Properties.ValueCount++;
            }

            // Update the in-memory metadata about which file this key-value exists in on disk.
            item.FileId = this.FileId;
        }
Пример #3
0
        private IReliableDictionary <string, byte[]> GetReminderDictionary(ActorId actorId)
        {
            var bytes      = Encoding.UTF8.GetBytes(actorId.GetStorageKey());
            var storageIdx = CRC64.ToCRC64(bytes) % (ulong)this.reminderDictionaries.Length;

            return(this.reminderDictionaries[storageIdx]);
        }
Пример #4
0
        public async Task WriteAsync()
        {
            var filePath = Path.Combine(this.directory, this.FileName);

            using (var stream = FabricFile.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.SequentialScan))
            {
                var versionArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(FileVersion));

                if (this.CurrentCheckpoint != null)
                {
                    ArraySegment <byte> currentCheckpointFileNameArraySegment;
                    using (var writer = new InMemoryBinaryWriter(new MemoryStream()))
                    {
                        writer.Write(this.CurrentCheckpoint.FileName);
                        currentCheckpointFileNameArraySegment = new ArraySegment <byte>(writer.BaseStream.GetBuffer(), 0, (int)writer.BaseStream.Position);
                    }

                    var currentCheckpointFileNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(currentCheckpointFileNameArraySegment.Count));

                    var crc             = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointFileNameLengthArraySegment, currentCheckpointFileNameArraySegment });
                    var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc));

                    await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false);

                    await
                    stream.WriteAsync(
                        currentCheckpointFileNameLengthArraySegment.Array,
                        currentCheckpointFileNameLengthArraySegment.Offset,
                        currentCheckpointFileNameLengthArraySegment.Count).ConfigureAwait(false);

                    await
                    stream.WriteAsync(
                        currentCheckpointFileNameArraySegment.Array,
                        currentCheckpointFileNameArraySegment.Offset,
                        currentCheckpointFileNameArraySegment.Count).ConfigureAwait(false);

                    await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false);
                }
                else
                {
                    var currentCheckpointNameLengthArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(0));

                    var crc             = CRC64.ToCRC64(new[] { versionArraySegment, currentCheckpointNameLengthArraySegment });
                    var crcArraySegment = new ArraySegment <byte>(BitConverter.GetBytes(crc));

                    await stream.WriteAsync(versionArraySegment.Array, versionArraySegment.Offset, versionArraySegment.Count).ConfigureAwait(false);

                    await
                    stream.WriteAsync(
                        currentCheckpointNameLengthArraySegment.Array,
                        currentCheckpointNameLengthArraySegment.Offset,
                        currentCheckpointNameLengthArraySegment.Count).ConfigureAwait(false);

                    await stream.WriteAsync(crcArraySegment.Array, crcArraySegment.Offset, crcArraySegment.Count).ConfigureAwait(false);
                }
            }
        }
Пример #5
0
        private static async Task CopyListElementsFrameAsync(Stream inputStream, Stream outputStream, string traceType)
        {
            var listElementsCountSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
            var listElementsBytesSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
            var crcSegment = new ArraySegment <byte>(new byte[sizeof(ulong)]);

            var listElementsCount = await SerializationHelper.ReadIntAsync(listElementsCountSegment, inputStream).ConfigureAwait(false);

            if (listElementsCount < 0)
            {
                var exc = new InvalidDataException(string.Format("CheckpointFileHelper.CopyListElementsFrameAsync : Unexpected listElementsCount: {0}", listElementsCount));
                FabricEvents.Events.ReliableConcurrentQueue_ExceptionError(traceType, exc.ToString());
                throw exc;
            }

            var listElementsBytes = await SerializationHelper.ReadIntAsync(listElementsBytesSegment, inputStream).ConfigureAwait(false);

            if (listElementsBytes < 0)
            {
                var exc = new InvalidDataException(string.Format("CheckpointFileHelper.CopyListElementsFrameAsync : Unexpected listElementsBytes: {0}", listElementsBytes));
                FabricEvents.Events.ReliableConcurrentQueue_ExceptionError(traceType, exc.ToString());
                throw exc;
            }

            var listElementsSegment = new ArraySegment <byte>(new byte[listElementsBytes]);
            await SerializationHelper.ReadBytesAsync(listElementsSegment, listElementsBytes, inputStream).ConfigureAwait(false);

            await SerializationHelper.ReadBytesAsync(crcSegment, sizeof(ulong), inputStream).ConfigureAwait(false);

            var readCRC = BitConverter.ToUInt64(crcSegment.Array, crcSegment.Offset);

            var calcCrc = CRC64.ToCRC64(new[] { listElementsCountSegment, listElementsBytesSegment, listElementsSegment });

            if (calcCrc != readCRC)
            {
                var exc = new InvalidDataException(string.Format("CheckpointFileHelper.CopyListElementsFrameAsync => CRC mismatch.  Read: {0} Calculated: {1}", readCRC, calcCrc));
                FabricEvents.Events.ReliableConcurrentQueue_ExceptionError(traceType, exc.ToString());
                throw exc;
            }

            await outputStream.WriteAsync(listElementsCountSegment.Array, listElementsCountSegment.Offset, listElementsCountSegment.Count).ConfigureAwait(false);

            await outputStream.WriteAsync(listElementsBytesSegment.Array, listElementsBytesSegment.Offset, listElementsBytesSegment.Count).ConfigureAwait(false);

            await outputStream.WriteAsync(listElementsSegment.Array, listElementsSegment.Offset, listElementsSegment.Count).ConfigureAwait(false);

            await outputStream.WriteAsync(crcSegment.Array, crcSegment.Offset, crcSegment.Count).ConfigureAwait(false);
        }
Пример #6
0
        public static async Task <IEnumerable <IListElement <T> > > ReadAsync(Stream stream, IStateSerializer <T> stateSerializer, string traceType)
        {
            var listElementsCountSegment = new ArraySegment <byte>(new byte[sizeof(int)]);
            var listElementsBytesSegment = new ArraySegment <byte>(new byte[sizeof(int)]);

            var listElementsCount = await SerializationHelper.ReadIntAsync(listElementsCountSegment, stream).ConfigureAwait(false);

            if (listElementsCount < 0)
            {
                throw new InvalidDataException(string.Format("Unexpected listElementsCount: {0}", listElementsCount));
            }

            var listElementsBytes = await SerializationHelper.ReadIntAsync(listElementsBytesSegment, stream).ConfigureAwait(false);

            if (listElementsBytes < 0)
            {
                throw new InvalidDataException(string.Format("Unexpected listElementsBytes: {0}", listElementsBytes));
            }

            using (var reader = new InMemoryBinaryReader(new MemoryStream()))
            {
                reader.BaseStream.SetLength(listElementsBytes + sizeof(ulong));
                await
                SerializationHelper.ReadBytesAsync(new ArraySegment <byte>(reader.BaseStream.GetBuffer()), listElementsBytes + sizeof(ulong), stream)
                .ConfigureAwait(false);

                var listElements = new IListElement <T> [listElementsCount];
                for (var i = 0; i < listElementsCount; i++)
                {
                    var id    = reader.ReadInt64();
                    var value = stateSerializer.Read(reader); // if this tries to read beyond the end of the stream, listElementsBytes was incorrect (too small)
                    listElements[i] = DataStore <T> .CreateQueueListElement(id, value, traceType, ListElementState.EnqueueApplied);
                }

                var readCRC = reader.ReadUInt64();
                var calcCRC =
                    CRC64.ToCRC64(new[] { listElementsCountSegment, listElementsBytesSegment, new ArraySegment <byte>(reader.BaseStream.GetBuffer(), 0, listElementsBytes), });
                if (readCRC != calcCRC)
                {
                    throw new InvalidDataException(string.Format("CRC mismatch.  Read: {0} Calculated: {1}", readCRC, calcCRC));
                }

                return(listElements);
            }
        }
Пример #7
0
        public async Task <byte[]> ReadValueAsync <TValue>(TVersionedItem <TValue> item)
        {
            Stream fileStream = null;

            try
            {
                // Acquire a re-usable file stream for exclusive use during this read.
                fileStream = this.ReaderPool.AcquireStream();

                var snapFileStream = fileStream as FileStream;
                Diagnostics.Assert(snapFileStream != null, this.traceType, "fileStream must be a FileStream");
                Microsoft.ServiceFabric.Replicator.Utility.SetIoPriorityHint(snapFileStream.SafeFileHandle, this.priorityHint);

                // TODO: use memory stream pool here.
                using (var stream = new MemoryStream(capacity: item.ValueSize))
                {
                    // Read the value bytes and the checksum into memory.
                    fileStream.Position = item.Offset;
                    stream.SetLength(item.ValueSize);
                    await fileStream.ReadAsync(stream.GetBuffer(), 0, item.ValueSize).ConfigureAwait(false);

                    // Read the checksum from memory.
                    stream.Position = item.ValueSize;
                    var checksum = item.ValueChecksum;

                    // Re-compute the checksum.
                    var expectedChecksum = CRC64.ToCRC64(stream.GetBuffer(), 0, item.ValueSize);
                    if (checksum != expectedChecksum)
                    {
                        throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FailedReadValue_ChecksumMismatch_TwoArgs, checksum, expectedChecksum));
                    }

                    return(stream.GetBuffer());
                }
            }
            finally
            {
                // Return the file stream to the pool for re-use.
                this.ReaderPool.ReleaseStream(fileStream);
            }
        }
Пример #8
0
        public async Task WriteAsync(Stream stream)
        {
            // chunk is full, write it to the stream
            var listElementsCountSegment = new ArraySegment <byte>(BitConverter.GetBytes(this.ListElementsCount));
            var listElementsBytesSegment = new ArraySegment <byte>(BitConverter.GetBytes(this.listElementsBytes));

            var crc =
                CRC64.ToCRC64(
                    new[]
            {
                listElementsCountSegment,
                listElementsBytesSegment,
                new ArraySegment <byte>(this.writer.BaseStream.GetBuffer(), 0, (int)this.writer.BaseStream.Position),
            });

            // write the crc to the end of the values memorystream to avoid the extra stream write
            this.writer.Write(crc);

            await stream.WriteAsync(listElementsCountSegment.Array, listElementsCountSegment.Offset, listElementsCountSegment.Count).ConfigureAwait(false);

            await stream.WriteAsync(listElementsBytesSegment.Array, listElementsBytesSegment.Offset, listElementsBytesSegment.Count).ConfigureAwait(false);

            await stream.WriteAsync(this.writer.BaseStream.GetBuffer(), 0, (int)this.writer.BaseStream.Position).ConfigureAwait(false);
        }
Пример #9
0
 internal static int ComputeIdWithCRC(string typeName)
 {
     return((int)CRC64.ToCRC64(Encoding.UTF8.GetBytes(typeName)));
 }
Пример #10
0
        /// <summary>
        /// Read the given value from disk.
        /// </summary>
        /// <typeparam name="TValue"></typeparam>
        /// <param name="item"></param>
        /// <param name="valueSerializer"></param>
        /// <returns></returns>
        public async Task <TValue> ReadValueAsync <TValue>(TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer)
        {
            // Validate the item has a value that can be read from disk.
            if (item == null)
            {
                throw new ArgumentNullException(SR.Error_Item);
            }
            if (item.Kind == RecordKind.DeletedVersion)
            {
                throw new ArgumentException(SR.Error_ValueCheckpoint_DeletedItemValue, SR.Error_Item);
            }

            // Validate that the item's disk properties are valid.
            if (item.Offset < this.Properties.ValuesHandle.Offset)
            {
                throw new ArgumentOutOfRangeException(SR.Error_Item, SR.Error_ValueCheckpoint_TVersionedItem_Offset_Negative);
            }
            if (item.ValueSize < 0)
            {
                throw new ArgumentOutOfRangeException(string.Format(CultureInfo.CurrentCulture, SR.Error_Item, SR.Error_ValueCheckpoint_TVersionedItem_ValueSize_Negative, item.ValueSize));
            }
            if (item.Offset + item.ValueSize > this.Properties.ValuesHandle.EndOffset)
            {
                throw new ArgumentOutOfRangeException(SR.Error_Item, SR.Error_ValueCheckpoint_StreamRangeExceeded);
            }

            Stream fileStream = null;

            try
            {
                // Acquire a re-usable file stream for exclusive use during this read.
                fileStream = this.ReaderPool.AcquireStream();

                var snapFileStream = fileStream as FileStream;
                Diagnostics.Assert(snapFileStream != null, this.traceType, "fileStream must be a FileStream");
                Microsoft.ServiceFabric.Replicator.Utility.SetIoPriorityHint(snapFileStream.SafeFileHandle, this.priorityHint);

                // TODO: use memory stream pool here.
                using (var stream = new MemoryStream(capacity: item.ValueSize))
                    using (var reader = new BinaryReader(stream))
                    {
                        // Read the value bytes and the checksum into memory.
                        fileStream.Position = item.Offset;
                        stream.SetLength(item.ValueSize);
                        await fileStream.ReadAsync(stream.GetBuffer(), 0, item.ValueSize).ConfigureAwait(false);

                        // Read the checksum from memory.
                        stream.Position = item.ValueSize;
                        var checksum = item.ValueChecksum;

                        // Re-compute the checksum.
                        var expectedChecksum = CRC64.ToCRC64(stream.GetBuffer(), 0, item.ValueSize);
                        if (checksum != expectedChecksum)
                        {
                            throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FailedReadValue_ChecksumMismatch_TwoArgs, checksum, expectedChecksum));
                        }

                        // Deserialize the value into memory.
                        stream.Position = 0;
                        return(valueSerializer.Read(reader));
                    }
            }
            finally
            {
                // Return the file stream to the pool for re-use.
                this.ReaderPool.ReleaseStream(fileStream);
            }
        }
        // TODO: 3.6 - Receiver Side Distribution
        public async Task <string> OpenAsync(CancellationToken cancellationToken)
        {
            endpointConfiguration = new EndpointConfiguration("back-stateful");
            var persistence = endpointConfiguration.UsePersistence <ServiceFabricPersistence>();

            persistence.StateManager(stateManager);

            #region Not Important

            endpointConfiguration.SendFailedMessagesTo("error");
            endpointConfiguration.AuditProcessedMessagesTo("audit");
            endpointConfiguration.UseSerialization <JsonSerializer>();
            endpointConfiguration.EnableInstallers();


            var recoverability = endpointConfiguration.Recoverability();
            recoverability.DisableLegacyRetriesSatellite();
            // for demo purposes
            recoverability.Immediate(d => d.NumberOfRetries(5));
            recoverability.Delayed(d => d.NumberOfRetries(0));

            var connectionString = context.GetTransportConnectionString();

            var transport = endpointConfiguration.UseTransport <RabbitMQTransport>();
            transport.ConnectionString(connectionString);

            var delayedDelivery = transport.DelayedDelivery();
            delayedDelivery.DisableTimeoutManager();

            var routing = transport.Routing();
            routing.RouteToEndpoint(typeof(UpdateOrderColdStorage), "back-cold");

            endpointConfiguration.Notifications.Errors.MessageHasFailedAnImmediateRetryAttempt += (sender, args) =>
            {
                args.Headers.TryGetValue(Headers.ProcessingEndpoint, out var endpointName);
                ServiceEventSource.Current.ServiceMessage(context, "{0} - {1}", endpointName ?? "back-stateful", args.Exception.Message);
            };

            var assemblyScanner = endpointConfiguration.AssemblyScanner();
            assemblyScanner.ExcludeAssemblies("netstandard");

            #endregion

            var partitionInfo =
                await ServicePartitionQueryHelper.QueryServicePartitions(context.ServiceName, context.PartitionId);

            endpointConfiguration.RegisterPartitionsForThisEndpoint(partitionInfo.LocalPartitionKey?.ToString(), partitionInfo.Partitions.Select(k => k.LowKey.ToString()).ToArray());

            endpointConfiguration.SendHeartbeatTo(
                serviceControlQueue: "Particular.ServiceControl.Rabbit",
                frequency: TimeSpan.FromSeconds(5),
                timeToLive: TimeSpan.FromSeconds(15));

            var instanceId = partitionInfo.LocalPartitionKey.HasValue ? $"back-stateful-{partitionInfo.LocalPartitionKey}" : "back-stateful";

            var hostInfo = endpointConfiguration.UniquelyIdentifyRunningInstance();
            hostInfo.UsingCustomDisplayName(instanceId);
            hostInfo.UsingCustomIdentifier(DeterministicIdBuilder.ToGuid(instanceId));

            var metrics = endpointConfiguration.EnableMetrics();

            metrics.SendMetricDataToServiceControl(
                serviceControlMetricsAddress: "Particular.Monitoring.RabbitMQ",
                interval: TimeSpan.FromSeconds(5),
                instanceId: instanceId);

            string ConvertOrderIdToPartitionLowKey(Guid orderId)
            {
                var key = CRC64.ToCRC64(orderId.ToByteArray());

                var partition = partitionInfo.Partitions.Single(p => p.LowKey <= key && p.HighKey >= key);

                return(partition.LowKey.ToString());
            }

            var receiverSideDistribution = routing.EnableReceiverSideDistribution(partitionInfo.Partitions.Select(k => k.LowKey.ToString()).ToArray());
            receiverSideDistribution.AddPartitionMappingForMessageType <OrderAccepted>(msg => ConvertOrderIdToPartitionLowKey(msg.OrderId));
            receiverSideDistribution.AddPartitionMappingForMessageType <OrderCanceled>(msg => ConvertOrderIdToPartitionLowKey(msg.OrderId));
            receiverSideDistribution.AddPartitionMappingForMessageType <OrderCreated>(msg => ConvertOrderIdToPartitionLowKey(msg.OrderId));

            return(string.Empty);
        }
Пример #12
0
        /// <summary>
        /// Read all file metadata from the metadata file.
        /// </summary>
        /// <param name="metadataTable">The metadata table.</param>
        /// <param name="filestream">The file stream to read from.</param>
        /// <param name="properties">The metadata manager file properties.</param>
        /// <param name="traceType">Tracing information.</param>
        /// <returns></returns>
        private static async Task <int> ReadDiskMetadataAsync(
            Dictionary <uint, FileMetadata> metadataTable, Stream filestream, MetadataManagerFileProperties properties, string traceType)
        {
            var startOffset   = properties.MetadataHandle.Offset;
            var endOffset     = properties.MetadataHandle.EndOffset;
            var metadataCount = 0;
            var fileId        = 0;

            // No metadata to read (there are no metadata chunks).
            if (startOffset + sizeof(int) >= endOffset)
            {
                return(fileId);
            }

            filestream.Position = startOffset;

            using (var metadataStream = new MemoryStream(capacity: 64 * 1024))
                using (var metadataReader = new InMemoryBinaryReader(metadataStream))
                {
                    // Read the first key chunk size into memory.
                    metadataStream.SetLength(64 * 1024);
                    await filestream.ReadAsync(metadataStream.GetBuffer(), 0, PropertyChunkMetadata.Size).ConfigureAwait(false);

                    var propertyChunkMetadata = PropertyChunkMetadata.Read(metadataReader);
                    var chunkSize             = propertyChunkMetadata.BlockSize;
                    filestream.Position -= PropertyChunkMetadata.Size;

                    while (filestream.Position + chunkSize + sizeof(ulong) <= endOffset)
                    {
                        // Consistency checks.
                        if (chunkSize < 0)
                        {
                            throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_NegativeSize_OneArgs, chunkSize));
                        }

                        // Read the entire chunk (plus the checksum and next chunk size) into memory.
                        metadataStream.SetLength(chunkSize + sizeof(ulong) + sizeof(int));
                        await filestream.ReadAsync(metadataStream.GetBuffer(), 0, chunkSize + sizeof(ulong) + sizeof(int)).ConfigureAwait(false);

                        // Read the checksum.
                        metadataStream.Position = chunkSize;
                        var checksum = metadataReader.ReadUInt64();

                        // Re-compute the checksum.
                        var expectedChecksum = CRC64.ToCRC64(metadataStream.GetBuffer(), 0, chunkSize);
                        if (checksum != expectedChecksum)
                        {
                            throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_ChecksumMismatch_TwoArgs, checksum, expectedChecksum));
                        }

                        // Deserialize the value into memory.
                        metadataStream.Position = sizeof(int);
                        metadataReader.ReadPaddingUntilAligned(true);
                        while (metadataStream.Position < chunkSize)
                        {
                            var fileMetadata = FileMetadata.Read(metadataReader, traceType);
                            if (metadataTable.ContainsKey(fileMetadata.FileId))
                            {
                                throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_DuplicateFileId_Found_OneArgs, fileMetadata.FileId));
                            }

                            metadataTable.Add(fileMetadata.FileId, fileMetadata);
                            metadataCount++;
                        }

                        // Read the next chunk size.
                        chunkSize            = BitConverter.ToInt32(metadataStream.GetBuffer(), chunkSize + sizeof(ulong));
                        filestream.Position -= sizeof(int);
                    }

                    // Consistency checks.
                    if (filestream.Position != endOffset)
                    {
                        throw new InvalidDataException(SR.Error_Metadata_Corrupt_IncorrectSize);
                    }

                    if (metadataCount != properties.FileCount)
                    {
                        throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_Metadata_Corrupt_FileCountMismatch_TwoArgs, metadataCount, properties.FileCount));
                    }

                    return(fileId);
                }
        }
Пример #13
0
 public long GetPartitionKey()
 {
     return((long)CRC64.ToCRC64(this.id.ToByteArray()));
 }
        // TODO: 3.5 - Sender Side Distribution
        public static void AddNServiceBus(this IServiceCollection services)
        {
            var endpointConfiguration = new EndpointConfiguration("front-stateful");

            #region Not Important

            endpointConfiguration.UseSerialization <JsonSerializer>();
            endpointConfiguration.EnableInstallers();
            endpointConfiguration.SendOnly();

            var provider = services.BuildServiceProvider();
            var context  = provider.GetService <StatelessServiceContext>();

            var connectionString = context.GetTransportConnectionString();

            var transport = endpointConfiguration.UseTransport <RabbitMQTransport>();
            transport.ConnectionString(connectionString);

            var delayedDelivery = transport.DelayedDelivery();
            delayedDelivery.DisableTimeoutManager();

            var assemblyScanner = endpointConfiguration.AssemblyScanner();
            assemblyScanner.ExcludeAssemblies("netstandard");

            #endregion

            var routing      = transport.Routing();
            var backStateful = "back-stateful";
            routing.RouteToEndpoint(typeof(SubmitOrder), backStateful);
            routing.RouteToEndpoint(typeof(CancelOrder), backStateful);

            var uriBuilder     = new ServiceUriBuilder("Back_Stateful");
            var backServiceUri = uriBuilder.Build();

            var partitionInfo = ServicePartitionQueryHelper.QueryServicePartitions(backServiceUri, Guid.Empty).GetAwaiter().GetResult();

            endpointConfiguration.SendHeartbeatTo(
                serviceControlQueue: "Particular.ServiceControl.Rabbit",
                frequency: TimeSpan.FromSeconds(5),
                timeToLive: TimeSpan.FromSeconds(15));

            var instanceId = partitionInfo.LocalPartitionKey.HasValue ? $"front-stateful-{partitionInfo.LocalPartitionKey}" : "front-stateful";

            var hostInfo = endpointConfiguration.UniquelyIdentifyRunningInstance();
            hostInfo.UsingCustomDisplayName(instanceId);
            hostInfo.UsingCustomIdentifier(DeterministicIdBuilder.ToGuid(instanceId));

            string ConvertOrderIdToPartitionLowKey(Guid orderId)
            {
                var key = CRC64.ToCRC64(orderId.ToByteArray());

                var partition = partitionInfo.Partitions.Single(p => p.LowKey <= key && p.HighKey >= key);

                return(partition.LowKey.ToString());
            }

            var senderSideDistribution =
                routing.RegisterPartitionedDestinationEndpoint(backStateful,
                                                               partitionInfo.Partitions.Select(k => k.LowKey.ToString()).ToArray());

            senderSideDistribution.AddPartitionMappingForMessageType <SubmitOrder>(msg => ConvertOrderIdToPartitionLowKey(msg.OrderId));
            senderSideDistribution.AddPartitionMappingForMessageType <CancelOrder>(msg => ConvertOrderIdToPartitionLowKey(msg.OrderId));

            var endpointInstance = Endpoint.Start(endpointConfiguration).GetAwaiter().GetResult();
            services.AddSingleton <IMessageSession>(endpointInstance);
        }