public override StreamPosition GetStreamPosition(EventData queueMessage) { IStreamIdentity stremIdentity = new StreamIdentity(partitionStreamGuid, null); StreamSequenceToken token = new EventSequenceTokenV2(queueMessage.SequenceNumber, 0); return(new StreamPosition(stremIdentity, token)); }
public static KafkaBatchContainer ToBatchContainer( this ConsumeResult <byte[], byte[]> result, SerializationManager serializationManager, QueueProperties queueProperties ) { var sequence = new EventSequenceTokenV2(result.Offset.Value); if (queueProperties.IsExternal) { var key = Encoding.UTF8.GetString(result.Key); return(new KafkaBatchContainer( StreamProviderUtils.GenerateStreamGuid(key), queueProperties.Namespace, new List <object> { Encoding.UTF8.GetString(result.Value) }, null, isExternalBatch: true, sequence, result.TopicPartitionOffset )); } var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Value); if (batchContainer.SequenceToken == null) { batchContainer.SequenceToken = sequence; } batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset; return(batchContainer); }
public GeneratedBatchContainer(StreamId streamId, object payload, EventSequenceTokenV2 token) { StreamId = streamId; EnqueueTimeUtc = DateTime.UtcNow; this.Payload = payload; this.RealToken = token; }
public StreamPosition GetStreamPosition(TestQueueMessage queueMessage) { IStreamIdentity streamIdentity = new StreamIdentity(queueMessage.StreamGuid, queueMessage.StreamNamespace); StreamSequenceToken sequenceToken = new EventSequenceTokenV2(queueMessage.SequenceNumber); return(new StreamPosition(streamIdentity, sequenceToken)); }
internal AzureQueueBatchContainerV2( Guid streamGuid, string streamNamespace, List <object> events, Dictionary <string, object> requestContext, EventSequenceTokenV2 sequenceToken) : base(streamGuid, streamNamespace, events, requestContext, sequenceToken) { }
private SQSBatchContainer( StreamId streamId, List <object> events, Dictionary <string, object> requestContext, EventSequenceTokenV2 sequenceToken) : this(streamId, events, requestContext) { this.sequenceToken = sequenceToken; }
public StreamTypeSerializationTests() { // FakeSerializer definied in ExternalSerializerTest.cs SerializationManager.InitializeForTesting(new List <TypeInfo> { typeof(FakeSerializer).GetTypeInfo() }); EventSequenceTokenV2.Register(); AzureQueueBatchContainerV2.Register(); }
public AzureQueueBatchContainerV2( StreamId streamId, List <object> events, Dictionary <string, object> requestContext, EventSequenceTokenV2 sequenceToken) : this(streamId, events, requestContext) { this.sequenceToken = sequenceToken; }
private SQSBatchContainer( Guid streamGuid, String streamNamespace, List <object> events, Dictionary <string, object> requestContext, EventSequenceTokenV2 sequenceToken) : this(streamGuid, streamNamespace, events, requestContext) { this.sequenceToken = sequenceToken; }
public override StreamPosition GetStreamPosition(EventData queueMessage) { IStreamIdentity stremIdentity = new StreamIdentity(partitionStreamGuid, null); StreamSequenceToken token = #if NETSTANDARD new EventSequenceTokenV2(queueMessage.SystemProperties.SequenceNumber, 0); #else new EventSequenceTokenV2(queueMessage.SequenceNumber, 0); #endif return(new StreamPosition(stremIdentity, token)); }
public KafkaExternalBatchContainer( Guid streamGuid, QueueProperties queueProps, List <byte[]> events, EventSequenceTokenV2 streamSequenceToken, TopicPartitionOffset offset, IExternalStreamDeserializer deserializer ) : base(streamGuid, queueProps.Namespace, events.Cast <object>().ToList(), null, streamSequenceToken, offset) { _queueProps = queueProps; _deserializer = deserializer; }
public KafkaBatchContainer( Guid streamGuid, string streamNamespace, List <object> events, Dictionary <string, object> requestContext, bool isExternalBatch, EventSequenceTokenV2 streamSequenceToken, TopicPartitionOffset offset ) : this(streamGuid, streamNamespace, events, requestContext, isExternalBatch) { SequenceToken = streamSequenceToken; TopicPartitionOffSet = offset; }
public void EventSequenceTokenV2_Serialize_IfNotNull() { var writer = new BinaryTokenStreamWriter(); var token = new EventSequenceTokenV2(long.MaxValue, int.MaxValue); EventSequenceTokenV2.Serialize(token, writer, null); var reader = new BinaryTokenStreamReader(writer.ToByteArray()); var deserialized = EventSequenceTokenV2.Deserialize(typeof(EventSequenceTokenV2), reader) as EventSequenceTokenV2; Assert.NotNull(deserialized); Assert.NotSame(token, deserialized); Assert.Equal(token.EventIndex, deserialized.EventIndex); Assert.Equal(token.SequenceNumber, deserialized.SequenceNumber); }
public KafkaBatchContainer( Guid streamGuid, string streamNamespace, List <object> events, Dictionary <string, object> requestContext, EventSequenceTokenV2 streamSequenceToken, TopicPartitionOffset offset ) { Events = events ?? throw new ArgumentNullException(nameof(events), "Message contains no events."); StreamGuid = streamGuid; StreamNamespace = streamNamespace; SequenceToken = streamSequenceToken; TopicPartitionOffSet = offset; _requestContext = requestContext; }
/// <summary> /// Gets the stream position from a queue message /// </summary> /// <param name="queueMessage"></param> /// <returns></returns> public virtual StreamPosition GetStreamPosition(EventData queueMessage) { Guid streamGuid = #if NETSTANDARD Guid.Parse(queueMessage.SystemProperties.PartitionKey); #else Guid.Parse(queueMessage.PartitionKey); #endif string streamNamespace = queueMessage.GetStreamNamespaceProperty(); IStreamIdentity stremIdentity = new StreamIdentity(streamGuid, streamNamespace); StreamSequenceToken token = #if NETSTANDARD new EventSequenceTokenV2(queueMessage.SystemProperties.SequenceNumber, 0); #else new EventSequenceTokenV2(queueMessage.SequenceNumber, 0); #endif return(new StreamPosition(stremIdentity, token)); }
public static KafkaBatchContainer ToBatchContainer( this ConsumeResult <byte[], byte[]> result, SerializationManager serializationManager, KafkaStreamOptions options, string streamNamespace ) { var externalHeader = result.Headers.FirstOrDefault(header => header.Key == options.ExternalMessageIdentifier); var sequence = new EventSequenceTokenV2(result.Offset.Value); if (externalHeader != null) { var isExternal = BitConverter.ToBoolean(externalHeader.GetValueBytes(), 0); if (isExternal) { var key = Encoding.UTF8.GetString(result.Key); return(new KafkaBatchContainer( StreamProviderUtils.GenerateStreamGuid(key), streamNamespace, new List <object> { Encoding.UTF8.GetString(result.Value) }, null, isExternalBatch: true, sequence, result.TopicPartitionOffset )); } } var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Value); if (batchContainer.SequenceToken == null) { batchContainer.SequenceToken = sequence; } batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset; return(batchContainer); }
public void EventSequenceTokenV2_DeepCopy_IfNotNull() { var token = new EventSequenceTokenV2(long.MaxValue, int.MaxValue); var copy = EventSequenceTokenV2.DeepCopy(token) as EventSequenceToken; Assert.NotNull(copy); Assert.NotSame(token, copy); Assert.Equal(token.EventIndex, copy.EventIndex); Assert.Equal(token.SequenceNumber, copy.SequenceNumber); var writer = new BinaryTokenStreamWriter(); SerializationManager.Serialize(token, writer); var bytes = writer.ToByteArray(); var reader = new BinaryTokenStreamReader(bytes); copy = SerializationManager.Deserialize(reader) as EventSequenceToken; Assert.NotNull(copy); Assert.NotSame(token, copy); Assert.Equal(token.EventIndex, copy.EventIndex); Assert.Equal(token.SequenceNumber, copy.SequenceNumber); }
public static KafkaBatchContainer ToBatchContainer( this ConsumeResult <byte[], byte[]> result, SerializationContext serializationContext, QueueProperties queueProperties ) { var sequence = new EventSequenceTokenV2(result.Offset.Value); if (queueProperties.IsExternal) { var key = Encoding.UTF8.GetString(result.Message.Key); var message = serializationContext .ExternalStreamDeserializer .Deserialize(queueProperties, queueProperties.ExternalContractType, result.Message.Value); return(new KafkaBatchContainer( StreamProviderUtils.GenerateStreamGuid(key), queueProperties.Namespace, new List <object> { message }, null, sequence, result.TopicPartitionOffset )); } var serializationManager = serializationContext.SerializationManager; var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Message.Value); batchContainer.SequenceToken ??= sequence; batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset; return(batchContainer); }
private Tuple <T, StreamSequenceToken> DeserializeExternalEvent <T>( EventSequenceTokenV2 sequenceToken, object @event, int iteration ) { T message; var messageType = typeof(T); if (messageType == typeof(byte[])) { message = (T)@event; } else if (messageType.IsPrimitive || messageType == typeof(string) || messageType == typeof(decimal)) { message = (T)Convert.ChangeType(@event, typeof(T)); } else { message = _deserializer.Deserialize <T>(_queueProps, (byte[])@event); } return(Tuple.Create <T, StreamSequenceToken>(message, sequenceToken.CreateSequenceTokenForEvent(iteration))); }
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory, IProviderConfiguration config) { IQueueAdapter adapter = await adapterFactory.CreateAdapter(); IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache(); // Create receiver per queue IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper(); Dictionary <QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver); Dictionary <QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache); await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5)))); // test using 2 streams Guid streamId1 = Guid.NewGuid(); Guid streamId2 = Guid.NewGuid(); int receivedBatches = 0; var streamsPerQueue = new ConcurrentDictionary <QueueId, HashSet <IStreamIdentity> >(); // reader threads (at most 2 active queues because only two streams) var work = new List <Task>(); foreach (KeyValuePair <QueueId, IQueueAdapterReceiver> receiverKvp in receivers) { QueueId queueId = receiverKvp.Key; var receiver = receiverKvp.Value; var qCache = caches[queueId]; Task task = Task.Factory.StartNew(() => { while (receivedBatches < NumBatches) { var messages = receiver.GetQueueMessagesAsync(CloudQueueMessage.MaxNumberOfMessagesToPeek).Result.ToArray(); if (!messages.Any()) { continue; } foreach (IBatchContainer message in messages) { streamsPerQueue.AddOrUpdate(queueId, id => new HashSet <IStreamIdentity> { new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()) }, (id, set) => { set.Add(new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString())); return(set); }); output.WriteLine("Queue {0} received message on stream {1}", queueId, message.StreamGuid); Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <int>().Count()); // "Half the events were ints" Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <string>().Count()); // "Half the events were strings" } Interlocked.Add(ref receivedBatches, messages.Length); qCache.AddToCache(messages); } }); work.Add(task); } // send events List <object> events = CreateEvents(NumMessagesPerBatch); work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches) .Select(i => i % 2 == 0 ? streamId1 : streamId2) .ToList() .ForEach(streamId => adapter.QueueMessageBatchAsync(streamId, streamId.ToString(), events.Take(NumMessagesPerBatch).ToArray(), null, RequestContextExtensions.Export(this.fixture.SerializationManager)).Wait()))); await Task.WhenAll(work); // Make sure we got back everything we sent Assert.Equal(NumBatches, receivedBatches); // check to see if all the events are in the cache and we can enumerate through them StreamSequenceToken firstInCache = new EventSequenceTokenV2(0); foreach (KeyValuePair <QueueId, HashSet <IStreamIdentity> > kvp in streamsPerQueue) { var receiver = receivers[kvp.Key]; var qCache = caches[kvp.Key]; foreach (IStreamIdentity streamGuid in kvp.Value) { // read all messages in cache for stream IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache); int messageCount = 0; StreamSequenceToken tenthInCache = null; StreamSequenceToken lastToken = firstInCache; while (cursor.MoveNext()) { Exception ex; messageCount++; IBatchContainer batch = cursor.GetCurrent(out ex); output.WriteLine("Token: {0}", batch.SequenceToken); Assert.True(batch.SequenceToken.CompareTo(lastToken) >= 0, $"order check for event {messageCount}"); lastToken = batch.SequenceToken; if (messageCount == 10) { tenthInCache = batch.SequenceToken; } } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); Assert.Equal(NumBatches / 2, messageCount); Assert.NotNull(tenthInCache); // read all messages from the 10th cursor = qCache.GetCacheCursor(streamGuid, tenthInCache); messageCount = 0; while (cursor.MoveNext()) { messageCount++; } output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid); const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1) Assert.Equal(expected, messageCount); } } }
private void SetValues(Guid streamGuid, string streamNamespace, List <object> events, Dictionary <string, object> requestContext, EventSequenceTokenV2 sequenceToken) { this.StreamGuid = streamGuid; this.StreamNamespace = streamNamespace; this.events = events; this.requestContext = requestContext; this.sequenceToken = sequenceToken; }
public GeneratedBatchContainer(Guid streamGuid, string streamNamespace, object payload, EventSequenceTokenV2 token) { StreamGuid = streamGuid; StreamNamespace = streamNamespace; this.Payload = payload; this.RealToken = token; }
private StreamPosition GetStreamPosition(TestQueueMessage queueMessage) { StreamSequenceToken sequenceToken = new EventSequenceTokenV2(queueMessage.SequenceNumber); return(new StreamPosition(queueMessage.StreamId, sequenceToken)); }
public int Compare(TestCachedMessage cachedMessage, StreamSequenceToken token) { var myToken = new EventSequenceTokenV2(cachedMessage.SequenceNumber, cachedMessage.EventIndex); return(myToken.CompareTo(token)); }
private IEnumerable <Tuple <T, StreamSequenceToken> > SerializeExternalEvents <T>(EventSequenceTokenV2 sequenceToken) { var serializedEvents = _events .Select((@event, iteration) => { try { T message; var messageType = typeof(T); if (messageType.IsPrimitive || messageType == typeof(string) || messageType == typeof(decimal)) { message = (T)Convert.ChangeType(@event, typeof(T)); } else { message = JsonConvert .DeserializeObject <T>((string)@event); // todo: support for multiple serializer } return(Tuple.Create <T, StreamSequenceToken>(message, sequenceToken.CreateSequenceTokenForEvent(iteration))); } catch (Exception) { return(null); } }) .Where(@event => @event != null); return(serializedEvents); }