public override StreamPosition GetStreamPosition(EventData queueMessage)
{
IStreamIdentity stremIdentity = new StreamIdentity(partitionStreamGuid, null);
StreamSequenceToken token = new EventSequenceTokenV2(queueMessage.SequenceNumber, 0);
return(new StreamPosition(stremIdentity, token));
}
public static KafkaBatchContainer ToBatchContainer(
this ConsumeResult <byte[], byte[]> result,
SerializationManager serializationManager,
QueueProperties queueProperties
)
{
var sequence = new EventSequenceTokenV2(result.Offset.Value);
if (queueProperties.IsExternal)
{
var key = Encoding.UTF8.GetString(result.Key);
return(new KafkaBatchContainer(
StreamProviderUtils.GenerateStreamGuid(key),
queueProperties.Namespace,
new List <object> {
Encoding.UTF8.GetString(result.Value)
},
null,
isExternalBatch: true,
sequence,
result.TopicPartitionOffset
));
}
var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Value);
if (batchContainer.SequenceToken == null)
{
batchContainer.SequenceToken = sequence;
}
batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset;
return(batchContainer);
}
public GeneratedBatchContainer(StreamId streamId, object payload, EventSequenceTokenV2 token)
{
StreamId = streamId;
EnqueueTimeUtc = DateTime.UtcNow;
this.Payload = payload;
this.RealToken = token;
}
public StreamPosition GetStreamPosition(TestQueueMessage queueMessage)
{
IStreamIdentity streamIdentity = new StreamIdentity(queueMessage.StreamGuid, queueMessage.StreamNamespace);
StreamSequenceToken sequenceToken = new EventSequenceTokenV2(queueMessage.SequenceNumber);
return(new StreamPosition(streamIdentity, sequenceToken));
}
internal AzureQueueBatchContainerV2(
Guid streamGuid,
string streamNamespace,
List <object> events,
Dictionary <string, object> requestContext,
EventSequenceTokenV2 sequenceToken)
: base(streamGuid, streamNamespace, events, requestContext, sequenceToken)
{
}
private SQSBatchContainer(
StreamId streamId,
List <object> events,
Dictionary <string, object> requestContext,
EventSequenceTokenV2 sequenceToken)
: this(streamId, events, requestContext)
{
this.sequenceToken = sequenceToken;
}
public StreamTypeSerializationTests()
{
// FakeSerializer definied in ExternalSerializerTest.cs
SerializationManager.InitializeForTesting(new List <TypeInfo> {
typeof(FakeSerializer).GetTypeInfo()
});
EventSequenceTokenV2.Register();
AzureQueueBatchContainerV2.Register();
}
public AzureQueueBatchContainerV2(
StreamId streamId,
List <object> events,
Dictionary <string, object> requestContext,
EventSequenceTokenV2 sequenceToken)
: this(streamId, events, requestContext)
{
this.sequenceToken = sequenceToken;
}
private SQSBatchContainer(
Guid streamGuid,
String streamNamespace,
List <object> events,
Dictionary <string, object> requestContext,
EventSequenceTokenV2 sequenceToken)
: this(streamGuid, streamNamespace, events, requestContext)
{
this.sequenceToken = sequenceToken;
}
public override StreamPosition GetStreamPosition(EventData queueMessage)
{
IStreamIdentity stremIdentity = new StreamIdentity(partitionStreamGuid, null);
StreamSequenceToken token =
#if NETSTANDARD
new EventSequenceTokenV2(queueMessage.SystemProperties.SequenceNumber, 0);
#else
new EventSequenceTokenV2(queueMessage.SequenceNumber, 0);
#endif
return(new StreamPosition(stremIdentity, token));
}
public KafkaExternalBatchContainer(
Guid streamGuid,
QueueProperties queueProps,
List <byte[]> events,
EventSequenceTokenV2 streamSequenceToken,
TopicPartitionOffset offset,
IExternalStreamDeserializer deserializer
) : base(streamGuid, queueProps.Namespace, events.Cast <object>().ToList(), null, streamSequenceToken, offset)
{
_queueProps = queueProps;
_deserializer = deserializer;
}
public KafkaBatchContainer(
Guid streamGuid,
string streamNamespace,
List <object> events,
Dictionary <string, object> requestContext,
bool isExternalBatch,
EventSequenceTokenV2 streamSequenceToken,
TopicPartitionOffset offset
) : this(streamGuid, streamNamespace, events, requestContext, isExternalBatch)
{
SequenceToken = streamSequenceToken;
TopicPartitionOffSet = offset;
}
public void EventSequenceTokenV2_Serialize_IfNotNull()
{
var writer = new BinaryTokenStreamWriter();
var token = new EventSequenceTokenV2(long.MaxValue, int.MaxValue);
EventSequenceTokenV2.Serialize(token, writer, null);
var reader = new BinaryTokenStreamReader(writer.ToByteArray());
var deserialized = EventSequenceTokenV2.Deserialize(typeof(EventSequenceTokenV2), reader) as EventSequenceTokenV2;
Assert.NotNull(deserialized);
Assert.NotSame(token, deserialized);
Assert.Equal(token.EventIndex, deserialized.EventIndex);
Assert.Equal(token.SequenceNumber, deserialized.SequenceNumber);
}
public KafkaBatchContainer(
Guid streamGuid,
string streamNamespace,
List <object> events,
Dictionary <string, object> requestContext,
EventSequenceTokenV2 streamSequenceToken,
TopicPartitionOffset offset
)
{
Events = events ?? throw new ArgumentNullException(nameof(events), "Message contains no events.");
StreamGuid = streamGuid;
StreamNamespace = streamNamespace;
SequenceToken = streamSequenceToken;
TopicPartitionOffSet = offset;
_requestContext = requestContext;
}
/// <summary>
/// Gets the stream position from a queue message
/// </summary>
/// <param name="queueMessage"></param>
/// <returns></returns>
public virtual StreamPosition GetStreamPosition(EventData queueMessage)
{
Guid streamGuid =
#if NETSTANDARD
Guid.Parse(queueMessage.SystemProperties.PartitionKey);
#else
Guid.Parse(queueMessage.PartitionKey);
#endif
string streamNamespace = queueMessage.GetStreamNamespaceProperty();
IStreamIdentity stremIdentity = new StreamIdentity(streamGuid, streamNamespace);
StreamSequenceToken token =
#if NETSTANDARD
new EventSequenceTokenV2(queueMessage.SystemProperties.SequenceNumber, 0);
#else
new EventSequenceTokenV2(queueMessage.SequenceNumber, 0);
#endif
return(new StreamPosition(stremIdentity, token));
}
public static KafkaBatchContainer ToBatchContainer(
this ConsumeResult <byte[], byte[]> result,
SerializationManager serializationManager,
KafkaStreamOptions options,
string streamNamespace
)
{
var externalHeader = result.Headers.FirstOrDefault(header => header.Key == options.ExternalMessageIdentifier);
var sequence = new EventSequenceTokenV2(result.Offset.Value);
if (externalHeader != null)
{
var isExternal = BitConverter.ToBoolean(externalHeader.GetValueBytes(), 0);
if (isExternal)
{
var key = Encoding.UTF8.GetString(result.Key);
return(new KafkaBatchContainer(
StreamProviderUtils.GenerateStreamGuid(key),
streamNamespace,
new List <object> {
Encoding.UTF8.GetString(result.Value)
},
null,
isExternalBatch: true,
sequence,
result.TopicPartitionOffset
));
}
}
var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Value);
if (batchContainer.SequenceToken == null)
{
batchContainer.SequenceToken = sequence;
}
batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset;
return(batchContainer);
}
public void EventSequenceTokenV2_DeepCopy_IfNotNull()
{
var token = new EventSequenceTokenV2(long.MaxValue, int.MaxValue);
var copy = EventSequenceTokenV2.DeepCopy(token) as EventSequenceToken;
Assert.NotNull(copy);
Assert.NotSame(token, copy);
Assert.Equal(token.EventIndex, copy.EventIndex);
Assert.Equal(token.SequenceNumber, copy.SequenceNumber);
var writer = new BinaryTokenStreamWriter();
SerializationManager.Serialize(token, writer);
var bytes = writer.ToByteArray();
var reader = new BinaryTokenStreamReader(bytes);
copy = SerializationManager.Deserialize(reader) as EventSequenceToken;
Assert.NotNull(copy);
Assert.NotSame(token, copy);
Assert.Equal(token.EventIndex, copy.EventIndex);
Assert.Equal(token.SequenceNumber, copy.SequenceNumber);
}
public static KafkaBatchContainer ToBatchContainer(
this ConsumeResult <byte[], byte[]> result,
SerializationContext serializationContext,
QueueProperties queueProperties
)
{
var sequence = new EventSequenceTokenV2(result.Offset.Value);
if (queueProperties.IsExternal)
{
var key = Encoding.UTF8.GetString(result.Message.Key);
var message = serializationContext
.ExternalStreamDeserializer
.Deserialize(queueProperties, queueProperties.ExternalContractType, result.Message.Value);
return(new KafkaBatchContainer(
StreamProviderUtils.GenerateStreamGuid(key),
queueProperties.Namespace,
new List <object> {
message
},
null,
sequence,
result.TopicPartitionOffset
));
}
var serializationManager = serializationContext.SerializationManager;
var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Message.Value);
batchContainer.SequenceToken ??= sequence;
batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset;
return(batchContainer);
}
private Tuple <T, StreamSequenceToken> DeserializeExternalEvent <T>(
EventSequenceTokenV2 sequenceToken,
object @event,
int iteration
)
{
T message;
var messageType = typeof(T);
if (messageType == typeof(byte[]))
{
message = (T)@event;
}
else if (messageType.IsPrimitive || messageType == typeof(string) || messageType == typeof(decimal))
{
message = (T)Convert.ChangeType(@event, typeof(T));
}
else
{
message = _deserializer.Deserialize <T>(_queueProps, (byte[])@event);
}
return(Tuple.Create <T, StreamSequenceToken>(message, sequenceToken.CreateSequenceTokenForEvent(iteration)));
}
private async Task SendAndReceiveFromQueueAdapter(IQueueAdapterFactory adapterFactory, IProviderConfiguration config)
{
IQueueAdapter adapter = await adapterFactory.CreateAdapter();
IQueueAdapterCache cache = adapterFactory.GetQueueAdapterCache();
// Create receiver per queue
IStreamQueueMapper mapper = adapterFactory.GetStreamQueueMapper();
Dictionary <QueueId, IQueueAdapterReceiver> receivers = mapper.GetAllQueues().ToDictionary(queueId => queueId, adapter.CreateReceiver);
Dictionary <QueueId, IQueueCache> caches = mapper.GetAllQueues().ToDictionary(queueId => queueId, cache.CreateQueueCache);
await Task.WhenAll(receivers.Values.Select(receiver => receiver.Initialize(TimeSpan.FromSeconds(5))));
// test using 2 streams
Guid streamId1 = Guid.NewGuid();
Guid streamId2 = Guid.NewGuid();
int receivedBatches = 0;
var streamsPerQueue = new ConcurrentDictionary <QueueId, HashSet <IStreamIdentity> >();
// reader threads (at most 2 active queues because only two streams)
var work = new List <Task>();
foreach (KeyValuePair <QueueId, IQueueAdapterReceiver> receiverKvp in receivers)
{
QueueId queueId = receiverKvp.Key;
var receiver = receiverKvp.Value;
var qCache = caches[queueId];
Task task = Task.Factory.StartNew(() =>
{
while (receivedBatches < NumBatches)
{
var messages = receiver.GetQueueMessagesAsync(CloudQueueMessage.MaxNumberOfMessagesToPeek).Result.ToArray();
if (!messages.Any())
{
continue;
}
foreach (IBatchContainer message in messages)
{
streamsPerQueue.AddOrUpdate(queueId,
id => new HashSet <IStreamIdentity> {
new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString())
},
(id, set) =>
{
set.Add(new StreamIdentity(message.StreamGuid, message.StreamGuid.ToString()));
return(set);
});
output.WriteLine("Queue {0} received message on stream {1}", queueId,
message.StreamGuid);
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <int>().Count()); // "Half the events were ints"
Assert.Equal(NumMessagesPerBatch / 2, message.GetEvents <string>().Count()); // "Half the events were strings"
}
Interlocked.Add(ref receivedBatches, messages.Length);
qCache.AddToCache(messages);
}
});
work.Add(task);
}
// send events
List <object> events = CreateEvents(NumMessagesPerBatch);
work.Add(Task.Factory.StartNew(() => Enumerable.Range(0, NumBatches)
.Select(i => i % 2 == 0 ? streamId1 : streamId2)
.ToList()
.ForEach(streamId =>
adapter.QueueMessageBatchAsync(streamId, streamId.ToString(),
events.Take(NumMessagesPerBatch).ToArray(), null, RequestContextExtensions.Export(this.fixture.SerializationManager)).Wait())));
await Task.WhenAll(work);
// Make sure we got back everything we sent
Assert.Equal(NumBatches, receivedBatches);
// check to see if all the events are in the cache and we can enumerate through them
StreamSequenceToken firstInCache = new EventSequenceTokenV2(0);
foreach (KeyValuePair <QueueId, HashSet <IStreamIdentity> > kvp in streamsPerQueue)
{
var receiver = receivers[kvp.Key];
var qCache = caches[kvp.Key];
foreach (IStreamIdentity streamGuid in kvp.Value)
{
// read all messages in cache for stream
IQueueCacheCursor cursor = qCache.GetCacheCursor(streamGuid, firstInCache);
int messageCount = 0;
StreamSequenceToken tenthInCache = null;
StreamSequenceToken lastToken = firstInCache;
while (cursor.MoveNext())
{
Exception ex;
messageCount++;
IBatchContainer batch = cursor.GetCurrent(out ex);
output.WriteLine("Token: {0}", batch.SequenceToken);
Assert.True(batch.SequenceToken.CompareTo(lastToken) >= 0, $"order check for event {messageCount}");
lastToken = batch.SequenceToken;
if (messageCount == 10)
{
tenthInCache = batch.SequenceToken;
}
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
Assert.Equal(NumBatches / 2, messageCount);
Assert.NotNull(tenthInCache);
// read all messages from the 10th
cursor = qCache.GetCacheCursor(streamGuid, tenthInCache);
messageCount = 0;
while (cursor.MoveNext())
{
messageCount++;
}
output.WriteLine("On Queue {0} we received a total of {1} message on stream {2}", kvp.Key, messageCount, streamGuid);
const int expected = NumBatches / 2 - 10 + 1; // all except the first 10, including the 10th (10 + 1)
Assert.Equal(expected, messageCount);
}
}
}
private void SetValues(Guid streamGuid, string streamNamespace, List <object> events, Dictionary <string, object> requestContext, EventSequenceTokenV2 sequenceToken)
{
this.StreamGuid = streamGuid;
this.StreamNamespace = streamNamespace;
this.events = events;
this.requestContext = requestContext;
this.sequenceToken = sequenceToken;
}
public GeneratedBatchContainer(Guid streamGuid, string streamNamespace, object payload, EventSequenceTokenV2 token)
{
StreamGuid = streamGuid;
StreamNamespace = streamNamespace;
this.Payload = payload;
this.RealToken = token;
}
private StreamPosition GetStreamPosition(TestQueueMessage queueMessage)
{
StreamSequenceToken sequenceToken = new EventSequenceTokenV2(queueMessage.SequenceNumber);
return(new StreamPosition(queueMessage.StreamId, sequenceToken));
}
public int Compare(TestCachedMessage cachedMessage, StreamSequenceToken token)
{
var myToken = new EventSequenceTokenV2(cachedMessage.SequenceNumber, cachedMessage.EventIndex);
return(myToken.CompareTo(token));
}
private IEnumerable <Tuple <T, StreamSequenceToken> > SerializeExternalEvents <T>(EventSequenceTokenV2 sequenceToken)
{
var serializedEvents = _events
.Select((@event, iteration) =>
{
try
{
T message;
var messageType = typeof(T);
if (messageType.IsPrimitive || messageType == typeof(string) || messageType == typeof(decimal))
{
message = (T)Convert.ChangeType(@event, typeof(T));
}
else
{
message = JsonConvert
.DeserializeObject <T>((string)@event); // todo: support for multiple serializer
}
return(Tuple.Create <T, StreamSequenceToken>(message, sequenceToken.CreateSequenceTokenForEvent(iteration)));
}
catch (Exception)
{
return(null);
}
})
.Where(@event => @event != null);
return(serializedEvents);
}
