Ejemplo n.º 1
0
        private static async Task <IEventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger, ITelemetryProducer telemetryProducer)
        {
            bool offsetInclusive         = true;
            var  connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString)
            {
                EntityPath = partitionSettings.Hub.Path
            };
            EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());

            // if we have a starting offset or if we're not configured to start reading from utc now, read from offset
            if (!partitionSettings.Hub.StartFromNow ||
                offset != EventHubConstants.StartOfStream)
            {
                logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }
            else
            {
                // to start reading from most recent data, we get the latest offset from the partition.
                EventHubPartitionRuntimeInformation partitionInfo =
                    await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);

                offset          = partitionInfo.LastEnqueuedOffset;
                offsetInclusive = false;
                logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }

            PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive);

            if (partitionSettings.Hub.PrefetchCount.HasValue)
            {
                receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
            }

            return(new EventHubReceiverProxy(receiver));
        }
Ejemplo n.º 2
0
        private static async Task <EventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger)
        {
            bool                  offsetInclusive = true;
            EventHubClient        client          = EventHubClient.CreateFromConnectionString(partitionSettings.Hub.ConnectionString, partitionSettings.Hub.Path);
            EventHubConsumerGroup consumerGroup   = client.GetConsumerGroup(partitionSettings.Hub.ConsumerGroup);

            if (partitionSettings.Hub.PrefetchCount.HasValue)
            {
                consumerGroup.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
            }
            // if we have a starting offset or if we're not configured to start reading from utc now, read from offset
            if (!partitionSettings.Hub.StartFromNow || offset != EventHubConsumerGroup.StartOfStream)
            {
                logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }
            else
            {
                // to start reading from most recent data, we get the latest offset from the partition.
                PartitionRuntimeInformation patitionInfo =
                    await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);

                offset          = patitionInfo.LastEnqueuedOffset;
                offsetInclusive = false;
                logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }
            return(await consumerGroup.CreateReceiverAsync(partitionSettings.Partition, offset, offsetInclusive));
        }
        internal static async Task <Tuple <string, DateTime, string> > DiscoverEndOfStreamForPartitionAsync(EventHubClient ehClient, string partitionId)
        {
            TestUtility.Log($"Getting partition information for {partitionId}.");
            var pInfo = await ehClient.GetPartitionRuntimeInformationAsync(partitionId);

            return(Tuple.Create(pInfo.LastEnqueuedOffset, pInfo.LastEnqueuedTimeUtc, pInfo.LastEnqueuedOffset));
        }
        public async Task SendDataAsync(CoffeeMachineData data)
        {
            EventData evData     = CreateEventData(data);
            var       partitions = await _ehClient.GetRuntimeInformationAsync();

            var partitionId = await _ehClient.GetPartitionRuntimeInformationAsync(partitions.PartitionIds.First());

            await _ehClient.SendAsync(evData);
        }
Ejemplo n.º 5
0
        private static async Task <EventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger)
#endif
        {
            bool offsetInclusive = true;

#if NETSTANDARD
            var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString)
            {
                EntityPath = partitionSettings.Hub.Path
            };
            EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
#else
            EventHubClient client = EventHubClient.CreateFromConnectionString(partitionSettings.Hub.ConnectionString, partitionSettings.Hub.Path);

            EventHubConsumerGroup consumerGroup = client.GetConsumerGroup(partitionSettings.Hub.ConsumerGroup);
            if (partitionSettings.Hub.PrefetchCount.HasValue)
            {
                consumerGroup.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
            }
#endif
            // if we have a starting offset or if we're not configured to start reading from utc now, read from offset
            if (!partitionSettings.Hub.StartFromNow ||
                offset != EventHubConstants.StartOfStream)
            {
                logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }
            else
            {
                // to start reading from most recent data, we get the latest offset from the partition.
#if NETSTANDARD
                EventHubPartitionRuntimeInformation partitionInfo =
#else
                PartitionRuntimeInformation partitionInfo =
#endif
                    await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);

                offset          = partitionInfo.LastEnqueuedOffset;
                offsetInclusive = false;
                logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }
#if NETSTANDARD
            PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive);

            if (partitionSettings.Hub.PrefetchCount.HasValue)
            {
                receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
            }

            return(receiver);
#else
            return(await consumerGroup.CreateReceiverAsync(partitionSettings.Partition, offset, offsetInclusive));
#endif
        }
Ejemplo n.º 6
0
        internal static async Task <Dictionary <string, string> > DiscoverEndOfStreamForPartitionsAsync(EventHubClient ehClient, string[] partitionIds)
        {
            // Mark offsets in all partitions so we can attempt to receive from that point.
            var partitionOffsets = new Dictionary <string, string>();

            // Discover the end of stream on each partition.
            TestUtility.Log("Discovering end of stream on each partition.");
            foreach (var partitionId in partitionIds)
            {
                var lastEvent = await ehClient.GetPartitionRuntimeInformationAsync(partitionId);

                partitionOffsets.Add(partitionId, lastEvent.LastEnqueuedOffset);
                TestUtility.Log($"Partition {partitionId} has last message with offset {lastEvent.LastEnqueuedOffset}");
            }

            return(partitionOffsets);
        }
Ejemplo n.º 7
0
        public Task OpenEventHubAsync(string eventHubConnectionString, string hubName)
        {
            return(Task.Factory.StartNew(async() => {
                if (string.IsNullOrWhiteSpace(eventHubConnectionString))
                {
                    throw new ArgumentException("invalid event hub connection string");
                }

                if (string.IsNullOrWhiteSpace(hubName))
                {
                    throw new ArgumentException("invalid hubname");
                }
                this.IsOpen = true;

                var builder = new ServiceBusConnectionStringBuilder(eventHubConnectionString)
                {
                    TransportType = TransportType.Amqp
                };
                var s = builder.ToString();
                _factory = MessagingFactory.CreateFromConnectionString(builder.ToString());

                _client = _factory.CreateEventHubClient(hubName);

                var runtimeInfo = await _client.GetRuntimeInformationAsync();

                _partitions.Clear();
                foreach (var p in runtimeInfo.PartitionIds)
                {
                    var partition = await _client.GetPartitionRuntimeInformationAsync(p);
                    var count = partition.LastEnqueuedSequenceNumber - partition.BeginSequenceNumber;
                    if (count != 0)
                    {
                        count++;
                    }
                    this.MessageCount = count;
                    _partitions.Add(new Partition(p, _messages));
                    _foundPartitions.OnNext(partition);
                }

                _foundPartitions.OnCompleted();
            }));
        }
Ejemplo n.º 8
0
        static async Task MonitorEventHub()
        {
            EventHubRuntimeInformation runtimeInfo = await _client.GetRuntimeInformationAsync();

            Console.WriteLine($"Partition count: {runtimeInfo.PartitionCount}");

            long[] sequenceNumbers = new long[runtimeInfo.PartitionCount];

            for (;;)
            {
                var tasks = new Task <PartitionRuntimeInformation> [runtimeInfo.PartitionCount];

                for (int i = 0; i < runtimeInfo.PartitionCount; i++)
                {
                    tasks[i] = _client.GetPartitionRuntimeInformationAsync(i.ToString());
                }

                // Wait for all of them
                await Task.WhenAll(tasks);

                for (int i = 0; i < runtimeInfo.PartitionCount; i++)
                {
                    PartitionRuntimeInformation part = tasks[i].GetAwaiter().GetResult();
                    Console.WriteLine($"BeginSequenceNumber: {part.BeginSequenceNumber}, LastEnqueuedOffset: {part.LastEnqueuedOffset}, LastEnqueuedSequenceNumber: {part.LastEnqueuedSequenceNumber}, LastEnqueuedTime: {part.LastEnqueuedTimeUtc}");

                    if (sequenceNumbers[i] != 0)
                    {
                        if (part.LastEnqueuedSequenceNumber != sequenceNumbers[i])
                        {
                            Console.WriteLine($"Sequence when up by {part.LastEnqueuedSequenceNumber - sequenceNumbers[i]} for partition {i}!");
                        }
                    }

                    sequenceNumbers[i] = part.LastEnqueuedSequenceNumber;
                }

                System.Threading.Thread.Sleep(5000);
            }
        }
        private static async Task <IReadOnlyList <PartitionReceiver> > GetReceivers(EventHubClient eventHubClient, string consumerGroupName)
        {
            EventHubRuntimeInformation runtimeInformation = await eventHubClient.GetRuntimeInformationAsync();

            var receivers = new List <PartitionReceiver>();

            foreach (string partitionId in runtimeInformation.PartitionIds)
            {
                EventHubPartitionRuntimeInformation partitionRuntimeInformation =
                    await eventHubClient.GetPartitionRuntimeInformationAsync(partitionId);

                PartitionReceiver receiver = eventHubClient.CreateReceiver(
                    consumerGroupName,
                    partitionId,
                    partitionRuntimeInformation.LastEnqueuedOffset,
                    offsetInclusive: false);

                receivers.Add(receiver);
            }

            return(receivers);
        }
Ejemplo n.º 10
0
        public  Task OpenEventHubAsync(string eventHubConnectionString, string hubName)
        {
            
            return Task.Factory.StartNew(async () => {
                if (string.IsNullOrWhiteSpace(eventHubConnectionString))
                    throw new ArgumentException("invalid event hub connection string");

                if (string.IsNullOrWhiteSpace(hubName))
                    throw new ArgumentException("invalid hubname");
                this.IsOpen = true;

                var builder = new ServiceBusConnectionStringBuilder(eventHubConnectionString) { TransportType = TransportType.Amqp };
                var s = builder.ToString();
                _factory = MessagingFactory.CreateFromConnectionString(builder.ToString());
       
                _client = _factory.CreateEventHubClient(hubName);
                
                var runtimeInfo = await _client.GetRuntimeInformationAsync();

                _partitions.Clear();
                foreach (var p in runtimeInfo.PartitionIds)
                {
                
                    var partition = await _client.GetPartitionRuntimeInformationAsync(p);
                    var count = partition.LastEnqueuedSequenceNumber - partition.BeginSequenceNumber;
                    if (count != 0)
                        count++;
                    this.MessageCount = count;
                    _partitions.Add(new Partition(p, _messages));
                    _foundPartitions.OnNext(partition);
                }

                _foundPartitions.OnCompleted();
                
            });

        }
Ejemplo n.º 11
0
        protected override async Task InternalRunAsync()
        {
            IEventHubClient eventHubClient =
                new EventHubClient(
                    this.EventHubConnectionString,
                    this.EventHubEntityPath);

            this.PrintObjectInfo(eventHubClient);

            IEventHubRuntimeInformation runtimeInformation =
                await eventHubClient.GetEventHubRuntimeInformationAsync()
                .ConfigureAwait(false);

            this.PrintObjectInfo(runtimeInformation);

            IEnumerable <IEventHubPartitionRuntimeInformation> partionInfos =
                runtimeInformation.PartitionIds
                .Select(
                    pid => eventHubClient
                    .GetPartitionRuntimeInformationAsync(pid)
                    .ConfigureAwait(false).GetAwaiter().GetResult())
                .ToArray();

            this.PrintObjectInfo(partionInfos);

            int batchCount = 500;
            int batchSize  = 100;

            for (int batchIdx = 0; batchIdx < batchCount; ++batchIdx)
            {
                await eventHubClient
                .BatchSendAsync(
                    new int[batchSize].Select(
                        (v, i) => $"Data {batchIdx}-{i} at {DateTime.UtcNow:o} dummy: {new string('$', 256)}"))
                .ConfigureAwait(false);
            }
            Console.WriteLine($"Total sent: {batchCount * batchSize}");

            IEventProcessor eventProcessor =
                new EventProcessor(
                    this.EventHubConsumerGroup,
                    this.StorageAccountConnectionString,
                    this.StorageAccountContainer)
            {
                Options = new EventProcessorOptions
                {
                    MaxBatchSize  = 50,
                    PrefetchCount = 500
                }
            };

            int totalReceived = 0;

            Dictionary <string, List <IEventData> > bufferDic =
                new Dictionary <string, List <IEventData> >();

            Dictionary <string, DateTimeOffset> lastDumpTimeDic =
                new Dictionary <string, DateTimeOffset>();

            Dictionary <string, IEventHubPartitionContext> partitionContextDic =
                new Dictionary <string, IEventHubPartitionContext>();

            CancellationTokenSource cancellationTokenSource =
                new CancellationTokenSource();
            TaskFactory taskFactory =
                new TaskFactory(cancellationTokenSource.Token);
            List <Task> tasksList = new List <Task>();

            DateTimeOffset initialTime =
                DateTimeOffset.FromUnixTimeSeconds(
                    DateTimeOffset.UtcNow.ToUnixTimeSeconds());

            foreach (var partitionInfo in partionInfos)
            {
                string partitionId = partitionInfo.PartitionId;
                bufferDic[partitionId]       = new List <IEventData>();
                lastDumpTimeDic[partitionId] =
                    initialTime.AddSeconds(
                        int.Parse(partitionId) % partionInfos.Count());

                Console.WriteLine($"last dump time for partition {partitionId}: {lastDumpTimeDic[partitionId]}");

                Task task = taskFactory.StartNew(
                    async() =>
                {
                    while (true)
                    {
                        DateTimeOffset checkTime =
                            DateTimeOffset.FromUnixTimeSeconds(
                                DateTimeOffset.UtcNow.ToUnixTimeSeconds());

                        List <IEventData> buffer = bufferDic[partitionId];

                        if (cancellationTokenSource.Token.IsCancellationRequested)
                        {
                            if (buffer.Any())
                            {
                                Console.WriteLine($"Dump {buffer.Count} messages from partition {partitionId} at {checkTime:o}.");
                                totalReceived += buffer.Count();
                                buffer.Clear();
                            }
                            break;
                        }

                        DateTimeOffset lastDumpTime =
                            lastDumpTimeDic[partitionId];

                        double secondsSinceLastDump =
                            checkTime.Subtract(lastDumpTime).TotalSeconds;

                        if (buffer.Any() &&
                            secondsSinceLastDump >= partionInfos.Count() &&
                            (secondsSinceLastDump % partionInfos.Count()) == 0)
                        {
                            IEventData checkpointData = null;

                            lock (buffer)
                            {
                                Console.WriteLine($"Dump {buffer.Count} messages from partition {partitionId} at {checkTime:o}.");
                                totalReceived += buffer.Count();

                                checkpointData =
                                    buffer
                                    .OrderByDescending(e => e.SequenceNumber)
                                    .FirstOrDefault();
                                buffer.Clear();
                            }

                            if (checkpointData != null)
                            {
                                IEventHubPartitionContext partitionContext =
                                    partitionContextDic[partitionId];
                                await partitionContext
                                .CheckpointAsync(checkpointData).ConfigureAwait(false);
                            }

                            lastDumpTimeDic[partitionId] = checkTime;

                            // Console.WriteLine($"last dump time for partition {partitionId}: {lastDumpTimeDic[partitionId]}");
                        }
                        else
                        {
                            Thread.Sleep(TimeSpan.FromSeconds(1));
                        }
                    }
                });

                tasksList.Add(task);
            }

            Task eventProcessorHandler(object obj, EventProcessorEventArgs args)
            {
                // Console.WriteLine($"[Partition - {args.PartitionContext.PartitionId}] [{args.EventType}]");
                // this.PrintObjectInfo(args);

                if (args.EventType == EventProcessorEventType.DataReceived)
                {
                    string partitionId = args.PartitionContext.PartitionId;

                    partitionContextDic[partitionId] = args.PartitionContext;

                    List <IEventData> buffer = bufferDic[partitionId];

                    lock (buffer)
                    {
                        Console.WriteLine($"Buffered {args.DataCollection.Count()} messages from partition {partitionId} at {DateTimeOffset.UtcNow:o}.");
                        buffer.AddRange(args.DataCollection);
                    }
                }

                return(Task.CompletedTask);
            }

            eventProcessor.OnClosed        += eventProcessorHandler;
            eventProcessor.OnOpened        += eventProcessorHandler;
            eventProcessor.OnErrorOccurred += eventProcessorHandler;
            eventProcessor.OnDataReceived  += eventProcessorHandler;

            IEventProcessorRegistration registration =
                await eventHubClient.RegisterAsync(eventProcessor)
                .ConfigureAwait(false);

            char c;

            do
            {
                Console.WriteLine("Receiving event data, press 'q' to exit.");
                c = Console.ReadKey(true).KeyChar;
            } while(c != 'q');

            cancellationTokenSource.Cancel();
            Task.WaitAll(tasksList.ToArray());

            Console.WriteLine($"Total received: {totalReceived}");

            await registration.UnregisterAsync().ConfigureAwait(false);
        }