public static async Task Start(string hostName, string eventHubName, string consumerGroupName, string containerName, string blobPrefix, string serviceBusConnectionString, string storageConnectionString) { // Create Consumer Group if it doesn't exist if (consumerGroupName != null) { NamespaceManager manager = NamespaceManager.CreateFromConnectionString(serviceBusConnectionString); ConsumerGroupDescription description = new ConsumerGroupDescription(eventHubName, consumerGroupName); manager.CreateConsumerGroupIfNotExists(description); } // Create the Event Processor Host var host = new EventProcessorHost( hostName, eventHubName, consumerGroupName == null ? EventHubConsumerGroup.DefaultGroupName : consumerGroupName, serviceBusConnectionString, storageConnectionString, eventHubName); // Create the Factory var factory = new BlobEventProcessorHostFactory(storageConnectionString, containerName, blobPrefix); // Register the Factory await host.RegisterEventProcessorFactoryAsync(factory); }
public void MessageProcessingWithPartitionDistribution(ConsumerGroupDescription group) { EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(eventHubConnectionString, this.eventHubName); // Get the default Consumer Group //defaultConsumerGroup = eventHubClient.GetDefaultConsumerGroup(); string blobConnectionString = ConfigurationManager.AppSettings["AzureStorageConnectionString"]; // Required for checkpoint/state if (null == group) { //Use default consumer group EventHubConsumerGroup defaultConsumerGroup = eventHubClient.GetDefaultConsumerGroup(); eventProcessorHost = new EventProcessorHost("singleworker", eventHubClient.Path, defaultConsumerGroup.GroupName, this.eventHubConnectionString, blobConnectionString); } else { //Use custom consumer group eventProcessorHost = new EventProcessorHost("singleworker", eventHubClient.Path, group.Name, this.eventHubConnectionString, blobConnectionString); } //Only use events from the time the sender is started. EventProcessorOptions options = new EventProcessorOptions(); options.InitialOffsetProvider = (partitionId) => DateTime.UtcNow; eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(options).Wait(); }
public void TestConsumerGroup() { string name = "testConsumerGroupEventHub"; string consumerGroupName = "consumergroup1"; NamespaceManager ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString); EventHubDescription description = ns.CreateEventHub(name); Assert.IsTrue(null != description); ConsumerGroupDescription cgDescription = ns.CreateConsumerGroup(name, consumerGroupName); if (!ns.ConsumerGroupExists(name, consumerGroupName, out cgDescription)) { Assert.Fail("Consumer Group did not exist"); } else { Assert.IsTrue(null != cgDescription); ns.DeleteConsumerGroup(name, consumerGroupName); if (ns.ConsumerGroupExists(name, consumerGroupName, out cgDescription)) { Assert.Fail("Consumer Group was not deleted"); } ns.DeleteEventHub(name); if (ns.EventHubExists(name, out description)) { Assert.Fail("EventHub was not deleted"); } } }
public void Run(string connectionString, string hubName) { NamespaceManager nsmgr = NamespaceManager.CreateFromConnectionString(connectionString); EventHubDescription desc = nsmgr.GetEventHub(hubName); string consumerGroupName = _consumerGroupPrefix + DateTime.UtcNow.Ticks.ToString(); ConsumerGroupDescription consumerGroupDesc = nsmgr.CreateConsumerGroupIfNotExists(new ConsumerGroupDescription(hubName, consumerGroupName)); EventHubClient client = EventHubClient.CreateFromConnectionString(connectionString, hubName); int numPartitions = desc.PartitionCount; _receivers = new EventHubReceiver[numPartitions]; //_receiversLastUpdate = new DateTime[numPartitions]; //for (int i = 0; i < numPartitions; i++) //{ // _receiversLastUpdate[i] = DateTime.UtcNow; //} _tasks = new Task[numPartitions]; _buffers = new Dictionary <string, CircularBuffer <SensorDataContract> >(); for (int iPart = 0; iPart < desc.PartitionCount; iPart++) { EventHubReceiver receiver = client.GetConsumerGroup(consumerGroupName).CreateReceiver( desc.PartitionIds[iPart], DateTime.UtcNow - TimeSpan.FromMinutes(2)); _receivers[iPart] = receiver; Task <IEnumerable <EventData> > task = receiver.ReceiveAsync(1000, TimeSpan.FromSeconds(1)); int thisPart = iPart; task.ContinueWith(new Action <Task <IEnumerable <EventData> > >((t) => OnTaskComplete(t, thisPart))); _tasks[iPart] = task; } }
public HandleConsumerGroupControl(WriteToLogDelegate writeToLog, ServiceBusHelper serviceBusHelper, ConsumerGroupDescription consumerGroupDescription, string eventHubName) { this.writeToLog = writeToLog; this.serviceBusHelper = serviceBusHelper; this.consumerGroupDescription = consumerGroupDescription; this.eventHubName = eventHubName; InitializeComponent(); InitializeControls(); }
public void RefreshData(ConsumerGroupDescription consumerGroup) { try { consumerGroupDescription = consumerGroup; InitializeData(); } catch (Exception ex) { HandleException(ex); } }
public ContainerForm(ServiceBusHelper serviceBusHelper, MainForm mainForm, ConsumerGroupDescription consumerGroupDescription, IEnumerable <PartitionDescription> partitionDescriptions) { try { var descriptions = partitionDescriptions as IList <PartitionDescription> ?? partitionDescriptions.ToList(); if (!descriptions.Any()) { return; } InitializeComponent(); Task.Factory.StartNew(AsyncWriteToLog).ContinueWith(t => { if (t.IsFaulted && t.Exception != null) { WriteToLog(t.Exception.Message); } }); this.mainForm = mainForm; mainSplitterDistance = mainSplitContainer.SplitterDistance; SuspendLayout(); panelMain.SuspendDrawing(); panelMain.Controls.Clear(); panelMain.BackColor = SystemColors.GradientInactiveCaption; var partitionListenerControl = new PartitionListenerControl(WriteToLog, StopLog, StartLog, new ServiceBusHelper(WriteToLog, serviceBusHelper), consumerGroupDescription, descriptions) { Location = new Point(1, panelMain.HeaderHeight + 1), Size = new Size(panelMain.Size.Width - 3, panelMain.Size.Height - 26), Anchor = AnchorStyles.Bottom | AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right }; if (descriptions.Count == 1) { Text = string.Format(PartitionListenerFormat, descriptions[0].PartitionId, consumerGroupDescription.Name); panelMain.HeaderText = string.Format(HeaderTextPartitionListenerFormat, descriptions[0].PartitionId); } else { Text = string.Format(ConsumerGroupListenerFormat, consumerGroupDescription.Name); panelMain.HeaderText = string.Format(HeaderTextConsumerGroupListenerFormat, consumerGroupDescription.Name); } partitionListenerControl.Focus(); panelMain.Controls.Add(partitionListenerControl); SetStyle(ControlStyles.ResizeRedraw, true); } finally { panelMain.ResumeDrawing(); ResumeLayout(); } }
public void TestParition() { string name = "testpartitionEventHub"; string consumerGroupName = "consumergroup1"; NamespaceManager ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString); EventHubDescription description = ns.CreateEventHub(name); Assert.IsTrue(null != description); ConsumerGroupDescription cgDescription = ns.CreateConsumerGroup(name, consumerGroupName); PartitionDescription pd = ns.GetEventHubPartition(name, consumerGroupName, "1"); Assert.IsTrue(null != pd); ns.DeleteEventHub(name); }
public HandleConsumerGroupControl(WriteToLogDelegate writeToLog, ServiceBusHelper serviceBusHelper, ConsumerGroupDescription consumerGroupDescription, string eventHubName) { this.writeToLog = writeToLog; this.serviceBusHelper = serviceBusHelper; this.consumerGroupDescription = consumerGroupDescription; this.eventHubName = eventHubName; dataPointBindingList = new BindingList <MetricDataPoint> { AllowNew = true, AllowEdit = true, AllowRemove = true }; InitializeComponent(); InitializeControls(); }
public bool AddConsumerGroup(String groupname) { var connectionString = String.Format("Endpoint=sb://{0}.servicebus.windows.net/;SharedAccessKeyName={1};SharedAccessKey={2};TransportType=Amqp", _config.ServiceBusNamespace, _config.SasPolicyName, _config.SasPolicyKey); Console.WriteLine("Creating consumer group {0}", groupname); var manager = NamespaceManager.CreateFromConnectionString(connectionString); var description = new ConsumerGroupDescription(_config.EventHubName, groupname); var result = manager.CreateConsumerGroupIfNotExists(description); Console.WriteLine("Consumer group created : {0}", result != null); return result != null; }
public bool AddConsumerGroup(String groupname) { var connectionString = String.Format("Endpoint=sb://{0}.servicebus.windows.net/;SharedAccessKeyName={1};SharedAccessKey={2};TransportType=Amqp", _config.ServiceBusNamespace, _config.SasPolicyName, _config.SasPolicyKey); Console.WriteLine("Creating consumer group {0}", groupname); var manager = NamespaceManager.CreateFromConnectionString(connectionString); var description = new ConsumerGroupDescription(_config.EventHubName, groupname); var result = manager.CreateConsumerGroupIfNotExists(description); Console.WriteLine("Consumer group created : {0}", result != null); return(result != null); }
private void btnCreateDelete_Click(object sender, EventArgs e) { try { if (serviceBusHelper == null) { return; } if (btnCreateDelete.Text == DeleteText) { using (var deleteForm = new DeleteForm(consumerGroupDescription.Name, ConsumerGroupEntity.ToLower())) { if (deleteForm.ShowDialog() == DialogResult.OK) { serviceBusHelper.DeleteConsumerGroup(consumerGroupDescription); } } } else { if (string.IsNullOrWhiteSpace(txtName.Text)) { writeToLog(PathCannotBeNull); return; } //var description = new ConsumerGroupDescription(eventHubName, txtName.Text) // { // UserMetadata = txtUserMetadata.Text, // EnableCheckpoint = checkBoxEnableCheckpoint.Enabled // }; var description = new ConsumerGroupDescription(eventHubName, txtName.Text) { UserMetadata = txtUserMetadata.Text }; consumerGroupDescription = serviceBusHelper.CreateConsumerGroup(description); InitializeControls(); } } catch (Exception ex) { HandleException(ex); } }
public void Run(string connectionString, string hubName, string measureNameFilter) { NamespaceManager nsmgr = NamespaceManager.CreateFromConnectionString(connectionString); EventHubDescription desc = nsmgr.GetEventHub(hubName); //we use already defined consumerGroup name to not reach limit on CG count string consumerGroupName = _consumerGroupPrefix; ConsumerGroupDescription consumerGroupDesc = nsmgr.CreateConsumerGroupIfNotExists(new ConsumerGroupDescription(hubName, consumerGroupName)); EventHubClient client = EventHubClient.CreateFromConnectionString(connectionString, hubName); int numPartitions = desc.PartitionCount; _receivers = new EventHubReceiver[numPartitions]; _tasks = new Task[numPartitions]; for (int iPart = 0; iPart < desc.PartitionCount; iPart++) { EventHubReceiver receiver = client.GetConsumerGroup(consumerGroupName).CreateReceiver( desc.PartitionIds[iPart], DateTime.UtcNow - TimeSpan.FromMinutes(2)); _receivers[iPart] = receiver; int part = iPart; Task.Factory.StartNew((state) => { try { while (true) { var messages = _receivers[part].Receive(1000, TimeSpan.FromSeconds(1)); Process(messages); } } catch (Exception ex) { //FailureEvent.Set(); Trace.TraceError("Ignored invalid event data: {0}"); } }, iPart); } }
private async void btnCancelUpdate_Click(object sender, EventArgs e) { bool ok = true; if (btnCancelUpdate.Text == CancelText) { if (OnCancel != null) { OnCancel(); } } else { try { consumerGroupDescription.UserMetadata = txtUserMetadata.Text; //consumerGroupDescription.EnableCheckpoint = checkBoxEnableCheckpoint.Enabled; serviceBusHelper.UpdateConsumerGroup(consumerGroupDescription); } catch (Exception ex) { HandleException(ex); ok = false; } finally { InitializeControls(); } if (ok) { return; } try { consumerGroupDescription = await serviceBusHelper.NamespaceManager.GetConsumerGroupAsync(eventHubName, consumerGroupDescription.Name); } catch (Exception ex) { HandleException(ex); } } }
public void RegisterEventProcessor(ConsumerGroupDescription group, string blobConnectionString, string hostName) { EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(eventHubConnectionString, this.eventHubName); if (null == group) { //Use default consumer group EventHubConsumerGroup defaultConsumerGroup = eventHubClient.GetDefaultConsumerGroup(); eventProcessorHost = new EventProcessorHost(hostName, eventHubClient.Path, defaultConsumerGroup.GroupName, this.eventHubConnectionString, blobConnectionString); } else { //Use custom consumer group eventProcessorHost = new EventProcessorHost(hostName, eventHubClient.Path, group.Name, this.eventHubConnectionString, blobConnectionString); } Trace.TraceInformation("Registering event processor"); eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>().Wait(); }
public void RegisterEventProcessor(ConsumerGroupDescription group, string blobConnectionString, string hostName) { EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(eventHubConnectionString, this.eventHubName); if (null == group) { //Use default consumer group EventHubConsumerGroup defaultConsumerGroup = eventHubClient.GetDefaultConsumerGroup(); eventProcessorHost = new EventProcessorHost(hostName, eventHubClient.Path, defaultConsumerGroup.GroupName, this.eventHubConnectionString, blobConnectionString); } else { //Use custom consumer group eventProcessorHost = new EventProcessorHost(hostName, eventHubClient.Path, group.Name, this.eventHubConnectionString, blobConnectionString); } Trace.TraceInformation("Registering event processor"); eventProcessorHost.RegisterEventProcessorAsync<EventProcessor>().Wait(); }
private void ShowConsumerGroup(ConsumerGroupDescription notificationHub, string eventHubname) { HandleConsumerGroupControl notificationHubControl = null; try { panelMain.SuspendDrawing(); foreach (var userControl in panelMain.Controls.OfType<UserControl>()) { userControl.Dispose(); } panelMain.Controls.Clear(); panelMain.BackColor = SystemColors.GradientInactiveCaption; notificationHubControl = new HandleConsumerGroupControl(WriteToLog, serviceBusHelper, notificationHub, eventHubname); notificationHubControl.SuspendDrawing(); notificationHubControl.Location = new Point(1, panelLog.HeaderHeight + 1); panelMain.Controls.Add(notificationHubControl); SetControlSize(notificationHubControl); notificationHubControl.OnCancel += MainForm_OnCancel; notificationHubControl.OnRefresh += MainForm_OnRefresh; } catch (Exception ex) { HandleException(ex); } finally { panelMain.ResumeDrawing(); if (notificationHubControl != null) { notificationHubControl.ResumeDrawing(); } } }
private TreeNode CreateEventHubConsumerGroupNode(EventHubDescription eventHub, ConsumerGroupDescription consumerGroupDescription, IList<PartitionDescription> partitionDescriptions, TreeNode consumerGroupsNode) { if (consumerGroupsNode.Nodes.ContainsKey(consumerGroupDescription.Name)) { return null; } var consumerGroupNode = consumerGroupsNode.Nodes.Add(consumerGroupDescription.Name, consumerGroupDescription.Name, ConsumerGroupIconIndex, ConsumerGroupIconIndex); consumerGroupNode.ContextMenuStrip = consumerGroupContextMenuStrip; consumerGroupNode.Tag = consumerGroupDescription; if (partitionDescriptions == null || !partitionDescriptions.Any()) { return consumerGroupNode; } consumerGroupNode.Nodes.Clear(); var partitionsNode = consumerGroupNode.Nodes.Add(PartitionEntities, PartitionEntities, PartitionListIconIndex, PartitionListIconIndex); partitionsNode.ContextMenuStrip = partitionsContextMenuStrip; partitionsNode.Tag = eventHub; foreach (var partition in partitionDescriptions) { CreateEventHubPartitionNode(partition, partitionsNode); } return consumerGroupNode; }
// Please set the following connection strings in app.config for this WebJob to run: // AzureWebJobsDashboard and AzureWebJobsStorage static void Main() { // make sure consumer group exists NamespaceManager manager = NamespaceManager.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); ConsumerGroupDescription description = new ConsumerGroupDescription(ConfigurationManager.AppSettings["ServiceBus.Path"], ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); manager.CreateConsumerGroupIfNotExists(description); //get a handle on the consumer group for the event hub we want to read from var factory = MessagingFactory.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); var client = factory.CreateEventHubClient(ConfigurationManager.AppSettings["ServiceBus.Path"]); var group = client.GetConsumerGroup(ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); while (true) { Task.WaitAll(client.GetRuntimeInformation().PartitionIds.Select(id => Task.Run(() => { var receiver = @group.CreateReceiver(id); Trace.TraceInformation("Waiting for messages " + receiver.PartitionId); while (true) { try { //read the message var message = receiver.Receive(); if (message == null) continue; var body = Encoding.UTF8.GetString(message.GetBytes()); if (body == null) continue; var type = MessageType.None; switch (message.PartitionKey.ToLower()) { case "energy": type = MessageType.Energy; break; case "temperature": type = MessageType.Temperature; break; case "humidity": type = MessageType.Humidity; break; case "light": type = MessageType.Light; break; } if (type == MessageType.None) continue; var writer = new DocumentDBWriter(); var task = writer.WriteDocument(type, body); Task.WaitAll(task); // block while the task completes Console.WriteLine(task.Result); } catch (Exception e) { Console.WriteLine(e.Message); //suppress for simplicity } } })).ToArray()); } }
public ConsumerGroupDescription CreateConsumerGroup(ConsumerGroupDescription description) { return(_instace.CreateConsumerGroup(description)); }
/// <summary> /// Deletes the consumer group described by the relative name of the service namespace base address. /// </summary> /// <param name="consumerGroupDescription">The consumer group to delete.</param> public void DeleteConsumerGroup(ConsumerGroupDescription consumerGroupDescription) { if (consumerGroupDescription == null || string.IsNullOrWhiteSpace(consumerGroupDescription.Name)) { throw new ArgumentException(ConsumerGroupDescriptionCannotBeNull); } if (namespaceManager != null) { RetryHelper.RetryAction(() => namespaceManager.DeleteConsumerGroup(consumerGroupDescription.EventHubPath, consumerGroupDescription.Name), writeToLog); WriteToLogIf(traceEnabled, string.Format(CultureInfo.CurrentCulture, ConsumerGroupDeleted, consumerGroupDescription.Name)); OnDelete(new ServiceBusHelperEventArgs(consumerGroupDescription, EntityType.ConsumerGroup)); } else { throw new ApplicationException(ServiceBusIsDisconnected); } }
// Please set the following connection strings in app.config for this WebJob to run: // AzureWebJobsDashboard and AzureWebJobsStorage static void Main() { // make sure consumer group exists NamespaceManager manager = NamespaceManager.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); ConsumerGroupDescription description = new ConsumerGroupDescription(ConfigurationManager.AppSettings["ServiceBus.Path"], ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); manager.CreateConsumerGroupIfNotExists(description); //get a handle on the consumer group for the event hub we want to read from var factory = MessagingFactory.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); var client = factory.CreateEventHubClient(ConfigurationManager.AppSettings["ServiceBus.Path"]); var group = client.GetConsumerGroup(ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); while (true) { Task.WaitAll(client.GetRuntimeInformation().PartitionIds.Select(id => Task.Run(() => { var receiver = @group.CreateReceiver(id); Trace.TraceInformation("Waiting for messages " + receiver.PartitionId); while (true) { try { //read the message var message = receiver.Receive(); if (message == null) { continue; } var body = Encoding.UTF8.GetString(message.GetBytes()); if (body == null) { continue; } var type = MessageType.None; switch (message.PartitionKey.ToLower()) { case "energy": type = MessageType.Energy; break; case "temperature": type = MessageType.Temperature; break; case "humidity": type = MessageType.Humidity; break; case "light": type = MessageType.Light; break; } if (type == MessageType.None) { continue; } var writer = new DocumentDBWriter(); var task = writer.WriteDocument(type, body); Task.WaitAll(task); // block while the task completes Console.WriteLine(task.Result); } catch (Exception e) { Console.WriteLine(e.Message); //suppress for simplicity } } })).ToArray()); } }
private async Task RunAsync(CancellationToken cancellationToken) { // make sure consumer group exists NamespaceManager manager = NamespaceManager.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); ConsumerGroupDescription description = new ConsumerGroupDescription(ConfigurationManager.AppSettings["ServiceBus.Path"], ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); manager.CreateConsumerGroupIfNotExists(description); //get a handle on the consumer group for the event hub we want to read from var factory = MessagingFactory.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); var client = factory.CreateEventHubClient(ConfigurationManager.AppSettings["ServiceBus.Path"]); var group = client.GetConsumerGroup(ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); //get a handle on the container we want to write blobs to var storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["Storage.ConnectionString"]); var blobClient = storageAccount.CreateCloudBlobClient(); var container = blobClient.GetContainerReference(ConfigurationManager.AppSettings["Storage.Container"]); container.CreateIfNotExists(); while (!cancellationToken.IsCancellationRequested) { Task.WaitAll(client.GetRuntimeInformation().PartitionIds.Select(id => Task.Run(() => { var receiver = @group.CreateReceiver(id); var messageBuffer = new List <string>(); var startTime = DateTime.UtcNow; Trace.TraceInformation("Waiting for messages " + receiver.PartitionId); while (true) { try { //read the message var message = receiver.Receive(); if (message == null) { continue; } var body = Encoding.UTF8.GetString(message.GetBytes()); if (body == null) { continue; } var currentTime = DateTime.UtcNow; //add to the buffer messageBuffer.Add(body); //write out a file if a minute has passed and we have at least one message if ((currentTime - startTime).TotalMinutes >= 1 && messageBuffer.Count >= 1) { var now = DateTime.Now; var asString = String.Join("\n", messageBuffer); //upload the blob var blockBlob = container.GetBlockBlobReference(String.Format("{0}/{1}/{2}/message_{3}_{4}.log", now.Year, now.Month, now.Day, id, now.TimeOfDay)); blockBlob.UploadFromStream(new MemoryStream(Encoding.UTF8.GetBytes(asString))); //clear the buffer messageBuffer.Clear(); //start tracking anew startTime = currentTime; } } catch (Exception e) { Console.WriteLine(e.Message); //suppress for simplicity } } }, cancellationToken)).ToArray()); } }
/// <summary> /// Updates a consumer group in the service namespace with the given name. /// </summary> /// <param name="description">A ConsumerGroupDescription object describing the attributes with which the new consumer group will be updated.</param> /// <returns>Returns an updated ConsumerGroupDescription object.</returns> public ConsumerGroupDescription UpdateConsumerGroup(ConsumerGroupDescription description) { if (description == null) { throw new ArgumentException(DescriptionCannotBeNull); } if (namespaceManager != null) { var consumerGroup = RetryHelper.RetryFunc(() => namespaceManager.UpdateConsumerGroup(description), writeToLog); WriteToLogIf(traceEnabled, string.Format(CultureInfo.CurrentCulture, ConsumerGroupUpdated, description.Name)); OnCreate(new ServiceBusHelperEventArgs(consumerGroup, EntityType.ConsumerGroup)); return consumerGroup; } throw new ApplicationException(ServiceBusIsDisconnected); }
private async Task RunAsync(CancellationToken cancellationToken) { // make sure consumer group exists NamespaceManager manager = NamespaceManager.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); ConsumerGroupDescription description = new ConsumerGroupDescription(ConfigurationManager.AppSettings["ServiceBus.Path"], ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); manager.CreateConsumerGroupIfNotExists(description); //get a handle on the consumer group for the event hub we want to read from var factory = MessagingFactory.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp"); var client = factory.CreateEventHubClient(ConfigurationManager.AppSettings["ServiceBus.Path"]); var group = client.GetConsumerGroup(ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]); //get a handle on the container we want to write blobs to var storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["Storage.ConnectionString"]); var blobClient = storageAccount.CreateCloudBlobClient(); var container = blobClient.GetContainerReference(ConfigurationManager.AppSettings["Storage.Container"]); container.CreateIfNotExists(); while (!cancellationToken.IsCancellationRequested) { Task.WaitAll(client.GetRuntimeInformation().PartitionIds.Select(id => Task.Run(() => { var receiver = @group.CreateReceiver(id); var messageBuffer = new List<string>(); var startTime = DateTime.UtcNow; Trace.TraceInformation("Waiting for messages " + receiver.PartitionId); while (true) { try { //read the message var message = receiver.Receive(); if (message == null) continue; var body = Encoding.UTF8.GetString(message.GetBytes()); if (body == null) continue; var currentTime = DateTime.UtcNow; //add to the buffer messageBuffer.Add(body); //write out a file if a minute has passed and we have at least one message if ((currentTime - startTime).TotalMinutes >= 1 && messageBuffer.Count >= 1) { var now = DateTime.Now; var asString = String.Join("\n", messageBuffer); //upload the blob var blockBlob = container.GetBlockBlobReference(String.Format("{0}/{1}/{2}/message_{3}_{4}.log", now.Year, now.Month, now.Day, id, now.TimeOfDay)); blockBlob.UploadFromStream(new MemoryStream(Encoding.UTF8.GetBytes(asString))); //clear the buffer messageBuffer.Clear(); //start tracking anew startTime = currentTime; } } catch (Exception e) { Console.WriteLine(e.Message); //suppress for simplicity } } }, cancellationToken)).ToArray()); } }
public PartitionListenerControl(WriteToLogDelegate writeToLog, Func<Task> stopLog, Action startLog, ServiceBusHelper serviceBusHelper, ConsumerGroupDescription consumerGroupDescription, IEnumerable<PartitionDescription> partitionDescriptions) { Task.Factory.StartNew(AsyncTrackEventData).ContinueWith(t => { if (t.IsFaulted && t.Exception != null) { writeToLog(t.Exception.Message); } }); this.writeToLog = writeToLog; this.stopLog = stopLog; this.startLog = startLog; this.serviceBusHelper = serviceBusHelper; eventHubDescription = serviceBusHelper.NamespaceManager.GetEventHub(consumerGroupDescription.EventHubPath); eventHubClient = EventHubClient.CreateFromConnectionString(GetAmqpConnectionString(serviceBusHelper.ConnectionString), consumerGroupDescription.EventHubPath); consumerGroup = string.Compare(consumerGroupDescription.Name, DefaultConsumerGroupName, StringComparison.InvariantCultureIgnoreCase) == 0 ? eventHubClient.GetDefaultConsumerGroup() : eventHubClient.GetConsumerGroup(consumerGroupDescription.Name); IList<string> partitionIdList = partitionDescriptions.Select(pd => pd.PartitionId).ToList(); foreach (var id in partitionIdList) { partitionRuntumeInformationList.Add(eventHubClient.GetPartitionRuntimeInformation(id)); } partitionCount = partitionRuntumeInformationList.Count; InitializeComponent(); InitializeControls(); Disposed += ListenerControl_Disposed; }
public HandleConsumerGroupControl(WriteToLogDelegate writeToLog, ServiceBusHelper serviceBusHelper, ConsumerGroupDescription consumerGroupDescription, string eventHubName) { this.writeToLog = writeToLog; this.serviceBusHelper = serviceBusHelper; this.consumerGroupDescription = consumerGroupDescription; this.eventHubName = eventHubName; dataPointBindingList = new BindingList<MetricDataPoint> { AllowNew = true, AllowEdit = true, AllowRemove = true }; InitializeComponent(); InitializeControls(); }
public ContainerForm(ServiceBusHelper serviceBusHelper, MainForm mainForm, ConsumerGroupDescription consumerGroupDescription, IEnumerable<PartitionDescription> partitionDescriptions) { try { var descriptions = partitionDescriptions as IList<PartitionDescription> ?? partitionDescriptions.ToList(); if (partitionDescriptions == null || !descriptions.Any()) { return; } InitializeComponent(); Task.Factory.StartNew(AsyncWriteToLog).ContinueWith(t => { if (t.IsFaulted && t.Exception != null) { WriteToLog(t.Exception.Message); } }); this.mainForm = mainForm; mainSplitterDistance = mainSplitContainer.SplitterDistance; SuspendLayout(); panelMain.SuspendDrawing(); panelMain.Controls.Clear(); panelMain.BackColor = SystemColors.GradientInactiveCaption; var partitionListenerControl = new PartitionListenerControl(WriteToLog, StopLog, StartLog, new ServiceBusHelper(WriteToLog, serviceBusHelper), consumerGroupDescription, descriptions) { Location = new Point(1, panelMain.HeaderHeight + 1), Size = new Size(panelMain.Size.Width - 3, panelMain.Size.Height - 26), Anchor = AnchorStyles.Bottom | AnchorStyles.Top | AnchorStyles.Left | AnchorStyles.Right }; if (descriptions.Count == 1) { Text = string.Format(PartitionListenerFormat, descriptions[0].PartitionId, consumerGroupDescription.Name); panelMain.HeaderText = string.Format(HeaderTextPartitionListenerFormat, descriptions[0].PartitionId); } else { Text = string.Format(ConsumerGroupListenerFormat, consumerGroupDescription.Name); panelMain.HeaderText = string.Format(HeaderTextConsumerGroupListenerFormat, consumerGroupDescription.Name); } partitionListenerControl.Focus(); panelMain.Controls.Add(partitionListenerControl); SetStyle(ControlStyles.ResizeRedraw, true); } finally { panelMain.ResumeDrawing(); ResumeLayout(); } }
public PartitionListenerControl(WriteToLogDelegate writeToLog, Action stopAndRestartLog, ServiceBusHelper serviceBusHelper, ConsumerGroupDescription consumerGroupDescription, IEnumerable<PartitionDescription> partitionDescriptions) { Task.Factory.StartNew(AsyncTrackEventData).ContinueWith(t => { if (t.IsFaulted && t.Exception != null) { writeToLog(t.Exception.Message); } }); this.writeToLog = writeToLog; this.stopAndRestartLog = stopAndRestartLog; this.serviceBusHelper = serviceBusHelper; this.consumerGroupDescription = consumerGroupDescription; var descriptions = partitionDescriptions as IList<PartitionDescription> ?? partitionDescriptions.ToList(); this.partitionDescriptions = descriptions; partitionCount = partitionDescriptions == null || descriptions.Count == 0? 0 : descriptions.Count; eventHubDescription = serviceBusHelper.NamespaceManager.GetEventHub(consumerGroupDescription.EventHubPath); InitializeComponent(); InitializeControls(); checkBoxCheckpoint_CheckedChanged(null, null); Disposed += ListenerControl_Disposed; }