public static async Task Start(string hostName, string eventHubName, string consumerGroupName, string containerName, string blobPrefix,
string serviceBusConnectionString, string storageConnectionString)
{
// Create Consumer Group if it doesn't exist
if (consumerGroupName != null)
{
NamespaceManager manager = NamespaceManager.CreateFromConnectionString(serviceBusConnectionString);
ConsumerGroupDescription description = new ConsumerGroupDescription(eventHubName, consumerGroupName);
manager.CreateConsumerGroupIfNotExists(description);
}
// Create the Event Processor Host
var host = new EventProcessorHost(
hostName,
eventHubName,
consumerGroupName == null ? EventHubConsumerGroup.DefaultGroupName : consumerGroupName,
serviceBusConnectionString,
storageConnectionString,
eventHubName);
// Create the Factory
var factory = new BlobEventProcessorHostFactory(storageConnectionString, containerName, blobPrefix);
// Register the Factory
await host.RegisterEventProcessorFactoryAsync(factory);
}
public void Run(string connectionString, string hubName)
{
NamespaceManager nsmgr = NamespaceManager.CreateFromConnectionString(connectionString);
EventHubDescription desc = nsmgr.GetEventHub(hubName);
string consumerGroupName = _consumerGroupPrefix + DateTime.UtcNow.Ticks.ToString();
ConsumerGroupDescription consumerGroupDesc = nsmgr.CreateConsumerGroupIfNotExists(new ConsumerGroupDescription(hubName, consumerGroupName));
EventHubClient client = EventHubClient.CreateFromConnectionString(connectionString, hubName);
int numPartitions = desc.PartitionCount;
_receivers = new EventHubReceiver[numPartitions];
//_receiversLastUpdate = new DateTime[numPartitions];
//for (int i = 0; i < numPartitions; i++)
//{
// _receiversLastUpdate[i] = DateTime.UtcNow;
//}
_tasks = new Task[numPartitions];
_buffers = new Dictionary <string, CircularBuffer <SensorDataContract> >();
for (int iPart = 0; iPart < desc.PartitionCount; iPart++)
{
EventHubReceiver receiver = client.GetConsumerGroup(consumerGroupName).CreateReceiver(
desc.PartitionIds[iPart], DateTime.UtcNow - TimeSpan.FromMinutes(2));
_receivers[iPart] = receiver;
Task <IEnumerable <EventData> > task = receiver.ReceiveAsync(1000, TimeSpan.FromSeconds(1));
int thisPart = iPart;
task.ContinueWith(new Action <Task <IEnumerable <EventData> > >((t) => OnTaskComplete(t, thisPart)));
_tasks[iPart] = task;
}
}
public void Run(string connectionString, string hubName, string measureNameFilter)
{
NamespaceManager nsmgr = NamespaceManager.CreateFromConnectionString(connectionString);
EventHubDescription desc = nsmgr.GetEventHub(hubName);
//we use already defined consumerGroup name to not reach limit on CG count
string consumerGroupName = _consumerGroupPrefix;
ConsumerGroupDescription consumerGroupDesc = nsmgr.CreateConsumerGroupIfNotExists(new ConsumerGroupDescription(hubName, consumerGroupName));
EventHubClient client = EventHubClient.CreateFromConnectionString(connectionString, hubName);
int numPartitions = desc.PartitionCount;
_receivers = new EventHubReceiver[numPartitions];
_tasks = new Task[numPartitions];
for (int iPart = 0; iPart < desc.PartitionCount; iPart++)
{
EventHubReceiver receiver = client.GetConsumerGroup(consumerGroupName).CreateReceiver(
desc.PartitionIds[iPart], DateTime.UtcNow - TimeSpan.FromMinutes(2));
_receivers[iPart] = receiver;
int part = iPart;
Task.Factory.StartNew((state) =>
{
try
{
while (true)
{
var messages = _receivers[part].Receive(1000, TimeSpan.FromSeconds(1));
Process(messages);
}
}
catch (Exception ex)
{
//FailureEvent.Set();
Trace.TraceError("Ignored invalid event data: {0}");
}
}, iPart);
}
}
public string GetCreateEHAndConsumerGroup(string ConsumenrGroup, ref string eventHubName, ref string eventHubConnectionString, ref string storageConnectionString)
{
try
{
//Set connection strings
eventHubConnectionString = "Endpoint=sb://[NAMESPACE].servicebus.windows.net/;SharedAccessKeyName=All;SharedAccessKey=[SharedAccessKey]";
string eventHubConnectionStringAdmin = "Endpoint=sb://[NAMESPACE].servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=[SharedAccessKey]";
eventHubName = "[EVENTHUBNAME]";
string storageAccountName = "[STORAGEACCOUNTNAME]";
string storageAccountKey = "[STORAGEACCOUNTKEY]";
storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageAccountName, storageAccountKey);
//Create connection string
var builder = new ServiceBusConnectionStringBuilder(eventHubConnectionString)
{
TransportType = Microsoft.ServiceBus.Messaging.TransportType.Amqp
};
ConsoleWriteLine("Create/Open the EH", ConsoleColor.Magenta);
NamespaceManager namespaceManager = NamespaceManager.CreateFromConnectionString(eventHubConnectionStringAdmin);
var evenhubDesc = new EventHubDescription(eventHubName);
namespaceManager.CreateEventHubIfNotExists(evenhubDesc);
if (ConsumenrGroup == null)
{
var client = EventHubClient.CreateFromConnectionString(builder.ToString(), eventHubName);
EventHubConsumerGroup eventHubConsumerGroup = client.GetDefaultConsumerGroup();
return(eventHubConsumerGroup.GroupName);
}
else
{
namespaceManager.CreateConsumerGroupIfNotExists(eventHubName, ConsumenrGroup);
return(ConsumenrGroup);
}
}
catch (Exception ex)
{
ConsoleWriteLine("ERROR-" + ex.Message, ConsoleColor.Red);
return(null);
}
}
// Please set the following connection strings in app.config for this WebJob to run:
// AzureWebJobsDashboard and AzureWebJobsStorage
static void Main()
{
string eventHubName = ConfigurationManager.AppSettings["EventHubName"];
string consumerGroup = ConfigurationManager.AppSettings["ConsumerGroup"];
string connectionString = GetServiceBusConnectionString();
NamespaceManager namespaceManager = NamespaceManager.CreateFromConnectionString(connectionString);
EventHubDescription ehd = namespaceManager.GetEventHub(eventHubName);
namespaceManager.CreateConsumerGroupIfNotExists(ehd.Path, consumerGroup);
var receiver = new Receiver(eventHubName, consumerGroup, connectionString);
receiver.MessageProcessingWithPartitionDistribution();
// The following code ensures that the WebJob will be running continuously
var host = new JobHost();
host.RunAndBlock();
}
private async Task RunAsync(CancellationToken cancellationToken)
{
// make sure consumer group exists
NamespaceManager manager = NamespaceManager.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp");
ConsumerGroupDescription description = new ConsumerGroupDescription(ConfigurationManager.AppSettings["ServiceBus.Path"], ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]);
manager.CreateConsumerGroupIfNotExists(description);
//get a handle on the consumer group for the event hub we want to read from
var factory = MessagingFactory.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp");
var client = factory.CreateEventHubClient(ConfigurationManager.AppSettings["ServiceBus.Path"]);
var group = client.GetConsumerGroup(ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]);
//get a handle on the container we want to write blobs to
var storageAccount = CloudStorageAccount.Parse(ConfigurationManager.AppSettings["Storage.ConnectionString"]);
var blobClient = storageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(ConfigurationManager.AppSettings["Storage.Container"]);
container.CreateIfNotExists();
while (!cancellationToken.IsCancellationRequested)
{
Task.WaitAll(client.GetRuntimeInformation().PartitionIds.Select(id => Task.Run(() =>
{
var receiver = @group.CreateReceiver(id);
var messageBuffer = new List <string>();
var startTime = DateTime.UtcNow;
Trace.TraceInformation("Waiting for messages " + receiver.PartitionId);
while (true)
{
try
{
//read the message
var message = receiver.Receive();
if (message == null)
{
continue;
}
var body = Encoding.UTF8.GetString(message.GetBytes());
if (body == null)
{
continue;
}
var currentTime = DateTime.UtcNow;
//add to the buffer
messageBuffer.Add(body);
//write out a file if a minute has passed and we have at least one message
if ((currentTime - startTime).TotalMinutes >= 1 && messageBuffer.Count >= 1)
{
var now = DateTime.Now;
var asString = String.Join("\n", messageBuffer);
//upload the blob
var blockBlob = container.GetBlockBlobReference(String.Format("{0}/{1}/{2}/message_{3}_{4}.log", now.Year, now.Month, now.Day, id, now.TimeOfDay));
blockBlob.UploadFromStream(new MemoryStream(Encoding.UTF8.GetBytes(asString)));
//clear the buffer
messageBuffer.Clear();
//start tracking anew
startTime = currentTime;
}
}
catch (Exception e)
{
Console.WriteLine(e.Message);
//suppress for simplicity
}
}
}, cancellationToken)).ToArray());
}
}
// Please set the following connection strings in app.config for this WebJob to run:
// AzureWebJobsDashboard and AzureWebJobsStorage
static void Main()
{
// make sure consumer group exists
NamespaceManager manager = NamespaceManager.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp");
ConsumerGroupDescription description = new ConsumerGroupDescription(ConfigurationManager.AppSettings["ServiceBus.Path"], ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]);
manager.CreateConsumerGroupIfNotExists(description);
//get a handle on the consumer group for the event hub we want to read from
var factory = MessagingFactory.CreateFromConnectionString(ConfigurationManager.AppSettings["ServiceBus.ConnectionString"] + ";TransportType=Amqp");
var client = factory.CreateEventHubClient(ConfigurationManager.AppSettings["ServiceBus.Path"]);
var group = client.GetConsumerGroup(ConfigurationManager.AppSettings["ServiceBus.ConsumerGroup"]);
while (true)
{
Task.WaitAll(client.GetRuntimeInformation().PartitionIds.Select(id => Task.Run(() =>
{
var receiver = @group.CreateReceiver(id);
Trace.TraceInformation("Waiting for messages " + receiver.PartitionId);
while (true)
{
try
{
//read the message
var message = receiver.Receive();
if (message == null)
{
continue;
}
var body = Encoding.UTF8.GetString(message.GetBytes());
if (body == null)
{
continue;
}
var type = MessageType.None;
switch (message.PartitionKey.ToLower())
{
case "energy":
type = MessageType.Energy;
break;
case "temperature":
type = MessageType.Temperature;
break;
case "humidity":
type = MessageType.Humidity;
break;
case "light":
type = MessageType.Light;
break;
}
if (type == MessageType.None)
{
continue;
}
var writer = new DocumentDBWriter();
var task = writer.WriteDocument(type, body);
Task.WaitAll(task); // block while the task completes
Console.WriteLine(task.Result);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
//suppress for simplicity
}
}
})).ToArray());
}
}
