private void CreateStreamAnalyticsJob(string nameSuffix, string query, string resourceGroupName, AzurePrepInputs azurePrepIn,
EventHubDescription ehInput, EventHubDescription ehOutput)
{
const string inputName = "DevicesInput";
const string outputName = "output";
string jobName = azurePrepIn.NamePrefix + nameSuffix;
string transformationName = jobName + "-tr";
var computeClient = new StreamAnalyticsManagementClient(azurePrepIn.Credentials);
var serialization = new JsonSerialization
{
Type = "JSON",
Properties = new JsonSerializationProperties
{
Encoding = "UTF8"
}
};
List <Input> jobInputs = new List <Input>
{
new Input
{
Name = inputName,
Properties = new StreamInputProperties
{
DataSource = new EventHubStreamInputDataSource
{
Properties = new EventHubStreamInputDataSourceProperties
{
EventHubName = ehInput.Path,
ServiceBusNamespace = azurePrepIn.SBNamespace,
SharedAccessPolicyName = "StreamingAnalytics",
SharedAccessPolicyKey = (ehInput.Authorization.First((d)
=> String.Equals(d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey,
}
},
Serialization = serialization
}
}
};
List <Output> jobOutputs = new List <Output>
{
new Output
{
Name = outputName,
Properties = new OutputProperties
{
DataSource = new EventHubOutputDataSource
{
Properties = new EventHubOutputDataSourceProperties
{
EventHubName = ehOutput.Path,
ServiceBusNamespace = azurePrepIn.SBNamespace,
SharedAccessPolicyName = "StreamingAnalytics",
SharedAccessPolicyKey = (ehOutput.Authorization.First((d)
=> String.Equals(d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey,
}
},
Serialization = serialization
}
}
};
bool created = true;
try
{
var jobCreateResponse = computeClient.StreamingJobs.CreateOrUpdateAsync(
resourceGroupName,
new JobCreateOrUpdateParameters
{
Job = new Job
{
Name = jobName,
Location = azurePrepIn.Location,
Properties = new JobProperties
{
Sku = new Sku
{
//should be "standart" according to https://msdn.microsoft.com/en-us/library/azure/dn834994.aspx
Name = "standard"
},
EventsOutOfOrderPolicy = "drop",
EventsOutOfOrderMaxDelayInSeconds = 10,
Inputs = jobInputs,
Outputs = jobOutputs,
Transformation = new Transformation
{
Name = transformationName,
Properties = new TransformationProperties
{
Query = query,
StreamingUnits = 1
}
}
}
}
}
).Result;
}
catch (Exception ex)
{
_ConsoleBuffer.Add("Exception on creation Stream Analytics Job " + jobName + ": " + ex.Message);
_ConsoleBuffer.Add("Inner exception message: " + ex.InnerException.Message);
created = false;
}
if (created)
{
_ConsoleBuffer.Add("Stream Analytics job " + jobName + " created.");
}
}
private AzurePrepOutputs CreateEventHub(AzurePrepInputs inputs)
{
AzurePrepOutputs result = new AzurePrepOutputs
{
SBNamespace = inputs.SBNamespace
};
// Create Namespace
var sbMgmt = new ServiceBusManagementClient(inputs.Credentials);
ServiceBusNamespaceResponse nsResponse = null;
_ConsoleBuffer.Add(string.Format("Creating Service Bus namespace {0} in location {1}", inputs.SBNamespace, inputs.Location));
try
{
// There is (currently) no clean error code returned when the namespace already exists
// Check if it does
nsResponse = sbMgmt.Namespaces.Create(inputs.SBNamespace, inputs.Location);
_ConsoleBuffer.Add(string.Format("Service Bus namespace {0} created.", inputs.SBNamespace));
}
catch (Exception)
{
nsResponse = null;
_ConsoleBuffer.Add(string.Format("Service Bus namespace {0} already existed.", inputs.SBNamespace));
}
int triesCount = 0;
// Wait until the namespace is active
while (nsResponse == null || nsResponse.Namespace.Status != "Active")
{
nsResponse = sbMgmt.Namespaces.Get(inputs.SBNamespace);
if (nsResponse.Namespace.Status == "Active")
{
break;
}
triesCount += 1;
if (triesCount % 10 == 0)
{
_ConsoleBuffer.Add("Please note that activation could last about an hour if namespace with the same name prefix was deleted recently...");
}
else
{
_ConsoleBuffer.Add(string.Format("Namespace {0} in state {1}. Waiting...", inputs.SBNamespace, nsResponse.Namespace.Status));
}
System.Threading.Thread.Sleep(5000);
}
// Get the namespace connection string
var nsDescription = sbMgmt.Namespaces.GetNamespaceDescription(inputs.SBNamespace);
result.nsConnectionString = nsDescription.NamespaceDescriptions.First(
(d) => String.Equals(d.AuthorizationType, "SharedAccessAuthorization")
).ConnectionString;
// Create EHs + device keys + consumer keys (WebSite*)
var nsManager = NamespaceManager.CreateFromConnectionString(result.nsConnectionString);
var ehDescriptionDevices = new EventHubDescription(inputs.EventHubNameDevices)
{
PartitionCount = 8,
};
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D1", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D2", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D3", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D4", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("WebSite", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("StreamingAnalytics", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
_ConsoleBuffer.Add(string.Format("Creating Event Hub {0}...", inputs.EventHubNameDevices));
result.ehDevices = null;
do
{
try
{
result.ehDevices = nsManager.CreateEventHubIfNotExists(ehDescriptionDevices);
}
catch (UnauthorizedAccessException)
{
_ConsoleBuffer.Add("Service Bus connection string not valid yet. Waiting...");
System.Threading.Thread.Sleep(5000);
}
} while (result.ehDevices == null);
ConsoleHelper.AskAndPerformAction(
"Do you want to create " + inputs.EventHubNameAlerts + " Event Hub?",
"Are you sure you want to create " + inputs.EventHubNameAlerts + " Event Hub?",
"Are you sure you do not want to create " + inputs.EventHubNameAlerts + " Event Hub?",
() =>
{
var ehDescriptionAlerts = new EventHubDescription(inputs.EventHubNameAlerts)
{
PartitionCount = 8,
};
ehDescriptionAlerts.Authorization.Add(new SharedAccessAuthorizationRule("WebSite", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
ehDescriptionAlerts.Authorization.Add(new SharedAccessAuthorizationRule("StreamingAnalytics", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
_ConsoleBuffer.Add(string.Format("Creating Event Hub {0}...", inputs.EventHubNameAlerts));
result.ehAlerts = null;
do
{
try
{
result.ehAlerts = nsManager.CreateEventHubIfNotExists(ehDescriptionAlerts);
}
catch (UnauthorizedAccessException)
{
_ConsoleBuffer.Add("Service Bus connection string not valid yet. Waiting...");
System.Threading.Thread.Sleep(5000);
}
} while (result.ehAlerts == null);
},
_ConsoleBuffer);
// Create Storage Account for Event Hub Processor
var stgMgmt = new StorageManagementClient(inputs.Credentials);
try
{
_ConsoleBuffer.Add(string.Format("Creating Storage Account {0} in location {1}...",
inputs.StorageAccountName, inputs.Location));
var resultStg = stgMgmt.StorageAccounts.Create(
new StorageAccountCreateParameters {
Name = inputs.StorageAccountName.ToLowerInvariant(), Location = inputs.Location, AccountType = "Standard_LRS"
});
if (resultStg.StatusCode != System.Net.HttpStatusCode.OK)
{
_ConsoleBuffer.Add(string.Format("Error creating storage account {0} in Location {1}: {2}",
inputs.StorageAccountName, inputs.Location, resultStg.StatusCode));
return(null);
}
}
catch (CloudException ce)
{
if (String.Equals(ce.Error.Code, "ConflictError", StringComparison.InvariantCultureIgnoreCase))
{
_ConsoleBuffer.Add(string.Format("Storage account {0} already existed.", inputs.StorageAccountName));
}
else
{
throw;
}
}
return(result);
}
