private void CreateStreamAnalyticsJobs(AzurePrepInputs azurePrepIn, AzurePrepOutputs azurePrepOut)
{
string resourceGroupName = SelectResourceGroup(azurePrepIn);
string path = Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly( ).Location);
path += "\\..\\..\\..\\..\\StreamAnalyticsQueries";
foreach (string filename in Directory.GetFiles(path))
{
string extension = Path.GetExtension(filename);
if (extension != null && extension.Contains("sql"))
{
string nameWithoutExtension = Path.GetFileNameWithoutExtension(filename);
EventHubDescription ehOutput = (filename.ToLower( ).Contains("aggregates") || azurePrepOut.ehAlerts == null)
? azurePrepOut.ehDevices
: azurePrepOut.ehAlerts;
if (ehOutput == null)
{
_ConsoleBuffer.Add(string.Format(" Skip creating {0} Stream Analytics job because there is no output Event Hub...", nameWithoutExtension));
continue;
}
string queryFilename = filename;
ConsoleHelper.AskAndPerformAction(
"Do you want to create " + nameWithoutExtension + " job?",
"Are you sure you want to create " + nameWithoutExtension + " job?",
"Are you sure you do not want to create " + nameWithoutExtension + " job?",
() =>
{
string query = File.ReadAllText(queryFilename);
_ConsoleBuffer.Add(string.Format("Creating {0} Stream Analytics job...", nameWithoutExtension));
CreateStreamAnalyticsJob(nameWithoutExtension, query, resourceGroupName,
azurePrepIn, azurePrepOut.ehDevices, ehOutput);
},
_ConsoleBuffer);
}
}
}
public bool Run( )
{
AzurePrepInputs inputs;
if (!GetInputs(out inputs))
{
Console.WriteLine("Error while getting inputs.");
Console.WriteLine("Press Enter to continue...");
Console.ReadLine( );
return(false);
}
AzurePrepOutputs createResults = CreateEventHub(inputs);
if (createResults == null)
{
Console.WriteLine("Error while creating Event Hubs.");
Console.WriteLine("Press Enter to continue...");
Console.ReadLine( );
return(false);
}
for ( ;;)
{
Console.WriteLine("Do you want to create Stream Analytics jobs? (y/n)");
string answer = Console.ReadLine( );
bool create;
string request;
if (!string.IsNullOrEmpty(answer) && answer.ToLower( ).StartsWith("y"))
{
create = true;
request = "";
}
else
{
create = false;
request = "do not ";
}
if (ConsoleHelper.Confirm("Are you sure you " + request + "want to create Stream Analytics jobs?"))
{
if (create)
{
CreateStreamAnalyticsJobs(inputs, createResults);
}
break;
}
}
#region print results
_ConsoleBuffer.Add("");
_ConsoleBuffer.Add("Service Bus management connection string (i.e. for use in Service Bus Explorer):");
_ConsoleBuffer.Add(createResults.nsConnectionString);
_ConsoleBuffer.Add("");
_ConsoleBuffer.Add("Device AMQP address strings (for Raspberry PI/devices):");
for (int i = 1; i <= 4; i++)
{
var deviceKeyName = String.Format("D{0}", i);
var deviceKey = (createResults.ehDevices.Authorization.First((d)
=> String.Equals(d.KeyName, deviceKeyName, StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey;
_ConsoleBuffer.Add(string.Format("amqps://{0}:{1}@{2}.servicebus.windows.net",
deviceKeyName, Uri.EscapeDataString(deviceKey), createResults.SBNamespace));
}
Console.WriteLine("");
Console.WriteLine("");
string fileName = createResults.SBNamespace + DateTime.UtcNow.ToString("_d_MMM_h_mm") + ".log";
string filePath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
string fileFullName = filePath + @"\" + fileName;
if (_ConsoleBuffer.FlushToFile(fileFullName))
{
Console.WriteLine("Output was saved to your desktop, at " + fileFullName + " file.");
}
Console.WriteLine("Please hit enter to close.");
Console.ReadLine( );
#endregion
return(true);
}
private AzurePrepOutputs CreateEventHub(AzurePrepInputs inputs)
{
AzurePrepOutputs result = new AzurePrepOutputs
{
SBNamespace = inputs.SBNamespace
};
// Create Namespace
var sbMgmt = new ServiceBusManagementClient(inputs.Credentials);
ServiceBusNamespaceResponse nsResponse = null;
_ConsoleBuffer.Add(string.Format("Creating Service Bus namespace {0} in location {1}", inputs.SBNamespace, inputs.Location));
try
{
// There is (currently) no clean error code returned when the namespace already exists
// Check if it does
nsResponse = sbMgmt.Namespaces.Create(inputs.SBNamespace, inputs.Location);
_ConsoleBuffer.Add(string.Format("Service Bus namespace {0} created.", inputs.SBNamespace));
}
catch (Exception)
{
nsResponse = null;
_ConsoleBuffer.Add(string.Format("Service Bus namespace {0} already existed.", inputs.SBNamespace));
}
// Wait until the namespace is active
while (nsResponse == null || nsResponse.Namespace.Status != "Active")
{
nsResponse = sbMgmt.Namespaces.Get(inputs.SBNamespace);
if (nsResponse.Namespace.Status == "Active")
{
break;
}
_ConsoleBuffer.Add(string.Format("Namespace {0} in state {1}. Waiting...", inputs.SBNamespace, nsResponse.Namespace.Status));
System.Threading.Thread.Sleep(5000);
}
// Get the namespace connection string
var nsDescription = sbMgmt.Namespaces.GetNamespaceDescription(inputs.SBNamespace);
result.nsConnectionString = nsDescription.NamespaceDescriptions.First(
(d) => String.Equals(d.AuthorizationType, "SharedAccessAuthorization")
).ConnectionString;
// Create EHs + device keys + consumer keys (WebSite*)
var nsManager = NamespaceManager.CreateFromConnectionString(result.nsConnectionString);
var ehDescriptionDevices = new EventHubDescription(inputs.EventHubNameDevices)
{
PartitionCount = 8,
};
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D1", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D2", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D3", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("D4", new List <AccessRights> {
AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("WebSite", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
ehDescriptionDevices.Authorization.Add(new SharedAccessAuthorizationRule("StreamingAnalytics", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
_ConsoleBuffer.Add(string.Format("Creating Event Hub {0}...", inputs.EventHubNameDevices));
result.ehDevices = null;
do
{
try
{
result.ehDevices = nsManager.CreateEventHubIfNotExists(ehDescriptionDevices);
}
catch (UnauthorizedAccessException)
{
_ConsoleBuffer.Add("Service Bus connection string not valid yet. Waiting...");
System.Threading.Thread.Sleep(5000);
}
} while (result.ehDevices == null);
ConsoleHelper.AskAndPerformAction(
"Do you want to create " + inputs.EventHubNameAlerts + " Event Hub?",
"Are you sure you want to create " + inputs.EventHubNameAlerts + " Event Hub?",
"Are you sure you do not want to create " + inputs.EventHubNameAlerts + " Event Hub?",
() =>
{
var ehDescriptionAlerts = new EventHubDescription(inputs.EventHubNameAlerts)
{
PartitionCount = 8,
};
ehDescriptionAlerts.Authorization.Add(new SharedAccessAuthorizationRule("WebSite", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
ehDescriptionAlerts.Authorization.Add(new SharedAccessAuthorizationRule("StreamingAnalytics", new List <AccessRights> {
AccessRights.Manage, AccessRights.Listen, AccessRights.Send
}));
_ConsoleBuffer.Add(string.Format("Creating Event Hub {0}...", inputs.EventHubNameAlerts));
result.ehAlerts = null;
do
{
try
{
result.ehAlerts = nsManager.CreateEventHubIfNotExists(ehDescriptionAlerts);
}
catch (UnauthorizedAccessException)
{
_ConsoleBuffer.Add("Service Bus connection string not valid yet. Waiting...");
System.Threading.Thread.Sleep(5000);
}
} while (result.ehAlerts == null);
},
_ConsoleBuffer);
// Create Storage Account for Event Hub Processor
var stgMgmt = new StorageManagementClient(inputs.Credentials);
try
{
_ConsoleBuffer.Add(string.Format("Creating Storage Account {0} in location {1}...",
inputs.StorageAccountName, inputs.Location));
var resultStg = stgMgmt.StorageAccounts.Create(
new StorageAccountCreateParameters {
Name = inputs.StorageAccountName.ToLowerInvariant(), Location = inputs.Location, AccountType = "Standard_LRS"
});
if (resultStg.StatusCode != System.Net.HttpStatusCode.OK)
{
_ConsoleBuffer.Add(string.Format("Error creating storage account {0} in Location {1}: {2}",
inputs.StorageAccountName, inputs.Location, resultStg.StatusCode));
return(null);
}
}
catch (CloudException ce)
{
if (String.Equals(ce.Error.Code, "ConflictError", StringComparison.InvariantCultureIgnoreCase))
{
_ConsoleBuffer.Add(string.Format("Storage account {0} already existed.", inputs.StorageAccountName));
}
else
{
throw;
}
}
return(result);
}
public bool Run( )
{
AzurePrepInputs inputs;
if (!GetInputs(out inputs))
{
Console.WriteLine("Error while getting inputs.");
Console.WriteLine("Press Enter to continue...");
Console.ReadLine();
return(false);
}
AzurePrepOutputs createResults = CreateEventHub(inputs);
if (createResults == null)
{
Console.WriteLine("Error while creating Event Hubs.");
Console.WriteLine("Press Enter to continue...");
Console.ReadLine();
return(false);
}
#region print results
Console.WriteLine( );
Console.WriteLine("Service Bus management connection string (i.e. for use in Service Bus Explorer):");
Console.WriteLine(createResults.nsConnectionString);
Console.WriteLine( );
Console.WriteLine("Device AMQP address strings (for Raspberry PI/devices):");
for (int i = 1; i <= 4; i++)
{
var deviceKeyName = String.Format("D{0}", i);
var deviceKey = (createResults.ehDevices.Authorization.First((d)
=> String.Equals(d.KeyName, deviceKeyName, StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey;
Console.WriteLine("amqps://{0}:{1}@{2}.servicebus.windows.net", deviceKeyName, Uri.EscapeDataString(deviceKey), createResults.SBNamespace);
}
Console.ReadLine();
#endregion
#if AZURESTREAMANALYTICS
// Create StreamAnalyticsJobs + inputs + outputs + enter keys
// Untested code. May require AAD authentication, no support for management cert?
// Create Resource Group for the Stream Analytics jobs
var groupCreateRequest = WebRequest.Create(String.Format("https://management.azure.com/subscriptions/{0}/resourcegroups/{1}?api-version=2014-04-01-preview",
SubscriptionId, StreamAnalyticsGroup)) as HttpWebRequest;
groupCreateRequest.ClientCertificates.Add(creds.ManagementCertificate);
groupCreateRequest.ContentType = "application/json";
groupCreateRequest.Method = "PUT";
groupCreateRequest.KeepAlive = true;
var bytesGroup = Encoding.UTF8.GetBytes("{\"location\":\"Central US\"}");
groupCreateRequest.ContentLength = bytesGroup.Length;
groupCreateRequest.GetRequestStream().Write(bytesGroup, 0, bytesGroup.Length);
var groupCreateResponse = groupCreateRequest.GetResponse();
//var streamMgmt = new ManagementClient(creds); //, new Uri("https://management.azure.com"));
//HttpClient client = streamMgmt.HttpClient;
var createJob = new StreamAnalyticsJob()
{
location = Location,
inputs = new List <StreamAnalyticsEntity>
{
new StreamAnalyticsEntity
{
name = "devicesInput",
properties = new Dictionary <string, object>
{
{ "type", "stream" },
{ "serialization", new Dictionary <string, object>
{
{ "type", "JSON" },
{ "properties", new Dictionary <string, object>
{
{ "encoding", "UTF8" },
} }
} },
{ "datasource", new Dictionary <string, object>
{
{ "type", "Microsoft.ServiceBus/EventHub" },
{ "properties", new Dictionary <string, object>
{
{ "eventHubNamespace", Namespace },
{ "eventHubName", EventHubDevices },
{ "sharedAccessPolicyName", "StreamingAnalytics" },
{ "sharedAccessPolicyKey",
(ehDevices.Authorization.First((d)
=> String.Equals(d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey },
} }
} }
},
},
},
transformation = new StreamAnalyticsEntity()
{
name = "Aggregates",
properties = new Dictionary <string, object>
{
{ "streamingUnits", 1 },
{ "query", "select * from devicesInput" },
}
},
outputs = new List <StreamAnalyticsEntity>
{
new StreamAnalyticsEntity
{
name = "output",
properties = new Dictionary <string, object>
{
{ "datasource", new Dictionary <string, object>
{
{ "type", "Microsoft.ServiceBus/EventHub" },
{ "properties", new Dictionary <string, object>
{
{ "eventHubNamespace", Namespace },
{ "eventHubName", EventHubAlerts },
{ "sharedAccessPolicyName", "StreamingAnalytics" },
{ "sharedAccessPolicyKey",
(ehAlerts.Authorization.First((d) => String.Equals(d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey },
} }
} },
{ "serialization", new Dictionary <string, object>
{
{ "type", "JSON" },
{ "properties", new Dictionary <string, object>
{
{ "encoding", "UTF8" },
} }
} },
},
},
}
};
var jobCreateRequest = WebRequest.Create(String.Format("https://management.azure.com/subscriptions/{0}/resourcegroups/{1}/Microsoft.StreamAnalytics/streamingjobs/{2}?api-version=2014-10-01",
SubscriptionId, StreamAnalyticsGroup, JobAggregates)) as HttpWebRequest;
jobCreateRequest.ClientCertificates.Add(creds.ManagementCertificate);
jobCreateRequest.ContentType = "application/json";
jobCreateRequest.Method = "PUT";
jobCreateRequest.KeepAlive = true;
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(createJob));
jobCreateRequest.ContentLength = bytes.Length;
jobCreateRequest.GetRequestStream().Write(bytes, 0, bytes.Length);
var jobCreateResponse = jobCreateRequest.GetResponse();
//var jobCreateTask = streamMgmt.HttpClient.PutAsync(
// String.Format("https://management.azure.com/subscriptions/{0}/resourcegroups/{1}/Microsoft.StreamAnalytics/streamingjobs/{2}?api-version=2014-10-01",
// SubscriptionId, StreamAnalyticsGroup, JobAggregates),
// new StringContent(JsonConvert.SerializeObject(createJob)));
//jobCreateTask.Wait();
//var jobCreateResponse = jobCreateTask.Result;
#endif
return(true);
}
