public virtual async Task <IStreamAnalyticsManagementClient> GetClientAsync()
{
try
{
string authToken = string.Empty;
try
{
authToken = await this.tokenHelper.GetTokenAsync();
if (string.IsNullOrEmpty(authToken))
{
throw new Exception("Auth Token from tokenHelper returned a null response.");
}
}
catch (Exception e)
{
throw new Exception("Unable to get an authorization token for creating a Stream Analytics Management Client.", e);
}
TokenCredentials credentials = new TokenCredentials(authToken);
StreamAnalyticsManagementClient client = new StreamAnalyticsManagementClient(credentials)
{
SubscriptionId = this.config.Global.SubscriptionId,
};
return(client);
}
catch (Exception e)
{
throw new Exception("Unable to get a new Stream Analytics Management Client.", e);
}
}
static void Main(string[] args)
{
string resourceGroupName = "<YOUR AZURE RESOURCE GROUP NAME>";
string streamAnalyticsJobName = "<YOUR STREAM ANALYTICS JOB NAME>";
// Get authentication token
TokenCloudCredentials aadTokenCredentials =
new TokenCloudCredentials(
ConfigurationManager.AppSettings["SubscriptionId"],
GetAuthorizationHeader());
Uri resourceManagerUri = new Uri(ConfigurationManager.AppSettings["ResourceManagerEndpoint"]);
// Create Stream Analytics and Insights management client
StreamAnalyticsManagementClient streamAnalyticsClient = new
StreamAnalyticsManagementClient(aadTokenCredentials, resourceManagerUri);
InsightsManagementClient insightsClient = new
InsightsManagementClient(aadTokenCredentials, resourceManagerUri);
// Get an existing Stream Analytics job
JobGetParameters jobGetParameters = new JobGetParameters()
{
PropertiesToExpand = "inputs,transformation,outputs"
};
JobGetResponse jobGetResponse = streamAnalyticsClient.StreamingJobs.Get(resourceGroupName, streamAnalyticsJobName, jobGetParameters);
// Enable monitoring
ServiceDiagnosticSettingsPutParameters insightPutParameters = new ServiceDiagnosticSettingsPutParameters()
{
Properties = new ServiceDiagnosticSettings()
{
StorageAccountName = "<YOUR STORAGE ACCOUNT NAME>"
}
};
InsightsClient.ServiceDiagnosticSettingsOperations.Put(jobGetResponse.Job.Id, insightPutParameters);
}
private void CreateStreamAnalyticsJob(string nameSuffix, string query, string resourceGroupName, AzurePrepInputs azurePrepIn,
EventHubDescription ehInput, EventHubDescription ehOutput)
{
const string inputName = "DevicesInput";
const string outputName = "output";
string jobName = azurePrepIn.NamePrefix + nameSuffix;
string transformationName = jobName + "-tr";
var computeClient = new StreamAnalyticsManagementClient(azurePrepIn.Credentials);
var serialization = new JsonSerialization
{
Type = "JSON",
Properties = new JsonSerializationProperties
{
Encoding = "UTF8"
}
};
List <Input> jobInputs = new List <Input>
{
new Input
{
Name = inputName,
Properties = new StreamInputProperties
{
DataSource = new EventHubStreamInputDataSource
{
Properties = new EventHubStreamInputDataSourceProperties
{
EventHubName = ehInput.Path,
ServiceBusNamespace = azurePrepIn.SBNamespace,
SharedAccessPolicyName = "StreamingAnalytics",
SharedAccessPolicyKey = (ehInput.Authorization.First((d)
=> String.Equals(d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey,
}
},
Serialization = serialization
}
}
};
List <Output> jobOutputs = new List <Output>
{
new Output
{
Name = outputName,
Properties = new OutputProperties
{
DataSource = new EventHubOutputDataSource
{
Properties = new EventHubOutputDataSourceProperties
{
EventHubName = ehOutput.Path,
ServiceBusNamespace = azurePrepIn.SBNamespace,
SharedAccessPolicyName = "StreamingAnalytics",
SharedAccessPolicyKey = (ehOutput.Authorization.First((d)
=> String.Equals(d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase)) as SharedAccessAuthorizationRule).PrimaryKey,
}
},
Serialization = serialization
}
}
};
bool created = true;
try
{
var jobCreateResponse = computeClient.StreamingJobs.CreateOrUpdateAsync(
resourceGroupName,
new JobCreateOrUpdateParameters
{
Job = new Job
{
Name = jobName,
Location = azurePrepIn.Location,
Properties = new JobProperties
{
Sku = new Sku
{
//should be "standart" according to https://msdn.microsoft.com/en-us/library/azure/dn834994.aspx
Name = "standard"
},
EventsOutOfOrderPolicy = "drop",
EventsOutOfOrderMaxDelayInSeconds = 10,
Inputs = jobInputs,
Outputs = jobOutputs,
Transformation = new Transformation
{
Name = transformationName,
Properties = new TransformationProperties
{
Query = query,
StreamingUnits = 1
}
}
}
}
}
).Result;
}
catch (Exception ex)
{
_ConsoleBuffer.Add("Exception on creation Stream Analytics Job " + jobName + ": " + ex.Message);
created = false;
}
if (created)
{
_ConsoleBuffer.Add("Stream Analytics job " + jobName + " created.");
}
}
public AsaManagementClient(StreamAnalyticsManagementClient client, AppConfig config)
{
this.client = client;
this.config = config;
this.client.SubscriptionId = this.config.Global.SubscriptionId;
}
public static IActionResult Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = "stopjob")]
HttpRequest req,
ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
var jobName = "-";
try
{
string resourceGroupName = Environment.GetEnvironmentVariable("DefaultASAJobRG");
string streamingJobName = Environment.GetEnvironmentVariable("DefaultASAJobName");
var subscriptionId = Environment.GetEnvironmentVariable("SDK-SubscriptionId");
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
// Get credentials
ServiceClientCredentials credentials = GetCredentials();
// Create Stream Analytics management client
StreamAnalyticsManagementClient streamAnalyticsManagementClient = new StreamAnalyticsManagementClient(credentials)
{
SubscriptionId = subscriptionId
};
var asaJobs = streamAnalyticsManagementClient.StreamingJobs.List();
var asaJobList = new List <StreamingJob>(asaJobs);
var myJob = asaJobList.Where(j => j.Name == streamingJobName).FirstOrDefault();
if (myJob == null)
{
var message = $"No Azure Stream Analytics Job found with name: {jobName}";
log.LogError($"{message} in subscription: {subscriptionId}");
return(new BadRequestObjectResult(message));
}
if (myJob.JobState != "Running")
{
var message = $"Azure Stream Analytics Job '{jobName}' is not running.";
log.LogError($"{message} - Job state is {myJob.JobState}");
return(new BadRequestObjectResult(message));
}
// Stop a streaming job
streamAnalyticsManagementClient.StreamingJobs.Stop(resourceGroupName, streamingJobName);
}
catch (CloudException ex)
{
log.LogError($"Error occurred: {ex.ToString()}");
return(new BadRequestObjectResult("Error occured, check the logs on the server"));
}
return((ActionResult) new OkObjectResult($"Job successfully stopped : {jobName}"));
}
static void Main(string[] args)
{
string AzureresourceGroupName = "<<Your Azure Resource Group Name>>";
string AzurestreamingJobName = "<<Your Azure ASA job name>>";
string ASAinputName = "<<Your ASA job Input>>";
string ASAtransformationName = "<<Your ASA Job Transformation Name>>";
string ASAoutputName = "<<Your ASA job output Name>>";
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
// Get credentials
ServiceClientCredentials credentials = GetAzureCredentials().Result;
// Create Stream Analytics management client
StreamAnalyticsManagementClient streamAnalyticsManagementClient = new StreamAnalyticsManagementClient(credentials)
{
SubscriptionId = ConfigurationManager.AppSettings["SubscriptionId"]
};
// Create Azure streaming job
StreamingJob streamingJob = new StreamingJob()
{
Tags = new Dictionary <string, string>()
{
{ "Origin", ".NET SDK" },
{ "ReasonCreated", "Stream Analytics Job Provision Using .NET Management SDK" }
},
Location = "East Asia",
EventsOutOfOrderPolicy = EventsOutOfOrderPolicy.Adjust,
EventsOutOfOrderMaxDelayInSeconds = 2,
EventsLateArrivalMaxDelayInSeconds = 16,
OutputErrorPolicy = OutputErrorPolicy.Stop,
DataLocale = "en-US",
CompatibilityLevel = CompatibilityLevel.OneFullStopZero,
Sku = new Sku()
{
Name = SkuName.Standard
}
};
StreamingJob createNewStreamingJobResult = streamAnalyticsManagementClient.StreamingJobs.CreateOrReplace(streamingJob, AzureresourceGroupName, AzurestreamingJobName);
// Create an input
StorageAccount storageAccount = new StorageAccount()
{
AccountName = "<<Your Azure Storage Account Name>>",
AccountKey = "<<Azure Storage Account Key>>"
};
Input input = new Input()
{
Properties = new StreamInputProperties()
{
Serialization = new CsvSerialization()
{
FieldDelimiter = ",",
Encoding = Microsoft.Azure.Management.StreamAnalytics.Models.Encoding.UTF8
},
Datasource = new BlobStreamInputDataSource()
{
StorageAccounts = new[] { storageAccount },
Container = "<<Your Storage Account Container>>",
PathPattern = "{date}/{time}",
DateFormat = "yyyy/MM/dd",
TimeFormat = "HH",
SourcePartitionCount = 16
}
}
};
Input createInputResult = streamAnalyticsManagementClient.Inputs.CreateOrReplace(input, AzureresourceGroupName, AzurestreamingJobName, ASAinputName);
// Test the connection to the input
ResourceTestStatus testInputResult = streamAnalyticsManagementClient.Inputs.Test(AzureresourceGroupName, AzurestreamingJobName, ASAinputName);
// Create an output
Output output = new Output()
{
Datasource = new AzureSqlDatabaseOutputDataSource()
{
Server = "<<Your Azure SQL Database Server>>",
Database = "<<Your Azure SQL db name>>",
User = "******",
Password = "******",
Table = "<<Your Azure SQL db table>>"
}
};
Output createOutputResult = streamAnalyticsManagementClient.Outputs.CreateOrReplace(output, AzureresourceGroupName, AzurestreamingJobName, ASAoutputName);
// Create a transformation
Transformation transformation = new Transformation()
{
Query = "Select Id, Name from input", // '<your input name>' should be replaced with the value you put for the 'inputName' variable above or in a previous step
StreamingUnits = 1
};
Transformation createTransformationResult = streamAnalyticsManagementClient.Transformations.CreateOrReplace(transformation, AzureresourceGroupName, AzurestreamingJobName, ASAtransformationName);
// Start a streaming job
StartStreamingJobParameters startStreamingJobParameters = new StartStreamingJobParameters()
{
OutputStartMode = OutputStartMode.CustomTime,
OutputStartTime = new DateTime(2017, 10, 12, 12, 12, 12, DateTimeKind.Utc)
};
streamAnalyticsManagementClient.StreamingJobs.Start(AzureresourceGroupName, AzurestreamingJobName, startStreamingJobParameters);
// Stop a streaming job
streamAnalyticsManagementClient.StreamingJobs.Stop(AzureresourceGroupName, AzurestreamingJobName);
// Delete a streaming job
// streamAnalyticsManagementClient.StreamingJobs.Delete(resourceGroupName, streamingJobName);
}
private void DeleteResources(ClearResourcesInputs inputs)
{
if (inputs.NamespaceExists)
{
_ConsoleBuffer.Add("Connecting to Service Bus...");
ServiceBusManagementClient sbMgmt = new ServiceBusManagementClient(inputs.Credentials);
bool deleteNamespace = ConsoleHelper.AskAndPerformAction(
"Do you want to delete whole namespace " + inputs.SBNamespace + " including all entities under it?",
"Are you sure you want to delete namespace " + inputs.SBNamespace + "?",
"Are you sure you do not want to delete namespace " + inputs.SBNamespace + "?",
() =>
{
_ConsoleBuffer.Add("Sending request to delete " + inputs.SBNamespace + " namespace...");
AzureOperationResponse nsResponse = sbMgmt.Namespaces.Delete(inputs.SBNamespace);
if (nsResponse.StatusCode == HttpStatusCode.OK)
{
_ConsoleBuffer.Add(inputs.SBNamespace + " namespace was deleted.");
}
},
_ConsoleBuffer);
//if we did not delete whole Namespace, maybe we want to delete some of its Event Hubs?
if (!deleteNamespace)
{
_ConsoleBuffer.Add("Reading list of Event Hubs from " + inputs.SBNamespace + " namespace...");
var nsDescription = sbMgmt.Namespaces.GetNamespaceDescription(inputs.SBNamespace);
var nsConnectionString = nsDescription.NamespaceDescriptions.First(
(d) => String.Equals(d.AuthorizationType, "SharedAccessAuthorization")
).ConnectionString;
var nsManager = NamespaceManager.CreateFromConnectionString(nsConnectionString);
var eventHubs = nsManager.GetEventHubs( );
foreach (var eventHubDescription in eventHubs)
{
EventHubDescription description = eventHubDescription;
ConsoleHelper.AskAndPerformAction(
"Do you want to delete Event Hub " + eventHubDescription.Path +
" including all messages under it?",
"Are you sure you want to delete Event Hub " + eventHubDescription.Path + "?",
"Are you sure you do not want to delete Event Hub " + eventHubDescription.Path + "?",
() =>
{
_ConsoleBuffer.Add("Sending request to delete " + description.Path + " Event Hub...");
nsManager.DeleteEventHub(description.Path);
_ConsoleBuffer.Add("Request to delete " + description.Path + " Event Hub was accepted.");
},
_ConsoleBuffer);
}
}
}
//Deleting Storage
_ConsoleBuffer.Add("Reading list of Storage Accounts...");
StorageManagementClient stgMgmt = new StorageManagementClient(inputs.Credentials);
HashSet <string> storageAccounts = new HashSet <string>( );
foreach (var storageAccount in stgMgmt.StorageAccounts.List( ))
{
storageAccounts.Add(storageAccount.Name);
}
int deletedCount = 0;
if (storageAccounts.Contains(inputs.StorageAccountName))
{
ConsoleHelper.AskAndPerformAction(
"Do you want to delete " + inputs.StorageAccountName + " storage account?",
"Are you sure you want to delete " + inputs.StorageAccountName + " storage account?",
"Are you sure you do not want to delete " + inputs.StorageAccountName + " storage account?",
() =>
{
_ConsoleBuffer.Add("Sending request to delete " + inputs.StorageAccountName + " Storage account...");
AzureOperationResponse resultStg = stgMgmt.StorageAccounts.Delete(inputs.StorageAccountName);
deletedCount += 1;
if (resultStg.StatusCode == System.Net.HttpStatusCode.OK)
{
_ConsoleBuffer.Add("Storage account " + inputs.StorageAccountName + " was deleted.");
}
},
_ConsoleBuffer);
}
if (deletedCount == 0)
{
_ConsoleBuffer.Add("No related Storage account was detected.");
}
//Deleting Stream Analytics jobs
_ConsoleBuffer.Add("Reading list of Stream Analytics jobs...");
StreamAnalyticsManagementClient saMgmt = new StreamAnalyticsManagementClient(inputs.Credentials);
JobListResponse jobListResponse = saMgmt.StreamingJobs.ListJobsInSubscription(new JobListParameters {
PropertiesToExpand = string.Empty
});
deletedCount = 0;
foreach (var job in jobListResponse.Value)
{
if (job.Name.StartsWith(inputs.NamePrefix))
{
Job jobToAsk = job;
ConsoleHelper.AskAndPerformAction(
"Do you want to delete Stream Analytics job " + job.Name + "?",
"Are you sure you want to delete Stream Analytics job " + job.Name + "?",
"Are you sure you do not want to delete namespace " + job.Name + "?",
() =>
{
//we need to figure out wat resourceGroup this job belongs to
//--//
const string resourceGroupPath = "/resourceGroups/";
const string providersPath = "/providers/";
int resourceGroupPathIndex = jobToAsk.Id.IndexOf(resourceGroupPath, System.StringComparison.Ordinal);
int providersPathIndex = jobToAsk.Id.IndexOf(providersPath, System.StringComparison.Ordinal);
int resourceGroupIdStartIndex = resourceGroupPathIndex + resourceGroupPath.Length;
string resourceGroup = jobToAsk.Id.Substring(resourceGroupIdStartIndex, providersPathIndex - resourceGroupIdStartIndex);
//--//
deletedCount += 1;
_ConsoleBuffer.Add("Sending request to delete " + jobToAsk.Name + " Stream Analytics job...");
LongRunningOperationResponse response = saMgmt.StreamingJobs.Delete(resourceGroup, jobToAsk.Name);
if (response.Status == OperationStatus.Succeeded)
{
_ConsoleBuffer.Add("Stream Analytics job " + jobToAsk.Name + " was deleted.");
}
},
_ConsoleBuffer);
}
}
if (deletedCount == 0)
{
_ConsoleBuffer.Add("No Stream Analytics job was deleted.");
}
}
private void DeployASJob(SAJobConfigModel cfg)
{
// Get authentication token
TokenCloudCredentials aadTokenCredentials =
new TokenCloudCredentials(cfg.subscriptionID, GetAuthorizationHeader());
// Create Stream Analytics management client
StreamAnalyticsManagementClient client = new StreamAnalyticsManagementClient(aadTokenCredentials);
// Create a Stream Analytics job
JobCreateOrUpdateParameters jobCreateParameters = new JobCreateOrUpdateParameters()
{
Job = new Job()
{
Name = cfg.streamAnalyticsJobName,
Location = cfg.location,
Properties = new JobProperties()
{
EventsOutOfOrderPolicy = EventsOutOfOrderPolicy.Adjust,
Sku = new Sku()
{
Name = "Standard"
}
}
}
};
JobCreateOrUpdateResponse jobCreateResponse = client.StreamingJobs.CreateOrUpdate(cfg.resourceGroupName, jobCreateParameters);
TempData["jobName"] = jobCreateResponse.Job.Name;
TempData["jobCreationStatus"] = jobCreateResponse.StatusCode;
// Create a Stream Analytics input source
InputCreateOrUpdateParameters jobInputCreateParameters = new InputCreateOrUpdateParameters()
{
Input = new Input()
{
Name = cfg.streamAnalyticsInputName,
Properties = new StreamInputProperties()
{
Serialization = new CsvSerialization
{
Properties = new CsvSerializationProperties
{
Encoding = "UTF8",
FieldDelimiter = ","
}
},
DataSource = new EventHubStreamInputDataSource
{
Properties = new EventHubStreamInputDataSourceProperties
{
EventHubName = cfg.EventHubName,
ServiceBusNamespace = cfg.ServiceBusNamespace,
SharedAccessPolicyKey = cfg.SharedAccessPolicyKey,
SharedAccessPolicyName = cfg.SharedAccessPolicyName,
}
}
}
}
};
InputCreateOrUpdateResponse inputCreateResponse =
client.Inputs.CreateOrUpdate(cfg.resourceGroupName, cfg.streamAnalyticsJobName, jobInputCreateParameters);
TempData["jobInputName"] = inputCreateResponse.Input.Name;
TempData["jobInputCreationStatus"] = inputCreateResponse.StatusCode;
}
