public void Test_OutputOperations_ServiceBusTopic()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
string resourceName = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");
string serviceLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetStreamAnalyticsManagementClient(handler);
try
{
ResourceGroup resourceGroup = new ResourceGroup() { Location = serviceLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// Construct the JobCreateProperties
JobCreateOrUpdateParameters jobCreateOrUpdateParameters =
new JobCreateOrUpdateParameters(TestHelper.GetDefaultJob(resourceName, serviceLocation));
// Create a streaming job
JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);
// Get a streaming job to check
JobGetParameters jobGetParameters = new JobGetParameters(string.Empty);
JobGetResponse jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
Assert.Equal(resourceName, jobGetResponse.Job.Name);
// Construct the Output
OutputProperties outputProperties = new OutputProperties();
string outputName = TestUtilities.GenerateName("outputtest");
string topicName = "sdktopic";
ServiceBusTopicOutputDataSource serviceBusTopicOutputDataSource = new ServiceBusTopicOutputDataSource()
{
Properties = new ServiceBusTopicOutputDataSourceProperties()
{
ServiceBusNamespace = TestHelper.ServiceBusNamespace,
TopicName = topicName,
SharedAccessPolicyName = TestHelper.SharedAccessPolicyName,
SharedAccessPolicyKey = TestHelper.SharedAccessPolicyKey,
}
};
JsonSerialization jsonSerialization = new JsonSerialization()
{
Properties = new JsonSerializationProperties()
{
Encoding = "UTF8",
Format = Format.LineSeparated
}
};
outputProperties.DataSource = serviceBusTopicOutputDataSource;
outputProperties.Serialization = jsonSerialization;
Output output1 = new Output(outputName)
{
Properties = outputProperties
};
// Add an output
OutputCreateOrUpdateParameters outputCreateOrUpdateParameters = new OutputCreateOrUpdateParameters();
outputCreateOrUpdateParameters.Output = output1;
OutputCreateOrUpdateResponse outputCreateOrUpdateResponse = client.Outputs.CreateOrUpdate(resourceGroupName, resourceName, outputCreateOrUpdateParameters);
Assert.Equal(HttpStatusCode.OK, outputCreateOrUpdateResponse.StatusCode);
Assert.Equal(outputName, outputCreateOrUpdateResponse.Output.Name);
Assert.True(outputCreateOrUpdateResponse.Output.Properties.DataSource is ServiceBusTopicOutputDataSource);
ServiceBusTopicOutputDataSource serviceBusTopicOutputDataSourceInResponse1 = (ServiceBusTopicOutputDataSource)outputCreateOrUpdateResponse.Output.Properties.DataSource;
Assert.Equal(topicName, serviceBusTopicOutputDataSourceInResponse1.Properties.TopicName);
Assert.NotNull(outputCreateOrUpdateResponse.Output.Properties.Etag);
// Get the output
OutputGetResponse outputGetResponse = client.Outputs.Get(resourceGroupName, resourceName, outputName);
Assert.Equal(HttpStatusCode.OK, outputGetResponse.StatusCode);
Assert.Equal(outputName, outputGetResponse.Output.Name);
Assert.True(outputGetResponse.Output.Properties.DataSource is ServiceBusTopicOutputDataSource);
ServiceBusTopicOutputDataSource serviceBusTopicOutputDataSourceInResponse2 = (ServiceBusTopicOutputDataSource)outputGetResponse.Output.Properties.DataSource;
Assert.Equal(topicName, serviceBusTopicOutputDataSourceInResponse2.Properties.TopicName);
// Test output connectivity
DataSourceTestConnectionResponse response = client.Outputs.TestConnection(resourceGroupName, resourceName, outputName);
Assert.Equal(OperationStatus.Succeeded, response.Status);
Assert.Equal(DataSourceTestStatus.TestSucceeded, response.DataSourceTestStatus);
// Update the output
string newTopicName = TestUtilities.GenerateName("NewTopicName");
serviceBusTopicOutputDataSource.Properties.TopicName = newTopicName;
outputProperties.DataSource = serviceBusTopicOutputDataSource;
outputProperties.Etag = outputCreateOrUpdateResponse.Output.Properties.Etag;
OutputPatchParameters outputPatchParameters = new OutputPatchParameters(outputProperties);
OutputPatchResponse outputPatchResponse = client.Outputs.Patch(resourceGroupName, resourceName, outputName, outputPatchParameters);
Assert.Equal(HttpStatusCode.OK, outputPatchResponse.StatusCode);
Assert.True(outputPatchResponse.Properties.DataSource is ServiceBusTopicOutputDataSource);
ServiceBusTopicOutputDataSource serviceBusTopicOutputDataSourceInResponse3 = (ServiceBusTopicOutputDataSource)outputPatchResponse.Properties.DataSource;
Assert.Equal(newTopicName, serviceBusTopicOutputDataSourceInResponse3.Properties.TopicName);
Assert.NotNull(outputPatchResponse.Properties.Etag);
Assert.NotEqual(outputCreateOrUpdateResponse.Output.Properties.Etag, outputPatchResponse.Properties.Etag);
// Delete the output
AzureOperationResponse deleteOutputOperationResponse = client.Outputs.Delete(resourceGroupName, resourceName, outputName);
Assert.Equal(HttpStatusCode.OK, deleteOutputOperationResponse.StatusCode);
// Check there is no output any more
jobGetParameters = new JobGetParameters("outputs");
jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
Assert.Equal(0, jobGetResponse.Job.Properties.Outputs.Count);
}
finally
{
client.StreamingJobs.Delete(resourceGroupName, resourceName);
resourceClient.ResourceGroups.Delete(resourceGroupName);
}
}
}
public void Test_OutputOperations_EventHub()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
string resourceName = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");
string serviceLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetStreamAnalyticsManagementClient(handler);
try
{
ResourceGroup resourceGroup = new ResourceGroup() { Location = serviceLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// Construct the JobCreateProperties
JobCreateOrUpdateParameters jobCreateOrUpdateParameters =
new JobCreateOrUpdateParameters(TestHelper.GetDefaultJob(resourceName, serviceLocation));
// Create a streaming job
JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);
// Get a streaming job to check
JobGetParameters jobGetParameters = new JobGetParameters(string.Empty);
JobGetResponse jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
Assert.Equal(resourceName, jobGetResponse.Job.Name);
// Construct the Output
OutputProperties outputProperties = new OutputProperties();
string outputName = TestUtilities.GenerateName("outputtest");
string partitionKey = "partitionKey";
EventHubOutputDataSource eventHubOutputDataSource = new EventHubOutputDataSource()
{
Properties = new EventHubOutputDataSourceProperties()
{
ServiceBusNamespace = TestHelper.ServiceBusNamespace,
EventHubName = TestHelper.EventHubName,
SharedAccessPolicyName = TestHelper.SharedAccessPolicyName,
SharedAccessPolicyKey = TestHelper.SharedAccessPolicyKey,
PartitionKey = partitionKey
}
};
JsonSerialization jsonSerialization = new JsonSerialization()
{
Properties = new JsonSerializationProperties()
{
Encoding = "UTF8",
Format = Format.LineSeparated
}
};
outputProperties.DataSource = eventHubOutputDataSource;
outputProperties.Serialization = jsonSerialization;
Output output1 = new Output(outputName)
{
Properties = outputProperties
};
// Add an output
OutputCreateOrUpdateParameters outputCreateOrUpdateParameters = new OutputCreateOrUpdateParameters();
outputCreateOrUpdateParameters.Output = output1;
OutputCreateOrUpdateResponse outputCreateOrUpdateResponse = client.Outputs.CreateOrUpdate(resourceGroupName, resourceName, outputCreateOrUpdateParameters);
Assert.Equal(HttpStatusCode.OK, outputCreateOrUpdateResponse.StatusCode);
Assert.Equal(outputName, outputCreateOrUpdateResponse.Output.Name);
Assert.True(outputCreateOrUpdateResponse.Output.Properties.Serialization is JsonSerialization);
JsonSerialization jsonSerializationInResponse1 = (JsonSerialization)outputCreateOrUpdateResponse.Output.Properties.Serialization;
Assert.Equal(Format.LineSeparated, jsonSerializationInResponse1.Properties.Format);
Assert.True(outputCreateOrUpdateResponse.Output.Properties.DataSource is EventHubOutputDataSource);
EventHubOutputDataSource eventHubOutputDataSourceInResponse1 = (EventHubOutputDataSource)outputCreateOrUpdateResponse.Output.Properties.DataSource;
Assert.Equal(partitionKey, eventHubOutputDataSourceInResponse1.Properties.PartitionKey);
Assert.NotNull(outputCreateOrUpdateResponse.Output.Properties.Etag);
// Get the output
OutputGetResponse outputGetResponse = client.Outputs.Get(resourceGroupName, resourceName, outputName);
Assert.Equal(HttpStatusCode.OK, outputGetResponse.StatusCode);
Assert.Equal(outputName, outputGetResponse.Output.Name);
Assert.True(outputGetResponse.Output.Properties.Serialization is JsonSerialization);
JsonSerialization jsonSerializationInResponse2 = (JsonSerialization)outputGetResponse.Output.Properties.Serialization;
Assert.Equal(Format.LineSeparated, jsonSerializationInResponse2.Properties.Format);
Assert.True(outputGetResponse.Output.Properties.DataSource is EventHubOutputDataSource);
EventHubOutputDataSource eventHubOutputDataSourceInResponse2 = (EventHubOutputDataSource)outputGetResponse.Output.Properties.DataSource;
Assert.Equal(partitionKey, eventHubOutputDataSourceInResponse2.Properties.PartitionKey);
// Test output connectivity
ResourceTestConnectionResponse response = client.Outputs.TestConnection(resourceGroupName, resourceName, outputName);
Assert.Equal(OperationStatus.Succeeded, response.Status);
Assert.Equal(ResourceTestStatus.TestSucceeded, response.ResourceTestStatus);
// Update the output
jsonSerialization = new JsonSerialization()
{
Properties = new JsonSerializationProperties()
{
Encoding = "UTF8",
Format = Format.Array
}
};
string newPartitionKey = TestUtilities.GenerateName("NewPartitionKey");
eventHubOutputDataSource.Properties.PartitionKey = newPartitionKey;
outputProperties.DataSource = eventHubOutputDataSource;
outputProperties.Serialization = jsonSerialization;
outputProperties.Etag = outputCreateOrUpdateResponse.Output.Properties.Etag;
OutputPatchParameters outputPatchParameters = new OutputPatchParameters(outputProperties);
OutputPatchResponse outputPatchResponse = client.Outputs.Patch(resourceGroupName, resourceName, outputName, outputPatchParameters);
Assert.Equal(HttpStatusCode.OK, outputPatchResponse.StatusCode);
Assert.True(outputPatchResponse.Properties.Serialization is JsonSerialization);
JsonSerialization jsonSerializationInResponse3 = (JsonSerialization)outputPatchResponse.Properties.Serialization;
// TODO : Re-enable check once service bug is fixed
//Assert.Equal(Format.Array, jsonSerializationInResponse3.Properties.Format);
Assert.True(outputPatchResponse.Properties.DataSource is EventHubOutputDataSource);
EventHubOutputDataSource eventHubOutputDataSourceInResponse3 = (EventHubOutputDataSource)outputPatchResponse.Properties.DataSource;
Assert.Equal(newPartitionKey, eventHubOutputDataSourceInResponse3.Properties.PartitionKey);
Assert.NotNull(outputPatchResponse.Properties.Etag);
Assert.NotEqual(outputCreateOrUpdateResponse.Output.Properties.Etag, outputPatchResponse.Properties.Etag);
// Delete the output
AzureOperationResponse deleteOutputOperationResponse = client.Outputs.Delete(resourceGroupName, resourceName, outputName);
Assert.Equal(HttpStatusCode.OK, deleteOutputOperationResponse.StatusCode);
// Check that there are 0 outputs in the job
jobGetParameters = new JobGetParameters("outputs");
jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
Assert.Equal(0, jobGetResponse.Job.Properties.Outputs.Count);
}
finally
{
client.StreamingJobs.Delete(resourceGroupName, resourceName);
resourceClient.ResourceGroups.Delete(resourceGroupName);
}
}
}
public void Test_OutputOperations_Blob()
{
BasicDelegatingHandler handler = new BasicDelegatingHandler();
using (var undoContext = UndoContext.Current)
{
undoContext.Start();
string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
string resourceName = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");
string serviceLocation = TestHelper.GetDefaultLocation();
var resourceClient = TestHelper.GetResourceClient(handler);
var client = TestHelper.GetStreamAnalyticsManagementClient(handler);
try
{
ResourceGroup resourceGroup = new ResourceGroup() { Location = serviceLocation };
resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);
// Construct the JobCreateProperties
JobCreateOrUpdateParameters jobCreateOrUpdateParameters =
new JobCreateOrUpdateParameters(TestHelper.GetDefaultJob(resourceName, serviceLocation));
// Create a streaming job
JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);
// Get a streaming job to check
JobGetParameters jobGetParameters = new JobGetParameters(string.Empty);
JobGetResponse jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
Assert.Equal(resourceName, jobGetResponse.Job.Name);
// Construct the Output
OutputProperties outputProperties = new OutputProperties();
string outputName = TestUtilities.GenerateName("outputtest");
StorageAccount storageAccount = new StorageAccount();
storageAccount.AccountName = TestHelper.AccountName;
storageAccount.AccountKey = TestHelper.AccountKey;
BlobOutputDataSource blobOutputDataSource = new BlobOutputDataSource()
{
Properties = new BlobOutputDataSourceProperties()
{
StorageAccounts = new[] { storageAccount },
Container = "state",
PathPattern = "{date}",
DateFormat = "yyyy/MM/dd"
}
};
JsonSerialization jsonSerialization = new JsonSerialization()
{
Properties = new JsonSerializationProperties()
{
Encoding = "UTF8",
Format = Format.LineSeparated
}
};
outputProperties.DataSource = blobOutputDataSource;
outputProperties.Serialization = jsonSerialization;
Output output1 = new Output(outputName)
{
Properties = outputProperties
};
// Add an output
OutputCreateOrUpdateParameters outputCreateOrUpdateParameters = new OutputCreateOrUpdateParameters();
outputCreateOrUpdateParameters.Output = output1;
OutputCreateOrUpdateResponse outputCreateOrUpdateResponse = client.Outputs.CreateOrUpdate(resourceGroupName, resourceName, outputCreateOrUpdateParameters);
Assert.Equal(HttpStatusCode.OK, outputCreateOrUpdateResponse.StatusCode);
Assert.Equal(outputName, outputCreateOrUpdateResponse.Output.Name);
Assert.True(outputCreateOrUpdateResponse.Output.Properties.DataSource is BlobOutputDataSource);
BlobOutputDataSource blobOutputDataSourceInResponse1 = (BlobOutputDataSource)outputCreateOrUpdateResponse.Output.Properties.DataSource;
Assert.Equal("{date}", blobOutputDataSourceInResponse1.Properties.PathPattern);
Assert.Equal("yyyy/MM/dd", blobOutputDataSourceInResponse1.Properties.DateFormat);
Assert.NotNull(outputCreateOrUpdateResponse.Output.Properties.Etag);
// Get the output
OutputGetResponse outputGetResponse = client.Outputs.Get(resourceGroupName, resourceName, outputName);
Assert.Equal(HttpStatusCode.OK, outputGetResponse.StatusCode);
Assert.Equal(outputName, outputGetResponse.Output.Name);
Assert.True(outputGetResponse.Output.Properties.DataSource is BlobOutputDataSource);
BlobOutputDataSource blobOutputDataSourceInResponse2 = (BlobOutputDataSource)outputGetResponse.Output.Properties.DataSource;
Assert.Equal("{date}", blobOutputDataSourceInResponse2.Properties.PathPattern);
Assert.Equal("yyyy/MM/dd", blobOutputDataSourceInResponse2.Properties.DateFormat);
Assert.Equal(outputCreateOrUpdateResponse.Output.Properties.Etag, outputGetResponse.Output.Properties.Etag);
// Test output connectivity
DataSourceTestConnectionResponse response = client.Outputs.TestConnection(resourceGroupName, resourceName, outputName);
Assert.Equal(OperationStatus.Succeeded, response.Status);
Assert.Equal(DataSourceTestStatus.TestSucceeded, response.DataSourceTestStatus);
// Update the output
blobOutputDataSource.Properties.PathPattern = "test.csv";
outputProperties.DataSource = blobOutputDataSource;
outputProperties.Etag = outputCreateOrUpdateResponse.Output.Properties.Etag;
OutputPatchParameters outputPatchParameters = new OutputPatchParameters(outputProperties);
OutputPatchResponse outputPatchResponse = client.Outputs.Patch(resourceGroupName, resourceName, outputName, outputPatchParameters);
Assert.Equal(HttpStatusCode.OK, outputPatchResponse.StatusCode);
Assert.True(outputPatchResponse.Properties.DataSource is BlobOutputDataSource);
BlobOutputDataSource blobOutputDataSourceInResponse3 = (BlobOutputDataSource)outputPatchResponse.Properties.DataSource;
Assert.Equal("test.csv", blobOutputDataSourceInResponse3.Properties.PathPattern);
Assert.Equal("yyyy/MM/dd", blobOutputDataSourceInResponse3.Properties.DateFormat);
Assert.NotNull(outputPatchResponse.Properties.Etag);
Assert.NotEqual(outputCreateOrUpdateResponse.Output.Properties.Etag, outputPatchResponse.Properties.Etag);
// Delete the output
AzureOperationResponse deleteOutputOperationResponse = client.Outputs.Delete(resourceGroupName, resourceName, outputName);
Assert.Equal(HttpStatusCode.OK, deleteOutputOperationResponse.StatusCode);
// Check that there are 0 outputs in the job
jobGetParameters = new JobGetParameters("outputs");
jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
Assert.Equal(0, jobGetResponse.Job.Properties.Outputs.Count);
}
finally
{
client.StreamingJobs.Delete(resourceGroupName, resourceName);
resourceClient.ResourceGroups.Delete(resourceGroupName);
}
}
}
private void CreateStreamAnalyticsJob( string nameSuffix, string query, string resourceGroupName, AzurePrepInputs azurePrepIn,
EventHubDescription ehInput, EventHubDescription ehOutput )
{
const string inputName = "DevicesInput";
const string outputName = "output";
string jobName = azurePrepIn.NamePrefix + nameSuffix;
string transformationName = jobName + "-tr";
var computeClient = new StreamAnalyticsManagementClient( azurePrepIn.Credentials );
var serialization = new JsonSerialization
{
Type = "JSON",
Properties = new JsonSerializationProperties
{
Encoding = "UTF8"
}
};
List<Input> jobInputs = new List<Input>
{
new Input
{
Name = inputName,
Properties = new StreamInputProperties
{
DataSource = new EventHubStreamInputDataSource
{
Properties = new EventHubStreamInputDataSourceProperties
{
EventHubName = ehInput.Path,
ServiceBusNamespace = azurePrepIn.SBNamespace,
SharedAccessPolicyName = "StreamingAnalytics",
SharedAccessPolicyKey = ( ehInput.Authorization.First( ( d )
=> String.Equals( d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase) ) as SharedAccessAuthorizationRule ).PrimaryKey,
}
},
Serialization = serialization
}
}
};
List<Output> jobOutputs = new List<Output>
{
new Output
{
Name = outputName,
Properties = new OutputProperties
{
DataSource = new EventHubOutputDataSource
{
Properties = new EventHubOutputDataSourceProperties
{
EventHubName = ehOutput.Path,
ServiceBusNamespace = azurePrepIn.SBNamespace,
SharedAccessPolicyName = "StreamingAnalytics",
SharedAccessPolicyKey = ( ehOutput.Authorization.First( ( d )
=> String.Equals( d.KeyName, "StreamingAnalytics", StringComparison.InvariantCultureIgnoreCase) ) as SharedAccessAuthorizationRule ).PrimaryKey,
}
},
Serialization = serialization
}
}
};
bool created = true;
try
{
var jobCreateResponse = computeClient.StreamingJobs.CreateOrUpdateAsync(
resourceGroupName,
new JobCreateOrUpdateParameters
{
Job = new Job
{
Name = jobName,
Location = azurePrepIn.Location,
Properties = new JobProperties
{
Sku = new Sku
{
//should be "standart" according to https://msdn.microsoft.com/en-us/library/azure/dn834994.aspx
Name = "standard"
},
EventsOutOfOrderPolicy = "drop",
EventsOutOfOrderMaxDelayInSeconds = 10,
Inputs = jobInputs,
Outputs = jobOutputs,
Transformation = new Transformation
{
Name = transformationName,
Properties = new TransformationProperties
{
Query = query,
StreamingUnits = 1
}
}
}
}
}
).Result;
}
catch( Exception ex )
{
_ConsoleBuffer.Add( "Exception on creation Stream Analytics Job " + jobName + ": " + ex.Message );
_ConsoleBuffer.Add( "Inner exception message: " + ex.InnerException.Message );
created = false;
}
if( created )
{
_ConsoleBuffer.Add( "Stream Analytics job " + jobName + " created." );
}
}