public void Test_JobOperations_E2E() { BasicDelegatingHandler handler = new BasicDelegatingHandler(); using (var undoContext = UndoContext.Current) { undoContext.Start(); string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics"); string resourceName = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK"); string serviceLocation = TestHelper.GetDefaultLocation(); var resourceClient = TestHelper.GetResourceClient(handler); var client = TestHelper.GetStreamAnalyticsManagementClient(handler); try { ResourceGroup resourceGroup = new ResourceGroup() { Location = serviceLocation }; resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup); Job job = new Job(); job.Name = resourceName; job.Location = serviceLocation; // Construct the general properties for JobProperties JobProperties jobProperties = new JobProperties(); jobProperties.Sku = new Sku() { Name = "standard" }; jobProperties.EventsOutOfOrderPolicy = EventsOutOfOrderPolicy.Drop; jobProperties.EventsOutOfOrderMaxDelayInSeconds = 0; // Construct the Input StorageAccount storageAccount = new StorageAccount { AccountName = TestHelper.AccountName, AccountKey = TestHelper.AccountKey }; InputProperties inputProperties = new StreamInputProperties() { Serialization = new CsvSerialization() { Properties = new CsvSerializationProperties() { FieldDelimiter = ",", Encoding = "UTF8" } }, DataSource = new BlobStreamInputDataSource() { Properties = new BlobStreamInputDataSourceProperties() { StorageAccounts = new[] { storageAccount }, Container = "state", PathPattern = "" } } }; Input input1 = new Input("inputtest") { Properties = inputProperties }; jobProperties.Inputs = new[] { input1 }; // Construct the Output OutputProperties outputProperties = new OutputProperties(); SqlAzureOutputDataSource sqlAzureOutputDataSource = new SqlAzureOutputDataSource() { Properties = new SqlAzureOutputDataSourceProperties() { Server = TestHelper.Server, Database = TestHelper.Database, User = TestHelper.User, Password = TestHelper.Password, Table = "StateInfo" } }; outputProperties.DataSource = sqlAzureOutputDataSource; Output output1 = new Output("outputtest") { Properties = outputProperties }; jobProperties.Outputs = new Output[] { output1 }; // Construct the transformation Transformation transformation = new Transformation() { Name = "transformationtest", Properties = new TransformationProperties() { Query = "Select Id, Name from inputtest", StreamingUnits = 1 } }; jobProperties.Transformation = transformation; job.Properties = jobProperties; // Construct the JobCreateProperties JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters(); jobCreateOrUpdateParameters.Job = job; // Create a streaming job JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters); Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode); Assert.NotNull(jobCreateOrUpdateResponse.Job.Properties.Etag); // Get a streaming job to check JobGetParameters jobGetParameters = new JobGetParameters("inputs,transformation,outputs"); JobGetResponse jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(serviceLocation, jobGetResponse.Job.Location); Assert.Equal(resourceName, jobGetResponse.Job.Name); Assert.True(jobGetResponse.Job.Properties.Inputs[0].Properties is StreamInputProperties); StreamInputProperties streamInputProperties = jobGetResponse.Job.Properties.Inputs[0].Properties as StreamInputProperties; Assert.Equal("Stream", jobGetResponse.Job.Properties.Inputs[0].Properties.Type); Assert.Equal("Microsoft.Storage/Blob", streamInputProperties.DataSource.Type); Assert.Equal("Csv", streamInputProperties.Serialization.Type); Assert.NotNull(jobGetResponse.Job.Properties.Etag); Assert.Equal(jobCreateOrUpdateResponse.Job.Properties.Etag, jobGetResponse.Job.Properties.Etag); JobListParameters parameters = new JobListParameters(string.Empty); JobListResponse response = client.StreamingJobs.ListJobsInResourceGroup(resourceGroupName, parameters); Assert.Equal(HttpStatusCode.OK, response.StatusCode); // Start a streaming job JobStartParameters jobStartParameters = new JobStartParameters() { OutputStartMode = OutputStartMode.LastOutputEventTime }; CloudException cloudException = Assert.Throws <CloudException>(() => client.StreamingJobs.Start(resourceGroupName, resourceName, jobStartParameters)); Assert.Equal("LastOutputEventTime must be available when OutputStartMode is set to LastOutputEventTime. Please make sure at least one output event has been processed. ", cloudException.Error.Message); jobStartParameters.OutputStartMode = OutputStartMode.CustomTime; jobStartParameters.OutputStartTime = DateTime.Now; AzureOperationResponse jobStartOperationResponse = client.StreamingJobs.Start(resourceGroupName, resourceName, jobStartParameters); Assert.Equal(HttpStatusCode.OK, jobStartOperationResponse.StatusCode); // Get a streaming job to check jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.True(IsRunning(jobGetResponse.Job.Properties.JobState)); // Check diagnostics InputListResponse inputListResponse = client.Inputs.ListInputInJob(resourceGroupName, resourceName, new InputListParameters("*")); Assert.Equal(HttpStatusCode.OK, inputListResponse.StatusCode); Assert.NotEqual(0, inputListResponse.Value.Count); Assert.NotNull(inputListResponse.Value[0].Properties.Diagnostics); Assert.NotEqual(0, inputListResponse.Value[0].Properties.Diagnostics.Conditions.Count); Assert.NotNull(inputListResponse.Value[0].Properties.Diagnostics.Conditions[0].Code); Assert.NotNull(inputListResponse.Value[0].Properties.Diagnostics.Conditions[0].Message); // Stop a streaming job AzureOperationResponse jobStopOperationResponse = client.StreamingJobs.Stop(resourceGroupName, resourceName); Assert.Equal(HttpStatusCode.OK, jobStopOperationResponse.StatusCode); // Get a streaming job to check jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(JobRunningState.Stopped, jobGetResponse.Job.Properties.JobState); // Delete a streaming job AzureOperationResponse jobDeleteOperationResponse = client.StreamingJobs.Delete(resourceGroupName, resourceName); Assert.Equal(HttpStatusCode.OK, jobDeleteOperationResponse.StatusCode); } finally { client.StreamingJobs.Delete(resourceGroupName, resourceName); resourceClient.ResourceGroups.Delete(resourceGroupName); } } }
public void Test_InputOperations_E2E() { BasicDelegatingHandler handler = new BasicDelegatingHandler(); using (var undoContext = UndoContext.Current) { undoContext.Start(); string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics"); string resourceName = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK"); string serviceLocation = TestHelper.GetDefaultLocation(); var resourceClient = TestHelper.GetResourceClient(handler); var client = TestHelper.GetStreamAnalyticsManagementClient(handler); try { ResourceGroup resourceGroup = new ResourceGroup() { Location = serviceLocation }; resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup); Job job = new Job(); job.Name = resourceName; job.Location = serviceLocation; // Construct the general properties for JobProperties JobProperties jobProperties = new JobProperties(); jobProperties.Sku = new Sku() { Name = "standard" }; jobProperties.EventsOutOfOrderPolicy = EventsOutOfOrderPolicy.Drop; jobProperties.EventsOutOfOrderMaxDelayInSeconds = 0; job.Properties = jobProperties; // Construct the JobCreateProperties JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters(); jobCreateOrUpdateParameters.Job = job; // Create a streaming job JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters); Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode); // Get a streaming job to check JobGetParameters jobGetParameters = new JobGetParameters(string.Empty); JobGetResponse jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(serviceLocation, jobGetResponse.Job.Location); Assert.Equal(resourceName, jobGetResponse.Job.Name); // Construct the Input StorageAccount storageAccount = new StorageAccount(); storageAccount.AccountName = TestHelper.AccountName; storageAccount.AccountKey = TestHelper.AccountKey; InputProperties inputProperties = new StreamInputProperties() { Serialization = new CsvSerialization() { Properties = new CsvSerializationProperties() { FieldDelimiter = ",", Encoding = "UTF8" } }, DataSource = new BlobStreamInputDataSource() { Properties = new BlobStreamInputDataSourceProperties() { StorageAccounts = new[] { storageAccount }, Container = "state", PathPattern = "" } } }; string inputName = TestUtilities.GenerateName("inputtest"); Input input1 = new Input(inputName) { Properties = inputProperties }; // Add an input InputCreateOrUpdateParameters inputCreateOrUpdateParameters = new InputCreateOrUpdateParameters(); inputCreateOrUpdateParameters.Input = input1; InputCreateOrUpdateResponse inputCreateOrUpdateResponse = client.Inputs.CreateOrUpdate(resourceGroupName, resourceName, inputCreateOrUpdateParameters); Assert.Equal(HttpStatusCode.OK, inputCreateOrUpdateResponse.StatusCode); Assert.Equal(inputName, inputCreateOrUpdateResponse.Input.Name); Assert.Equal("Stream", inputCreateOrUpdateResponse.Input.Properties.Type); Assert.True(inputCreateOrUpdateResponse.Input.Properties.Serialization is CsvSerialization); CsvSerialization csvSerializationInResponse1 = (CsvSerialization)inputCreateOrUpdateResponse.Input.Properties.Serialization; Assert.Equal(",", csvSerializationInResponse1.Properties.FieldDelimiter); Assert.NotNull(inputCreateOrUpdateResponse.Input.Properties.Etag); // Get the input InputGetResponse inputGetResponse = client.Inputs.Get(resourceGroupName, resourceName, inputName); Assert.Equal(HttpStatusCode.OK, inputGetResponse.StatusCode); Assert.Equal(inputName, inputGetResponse.Input.Name); Assert.True(inputGetResponse.Input.Properties.Serialization is CsvSerialization); CsvSerialization csvSerializationInResponse2 = (CsvSerialization)inputGetResponse.Input.Properties.Serialization; Assert.Equal(",", csvSerializationInResponse2.Properties.FieldDelimiter); Assert.Equal(inputCreateOrUpdateResponse.Input.Properties.Etag, inputGetResponse.Input.Properties.Etag); // List inputs InputListResponse inputListResponse = client.Inputs.ListInputInJob(resourceGroupName, resourceName, new InputListParameters()); Assert.Equal(HttpStatusCode.OK, inputListResponse.StatusCode); Assert.Equal(1, inputListResponse.Value.Count); // Check that there is 1 input in the job jobGetParameters = new JobGetParameters("inputs"); jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(1, jobGetResponse.Job.Properties.Inputs.Count); // Test input connectivity DataSourceTestConnectionResponse response = client.Inputs.TestConnection(resourceGroupName, resourceName, inputName); Assert.Equal(OperationStatus.Succeeded, response.Status); Assert.Equal(DataSourceTestStatus.TestSucceeded, response.DataSourceTestStatus); // Update the input Serialization csvSerialization = new CsvSerialization() { Properties = new CsvSerializationProperties() { FieldDelimiter = "|", Encoding = "UTF8" } }; inputProperties.Serialization = csvSerialization; inputProperties.Etag = inputCreateOrUpdateResponse.Input.Properties.Etag; InputPatchParameters inputPatchParameters = new InputPatchParameters(inputProperties); InputPatchResponse inputPatchResponse = client.Inputs.Patch(resourceGroupName, resourceName, inputName, inputPatchParameters); Assert.Equal(HttpStatusCode.OK, inputPatchResponse.StatusCode); Assert.True(inputPatchResponse.Properties.Serialization is CsvSerialization); CsvSerialization csvSerializationInResponse3 = (CsvSerialization)inputPatchResponse.Properties.Serialization; Assert.Equal("|", csvSerializationInResponse3.Properties.FieldDelimiter); Assert.NotNull(inputPatchResponse.Properties.Etag); Assert.NotEqual(inputCreateOrUpdateResponse.Input.Properties.Etag, inputPatchResponse.Properties.Etag); // Add second input string inputName2 = TestUtilities.GenerateName("inputtest"); Input input2 = new Input(inputName2) { Properties = inputProperties }; inputCreateOrUpdateParameters.Input = input2; inputCreateOrUpdateResponse = client.Inputs.CreateOrUpdate(resourceGroupName, resourceName, inputCreateOrUpdateParameters); // List inputs inputListResponse = client.Inputs.ListInputInJob(resourceGroupName, resourceName, new InputListParameters()); Assert.Equal(HttpStatusCode.OK, inputListResponse.StatusCode); Assert.Equal(2, inputListResponse.Value.Count); // Check that there are 2 inputs in the job jobGetParameters = new JobGetParameters("inputs"); jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(2, jobGetResponse.Job.Properties.Inputs.Count); // Delete the inputs AzureOperationResponse deleteInputOperationResponse = client.Inputs.Delete(resourceGroupName, resourceName, inputName); Assert.Equal(HttpStatusCode.OK, deleteInputOperationResponse.StatusCode); deleteInputOperationResponse = client.Inputs.Delete(resourceGroupName, resourceName, inputName2); Assert.Equal(HttpStatusCode.OK, deleteInputOperationResponse.StatusCode); // Check that there are 0 inputs in the job jobGetParameters = new JobGetParameters("inputs"); jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(0, jobGetResponse.Job.Properties.Inputs.Count); } finally { client.StreamingJobs.Delete(resourceGroupName, resourceName); resourceClient.ResourceGroups.Delete(resourceGroupName); } } }
public void Test_InputOperations_ReferenceBlob() { BasicDelegatingHandler handler = new BasicDelegatingHandler(); using (var undoContext = UndoContext.Current) { undoContext.Start(); string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics"); string resourceName = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK"); string serviceLocation = TestHelper.GetDefaultLocation(); var resourceClient = TestHelper.GetResourceClient(handler); var client = TestHelper.GetStreamAnalyticsManagementClient(handler); try { ResourceGroup resourceGroup = new ResourceGroup() { Location = serviceLocation }; resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup); // Construct the JobCreateProperties JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters(TestHelper.GetDefaultJob(resourceName, serviceLocation)); // Create a streaming job JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters); Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode); // Get a streaming job to check JobGetParameters jobGetParameters = new JobGetParameters(string.Empty); JobGetResponse jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(serviceLocation, jobGetResponse.Job.Location); Assert.Equal(resourceName, jobGetResponse.Job.Name); // Construct the Input StorageAccount storageAccount = new StorageAccount(); storageAccount.AccountName = TestHelper.AccountName; storageAccount.AccountKey = TestHelper.AccountKey; InputProperties inputProperties = new ReferenceInputProperties() { Serialization = new CsvSerialization() { Properties = new CsvSerializationProperties() { FieldDelimiter = ",", Encoding = "UTF8" } }, DataSource = new BlobReferenceInputDataSource() { Properties = new BlobReferenceInputDataSourceProperties() { StorageAccounts = new[] { storageAccount }, Container = "state", PathPattern = "{date}", DateFormat = "yyyy/MM/dd" } } }; string inputName = TestUtilities.GenerateName("inputtest"); Input input1 = new Input(inputName) { Properties = inputProperties }; // Add an input InputCreateOrUpdateParameters inputCreateOrUpdateParameters = new InputCreateOrUpdateParameters(); inputCreateOrUpdateParameters.Input = input1; InputCreateOrUpdateResponse inputCreateOrUpdateResponse = client.Inputs.CreateOrUpdate(resourceGroupName, resourceName, inputCreateOrUpdateParameters); Assert.Equal(HttpStatusCode.OK, inputCreateOrUpdateResponse.StatusCode); Assert.Equal(inputName, inputCreateOrUpdateResponse.Input.Name); Assert.Equal("Reference", inputCreateOrUpdateResponse.Input.Properties.Type); Assert.True(inputCreateOrUpdateResponse.Input.Properties is ReferenceInputProperties); ReferenceInputProperties referenceInputPropertiesInResponse1 = (ReferenceInputProperties)inputCreateOrUpdateResponse.Input.Properties; Assert.True(referenceInputPropertiesInResponse1.DataSource is BlobReferenceInputDataSource); BlobReferenceInputDataSource blobReferenceInputDataSourceInResponse1 = (BlobReferenceInputDataSource)referenceInputPropertiesInResponse1.DataSource; Assert.Equal("{date}", blobReferenceInputDataSourceInResponse1.Properties.PathPattern); Assert.Equal("yyyy/MM/dd", blobReferenceInputDataSourceInResponse1.Properties.DateFormat); Assert.NotNull(inputCreateOrUpdateResponse.Input.Properties.Etag); // Get the input InputGetResponse inputGetResponse = client.Inputs.Get(resourceGroupName, resourceName, inputName); Assert.Equal(HttpStatusCode.OK, inputGetResponse.StatusCode); Assert.Equal(inputName, inputGetResponse.Input.Name); Assert.True(inputGetResponse.Input.Properties is ReferenceInputProperties); ReferenceInputProperties referenceInputPropertiesInResponse2 = (ReferenceInputProperties)inputGetResponse.Input.Properties; Assert.True(referenceInputPropertiesInResponse2.DataSource is BlobReferenceInputDataSource); BlobReferenceInputDataSource blobReferenceInputDataSourceInResponse2 = (BlobReferenceInputDataSource)referenceInputPropertiesInResponse2.DataSource; Assert.Equal("{date}", blobReferenceInputDataSourceInResponse2.Properties.PathPattern); Assert.Equal("yyyy/MM/dd", blobReferenceInputDataSourceInResponse2.Properties.DateFormat); Assert.Equal(inputCreateOrUpdateResponse.Input.Properties.Etag, inputGetResponse.Input.Properties.Etag); // List inputs InputListResponse inputListResponse = client.Inputs.ListInputInJob(resourceGroupName, resourceName, new InputListParameters()); Assert.Equal(HttpStatusCode.OK, inputListResponse.StatusCode); Assert.Equal(1, inputListResponse.Value.Count); // Test input connectivity DataSourceTestConnectionResponse response = client.Inputs.TestConnection(resourceGroupName, resourceName, inputName); Assert.Equal(OperationStatus.Succeeded, response.Status); Assert.Equal(DataSourceTestStatus.TestSucceeded, response.DataSourceTestStatus); // Update the input BlobReferenceInputDataSource blobReferenceInputDataSource = new BlobReferenceInputDataSource() { Properties = new BlobReferenceInputDataSourceProperties() { StorageAccounts = new[] { storageAccount }, Container = "state", PathPattern = "test.csv", DateFormat = "yyyy/MM/dd" } }; ((ReferenceInputProperties)inputProperties).DataSource = blobReferenceInputDataSource; inputProperties.Etag = inputCreateOrUpdateResponse.Input.Properties.Etag; InputPatchParameters inputPatchParameters = new InputPatchParameters(inputProperties); InputPatchResponse inputPatchResponse = client.Inputs.Patch(resourceGroupName, resourceName, inputName, inputPatchParameters); Assert.Equal(HttpStatusCode.OK, inputPatchResponse.StatusCode); Assert.True(inputPatchResponse.Properties is ReferenceInputProperties); ReferenceInputProperties referenceInputPropertiesInResponse3 = (ReferenceInputProperties)inputPatchResponse.Properties; Assert.True(referenceInputPropertiesInResponse3.DataSource is BlobReferenceInputDataSource); BlobReferenceInputDataSource blobReferenceInputDataSourceInResponse3 = (BlobReferenceInputDataSource)referenceInputPropertiesInResponse3.DataSource; Assert.Equal("test.csv", blobReferenceInputDataSourceInResponse3.Properties.PathPattern); Assert.Equal("yyyy/MM/dd", blobReferenceInputDataSourceInResponse3.Properties.DateFormat); Assert.NotNull(inputPatchResponse.Properties.Etag); Assert.NotEqual(inputCreateOrUpdateResponse.Input.Properties.Etag, inputPatchResponse.Properties.Etag); // Delete the inputs AzureOperationResponse deleteInputOperationResponse = client.Inputs.Delete(resourceGroupName, resourceName, inputName); Assert.Equal(HttpStatusCode.OK, deleteInputOperationResponse.StatusCode); // Check that there are 0 inputs in the job jobGetParameters = new JobGetParameters("inputs"); jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters); Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode); Assert.Equal(0, jobGetResponse.Job.Properties.Inputs.Count); } finally { client.StreamingJobs.Delete(resourceGroupName, resourceName); resourceClient.ResourceGroups.Delete(resourceGroupName); } } }