Пример #1
0
        static async System.Threading.Tasks.Task Main(string[] args)
        {
            var configuration = new ConfigurationBuilder()
                                .AddJsonFile("appsettings.json")
                                .AddEnvironmentVariables()
                                .Build();

            Console.WriteLine(configuration.GetConnectionString("DestinationIotHubConnectionString"));
            Console.WriteLine(configuration.GetConnectionString("BlobSasToken"));

            RegistryManager sourceRegistryManager      = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("SourceIotHubConnectionString"));
            RegistryManager destinationRegistryManager = RegistryManager.CreateFromConnectionString(configuration.GetConnectionString("DestinationIotHubConnectionString"));

            string containerSasUri = configuration.GetConnectionString("BlobSasToken");

            Console.WriteLine($"Start export of IoT Hub devices to blob");
            // Call an export job on the IoT Hub to retrieve all devices

            JobProperties exportJob =
                await sourceRegistryManager.ExportDevicesAsync(containerSasUri, false);

            var exportJobResult = await WaitForJobCompletion(sourceRegistryManager, exportJob);

            Console.WriteLine($"Start import of IoT Hub devices to blob");

            JobProperties importJob =
                await destinationRegistryManager.ImportDevicesAsync(containerSasUri, containerSasUri);

            var importJobResult = await WaitForJobCompletion(destinationRegistryManager, importJob);
        }
Пример #2
0
        public async Task <bool> ImportAllDevices(string connString, string containerSasUri)
        {
            bool bret = false;
            // Call an export job on the IoT Hub to retrieve all devices
            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(connString);
            JobProperties   exportJob       = await registryManager.ImportDevicesAsync(containerSasUri, containerSasUri);

            // Wait until job is finished
            while (true)
            {
                exportJob = await registryManager.GetJobAsync(exportJob.JobId);

                if (exportJob.Status == JobStatus.Completed ||
                    exportJob.Status == JobStatus.Failed ||
                    exportJob.Status == JobStatus.Cancelled)
                {
                    if (exportJob.Status == JobStatus.Completed)
                    {
                        bret = true;
                    }
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5));
            }
            return(bret);
        }
Пример #3
0
        public async Task <Response <JobProperties> > CreateImportExportJobAsync(JobProperties jobProperties, CancellationToken cancellationToken = default)
        {
            if (jobProperties == null)
            {
                throw new ArgumentNullException(nameof(jobProperties));
            }

            using var message = CreateCreateImportExportJobRequest(jobProperties);
            await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false);

            switch (message.Response.Status)
            {
            case 200:
            {
                JobProperties value = default;
                using var document = await JsonDocument.ParseAsync(message.Response.ContentStream, default, cancellationToken).ConfigureAwait(false);

                if (document.RootElement.ValueKind == JsonValueKind.Null)
                {
                    value = null;
                }
                else
                {
                    value = JobProperties.DeserializeJobProperties(document.RootElement);
                }
                return(Response.FromValue(value, message.Response));
            }
Пример #4
0
        /// Get the list of devices registered to the IoT Hub
        ///   and export it to a blob as deserialized objects.
        private async Task ExportDevices(string containerURI, string hubConnectionString)
        {
            try
            {
                Console.WriteLine("Creating and running registry manager job to retrieve the devices from the hub.");
                // Create an instance of the registry manager class.
                using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);

                // Call an export job on the IoT Hub to retrieve all devices.
                // This writes them to devices.txt in the container.
                // The second parameter indicates whether to export the keys or not.
                JobProperties exportJob = await registryManager.ExportDevicesAsync(containerURI, false).ConfigureAwait(false);

                // Poll every 5 seconds to see if the job has finished executing.
                while (true)
                {
                    exportJob = await registryManager.GetJobAsync(exportJob.JobId).ConfigureAwait(false);

                    if (exportJob.Status == JobStatus.Completed ||
                        exportJob.Status == JobStatus.Failed ||
                        exportJob.Status == JobStatus.Cancelled)
                    {
                        // Job has finished executing
                        break;
                    }

                    await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
                }
            }
            catch (Exception ex)
            {
                Debug.Print("Error exporting devices to blob storage. Exception message = {0}", ex.Message);
            }
        }
Пример #5
0
        /// Get the list of devices registered to the IoT Hub
        ///   and export it to a blob as deserialized objects.
        public static async Task ExportDevices(string containerURI, string hubConnectionString)
        {
            // Create an instance of the registry manager class.
            RegistryManager registryManager =
                RegistryManager.CreateFromConnectionString(hubConnectionString);

            // Call an export job on the IoT Hub to retrieve all devices.
            // This writes them to devices.txt in the container.
            // The second parameter indicates whether to export the keys or not.
            JobProperties exportJob = await
                                      registryManager.ExportDevicesAsync(containerURI, false);

            // Poll every 5 seconds to see if the job has finished executing.
            while (true)
            {
                exportJob = await registryManager.GetJobAsync(exportJob.JobId);

                if (exportJob.Status == JobStatus.Completed ||
                    exportJob.Status == JobStatus.Failed ||
                    exportJob.Status == JobStatus.Cancelled)
                {
                    // Job has finished executing
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            // Note: could add twin data here if you want to export it.
        }
        private async Task <JobProperties> CreateAndWaitForJobAsync(
            StorageAuthenticationType storageAuthenticationType,
            string devicesFileName,
            string configsFileName,
            RegistryManager registryManager,
            Uri containerUri,
            ManagedIdentity identity)
        {
            int           tryCount          = 0;
            JobProperties importJobResponse = null;

            JobProperties jobProperties = JobProperties.CreateForImportJob(
                containerUri.ToString(),
                containerUri.ToString(),
                devicesFileName,
                storageAuthenticationType,
                identity);

            jobProperties.ConfigurationsBlobName = configsFileName;
            jobProperties.IncludeConfigurations  = true;

            while (tryCount < MaxIterationWait)
            {
                try
                {
                    importJobResponse = await registryManager.ImportDevicesAsync(jobProperties).ConfigureAwait(false);

                    if (!string.IsNullOrWhiteSpace(importJobResponse.FailureReason))
                    {
                        Logger.Trace($"Job failed due to {importJobResponse.FailureReason}");
                    }
                    break;
                }
                // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
                catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                {
                    Logger.Trace($"JobQuotaExceededException... waiting.");
                    await Task.Delay(s_waitDuration).ConfigureAwait(false);

                    continue;
                }
            }

            // wait for job to complete
            for (int i = 0; i < MaxIterationWait; ++i)
            {
                await Task.Delay(1000).ConfigureAwait(false);

                importJobResponse = await registryManager.GetJobAsync(importJobResponse?.JobId).ConfigureAwait(false);

                Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%");
                if (!s_incompleteJobs.Contains(importJobResponse.Status))
                {
                    break;
                }
            }

            return(importJobResponse);
        }
Пример #7
0
        public JobProperties getDefaultJobProperties()
        {
            JobProperties properties = new JobProperties();

            properties.topMargin    = 0;
            properties.interspacing = "1/8";
            return(properties);
        }
Пример #8
0
 public void PrintDocumentToFile(String Filename, String Type, String Template, Int64 documentId, String connectionstring)
 {
     datacollection = new DataSet();
     port           = new FilePort(Filename);
     printer        = PrinterFactory.getPrinter(Type);
     port.SetDocumentName("CDS DOC " + documentId);
     job = printer.getDefaultJobProperties();
     PrintDocumentFromTemplate(Template, documentId, connectionstring);
     printer.endJob();
 }
Пример #9
0
        public JobWindow(JobObjectHandle jobHandle)
        {
            InitializeComponent();
            this.AddEscapeToClose();
            this.SetTopMost();

            _jobProps      = new JobProperties(jobHandle);
            _jobProps.Dock = DockStyle.Fill;

            panelJob.Controls.Add(_jobProps);
        }
        public async Task ImportDevicesAsync(string hubConnectionString,
                                             string blobContainerUri,
                                             string userDefinedManagedIdentityResourceId = null)
        {
            using RegistryManager destRegistryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);

            // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
            // not null, the jobs will use user defined managed identity. If the IoT hub is not
            // configured with the user defined managed identity specified in userAssignedIdentity,
            // the job will fail.
            // If StorageAuthenticationType is set to IdentityBased and userAssignedIdentity property is
            // null, the jobs will use system defined identity by default. If the IoT hub is configured with the
            // system defined managed identity, the job will succeed but will not use the user defined managed identity.
            // If the IoT hub is not configured with system defined managed identity, the job will fail.
            // If StorageAuthenticationType is set to IdentityBased and neither user defined nor system defined
            // managed identities are configured on the hub, the job will fail.
            JobProperties jobProperties = JobProperties.CreateForImportJob(
                inputBlobContainerUri: blobContainerUri,
                outputBlobContainerUri: blobContainerUri,
                storageAuthenticationType: StorageAuthenticationType.IdentityBased,
                identity: new ManagedIdentity
            {
                userAssignedIdentity = userDefinedManagedIdentityResourceId
            });

            JobProperties jobResult = await destRegistryManager
                                      .ImportDevicesAsync(jobProperties);

            // Poll every 5 seconds to see if the job has finished executing.
            while (true)
            {
                jobResult = await destRegistryManager.GetJobAsync(jobResult.JobId);

                if (jobResult.Status == JobStatus.Completed)
                {
                    break;
                }
                else if (jobResult.Status == JobStatus.Failed)
                {
                    throw new Exception("Import job failed.");
                }
                else if (jobResult.Status == JobStatus.Cancelled)
                {
                    throw new Exception("Import job was canceled.");
                }
                else
                {
                    await Task.Delay(TimeSpan.FromSeconds(5));
                }
            }
        }
        public static async void RunAsync([TimerTrigger("0 */2 * * * *")] TimerInfo myTimer, ILogger log, ExecutionContext context)
        {
            log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");

            var config = new ConfigurationBuilder()
                         .SetBasePath(context.FunctionAppDirectory)
                         .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
                         .AddEnvironmentVariables()
                         .Build();

            var BlobConStr    = config["blobConStr"];
            var BlobContainer = config["blobContainer"];
            var BlobURL       = config["blobURL"];
            var IotHubConStr  = config["iotHubConStr"];

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(BlobConStr);
            CloudBlobClient     blobClient     = storageAccount.CreateCloudBlobClient();
            CloudBlobContainer  container      = blobClient.GetContainerReference(BlobContainer);

            string storedPolicyName = null;

            string sasContainerToken;

            if (storedPolicyName == null)
            {
                SharedAccessBlobPolicy adHocPolicy = new SharedAccessBlobPolicy()
                {
                    SharedAccessExpiryTime = DateTime.UtcNow.AddHours(1),
                    Permissions            = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write | SharedAccessBlobPermissions.List | SharedAccessBlobPermissions.Delete
                };

                sasContainerToken = container.GetSharedAccessSignature(adHocPolicy, null);

                Console.WriteLine("SAS for blob container (ad hoc): {0}", sasContainerToken);
            }
            else
            {
                sasContainerToken = container.GetSharedAccessSignature(null, storedPolicyName);

                Console.WriteLine("SAS for blob container (stored access policy): {0}", sasContainerToken);
            }

            string containerSasUri = container.Uri + sasContainerToken;


            Console.WriteLine("Blob SAS URI:  {0}", containerSasUri);

            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(IotHubConStr);
            JobProperties   exportJob       = await registryManager.ExportDevicesAsync(containerSasUri, false);
        }
Пример #12
0
        static async Task <string> BulkCreateListAsync(IList <string> deviceIds)
        {
            var serializedDevices = deviceIds
                                    .Select(id => new ExportImportDevice {
                Id = id, ImportMode = ImportMode.CreateOrUpdate
            })
                                    .Select(JsonConvert.SerializeObject);

            CloudBlockBlob blob = await WriteDevicesToBlobAsync(serializedDevices);

            string        containerUri = blob.Container.StorageUri.PrimaryUri.AbsoluteUri + GetSasToken();
            JobProperties job          = await registry.ImportDevicesAsync(containerUri, containerUri, blob.Name);

            return(job.JobId);
        }
Пример #13
0
        /// <summary>
        /// Get job properties.
        /// </summary>
        /// <param name="updateJobParams">Job properties specified via PowerShell.</param>
        /// <param name="existingJobProperties">Exsting job properties.</param>
        /// <returns></returns>
        private JobProperties GetExistingJobPropertiesParams(PSJobParams updateJobParams, JobProperties existingJobProperties)
        {
            var newJobProperties = new JobProperties()
            {
                Action     = this.GetExistingJobAction(updateJobParams.JobAction, existingJobProperties.Action),
                Recurrence = this.GetExistingJobRecurrence(updateJobParams.JobRecurrence, existingJobProperties.Recurrence),
                StartTime  = updateJobParams.StartTime ?? existingJobProperties.StartTime,
            };

            newJobProperties.Action.ErrorAction = this.GetExistingJobErrorAction(updateJobParams.JobErrorAction, existingJobProperties.Action.ErrorAction);

            newJobProperties.State = updateJobParams.JobState.GetValueOrDefaultEnum <JobState?>(defaultValue: null);

            return(newJobProperties);
        }
        /// <summary>
        /// Get job properties.
        /// </summary>
        /// <param name="updateJobParams">Job properties specified via PowerShell.</param>
        /// <param name="existingJobProperties">Exsting job properties.</param>
        /// <returns></returns>
        private JobProperties GetExistingJobPropertiesParams(PSJobParams updateJobParams, JobProperties existingJobProperties)
        {
            var newJobProperties = new JobProperties()
            {
                Action = this.GetExistingJobAction(updateJobParams.JobAction, existingJobProperties.Action),
                Recurrence = this.GetExistingJobRecurrence(updateJobParams.JobRecurrence, existingJobProperties.Recurrence),
                StartTime = updateJobParams.StartTime ?? existingJobProperties.StartTime,
            };

            newJobProperties.Action.ErrorAction = this.GetExistingJobErrorAction(updateJobParams.JobErrorAction, existingJobProperties.Action.ErrorAction);

            newJobProperties.State = updateJobParams.JobState.GetValueOrDefaultEnum<JobState?>(defaultValue: null);

            return newJobProperties;
        }
Пример #15
0
        public void PrintDocument(String Printer, String Type, String Template, Int64 documentId, String connectionstring)
        {
            datacollection = new DataSet();
            port           = new WindowsPrinter(Printer);
            printer        = PrinterFactory.getPrinter(Type);
            port.SetDocumentName("CDS DOC " + documentId);
            job = printer.getDefaultJobProperties();

            job.draftQuality = true;
            job.pitch        = 10;
            //job.paperSize = PaperSize.LETTER;

            PrintDocumentFromTemplate(Template, documentId, connectionstring);

            printer.endJob();
        }
Пример #16
0
        static async Task WaitForJobToComplete(RegistryManager registryManager, JobProperties job)
        {
            while (true)
            {
                job = await registryManager.GetJobAsync(job.JobId);

                Console.WriteLine("\t Job " + job.Status);

                if (job.Status == JobStatus.Completed ||
                    job.Status == JobStatus.Failed ||
                    job.Status == JobStatus.Cancelled)
                {
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5));
            }
        }
Пример #17
0
        internal HttpMessage CreateCreateImportExportJobRequest(JobProperties jobProperties)
        {
            var message = _pipeline.CreateMessage();
            var request = message.Request;

            request.Method = RequestMethod.Post;
            var uri = new RawRequestUriBuilder();

            uri.Reset(endpoint);
            uri.AppendPath("/jobs/create", false);
            uri.AppendQuery("api-version", apiVersion, true);
            request.Uri = uri;
            request.Headers.Add("Content-Type", "application/json");
            var content = new Utf8JsonRequestContent();

            content.JsonWriter.WriteObjectValue(jobProperties);
            request.Content = content;
            return(message);
        }
Пример #18
0
        /// <summary>
        /// Creates a job to export device registrations to the container.
        /// </summary>
        /// <param name="outputBlobContainerUri">URI containing SAS token to a blob container. This is used to output the results of the export job.</param>
        /// <param name="excludeKeys">If true, authorization keys are NOT included in export output.</param>
        /// <param name="options">The optional settings for this request.</param>
        /// <param name="cancellationToken">Task cancellation token.</param>
        /// <returns>JobProperties of the newly created job.</returns>
        public virtual Task <Response <JobProperties> > CreateExportDevicesJobAsync(
            Uri outputBlobContainerUri,
            bool excludeKeys,
            ExportJobRequestOptions options     = default,
            CancellationToken cancellationToken = default)
        {
            Argument.AssertNotNull(outputBlobContainerUri, nameof(outputBlobContainerUri));

            var jobProperties = new JobProperties
            {
                Type = JobPropertiesType.Export,
                OutputBlobContainerUri    = outputBlobContainerUri.ToString(),
                ExcludeKeysInExport       = excludeKeys,
                StorageAuthenticationType = options?.AuthenticationType,
                OutputBlobName            = options?.OutputBlobName
            };

            return(_jobsRestClient.CreateImportExportJobAsync(jobProperties, cancellationToken));
        }
Пример #19
0
        /// <summary>
        /// Creates a job to import device registrations into the IoT Hub.
        /// </summary>
        /// <param name="importBlobContainerUri">URI containing SAS token to a blob container that contains registry data to sync.</param>
        /// <param name="outputBlobContainerUri">URI containing SAS token to a blob container. This is used to output the status of the job.</param>
        /// <param name="options">The optional settings for this request.</param>
        /// <param name="cancellationToken">Task cancellation token.</param>
        /// <returns>JobProperties of the newly created job.</returns>
        public virtual Response <JobProperties> CreateImportDevicesJob(
            Uri importBlobContainerUri,
            Uri outputBlobContainerUri,
            ImportJobRequestOptions options     = default,
            CancellationToken cancellationToken = default)
        {
            Argument.AssertNotNull(importBlobContainerUri, nameof(importBlobContainerUri));
            Argument.AssertNotNull(outputBlobContainerUri, nameof(outputBlobContainerUri));

            var jobProperties = new JobProperties
            {
                Type = JobPropertiesType.Import,
                InputBlobContainerUri     = importBlobContainerUri.ToString(),
                OutputBlobContainerUri    = outputBlobContainerUri.ToString(),
                StorageAuthenticationType = options?.AuthenticationType,
                InputBlobName             = options?.InputBlobName,
                OutputBlobName            = options?.OutputBlobName,
            };

            return(_jobsRestClient.CreateImportExportJob(jobProperties, cancellationToken));
        }
Пример #20
0
        public void Test_InputOperations_ReferenceBlob()
        {
            BasicDelegatingHandler handler = new BasicDelegatingHandler();

            using (var undoContext = UndoContext.Current)
            {
                undoContext.Start();

                string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
                string resourceName      = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");

                string serviceLocation = TestHelper.GetDefaultLocation();

                var resourceClient = TestHelper.GetResourceClient(handler);
                var client         = TestHelper.GetStreamAnalyticsManagementClient(handler);

                try
                {
                    ResourceGroup resourceGroup = new ResourceGroup()
                    {
                        Location = serviceLocation
                    };
                    resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);

                    Job job = new Job();
                    job.Name     = resourceName;
                    job.Location = serviceLocation;

                    // Construct the general properties for JobProperties
                    JobProperties jobProperties = new JobProperties();
                    jobProperties.Sku = new Sku()
                    {
                        Name = "standard"
                    };
                    jobProperties.EventsOutOfOrderPolicy            = EventsOutOfOrderPolicy.Drop;
                    jobProperties.EventsOutOfOrderMaxDelayInSeconds = 0;

                    job.Properties = jobProperties;

                    // Construct the JobCreateProperties
                    JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters();
                    jobCreateOrUpdateParameters.Job = job;

                    // Create a streaming job
                    JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);

                    // Get a streaming job to check
                    JobGetParameters jobGetParameters = new JobGetParameters(string.Empty);
                    JobGetResponse   jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
                    Assert.Equal(resourceName, jobGetResponse.Job.Name);

                    // Construct the Input
                    StorageAccount storageAccount = new StorageAccount();
                    storageAccount.AccountName = TestHelper.AccountName;
                    storageAccount.AccountKey  = TestHelper.AccountKey;
                    InputProperties inputProperties = new ReferenceInputProperties()
                    {
                        Serialization = new CsvSerialization()
                        {
                            Properties = new CsvSerializationProperties()
                            {
                                FieldDelimiter = ",",
                                Encoding       = "UTF8"
                            }
                        },
                        DataSource = new BlobReferenceInputDataSource()
                        {
                            Properties = new BlobReferenceInputDataSourceProperties()
                            {
                                StorageAccounts = new[] { storageAccount },
                                Container       = "state",
                                PathPattern     = "{date}",
                                DateFormat      = "yyyy/MM/dd"
                            }
                        }
                    };
                    string inputName = TestUtilities.GenerateName("inputtest");
                    Input  input1    = new Input(inputName)
                    {
                        Properties = inputProperties
                    };

                    // Add an input
                    InputCreateOrUpdateParameters inputCreateOrUpdateParameters = new InputCreateOrUpdateParameters();
                    inputCreateOrUpdateParameters.Input = input1;
                    InputCreateOrUpdateResponse inputCreateOrUpdateResponse = client.Inputs.CreateOrUpdate(resourceGroupName, resourceName, inputCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, inputCreateOrUpdateResponse.StatusCode);
                    Assert.Equal(inputName, inputCreateOrUpdateResponse.Input.Name);
                    Assert.Equal("Reference", inputCreateOrUpdateResponse.Input.Properties.Type);
                    Assert.True(inputCreateOrUpdateResponse.Input.Properties is ReferenceInputProperties);
                    ReferenceInputProperties referenceInputPropertiesInResponse1 = (ReferenceInputProperties)inputCreateOrUpdateResponse.Input.Properties;
                    Assert.True(referenceInputPropertiesInResponse1.DataSource is BlobReferenceInputDataSource);
                    BlobReferenceInputDataSource blobReferenceInputDataSourceInResponse1 = (BlobReferenceInputDataSource)referenceInputPropertiesInResponse1.DataSource;
                    Assert.Equal("{date}", blobReferenceInputDataSourceInResponse1.Properties.PathPattern);
                    Assert.Equal("yyyy/MM/dd", blobReferenceInputDataSourceInResponse1.Properties.DateFormat);
                    Assert.NotNull(inputCreateOrUpdateResponse.Input.Properties.Etag);

                    // Get the input
                    InputGetResponse inputGetResponse = client.Inputs.Get(resourceGroupName, resourceName, inputName);
                    Assert.Equal(HttpStatusCode.OK, inputGetResponse.StatusCode);
                    Assert.Equal(inputName, inputGetResponse.Input.Name);
                    Assert.True(inputGetResponse.Input.Properties is ReferenceInputProperties);
                    ReferenceInputProperties referenceInputPropertiesInResponse2 = (ReferenceInputProperties)inputGetResponse.Input.Properties;
                    Assert.True(referenceInputPropertiesInResponse2.DataSource is BlobReferenceInputDataSource);
                    BlobReferenceInputDataSource blobReferenceInputDataSourceInResponse2 = (BlobReferenceInputDataSource)referenceInputPropertiesInResponse2.DataSource;
                    Assert.Equal("{date}", blobReferenceInputDataSourceInResponse2.Properties.PathPattern);
                    Assert.Equal("yyyy/MM/dd", blobReferenceInputDataSourceInResponse2.Properties.DateFormat);
                    Assert.Equal(inputCreateOrUpdateResponse.Input.Properties.Etag, inputGetResponse.Input.Properties.Etag);

                    // List inputs
                    InputListResponse inputListResponse = client.Inputs.ListInputInJob(resourceGroupName, resourceName, new InputListParameters());
                    Assert.Equal(HttpStatusCode.OK, inputListResponse.StatusCode);
                    Assert.Equal(1, inputListResponse.Value.Count);

                    // Test input connectivity
                    DataSourceTestConnectionResponse response = client.Inputs.TestConnection(resourceGroupName, resourceName, inputName);
                    Assert.Equal(OperationStatus.Succeeded, response.Status);
                    Assert.Equal(DataSourceTestStatus.TestSucceeded, response.DataSourceTestStatus);

                    // Update the input
                    BlobReferenceInputDataSource blobReferenceInputDataSource = new BlobReferenceInputDataSource()
                    {
                        Properties = new BlobReferenceInputDataSourceProperties()
                        {
                            StorageAccounts = new[] { storageAccount },
                            Container       = "state",
                            PathPattern     = "test.csv",
                            DateFormat      = "yyyy/MM/dd"
                        }
                    };
                    ((ReferenceInputProperties)inputProperties).DataSource = blobReferenceInputDataSource;
                    inputProperties.Etag = inputCreateOrUpdateResponse.Input.Properties.Etag;
                    InputPatchParameters inputPatchParameters = new InputPatchParameters(inputProperties);
                    InputPatchResponse   inputPatchResponse   = client.Inputs.Patch(resourceGroupName, resourceName, inputName, inputPatchParameters);
                    Assert.Equal(HttpStatusCode.OK, inputPatchResponse.StatusCode);
                    Assert.True(inputPatchResponse.Properties is ReferenceInputProperties);
                    ReferenceInputProperties referenceInputPropertiesInResponse3 = (ReferenceInputProperties)inputPatchResponse.Properties;
                    Assert.True(referenceInputPropertiesInResponse3.DataSource is BlobReferenceInputDataSource);
                    BlobReferenceInputDataSource blobReferenceInputDataSourceInResponse3 = (BlobReferenceInputDataSource)referenceInputPropertiesInResponse3.DataSource;
                    Assert.Equal("test.csv", blobReferenceInputDataSourceInResponse3.Properties.PathPattern);
                    Assert.Equal("yyyy/MM/dd", blobReferenceInputDataSourceInResponse3.Properties.DateFormat);
                    Assert.NotNull(inputPatchResponse.Properties.Etag);
                    Assert.NotEqual(inputCreateOrUpdateResponse.Input.Properties.Etag, inputPatchResponse.Properties.Etag);

                    // Delete the inputs
                    AzureOperationResponse deleteInputOperationResponse = client.Inputs.Delete(resourceGroupName, resourceName, inputName);
                    Assert.Equal(HttpStatusCode.OK, deleteInputOperationResponse.StatusCode);

                    // Check that there are 0 inputs in the job
                    jobGetParameters = new JobGetParameters("inputs");
                    jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(0, jobGetResponse.Job.Properties.Inputs.Count);
                }
                finally
                {
                    client.StreamingJobs.Delete(resourceGroupName, resourceName);
                    resourceClient.ResourceGroups.Delete(resourceGroupName);
                }
            }
        }
Пример #21
0
        // This shows how to copy devices from one IoT Hub to another.
        // First, export the list from the Source hut to devices.txt (ExportDevices).
        // Next, read in that file. Each row is a serialized object;
        //   read them into the generic list serializedDevices.
        // Delete the devices.txt in blob storage, because you're going to recreate it.
        // For each serializedDevice, deserialize it, set ImportMode to CREATE,
        //   reserialize it, and write it to a StringBuilder. The ImportMode field is what
        //   tells the job framework to add each device.
        // Write the new StringBuilder to the block blob.
        //   This essentially replaces the list with a list of devices that have ImportJob = Delete.
        // Call ImportDevicesAsync, which will read in the list in devices.txt, then add each one
        //   because it doesn't already exist. If it already exists, it will write an entry to
        //   the import error log and not add the new one.
        private async Task CopyAllDevicesToNewHub(string sourceHubConnectionString, string destHubConnectionString, string containerUri, string deviceListFile)
        {
            Console.WriteLine("Exporting devices on current hub");

            // Read the devices from the hub and write them to devices.txt in blob storage.
            await ExportDevices(containerUri, sourceHubConnectionString).ConfigureAwait(false);

            // Read devices.txt which contains serialized objects.
            // Write each line to the serializedDevices list. (List<string>).
            CloudBlockBlob blockBlob = _cloudBlobContainer.GetBlockBlobReference(deviceListFile);

            // Get the URI for the blob.
            string blobUri = blockBlob.Uri.ToString();

            // Instantiate the generic list.
            var serializedDevices = new List <string>();

            Console.WriteLine("Read in list of devices from blob storage.");

            // Read the blob file of devices, import each row into serializedDevices.
            using Stream blobStream = await blockBlob.OpenReadAsync(AccessCondition.GenerateIfExistsCondition(), null, null).ConfigureAwait(false);

            using var streamReader = new StreamReader(blobStream, Encoding.UTF8);
            while (streamReader.Peek() != -1)
            {
                string line = await streamReader.ReadLineAsync().ConfigureAwait(false);

                serializedDevices.Add(line);
            }

            // Delete the blob containing the list of devices, because you're going to recreate it.
            CloudBlockBlob blobToDelete = _cloudBlobContainer.GetBlockBlobReference("devices.txt");

            Console.WriteLine("Update ImportMode to be Create.");

            // Step 1: Update each device's ImportMode to be Create
            var sb = new StringBuilder();

            serializedDevices.ForEach(serializedDevice =>
            {
                // Deserialize back to an ExportImportDevice.
                var device = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDevice);

                // Update the property.
                device.ImportMode = ImportMode.Create;

                // Re-serialize the object now that you're updated the property.
                sb.AppendLine(JsonConvert.SerializeObject(device));
            });

            // Step 2: Delete the blob if it already exists, then write the list in memory to the blob.
            await blobToDelete.DeleteIfExistsAsync().ConfigureAwait(false);

            using CloudBlobStream stream = await blobToDelete.OpenWriteAsync().ConfigureAwait(false);

            byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
            for (var i = 0; i < bytes.Length; i += 500)
            {
                int length = Math.Min(bytes.Length - i, 500);
                await stream.WriteAsync(bytes, i, length).ConfigureAwait(false);
            }

            Console.WriteLine("Creating and running registry manager job to import the entries from the text file to the new hub");

            // Step 3: Call import using the same blob to create all devices.
            // Loads devices.txt and adds the devices to the destination hub.
            using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(destHubConnectionString);
            JobProperties importJob = await registryManager.ImportDevicesAsync(containerUri, containerUri).ConfigureAwait(false);

            // Wait until job is finished
            while (true)
            {
                importJob = await registryManager.GetJobAsync(importJob.JobId).ConfigureAwait(false);

                Console.WriteLine($"Import job status is {importJob.Status}");
                if (importJob.Status == JobStatus.Completed ||
                    importJob.Status == JobStatus.Failed ||
                    importJob.Status == JobStatus.Cancelled)
                {
                    // Job has finished executing
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
            }
        }
        public void Test_OutputOperations_E2E()
        {
            BasicDelegatingHandler handler = new BasicDelegatingHandler();

            using (var undoContext = UndoContext.Current)
            {
                undoContext.Start();

                string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
                string resourceName      = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");

                string serviceLocation = TestHelper.GetDefaultLocation();

                var resourceClient = TestHelper.GetResourceClient(handler);
                var client         = TestHelper.GetStreamAnalyticsManagementClient(handler);

                try
                {
                    ResourceGroup resourceGroup = new ResourceGroup()
                    {
                        Location = serviceLocation
                    };
                    resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);

                    Job job = new Job();
                    job.Name     = resourceName;
                    job.Location = serviceLocation;

                    // Construct the general properties for JobProperties
                    JobProperties jobProperties = new JobProperties();
                    jobProperties.Sku = new Sku()
                    {
                        Name = "standard"
                    };
                    jobProperties.EventsOutOfOrderPolicy            = EventsOutOfOrderPolicy.Drop;
                    jobProperties.EventsOutOfOrderMaxDelayInSeconds = 0;

                    job.Properties = jobProperties;

                    // Construct the JobCreateProperties
                    JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters();
                    jobCreateOrUpdateParameters.Job = job;

                    // Create a streaming job
                    JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);

                    // Get a streaming job to check
                    JobGetParameters jobGetParameters = new JobGetParameters(string.Empty);
                    JobGetResponse   jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
                    Assert.Equal(resourceName, jobGetResponse.Job.Name);

                    // Construct the Output
                    OutputProperties         outputProperties         = new OutputProperties();
                    string                   outputName               = TestUtilities.GenerateName("outputtest");
                    string                   tableName                = "StateInfo";
                    SqlAzureOutputDataSource sqlAzureOutputDataSource = new SqlAzureOutputDataSource()
                    {
                        Properties = new SqlAzureOutputDataSourceProperties()
                        {
                            Server   = TestHelper.Server,
                            Database = TestHelper.Database,
                            User     = TestHelper.User,
                            Password = TestHelper.Password,
                            Table    = tableName
                        }
                    };
                    outputProperties.DataSource = sqlAzureOutputDataSource;
                    Output output1 = new Output(outputName)
                    {
                        Properties = outputProperties
                    };

                    // Add an output
                    OutputCreateOrUpdateParameters outputCreateOrUpdateParameters = new OutputCreateOrUpdateParameters();
                    outputCreateOrUpdateParameters.Output = output1;
                    OutputCreateOrUpdateResponse outputCreateOrUpdateResponse = client.Outputs.CreateOrUpdate(resourceGroupName, resourceName, outputCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, outputCreateOrUpdateResponse.StatusCode);
                    Assert.Equal(outputName, outputCreateOrUpdateResponse.Output.Name);
                    Assert.True(outputCreateOrUpdateResponse.Output.Properties.DataSource is SqlAzureOutputDataSource);
                    SqlAzureOutputDataSource sqlAzureOutputDataSourceInResponse1 = (SqlAzureOutputDataSource)outputCreateOrUpdateResponse.Output.Properties.DataSource;
                    Assert.Equal(tableName, sqlAzureOutputDataSourceInResponse1.Properties.Table);
                    Assert.NotNull(outputCreateOrUpdateResponse.Output.Properties.Etag);

                    // Get the output
                    OutputGetResponse outputGetResponse = client.Outputs.Get(resourceGroupName, resourceName, outputName);
                    Assert.Equal(HttpStatusCode.OK, outputGetResponse.StatusCode);
                    Assert.Equal(outputName, outputGetResponse.Output.Name);
                    Assert.True(outputGetResponse.Output.Properties.DataSource is SqlAzureOutputDataSource);
                    SqlAzureOutputDataSource sqlAzureOutputDataSourceInResponse2 = (SqlAzureOutputDataSource)outputGetResponse.Output.Properties.DataSource;
                    Assert.Equal(tableName, sqlAzureOutputDataSourceInResponse2.Properties.Table);

                    // List outputs
                    OutputListResponse outputListResponse = client.Outputs.ListOutputInJob(resourceGroupName, resourceName, new OutputListParameters());
                    Assert.Equal(HttpStatusCode.OK, outputListResponse.StatusCode);
                    Assert.Equal(1, outputListResponse.Value.Count);

                    // Check that there is 1 output in the job
                    jobGetParameters = new JobGetParameters("outputs");
                    jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(1, jobGetResponse.Job.Properties.Outputs.Count);

                    // Test output connectivity
                    DataSourceTestConnectionResponse response = client.Outputs.TestConnection(resourceGroupName, resourceName, outputName);
                    Assert.Equal(OperationStatus.Succeeded, response.Status);
                    Assert.Equal(DataSourceTestStatus.TestSucceeded, response.DataSourceTestStatus);

                    // Update the output
                    string newTableName = TestUtilities.GenerateName("NewTableName");
                    sqlAzureOutputDataSource.Properties.Table = newTableName;
                    outputProperties.DataSource = sqlAzureOutputDataSource;
                    outputProperties.Etag       = outputCreateOrUpdateResponse.Output.Properties.Etag;
                    OutputPatchParameters outputPatchParameters = new OutputPatchParameters(outputProperties);
                    OutputPatchResponse   outputPatchResponse   = client.Outputs.Patch(resourceGroupName, resourceName, outputName, outputPatchParameters);
                    Assert.Equal(HttpStatusCode.OK, outputPatchResponse.StatusCode);
                    Assert.True(outputPatchResponse.Properties.DataSource is SqlAzureOutputDataSource);
                    SqlAzureOutputDataSource sqlAzureOutputDataSourceInResponse3 = (SqlAzureOutputDataSource)outputPatchResponse.Properties.DataSource;
                    Assert.Equal(newTableName, sqlAzureOutputDataSourceInResponse3.Properties.Table);
                    Assert.NotNull(outputPatchResponse.Properties.Etag);
                    Assert.NotEqual(outputCreateOrUpdateResponse.Output.Properties.Etag, outputPatchResponse.Properties.Etag);

                    // Add second output
                    string outputName2 = TestUtilities.GenerateName("outputtest");
                    Output output2     = new Output(outputName2)
                    {
                        Properties = outputProperties
                    };
                    outputCreateOrUpdateParameters.Output = output2;
                    outputCreateOrUpdateResponse          = client.Outputs.CreateOrUpdate(resourceGroupName, resourceName, outputCreateOrUpdateParameters);

                    // List outputs
                    outputListResponse = client.Outputs.ListOutputInJob(resourceGroupName, resourceName, new OutputListParameters());
                    Assert.Equal(HttpStatusCode.OK, outputListResponse.StatusCode);
                    Assert.Equal(2, outputListResponse.Value.Count);

                    // Check that there are 2 outputs in the job
                    jobGetParameters = new JobGetParameters("outputs");
                    jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(2, jobGetResponse.Job.Properties.Outputs.Count);

                    // Delete the outputs
                    AzureOperationResponse deleteInputOperationResponse = client.Outputs.Delete(resourceGroupName, resourceName, outputName);
                    Assert.Equal(HttpStatusCode.OK, deleteInputOperationResponse.StatusCode);

                    deleteInputOperationResponse = client.Outputs.Delete(resourceGroupName, resourceName, outputName2);
                    Assert.Equal(HttpStatusCode.OK, deleteInputOperationResponse.StatusCode);

                    // Check that there are 0 outputs in the job
                    jobGetParameters = new JobGetParameters("outputs");
                    jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(0, jobGetResponse.Job.Properties.Outputs.Count);
                }
                finally
                {
                    client.StreamingJobs.Delete(resourceGroupName, resourceName);
                    resourceClient.ResourceGroups.Delete(resourceGroupName);
                }
            }
        }
Пример #23
0
        /// <summary>c
        /// Generate NumToAdd devices and add them to the hub.
        /// To do this, generate each identity.
        /// Include authentication keys.
        /// Write the device info to a block blob.
        /// Import the devices into the identity registry by calling the import job.
        /// </summary>
        private async Task GenerateAndAddDevices(string hubConnectionString, string containerURI, int NumToAdd, string devicesToAdd)
        {
            int interimProgressCount = 0;
            int displayProgressCount = 1000;
            int totalProgressCount   = 0;


            //generate reference for list of new devices you're going to add, will write list to this blob
            CloudBlockBlob generatedListBlob = _cloudBlobContainer.GetBlockBlobReference(devicesToAdd);

            // define serializedDevices as a generic list<string>
            List <string> serializedDevices = new List <string>();

            for (var i = 1; i <= NumToAdd; i++)
            {
                // Create device name with this format: Hub_00000000 + a new guid.
                // This should be large enough to display the largest number (1 million).
                //string deviceName = "Hub_" + i.ToString("D8") + "-" + Guid.NewGuid().ToString();
                string deviceName = $"Hub_{i.ToString("D8")}-{Guid.NewGuid().ToString()}";
                Debug.Print($"device = '{deviceName}'\n");

                // Create a new ExportImportDevice.
                // CryptoKeyGenerator is in the Microsoft.Azure.Devices.Common namespace.
                var deviceToAdd = new ExportImportDevice()
                {
                    Id             = deviceName,
                    Status         = DeviceStatus.Enabled,
                    Authentication = new AuthenticationMechanism
                    {
                        SymmetricKey = new SymmetricKey
                        {
                            PrimaryKey   = CryptoKeyGenerator.GenerateKey(32),
                            SecondaryKey = CryptoKeyGenerator.GenerateKey(32)
                        }
                    },
                    // This indicates that the entry should be added as a new device.
                    ImportMode = ImportMode.Create
                };

                // Add device to the list as a serialized object.
                serializedDevices.Add(JsonConvert.SerializeObject(deviceToAdd));

                // Not real progress as you write the new devices, but will at least show *some* progress.
                interimProgressCount++;
                totalProgressCount++;
                if (interimProgressCount >= displayProgressCount)
                {
                    Console.WriteLine("Added {0} messages.", totalProgressCount);
                    interimProgressCount = 0;
                }
            }

            // Now have a list of devices to be added, each one has been serialized.
            // Write the list to the blob.
            var sb = new StringBuilder();

            serializedDevices.ForEach(serializedDevice => sb.AppendLine(serializedDevice));

            // Before writing the new file, make sure there's not already one there.
            await generatedListBlob.DeleteIfExistsAsync().ConfigureAwait(false);

            // Write list of serialized objects to the blob.
            using (CloudBlobStream stream = await generatedListBlob.OpenWriteAsync().ConfigureAwait(false))
            {
                byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
                for (var i = 0; i < bytes.Length; i += 500)
                {
                    int length = Math.Min(bytes.Length - i, 500);
                    await stream.WriteAsync(bytes, i, length).ConfigureAwait(false);
                }
            }

            Console.WriteLine("Creating and running registry manager job to write the new devices.");

            // Should now have a file with all the new devices in it as serialized objects in blob storage.
            // generatedListBlob has the list of devices to be added as serialized objects.
            // Call import using the blob to add the new devices.
            // Log information related to the job is written to the same container.
            // This normally takes 1 minute per 100 devices (according to the docs).

            // First, initiate an import job.
            // This reads in the rows from the text file and writes them to IoT Devices.
            // If you want to add devices from a file, you can create a file and use this to import it.
            //   They have to be in the exact right format.
            JobProperties   importJob       = new JobProperties();
            RegistryManager registryManager = RegistryManager.CreateFromConnectionString(hubConnectionString);

            try
            {
                // First URL is the container to import from; the file must be called devices.txt
                // Second URL points to the container to write errors to as a block blob.
                // This lets you import the devices from any file name. Since we wrote the new
                //   devices to [devicesToAdd], need to read the list from there as well.
                importJob = await registryManager.ImportDevicesAsync(containerURI, containerURI, devicesToAdd).ConfigureAwait(false);

                // This will catch any errors if something bad happens to interrupt the job.
                while (true)
                {
                    importJob = await registryManager.GetJobAsync(importJob.JobId).ConfigureAwait(false);

                    if (importJob.Status == JobStatus.Completed ||
                        importJob.Status == JobStatus.Failed ||
                        importJob.Status == JobStatus.Cancelled)
                    {
                        // Job has finished executing
                        break;
                    }

                    await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
                }
            }
            catch (Exception ex)
            {
                Debug.Print("exception message {0}", ex.Message);
            }
        }
        public void Test_TransformationOperations_E2E()
        {
            BasicDelegatingHandler handler = new BasicDelegatingHandler();

            using (var undoContext = UndoContext.Current)
            {
                undoContext.Start();

                string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
                string resourceName      = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");

                string serviceLocation = TestHelper.GetDefaultLocation();

                var resourceClient = TestHelper.GetResourceClient(handler);
                var client         = TestHelper.GetStreamAnalyticsManagementClient(handler);

                try
                {
                    ResourceGroup resourceGroup = new ResourceGroup()
                    {
                        Location = serviceLocation
                    };
                    resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);

                    Job job = new Job();
                    job.Name     = resourceName;
                    job.Location = serviceLocation;

                    // Construct the general properties for JobProperties
                    JobProperties jobProperties = new JobProperties();
                    jobProperties.Sku = new Sku()
                    {
                        Name = "standard"
                    };
                    jobProperties.EventsOutOfOrderPolicy            = EventsOutOfOrderPolicy.Drop;
                    jobProperties.EventsOutOfOrderMaxDelayInSeconds = 0;

                    job.Properties = jobProperties;

                    // Construct the JobCreateProperties
                    JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters();
                    jobCreateOrUpdateParameters.Job = job;

                    // Create a streaming job
                    JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);

                    // Get a streaming job to check
                    JobGetParameters jobGetParameters = new JobGetParameters(string.Empty);
                    JobGetResponse   jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
                    Assert.Equal(resourceName, jobGetResponse.Job.Name);

                    // Construct the Transformation
                    string         transformationName     = TestUtilities.GenerateName("transformationtest");
                    int            numberOfStreamingUnits = 1;
                    Transformation transformation         = new Transformation()
                    {
                        Name       = transformationName,
                        Properties = new TransformationProperties()
                        {
                            Query          = "Select Id, Name from inputtest",
                            StreamingUnits = numberOfStreamingUnits
                        }
                    };

                    // Add an Transformation
                    TransformationCreateOrUpdateParameters transformationCreateOrUpdateParameters = new TransformationCreateOrUpdateParameters();
                    transformationCreateOrUpdateParameters.Transformation = transformation;
                    TransformationCreateOrUpdateResponse transformationCreateOrUpdateResponse = client.Transformations.CreateOrUpdate(resourceGroupName, resourceName, transformationCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, transformationCreateOrUpdateResponse.StatusCode);
                    Assert.Equal(numberOfStreamingUnits, transformationCreateOrUpdateResponse.Transformation.Properties.StreamingUnits);
                    Assert.NotNull(transformationCreateOrUpdateResponse.Transformation.Properties.Etag);

                    // Update the Transformation
                    transformation.Properties.StreamingUnits = 3;
                    transformation.Properties.Etag           = transformationCreateOrUpdateResponse.Transformation.Properties.Etag;
                    TransformationPatchParameters transformationPatchParameters = new TransformationPatchParameters(transformation.Properties);
                    TransformationPatchResponse   transformationPatchResponse   = client.Transformations.Patch(resourceGroupName, resourceName, transformationName, transformationPatchParameters);
                    Assert.Equal(HttpStatusCode.OK, transformationPatchResponse.StatusCode);
                    Assert.Equal(3, transformationPatchResponse.Properties.StreamingUnits);
                    Assert.NotNull(transformationPatchResponse.Properties.Etag);
                    Assert.NotEqual(transformationCreateOrUpdateResponse.Transformation.Properties.Etag, transformationPatchResponse.Properties.Etag);
                }
                finally
                {
                    client.StreamingJobs.Delete(resourceGroupName, resourceName);
                    resourceClient.ResourceGroups.Delete(resourceGroupName);
                }
            }
        }
        public async Task RegistryManager_ImportDevices(StorageAuthenticationType storageAuthenticationType, bool isUserAssignedMsi)
        {
            // arrange

            const string idPrefix = nameof(RegistryManager_ImportDevices);

            string deviceId = $"{idPrefix}-device-{StorageContainer.GetRandomSuffix(4)}";
            string configId = $"{idPrefix}-config-{StorageContainer.GetRandomSuffix(4)}".ToLower(); // Configuration Id characters must be all lower-case.

            Logger.Trace($"Using Ids {deviceId} and {configId}.");

            string devicesFileName = $"{idPrefix}-devices-{StorageContainer.GetRandomSuffix(4)}.txt";
            string configsFileName = $"{idPrefix}-configs-{StorageContainer.GetRandomSuffix(4)}.txt";

            using RegistryManager registryManager = RegistryManager.CreateFromConnectionString(TestConfiguration.IoTHub.ConnectionString);

            try
            {
                string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ImportDevices));
                using StorageContainer storageContainer = await StorageContainer.GetInstanceAsync(containerName).ConfigureAwait(false);

                Logger.Trace($"Using devices container {storageContainer.Uri}");

                Uri containerUri = storageAuthenticationType == StorageAuthenticationType.KeyBased
                    ? storageContainer.SasUri
                    : storageContainer.Uri;

                using Stream devicesStream = ImportExportHelpers.BuildImportStream(
                          new List <ExportImportDevice>
                {
                    new ExportImportDevice(
                        new Device(deviceId)
                    {
                        Authentication = new AuthenticationMechanism { Type = AuthenticationType.Sas }
                    },
                        ImportMode.Create),
                });
                await UploadFileAndConfirmAsync(storageContainer, devicesStream, devicesFileName).ConfigureAwait(false);

                using Stream configsStream = ImportExportHelpers.BuildImportStream(
                          new List <ImportConfiguration>
                {
                    new ImportConfiguration(configId)
                    {
                        ImportMode      = ConfigurationImportMode.CreateOrUpdateIfMatchETag,
                        Priority        = 3,
                        Labels          = { { "labelName", "labelValue" } },
                        TargetCondition = "*",
                        Content         =
                        {
                            DeviceContent = { { "properties.desired.x", 5L } },
                        },
                        Metrics =
                        {
                            Queries = { { "successfullyConfigured", "select deviceId from devices where properties.reported.x = 5" } }
                        },
                    },
                });
                await UploadFileAndConfirmAsync(storageContainer, configsStream, configsFileName).ConfigureAwait(false);

                ManagedIdentity identity = isUserAssignedMsi
                    ? new ManagedIdentity
                {
                    UserAssignedIdentity = TestConfiguration.IoTHub.UserAssignedMsiResourceId
                }
                    : null;

                // act

                JobProperties importJobResponse = await CreateAndWaitForJobAsync(
                    storageAuthenticationType,
                    devicesFileName,
                    configsFileName,
                    registryManager,
                    containerUri,
                    identity)
                                                  .ConfigureAwait(false);

                // assert

                importJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed");
                importJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed");

                // should not throw due to 404, but device may not immediately appear in registry
                Device        device = null;
                Configuration config = null;
                for (int i = 0; i < MaxIterationWait; ++i)
                {
                    await Task.Delay(s_waitDuration).ConfigureAwait(false);

                    try
                    {
                        device = await registryManager.GetDeviceAsync(deviceId).ConfigureAwait(false);

                        config = await registryManager.GetConfigurationAsync(configId).ConfigureAwait(false);

                        break;
                    }
                    catch (Exception ex)
                    {
                        Logger.Trace($"Could not find device/config on iteration {i} due to [{ex.Message}]");
                    }
                }
                if (device == null)
                {
                    Assert.Fail($"Device {deviceId} not found in registry manager");
                }
                if (config == null)
                {
                    Assert.Fail($"Config {configId} not found in registry manager");
                }
            }
            finally
            {
                try
                {
                    await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false);

                    await registryManager.RemoveConfigurationAsync(configId).ConfigureAwait(false);
                }
                catch (Exception ex)
                {
                    Logger.Trace($"Failed to clean up device/config due to {ex}");
                }
            }
        }
Пример #26
0
 public static Job CreateJob(IDatabaseSource src, IDatabaseSource dst, DataSynDef datasyn, string outFile, List <IJobReportConfiguration> reports, JobProperties jobProps, DataSynGuiEnv guienv)
 {
     //UsageStats.Usage("datasyn_job", "src", src.ToString(), "dst", dst.ToString());
     return(Job.FromCommand(new DataSynJobCommand(src, dst, datasyn, outFile, reports, guienv), jobProps));
 }
Пример #27
0
        private async Task CleanupDevices()
        {
            Console.WriteLine($"Using storage container {_blobContainerClient.Name}" +
                              $" for exporting devices identities to and importing device identities from.");

            // Retrieve the SAS Uri that will be used to grant access to the storage containers.
            string storageAccountSasUri = GetStorageAccountSasUriForCleanupJob(_blobContainerClient).ToString();

            // Step 1: Export all device identities.
            JobProperties exportAllDevicesProperties = JobProperties
                                                       .CreateForExportJob(
                outputBlobContainerUri: storageAccountSasUri,
                excludeKeysInExport: true,
                storageAuthenticationType: StorageAuthenticationType.KeyBased);

            JobProperties exportAllDevicesJob = null;

            int tryCount = 0;

            while (true)
            {
                try
                {
                    exportAllDevicesJob = await _registryManager.ExportDevicesAsync(exportAllDevicesProperties);

                    break;
                }
                // Wait for pending jobs to finish.
                catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                {
                    Console.WriteLine($"JobQuotaExceededException... waiting.");
                    await Task.Delay(WaitDuration);
                }
            }

            if (exportAllDevicesJob == null)
            {
                throw new Exception("Export devices job failed.");
            }

            // Wait until the export job is finished.
            while (true)
            {
                exportAllDevicesJob = await _registryManager.GetJobAsync(exportAllDevicesJob.JobId);

                if (s_completedJobs.Contains(exportAllDevicesJob.Status))
                {
                    // Job has finished executing.
                    break;
                }

                Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status} with progress {exportAllDevicesJob.Progress}%");
                await Task.Delay(s_waitDuration);
            }
            Console.WriteLine($"Job {exportAllDevicesJob.JobId} is {exportAllDevicesJob.Status}.");

            if (exportAllDevicesJob.Status != JobStatus.Completed)
            {
                throw new Exception("Exporting devices failed, exiting.");
            }

            // Step 2: Download the exported devices list from the blob create in Step 1.
            BlobClient       blobClient = _blobContainerClient.GetBlobClient(ImportExportDevicesFileName);
            BlobDownloadInfo download   = await blobClient.DownloadAsync();

            IEnumerable <ExportImportDevice> exportedDevices = ImportExportDevicesHelpers.BuildExportImportDeviceFromStream(download.Content);

            // Step 3: Collect the devices that need to be deleted and update their ImportMode to be Delete.
            // Thie step will create an ExportImportDevice identity for each device/ module identity registered on hub.
            // If you hub instance has IoT Hub module or Edge module instances registered, then they will be counted as separate entities
            // from the corresponding IoT Hub device/ Edge device that they are associated with.
            // As a result, the count of ExportImportDevice identities to be deleted might be greater than the
            // count of IoT hub devices retrieved in PrintDeviceCountAsync().
            var devicesToBeDeleted = new List <ExportImportDevice>();

            foreach (var device in exportedDevices)
            {
                string deviceId = device.Id;
                foreach (string prefix in _deleteDevicesWithPrefix)
                {
                    if (deviceId.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
                    {
                        devicesToBeDeleted.Add(device);
                    }
                }
            }
            devicesToBeDeleted
            .ForEach(device => device.ImportMode = ImportMode.Delete);
            Console.WriteLine($"Retrieved {devicesToBeDeleted.Count} devices for deletion.");

            if (devicesToBeDeleted.Count > 0)
            {
                // Step 3a: Write the new import data back to the blob.
                using Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(devicesToBeDeleted);
                await blobClient.UploadAsync(devicesFile, overwrite : true);

                // Step 3b: Call import using the same blob to delete all devices.
                JobProperties importDevicesToBeDeletedProperties = JobProperties
                                                                   .CreateForImportJob(
                    inputBlobContainerUri: storageAccountSasUri,
                    outputBlobContainerUri: storageAccountSasUri,
                    storageAuthenticationType: StorageAuthenticationType.KeyBased);

                JobProperties importDevicesToBeDeletedJob = null;

                tryCount = 0;
                while (true)
                {
                    try
                    {
                        importDevicesToBeDeletedJob = await _registryManager.ImportDevicesAsync(importDevicesToBeDeletedProperties);

                        break;
                    }
                    // Wait for pending jobs to finish.
                    catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                    {
                        Console.WriteLine($"JobQuotaExceededException... waiting.");
                        await Task.Delay(WaitDuration);
                    }
                }

                if (importDevicesToBeDeletedJob == null)
                {
                    throw new Exception("Import devices job failed.");
                }

                // Wait until job is finished.
                while (true)
                {
                    importDevicesToBeDeletedJob = await _registryManager.GetJobAsync(importDevicesToBeDeletedJob.JobId);

                    if (s_completedJobs.Contains(importDevicesToBeDeletedJob.Status))
                    {
                        // Job has finished executing.
                        break;
                    }

                    Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status} with progress {importDevicesToBeDeletedJob.Progress}%");
                    await Task.Delay(s_waitDuration);
                }
                Console.WriteLine($"Job {importDevicesToBeDeletedJob.JobId} is {importDevicesToBeDeletedJob.Status}.");
            }

            // Step 4: Delete the storage container created.
            await _blobContainerClient.DeleteAsync();

            Console.WriteLine($"Storage container {_blobContainerClient.Name} deleted.");
        }
Пример #28
0
        // This shows how to delete all of the devices for the IoT Hub.
        // First, export the list to devices.txt (ExportDevices).
        // Next, read in that file. Each row is a serialized object;
        //   read them into the generic list serializedDevices.
        // Delete the devices.txt in blob storage, because you're going to recreate it.
        // For each serializedDevice, deserialize it, set ImportMode to Delete,
        //   reserialize it, and write it to a StringBuilder. The ImportMode field is what
        //   tells the job framework to delete each one.
        // Write the new StringBuilder to the block blob.
        //   This essentially replaces the list with a list of devices that have ImportJob = Delete.
        // Call ImportDevicesAsync, which will read in the list in devices.txt, then delete each one.
        public static async Task DeleteAllDevicesFromHub(string hubConnectionString,
                                                         CloudBlobContainer cloudBlobContainer, string containerURI, string deviceListFile)
        {
            // Read the devices from the hub and write them to devices.txt in blob storage.
            await ExportDevices(containerURI, hubConnectionString);

            // Read devices.txt which contains serialized objects.
            // Write each line to the serializedDevices list. (List<string>).
            CloudBlockBlob blockBlob = cloudBlobContainer.GetBlockBlobReference(deviceListFile);

            // Get the URI for the blob.
            string blobURI = blockBlob.Uri.ToString();

            // Instantiate the generic list.
            var serializedDevices = new List <string>();

            // Read the blob file of devices, import each row into serializedDevices.
            using (var streamReader =
                       new StreamReader(await blockBlob.OpenReadAsync(AccessCondition.GenerateIfExistsCondition(),
                                                                      null, null), Encoding.UTF8))
            {
                while (streamReader.Peek() != -1)
                {
                    string line = await streamReader.ReadLineAsync();

                    serializedDevices.Add(line);
                }
            }

            // Delete the blob containing the list of devices,
            //   because you're going to recreate it.
            CloudBlockBlob blobToDelete = cloudBlobContainer.GetBlockBlobReference("devices.txt");

            // Step 1: Update each device's ImportMode to be Delete
            StringBuilder sb = new StringBuilder();

            serializedDevices.ForEach(serializedDevice =>
            {
                // Deserialize back to an ExportImportDevice.
                var device = JsonConvert.DeserializeObject <ExportImportDevice>(serializedDevice);

                // Update the property.
                device.ImportMode = ImportMode.Delete;

                // Re-serialize the object now that you're updated the property.
                sb.AppendLine(JsonConvert.SerializeObject(device));
            });

            // Step 2: Delete the blob if it already exists, then write the list in memory to the blob.
            await blobToDelete.DeleteIfExistsAsync();

            using (CloudBlobStream stream = await blobToDelete.OpenWriteAsync())
            {
                byte[] bytes = Encoding.UTF8.GetBytes(sb.ToString());
                for (var i = 0; i < bytes.Length; i += 500)
                {
                    int length = Math.Min(bytes.Length - i, 500);
                    await stream.WriteAsync(bytes, i, length);
                }
            }

            // Step 3: Call import using the same blob to delete all devices.
            // Loads devices.txt and applies that change.
            RegistryManager registryManager =
                RegistryManager.CreateFromConnectionString(hubConnectionString);
            JobProperties importJob =
                await registryManager.ImportDevicesAsync(containerURI, containerURI);

            // Wait until job is finished
            while (true)
            {
                importJob = await registryManager.GetJobAsync(importJob.JobId);

                if (importJob.Status == JobStatus.Completed ||
                    importJob.Status == JobStatus.Failed ||
                    importJob.Status == JobStatus.Cancelled)
                {
                    // Job has finished executing
                    break;
                }

                await Task.Delay(TimeSpan.FromSeconds(5));
            }
        }
        public void Test_OutputOperations_EventHub()
        {
            BasicDelegatingHandler handler = new BasicDelegatingHandler();

            using (var undoContext = UndoContext.Current)
            {
                undoContext.Start();

                string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
                string resourceName      = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");

                string serviceLocation = TestHelper.GetDefaultLocation();

                var resourceClient = TestHelper.GetResourceClient(handler);
                var client         = TestHelper.GetStreamAnalyticsManagementClient(handler);

                try
                {
                    ResourceGroup resourceGroup = new ResourceGroup()
                    {
                        Location = serviceLocation
                    };
                    resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);

                    Job job = new Job();
                    job.Name     = resourceName;
                    job.Location = serviceLocation;

                    // Construct the general properties for JobProperties
                    JobProperties jobProperties = new JobProperties();
                    jobProperties.Sku = new Sku()
                    {
                        Name = "standard"
                    };
                    jobProperties.EventsOutOfOrderPolicy            = EventsOutOfOrderPolicy.Drop;
                    jobProperties.EventsOutOfOrderMaxDelayInSeconds = 0;

                    job.Properties = jobProperties;

                    // Construct the JobCreateProperties
                    JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters();
                    jobCreateOrUpdateParameters.Job = job;

                    // Create a streaming job
                    JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);

                    // Get a streaming job to check
                    JobGetParameters jobGetParameters = new JobGetParameters(string.Empty);
                    JobGetResponse   jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
                    Assert.Equal(resourceName, jobGetResponse.Job.Name);

                    // Construct the Output
                    OutputProperties         outputProperties         = new OutputProperties();
                    string                   outputName               = TestUtilities.GenerateName("outputtest");
                    string                   partitionKey             = "partitionKey";
                    EventHubOutputDataSource eventHubOutputDataSource = new EventHubOutputDataSource()
                    {
                        Properties = new EventHubOutputDataSourceProperties()
                        {
                            ServiceBusNamespace    = "sdktest",
                            EventHubName           = "sdkeventhub",
                            SharedAccessPolicyName = TestHelper.SharedAccessPolicyName,
                            SharedAccessPolicyKey  = TestHelper.SharedAccessPolicyKey,
                            PartitionKey           = partitionKey
                        }
                    };
                    JsonSerialization jsonSerialization = new JsonSerialization()
                    {
                        Properties = new JsonSerializationProperties()
                        {
                            Encoding = "UTF8",
                            Format   = Format.LineSeparated
                        }
                    };
                    outputProperties.DataSource    = eventHubOutputDataSource;
                    outputProperties.Serialization = jsonSerialization;
                    Output output1 = new Output(outputName)
                    {
                        Properties = outputProperties
                    };

                    // Add an output
                    OutputCreateOrUpdateParameters outputCreateOrUpdateParameters = new OutputCreateOrUpdateParameters();
                    outputCreateOrUpdateParameters.Output = output1;
                    OutputCreateOrUpdateResponse outputCreateOrUpdateResponse = client.Outputs.CreateOrUpdate(resourceGroupName, resourceName, outputCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, outputCreateOrUpdateResponse.StatusCode);
                    Assert.Equal(outputName, outputCreateOrUpdateResponse.Output.Name);
                    Assert.True(outputCreateOrUpdateResponse.Output.Properties.Serialization is JsonSerialization);
                    JsonSerialization jsonSerializationInResponse1 = (JsonSerialization)outputCreateOrUpdateResponse.Output.Properties.Serialization;
                    Assert.Equal(Format.LineSeparated, jsonSerializationInResponse1.Properties.Format);
                    Assert.True(outputCreateOrUpdateResponse.Output.Properties.DataSource is EventHubOutputDataSource);
                    EventHubOutputDataSource eventHubOutputDataSourceInResponse1 = (EventHubOutputDataSource)outputCreateOrUpdateResponse.Output.Properties.DataSource;
                    Assert.Equal(partitionKey, eventHubOutputDataSourceInResponse1.Properties.PartitionKey);
                    Assert.NotNull(outputCreateOrUpdateResponse.Output.Properties.Etag);

                    // Get the output
                    OutputGetResponse outputGetResponse = client.Outputs.Get(resourceGroupName, resourceName, outputName);
                    Assert.Equal(HttpStatusCode.OK, outputGetResponse.StatusCode);
                    Assert.Equal(outputName, outputGetResponse.Output.Name);
                    Assert.True(outputGetResponse.Output.Properties.Serialization is JsonSerialization);
                    JsonSerialization jsonSerializationInResponse2 = (JsonSerialization)outputGetResponse.Output.Properties.Serialization;
                    Assert.Equal(Format.LineSeparated, jsonSerializationInResponse2.Properties.Format);
                    Assert.True(outputGetResponse.Output.Properties.DataSource is EventHubOutputDataSource);
                    EventHubOutputDataSource eventHubOutputDataSourceInResponse2 = (EventHubOutputDataSource)outputGetResponse.Output.Properties.DataSource;
                    Assert.Equal(partitionKey, eventHubOutputDataSourceInResponse2.Properties.PartitionKey);

                    // Test output connectivity
                    DataSourceTestConnectionResponse response = client.Outputs.TestConnection(resourceGroupName, resourceName, outputName);
                    Assert.Equal(OperationStatus.Succeeded, response.Status);
                    Assert.Equal(DataSourceTestStatus.TestSucceeded, response.DataSourceTestStatus);

                    // Update the output
                    jsonSerialization = new JsonSerialization()
                    {
                        Properties = new JsonSerializationProperties()
                        {
                            Encoding = "UTF8",
                            Format   = Format.Array
                        }
                    };
                    string newPartitionKey = TestUtilities.GenerateName("NewPartitionKey");
                    eventHubOutputDataSource.Properties.PartitionKey = newPartitionKey;
                    outputProperties.DataSource    = eventHubOutputDataSource;
                    outputProperties.Serialization = jsonSerialization;
                    outputProperties.Etag          = outputCreateOrUpdateResponse.Output.Properties.Etag;
                    OutputPatchParameters outputPatchParameters = new OutputPatchParameters(outputProperties);
                    OutputPatchResponse   outputPatchResponse   = client.Outputs.Patch(resourceGroupName, resourceName, outputName, outputPatchParameters);
                    Assert.Equal(HttpStatusCode.OK, outputPatchResponse.StatusCode);
                    Assert.True(outputPatchResponse.Properties.Serialization is JsonSerialization);
                    JsonSerialization jsonSerializationInResponse3 = (JsonSerialization)outputPatchResponse.Properties.Serialization;
                    Assert.Equal(Format.Array, jsonSerializationInResponse3.Properties.Format);
                    Assert.True(outputPatchResponse.Properties.DataSource is EventHubOutputDataSource);
                    EventHubOutputDataSource eventHubOutputDataSourceInResponse3 = (EventHubOutputDataSource)outputPatchResponse.Properties.DataSource;
                    Assert.Equal(newPartitionKey, eventHubOutputDataSourceInResponse3.Properties.PartitionKey);
                    Assert.NotNull(outputPatchResponse.Properties.Etag);
                    Assert.NotEqual(outputCreateOrUpdateResponse.Output.Properties.Etag, outputPatchResponse.Properties.Etag);

                    // Delete the output
                    AzureOperationResponse deleteInputOperationResponse = client.Outputs.Delete(resourceGroupName, resourceName, outputName);
                    Assert.Equal(HttpStatusCode.OK, deleteInputOperationResponse.StatusCode);

                    // Check that there are 0 outputs in the job
                    jobGetParameters = new JobGetParameters("outputs");
                    jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(0, jobGetResponse.Job.Properties.Outputs.Count);
                }
                finally
                {
                    client.StreamingJobs.Delete(resourceGroupName, resourceName);
                    resourceClient.ResourceGroups.Delete(resourceGroupName);
                }
            }
        }
        public void Test_JobOperations_E2E()
        {
            BasicDelegatingHandler handler = new BasicDelegatingHandler();

            using (var undoContext = UndoContext.Current)
            {
                undoContext.Start();

                string resourceGroupName = TestUtilities.GenerateName("StreamAnalytics");
                string resourceName      = TestUtilities.GenerateName("MyStreamingJobSubmittedBySDK");

                string serviceLocation = TestHelper.GetDefaultLocation();

                var resourceClient = TestHelper.GetResourceClient(handler);
                var client         = TestHelper.GetStreamAnalyticsManagementClient(handler);

                try
                {
                    ResourceGroup resourceGroup = new ResourceGroup()
                    {
                        Location = serviceLocation
                    };
                    resourceClient.ResourceGroups.CreateOrUpdate(resourceGroupName, resourceGroup);

                    Job job = new Job();
                    job.Name     = resourceName;
                    job.Location = serviceLocation;

                    // Construct the general properties for JobProperties
                    JobProperties jobProperties = new JobProperties();
                    jobProperties.Sku = new Sku()
                    {
                        Name = "standard"
                    };
                    jobProperties.EventsOutOfOrderPolicy            = EventsOutOfOrderPolicy.Drop;
                    jobProperties.EventsOutOfOrderMaxDelayInSeconds = 0;

                    // Construct the Input
                    StorageAccount storageAccount = new StorageAccount
                    {
                        AccountName = TestHelper.AccountName,
                        AccountKey  = TestHelper.AccountKey
                    };
                    InputProperties inputProperties = new StreamInputProperties()
                    {
                        Serialization = new CsvSerialization()
                        {
                            Properties = new CsvSerializationProperties()
                            {
                                FieldDelimiter = ",",
                                Encoding       = "UTF8"
                            }
                        },
                        DataSource = new BlobStreamInputDataSource()
                        {
                            Properties = new BlobStreamInputDataSourceProperties()
                            {
                                StorageAccounts = new[] { storageAccount },
                                Container       = "state",
                                PathPattern     = ""
                            }
                        }
                    };
                    Input input1 = new Input("inputtest")
                    {
                        Properties = inputProperties
                    };
                    jobProperties.Inputs = new[] { input1 };

                    // Construct the Output
                    OutputProperties         outputProperties         = new OutputProperties();
                    SqlAzureOutputDataSource sqlAzureOutputDataSource = new SqlAzureOutputDataSource()
                    {
                        Properties = new SqlAzureOutputDataSourceProperties()
                        {
                            Server   = TestHelper.Server,
                            Database = TestHelper.Database,
                            User     = TestHelper.User,
                            Password = TestHelper.Password,
                            Table    = "StateInfo"
                        }
                    };
                    outputProperties.DataSource = sqlAzureOutputDataSource;
                    Output output1 = new Output("outputtest")
                    {
                        Properties = outputProperties
                    };
                    jobProperties.Outputs = new Output[] { output1 };

                    // Construct the transformation
                    Transformation transformation = new Transformation()
                    {
                        Name       = "transformationtest",
                        Properties = new TransformationProperties()
                        {
                            Query          = "Select Id, Name from inputtest",
                            StreamingUnits = 1
                        }
                    };
                    jobProperties.Transformation = transformation;

                    job.Properties = jobProperties;

                    // Construct the JobCreateProperties
                    JobCreateOrUpdateParameters jobCreateOrUpdateParameters = new JobCreateOrUpdateParameters();
                    jobCreateOrUpdateParameters.Job = job;

                    // Create a streaming job
                    JobCreateOrUpdateResponse jobCreateOrUpdateResponse = client.StreamingJobs.CreateOrUpdate(resourceGroupName, jobCreateOrUpdateParameters);
                    Assert.Equal(HttpStatusCode.OK, jobCreateOrUpdateResponse.StatusCode);
                    Assert.NotNull(jobCreateOrUpdateResponse.Job.Properties.Etag);

                    // Get a streaming job to check
                    JobGetParameters jobGetParameters = new JobGetParameters("inputs,transformation,outputs");
                    JobGetResponse   jobGetResponse   = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(serviceLocation, jobGetResponse.Job.Location);
                    Assert.Equal(resourceName, jobGetResponse.Job.Name);
                    Assert.True(jobGetResponse.Job.Properties.Inputs[0].Properties is StreamInputProperties);
                    StreamInputProperties streamInputProperties = jobGetResponse.Job.Properties.Inputs[0].Properties as StreamInputProperties;
                    Assert.Equal("Stream", jobGetResponse.Job.Properties.Inputs[0].Properties.Type);
                    Assert.Equal("Microsoft.Storage/Blob", streamInputProperties.DataSource.Type);
                    Assert.Equal("Csv", streamInputProperties.Serialization.Type);
                    Assert.Equal(EventsOutOfOrderPolicy.Drop, jobGetResponse.Job.Properties.EventsOutOfOrderPolicy);
                    Assert.NotNull(jobGetResponse.Job.Properties.Etag);
                    Assert.Equal(jobCreateOrUpdateResponse.Job.Properties.Etag, jobGetResponse.Job.Properties.Etag);

                    // Patch the streaming job
                    JobPatchParameters jobPatchParameters = new JobPatchParameters()
                    {
                        JobPatchRequest = new JobPatchRequest()
                        {
                            Properties = new JobProperties()
                            {
                                EventsOutOfOrderPolicy = EventsOutOfOrderPolicy.Adjust
                            }
                        }
                    };
                    var jobPatchResponse = client.StreamingJobs.Patch(resourceGroupName, resourceName, jobPatchParameters);
                    jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobPatchResponse.StatusCode);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(EventsOutOfOrderPolicy.Adjust, jobPatchResponse.Job.Properties.EventsOutOfOrderPolicy);
                    Assert.Equal(EventsOutOfOrderPolicy.Adjust, jobGetResponse.Job.Properties.EventsOutOfOrderPolicy);

                    JobListParameters parameters = new JobListParameters(string.Empty);
                    JobListResponse   response   = client.StreamingJobs.ListJobsInResourceGroup(resourceGroupName, parameters);
                    Assert.Equal(HttpStatusCode.OK, response.StatusCode);

                    // Start a streaming job
                    JobStartParameters jobStartParameters = new JobStartParameters()
                    {
                        OutputStartMode = OutputStartMode.LastOutputEventTime
                    };
                    CloudException cloudException = Assert.Throws <CloudException>(() => client.StreamingJobs.Start(resourceGroupName, resourceName, jobStartParameters));
                    Assert.Equal("LastOutputEventTime must be available when OutputStartMode is set to LastOutputEventTime. Please make sure at least one output event has been processed. ", cloudException.Error.Message);

                    jobStartParameters.OutputStartMode = OutputStartMode.CustomTime;
                    jobStartParameters.OutputStartTime = DateTime.Now;
                    AzureOperationResponse jobStartOperationResponse = client.StreamingJobs.Start(resourceGroupName, resourceName, jobStartParameters);
                    Assert.Equal(HttpStatusCode.OK, jobStartOperationResponse.StatusCode);

                    // Get a streaming job to check
                    jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.True(IsRunning(jobGetResponse.Job.Properties.JobState));

                    // Check diagnostics
                    InputListResponse inputListResponse = client.Inputs.ListInputInJob(resourceGroupName, resourceName,
                                                                                       new InputListParameters("*"));
                    Assert.Equal(HttpStatusCode.OK, inputListResponse.StatusCode);
                    Assert.NotEqual(0, inputListResponse.Value.Count);
                    Assert.NotNull(inputListResponse.Value[0].Properties.Diagnostics);
                    Assert.NotEqual(0, inputListResponse.Value[0].Properties.Diagnostics.Conditions.Count);
                    Assert.NotNull(inputListResponse.Value[0].Properties.Diagnostics.Conditions[0].Code);
                    Assert.NotNull(inputListResponse.Value[0].Properties.Diagnostics.Conditions[0].Message);

                    // Stop a streaming job
                    AzureOperationResponse jobStopOperationResponse = client.StreamingJobs.Stop(resourceGroupName, resourceName);
                    Assert.Equal(HttpStatusCode.OK, jobStopOperationResponse.StatusCode);

                    // Get a streaming job to check
                    jobGetResponse = client.StreamingJobs.Get(resourceGroupName, resourceName, jobGetParameters);
                    Assert.Equal(HttpStatusCode.OK, jobGetResponse.StatusCode);
                    Assert.Equal(JobRunningState.Stopped, jobGetResponse.Job.Properties.JobState);

                    // Delete a streaming job
                    AzureOperationResponse jobDeleteOperationResponse = client.StreamingJobs.Delete(resourceGroupName, resourceName);
                    Assert.Equal(HttpStatusCode.OK, jobDeleteOperationResponse.StatusCode);
                }
                finally
                {
                    client.StreamingJobs.Delete(resourceGroupName, resourceName);
                    resourceClient.ResourceGroups.Delete(resourceGroupName);
                }
            }
        }
Пример #31
0
        public async Task RegistryManager_ImportDevices(StorageAuthenticationType storageAuthenticationType)
        {
            // arrange

            StorageContainer storageContainer = null;
            string           deviceId         = $"{nameof(RegistryManager_ImportDevices)}-{StorageContainer.GetRandomSuffix(4)}";
            var registryManager = RegistryManager.CreateFromConnectionString(Configuration.IoTHub.ConnectionString);

            Logger.Trace($"Using deviceId {deviceId}");

            try
            {
                string containerName = StorageContainer.BuildContainerName(nameof(RegistryManager_ImportDevices));
                storageContainer = await StorageContainer
                                   .GetInstanceAsync(containerName)
                                   .ConfigureAwait(false);

                Logger.Trace($"Using container {storageContainer.Uri}");

                Uri containerUri = storageAuthenticationType == StorageAuthenticationType.KeyBased
                    ? storageContainer.SasUri
                    : storageContainer.Uri;

                Stream devicesFile = ImportExportDevicesHelpers.BuildDevicesStream(
                    new List <ExportImportDevice>
                {
                    new ExportImportDevice(
                        new Device(deviceId)
                    {
                        Authentication = new AuthenticationMechanism {
                            Type = AuthenticationType.Sas
                        }
                    },
                        ImportMode.Create),
                });
                await UploadFileAndConfirmAsync(storageContainer, devicesFile).ConfigureAwait(false);

                // act

                JobProperties importJobResponse = null;
                int           tryCount          = 0;
                while (true)
                {
                    try
                    {
                        importJobResponse = await registryManager
                                            .ImportDevicesAsync(
                            JobProperties.CreateForImportJob(
                                containerUri.ToString(),
                                containerUri.ToString(),
                                null,
                                storageAuthenticationType))
                                            .ConfigureAwait(false);

                        break;
                    }
                    // Concurrent jobs can be rejected, so implement a retry mechanism to handle conflicts with other tests
                    catch (JobQuotaExceededException) when(++tryCount < MaxIterationWait)
                    {
                        Logger.Trace($"JobQuotaExceededException... waiting.");
                        await Task.Delay(s_waitDuration).ConfigureAwait(false);

                        continue;
                    }
                }

                // wait for job to complete
                for (int i = 0; i < MaxIterationWait; ++i)
                {
                    await Task.Delay(1000).ConfigureAwait(false);

                    importJobResponse = await registryManager.GetJobAsync(importJobResponse.JobId).ConfigureAwait(false);

                    Logger.Trace($"Job {importJobResponse.JobId} is {importJobResponse.Status} with progress {importJobResponse.Progress}%");
                    if (!s_incompleteJobs.Contains(importJobResponse.Status))
                    {
                        break;
                    }
                }

                // assert

                importJobResponse.Status.Should().Be(JobStatus.Completed, "Otherwise import failed");
                importJobResponse.FailureReason.Should().BeNullOrEmpty("Otherwise import failed");

                // should not throw due to 404, but device may not immediately appear in registry
                Device device = null;
                for (int i = 0; i < MaxIterationWait; ++i)
                {
                    await Task.Delay(s_waitDuration).ConfigureAwait(false);

                    try
                    {
                        device = await registryManager.GetDeviceAsync(deviceId).ConfigureAwait(false);

                        break;
                    }
                    catch (Exception ex)
                    {
                        Logger.Trace($"Could not find device on iteration {i} due to [{ex.Message}]");
                    }
                }
                if (device == null)
                {
                    Assert.Fail($"Device {deviceId} not found in registry manager");
                }
            }
            finally
            {
                try
                {
                    storageContainer?.Dispose();

                    await registryManager.RemoveDeviceAsync(deviceId).ConfigureAwait(false);
                }
                catch { }
            }
        }
        /// <summary>
        /// Create a job.
        /// </summary>
        /// <param name="createJobParams">Job properties entered via powershell.</param>
        /// <returns>The Job definition.</returns>
        public PSSchedulerJobDefinition CreateJob(PSJobParams createJobParams)
        {
            if (string.IsNullOrWhiteSpace(createJobParams.ResourceGroupName))
            {
                throw new PSManagement.PSArgumentNullException(paramName: "ResourceGroupName");
            }

            if (string.IsNullOrWhiteSpace(createJobParams.JobCollectionName))
            {
                throw new PSManagement.PSArgumentNullException(paramName: "JobCollectionName");
            }

            if (string.IsNullOrWhiteSpace(createJobParams.JobName))
            {
                throw new PSManagement.PSArgumentNullException(paramName: "JobName");
            }

            if (!DoesResourceGroupExists(createJobParams.ResourceGroupName))
            {
                throw new PSManagement.PSArgumentException(Resources.SchedulerInvalidResourceGroup);
            }

            IList<JobCollectionDefinition> jobCollection = ListJobCollection(createJobParams.ResourceGroupName, createJobParams.JobCollectionName);

            if (jobCollection == null || jobCollection.Count < 1)
            {
                throw new PSManagement.PSInvalidOperationException(string.Format(Resources.JobCollectionDoesnotExist, createJobParams.JobCollectionName, createJobParams.ResourceGroupName));
            }
            else
            {
                if (JobExists(createJobParams.ResourceGroupName, createJobParams.JobCollectionName, createJobParams.JobName))
                {
                    throw new PSManagement.PSArgumentException(string.Format(Resources.SchedulerExistingJob, createJobParams.JobName, createJobParams.JobCollectionName));
                }

                IList<JobDefinition> listOfJobs = ListJobs(createJobParams.ResourceGroupName, createJobParams.JobCollectionName, jobState: null);

                if (listOfJobs != null)
                {
                    Validate(jobCollection[0], listOfJobs.Count);
                }
            }

            JobAction jobAction = this.GetJobAction(createJobParams);

            JobRecurrence jobRecurrence = this.GetJobRecurrence(createJobParams.JobRecurrence);

            var properties = new JobProperties()
            {
                Action = jobAction,
                Recurrence = jobRecurrence,
                StartTime = createJobParams.StartTime,
                State = createJobParams.JobState.GetValueOrDefaultEnum<JobState?>(defaultValue: null)
            };

            var jobDefinition = new JobDefinition(name: createJobParams.JobName)
            {
                Properties = properties
            };

            JobDefinition jobDefinitionResult =  this.SchedulerManagementClient.Jobs.CreateOrUpdate(createJobParams.ResourceGroupName, createJobParams.JobCollectionName, createJobParams.JobName, jobDefinition);

            return Converter.ConvertJobDefinitionToPS(jobDefinitionResult);
        }