Example #1
0
        public async Task <IMessageHandle> SendDeploymentUpdateNotification(
            string channel,
            DeploymentUpdate deploymentUpdate,
            DeploymentUpdateStatus status
            )
        {
            if (channel == null)
            {
                throw new ArgumentNullException(nameof(channel));
            }
            if (deploymentUpdate == null)
            {
                throw new ArgumentNullException(nameof(deploymentUpdate));
            }

//            var channelsResponse = await _actualClient.APIRequestWithTokenAsync<ConversationsListResponse>(
//                new Tuple<string, string>("exclude_archived", "true"),
//                new Tuple<string, string>("types", "public_channel,private_channel")
//            );
//
//            var channelMetadata = channelsResponse.channels.FirstOrDefault(c =>
//                c.name.Equals(channel, StringComparison.OrdinalIgnoreCase)
//            );
//
//            if (channelMetadata == null)
//            {
//                throw new Exception($"Could not find channel with name {channel}");
//            }

            return(await PostMessageAsync(
                       channel,
                       BuildDeploymentUpdateMessage(deploymentUpdate, status)
                       ));
        }
Example #2
0
        public async Task AddDeploymentUpdate(Application application, Image image, string newTag)
        {
            var currentTags = _applicationService.GetCurrentImageTags(application);

            if (!currentTags.TryGetValue(image, out var currentTag))
            {
                currentTag = "<new image>";
            }

            var deploymentUpdate = new DeploymentUpdate(application, image, currentTag, newTag);

            if (!_deploymentUpdates.TryAdd(deploymentUpdate, DeploymentUpdateStatus.Pending))
            {
                _log.LogInformation(
                    "Image tag update operation already in queue for '{Repository}' with {Tag} for application {Application} with new tag {NewTag}",
                    image.Repository,
                    currentTags[image],
                    application.Name,
                    newTag
                    );

                return;
            }

            _log.LogInformation(
                "Adding image tag update operation for '{Repository}' with {Tag} for application {Application} with new tag {NewTag}",
                image.Repository,
                currentTags[image],
                application.Name,
                newTag
                );
            var queue = _pendingDeploymentUpdates.GetOrAdd(application, key => new ConcurrentQueue <DeploymentUpdate>());

            queue.Enqueue(deploymentUpdate);

            var channel = application.Notifications.Channels.FirstOrDefault();

            if (channel != null)
            {
                _log.LogInformation(
                    "Sending notification about image tag update operation for '{Repository}' with {Tag} for application {Application} with new tag {NewTag}",
                    image.Repository,
                    currentTags[image],
                    application.Name,
                    newTag
                    );
                try
                {
                    var handle = await _slackClient.SendDeploymentUpdateNotification(channel, deploymentUpdate, DeploymentUpdateStatus.Pending);

                    _notificationHandles.TryAdd(deploymentUpdate, handle);
                }
                catch (Exception e)
                {
                    _log.LogError(e, "Failed to send deployment update notification to slack");
                }
            }
        }
Example #3
0
        public async Task FinishDeploymentUpdate(
            DeploymentUpdate deploymentUpdate,
            DeploymentUpdateStatus finalStatus
            )
        {
            await ChangeDeploymentUpdateStatus(deploymentUpdate, finalStatus);

            _applicationService.SetCurrentImageTag(deploymentUpdate.Application, deploymentUpdate.Image, deploymentUpdate.TargetTag);
        }
Example #4
0
        public void VmssPublicIpAddressApiTest()
        {
            var handler1 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };
            var handler2 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };

            using (MockContext context = MockContext.Start(this.GetType()))
            {
                var resourcesClient         = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
                var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);

                var    location          = NetworkManagementTestUtilities.GetResourceLocation(resourcesClient, "Microsoft.Compute/virtualMachineScaleSets");
                string resourceGroupName = TestUtilities.GenerateName();
                string deploymentName    = TestUtilities.GenerateName("vmss");
                resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
                                                              new ResourceGroup
                {
                    Location = location
                });

                DeploymentUpdate.CreateVmss(resourcesClient, resourceGroupName, deploymentName);

                string virtualMachineScaleSetName = "vmssip";
                var    vmssListAllPageResult      = networkManagementClient.PublicIPAddresses.ListVirtualMachineScaleSetPublicIPAddresses(resourceGroupName, virtualMachineScaleSetName);
                var    vmssListAllResult          = vmssListAllPageResult.ToList();
                var    firstResult = vmssListAllResult.First();

                Assert.NotNull(vmssListAllResult);
                Assert.Equal("Succeeded", firstResult.ProvisioningState);
                Assert.NotNull(firstResult.ResourceGuid);

                var idItem       = firstResult.Id;
                var vmIndex      = GetNameById(idItem, "virtualMachines");
                var nicName      = GetNameById(idItem, "networkInterfaces");
                var ipConfigName = GetNameById(idItem, "ipConfigurations");
                var ipName       = GetNameById(idItem, "publicIPAddresses");

                var vmssListPageResult = networkManagementClient.PublicIPAddresses.ListVirtualMachineScaleSetVMPublicIPAddresses(
                    resourceGroupName, virtualMachineScaleSetName, vmIndex, nicName, ipConfigName);
                var vmssListResult = vmssListPageResult.ToList();

                Assert.Single(vmssListResult);

                var vmssGetResult = networkManagementClient.PublicIPAddresses.GetVirtualMachineScaleSetPublicIPAddress(
                    resourceGroupName, virtualMachineScaleSetName, vmIndex, nicName, ipConfigName, ipName);

                Assert.NotNull(vmssGetResult);
                Assert.Equal("Succeeded", vmssGetResult.ProvisioningState);
                Assert.NotNull(vmssGetResult.ResourceGuid);

                resourcesClient.ResourceGroups.Delete(resourceGroupName);
            }
        }
        public async Task StartDeploymentUpdateJob(DeploymentUpdate deploymentUpdate)
        {
            var jobkey = new JobKey($"gitwatch-{deploymentUpdate.Application.Name}", "gitrepowatcher");

            var data = new JobDataMap
            {
                ["DeploymentUpdate"] = deploymentUpdate
            };

            await _scheduler.TriggerJob(jobkey, data, CancellationToken.None);
        }
Example #6
0
 private SlackMessage BuildDeploymentUpdateMessage(DeploymentUpdate deploymentUpdate, DeploymentUpdateStatus status)
 {
     return(new SlackMessage(
                $"A new image of *{deploymentUpdate.Image.Repository}* was detected (tag *{deploymentUpdate.TargetTag}*).",
                new IBlock[]
     {
         new SectionBlock()
         {
             text = new Text()
             {
                 type = "mrkdwn",
                 text =
                     $"A new image of *{deploymentUpdate.Image.Repository}* was detected (tag *{deploymentUpdate.TargetTag}*)."
             }
         },
         new DividerBlock(),
         new SectionBlock()
         {
             fields = new Text[]
             {
                 new Text()
                 {
                     text = $"*From*\n{deploymentUpdate.CurrentTag}",
                     type = "mrkdwn"
                 },
                 new Text()
                 {
                     text = $"*To*\n{deploymentUpdate.TargetTag}",
                     type = "mrkdwn"
                 },
             }
         },
         new SectionBlock()
         {
             fields = new Text[]
             {
                 new Text()
                 {
                     text = $"*Application*\n{deploymentUpdate.Application}",
                     type = "mrkdwn"
                 },
                 new Text()
                 {
                     text = $"*Status*\n{status}",
                     type = "mrkdwn"
                 },
             }
         }
     }
                ));
 }
Example #7
0
        public async Task ChangeDeploymentUpdateStatus(DeploymentUpdate deploymentUpdate, DeploymentUpdateStatus status)
        {
            _deploymentUpdates[deploymentUpdate] = status;

            if (_notificationHandles.TryGetValue(deploymentUpdate, out var handle))
            {
                try
                {
                    _log.LogInformation("Submitting {@DeploymentUpdate} notification change to slack {@MessageHandle}. ", deploymentUpdate, handle);
                    var newHandle = await _slackClient.UpdateDeploymentUpdateNotification(handle, deploymentUpdate, status);

                    _notificationHandles.TryUpdate(deploymentUpdate, newHandle, handle);
                }
                catch (Exception e)
                {
                    _log.LogError(e, "Failed to submit {@DeploymentUpdate} notification {@MessageHandle}", deploymentUpdate, handle);
                }
            }
        }
Example #8
0
        public async Task <IMessageHandle> UpdateDeploymentUpdateNotification(
            IMessageHandle handle,
            DeploymentUpdate deploymentUpdate,
            DeploymentUpdateStatus status
            )
        {
            if (handle == null)
            {
                throw new ArgumentNullException(nameof(handle));
            }
            if (deploymentUpdate == null)
            {
                throw new ArgumentNullException(nameof(deploymentUpdate));
            }

            return(await UpdateMessageAsync(
                       handle as SingleMessageHandle,
                       BuildDeploymentUpdateMessage(deploymentUpdate, status)
                       ));
        }
Example #9
0
        public void DeleteConnectionMonitorTest()
        {
            var handler1 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };
            var handler2 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };
            var handler3 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };

            using (MockContext context = MockContext.Start(this.GetType().FullName))
            {
                var resourcesClient         = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
                var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);
                var computeManagementClient = NetworkManagementTestUtilities.GetComputeManagementClientWithHandler(context, handler3);

                string location = "centraluseuap";

                string resourceGroupName = TestUtilities.GenerateName();
                resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
                                                              new ResourceGroup
                {
                    Location = location
                });

                string virtualMachineName       = TestUtilities.GenerateName();
                string networkInterfaceName     = TestUtilities.GenerateName();
                string networkSecurityGroupName = virtualMachineName + "-nsg";

                //Deploy VM with a template
                DeploymentUpdate.CreateVm(
                    resourcesClient: resourcesClient,
                    resourceGroupName: resourceGroupName,
                    location: location,
                    virtualMachineName: virtualMachineName,
                    storageAccountName: TestUtilities.GenerateName(),
                    networkInterfaceName: networkInterfaceName,
                    networkSecurityGroupName: networkSecurityGroupName,
                    diagnosticsStorageAccountName: TestUtilities.GenerateName(),
                    deploymentName: TestUtilities.GenerateName()
                    );

                var getVm = computeManagementClient.VirtualMachines.Get(resourceGroupName, virtualMachineName);

                //Deploy networkWatcherAgent on VM
                VirtualMachineExtension parameters = new VirtualMachineExtension
                {
                    Publisher                   = "Microsoft.Azure.NetworkWatcher",
                    TypeHandlerVersion          = "1.4",
                    VirtualMachineExtensionType = "NetworkWatcherAgentWindows",
                    Location = location
                };

                var addExtension = computeManagementClient.VirtualMachineExtensions.CreateOrUpdate(resourceGroupName, getVm.Name, "NetworkWatcherAgent", parameters);

                string         networkWatcherName = TestUtilities.GenerateName();
                NetworkWatcher properties         = new NetworkWatcher
                {
                    Location = location
                };

                //Create network Watcher
                var createNetworkWatcher = networkManagementClient.NetworkWatchers.CreateOrUpdate(resourceGroupName, networkWatcherName, properties);

                string            connectionMonitorName1 = TestUtilities.GenerateName();
                string            connectionMonitorName2 = TestUtilities.GenerateName();
                ConnectionMonitor cm = new ConnectionMonitor
                {
                    Location = location,
                    Source   = new ConnectionMonitorSource
                    {
                        ResourceId = getVm.Id
                    },
                    Destination = new ConnectionMonitorDestination
                    {
                        Address = "bing.com",
                        Port    = 80
                    },
                    MonitoringIntervalInSeconds = 30,
                    AutoStart = false
                };

                var connectionMonitor1 = networkManagementClient.ConnectionMonitors.CreateOrUpdate(resourceGroupName, networkWatcherName, connectionMonitorName1, cm);
                var connectionMonitor2 = networkManagementClient.ConnectionMonitors.CreateOrUpdate(resourceGroupName, networkWatcherName, connectionMonitorName2, cm);

                var getConnectionMonitors1 = networkManagementClient.ConnectionMonitors.List(resourceGroupName, networkWatcherName);
                Assert.Equal(2, getConnectionMonitors1.Count());

                networkManagementClient.ConnectionMonitors.Delete(resourceGroupName, networkWatcherName, connectionMonitorName2);
                var getConnectionMonitors2 = networkManagementClient.ConnectionMonitors.List(resourceGroupName, networkWatcherName);
                Assert.Single(getConnectionMonitors2);
            }
        }
        public void CheckConnectivityVmToInternetTest()
        {
            var handler1 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };
            var handler2 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };
            var handler3 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };

            using (MockContext context = MockContext.Start(this.GetType()))
            {
                var resourcesClient         = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
                var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);
                var computeManagementClient = NetworkManagementTestUtilities.GetComputeManagementClientWithHandler(context, handler3);

                string location = "westcentralus";

                string resourceGroupName = TestUtilities.GenerateName();
                resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
                                                              new ResourceGroup
                {
                    Location = location
                });

                string virtualMachineName       = TestUtilities.GenerateName();
                string networkInterfaceName     = TestUtilities.GenerateName();
                string networkSecurityGroupName = virtualMachineName + "-nsg";

                //Deploy VM with a template
                DeploymentUpdate.CreateVm(
                    resourcesClient: resourcesClient,
                    resourceGroupName: resourceGroupName,
                    location: location,
                    virtualMachineName: virtualMachineName,
                    storageAccountName: TestUtilities.GenerateName(),
                    networkInterfaceName: networkInterfaceName,
                    networkSecurityGroupName: networkSecurityGroupName,
                    diagnosticsStorageAccountName: TestUtilities.GenerateName(),
                    deploymentName: TestUtilities.GenerateName()
                    );

                var getVm = computeManagementClient.VirtualMachines.Get(resourceGroupName, virtualMachineName);

                //Deploy networkWatcherAgent on VM
                VirtualMachineExtension parameters = new VirtualMachineExtension();
                parameters.Publisher                   = "Microsoft.Azure.NetworkWatcher.Edp";
                parameters.TypeHandlerVersion          = "1.4";
                parameters.VirtualMachineExtensionType = "NetworkWatcherAgentWindows";
                parameters.Location = location;

                var addExtension = computeManagementClient.VirtualMachineExtensions.CreateOrUpdate(resourceGroupName, getVm.Name, "NetworkWatcherAgent", parameters);

                string         networkWatcherName = TestUtilities.GenerateName();
                NetworkWatcher properties         = new NetworkWatcher();
                properties.Location = location;

                //Create network Watcher
                var createNetworkWatcher = networkManagementClient.NetworkWatchers.CreateOrUpdate(resourceGroupName, networkWatcherName, properties);

                ConnectivityParameters connectivityParameters = new ConnectivityParameters();
                connectivityParameters.Source              = new ConnectivitySource();
                connectivityParameters.Source.ResourceId   = getVm.Id;
                connectivityParameters.Destination         = new ConnectivityDestination();
                connectivityParameters.Destination.Address = "bing.com";
                connectivityParameters.Destination.Port    = 80;

                var connectivityCheck = networkManagementClient.NetworkWatchers.CheckConnectivity(resourceGroupName, networkWatcherName, connectivityParameters);

                //Validation
                Assert.Equal("Reachable", connectivityCheck.ConnectionStatus);
                Assert.Equal(0, connectivityCheck.ProbesFailed);
                Assert.Equal("Source", connectivityCheck.Hops.FirstOrDefault().Type);
                Assert.Equal("Internet", connectivityCheck.Hops.LastOrDefault().Type);
            }
        }
Example #11
0
        private async Task <bool> UpdateDeploymentManifests(
            Repository gitRepository,
            DeploymentUpdate deploymentUpdate,
            Dictionary <Image, string> imageToFilenameMap,
            string currentImageTag,
            string applicationSourcePath,
            YamlUtilities yamlUtilities,
            string relativePath)
        {
            if (!imageToFilenameMap.TryGetValue(deploymentUpdate.Image, out string file))
            {
                // TODO: warn that we have an image tag update but no corresponding file
                _log.LogWarning("Update to {Repository} cannot be applied since there isn't matching file.");
                return(false);
            }

            _log.LogInformation("Upgrading {Repository} to {NewTag} from {Tag}",
                                deploymentUpdate.Image.Repository,
                                deploymentUpdate.TargetTag,
                                currentImageTag
                                );

            var yaml     = new YamlStream();
            var filePath = Path.Combine(applicationSourcePath, file);

            yamlUtilities.ReadYamlStream(yaml, filePath);

            var image = deploymentUpdate.Image;

            var doCommit = false;

            foreach (var doc in yaml.Documents)
            {
                var tagInManifest = yamlUtilities.ExtractValueFromDoc(image.TagProperty.Path, doc);
                if (tagInManifest == null)
                {
                    continue;
                }

                if (tagInManifest == deploymentUpdate.TargetTag)
                {
                    _log.LogInformation("Tag for {Repository} matches new tag {NewTag}", image.Repository, deploymentUpdate.TargetTag);
                    continue;
                }

                _log.LogInformation("Setting current-tag for {Repository} to {Tag}", image.Repository,
                                    deploymentUpdate.TargetTag);
                yamlUtilities.SetValueInDoc(image.TagProperty.Path, doc, deploymentUpdate.TargetTag);

                doCommit = true;
            }

            if (!doCommit)
            {
                return(true);
            }

            yamlUtilities.WriteYamlStream(yaml, filePath);

            _log.LogInformation("Adding {Path} to repository staging", filePath);
            var gitFilePath = Path.Combine(relativePath, file);

            Commands.Stage(gitRepository, gitFilePath);

            gitRepository.Commit(
                $"[{deploymentUpdate.Application}] Updated deployment for {image.RepositoryName}; {deploymentUpdate.CurrentTag} to {deploymentUpdate.TargetTag}",
                new Signature("deploy-bot", "*****@*****.**", DateTimeOffset.Now),
                new Signature("deploy-bot", "*****@*****.**", DateTimeOffset.Now)
                );

            // add delay instruction due to the deployment update notification delivered too fast
            await Task.Delay(500);

            return(true);
        }
Example #12
0
        private async Task <bool> SynchronizeHelmApplicationSource(Repository gitRepository,
                                                                   ApplicationSourceTrackingContext context,
                                                                   HelmApplicationSource helmApplicationSource)
        {
            var relativePath          = helmApplicationSource.Path;
            var applicationSourcePath = Path.Combine(context.GitRepositoryPath, relativePath);

            var application   = context.Application;
            var yamlUtilities = new YamlUtilities();

            // build map of images -> yaml file that defines them and image -> current tag
            _log.LogInformation("Beginning to parse value files defined in application source ...");

            var imageToFilenameMap   = new Dictionary <Image, string>();
            var imageToTagInManifest = new Dictionary <Image, string>();

            foreach (var file in helmApplicationSource.ValuesFiles)
            {
                var yaml     = new YamlStream();
                var filePath = Path.Combine(applicationSourcePath, file);
                yamlUtilities.ReadYamlStream(yaml, filePath);

                foreach (var doc in yaml.Documents)
                {
                    foreach (var image in context.Application.Images)
                    {
                        var tagInManifest = yamlUtilities.ExtractValueFromDoc(image.TagProperty.Path, doc);

                        if (tagInManifest == null)
                        {
                            continue;
                        }

                        if (imageToFilenameMap.ContainsKey(image))
                        {
                            // TODO: handle situation where multiple files define the same image tag (ERROR and warn user)
                        }

                        imageToFilenameMap[image]   = file;
                        imageToTagInManifest[image] = tagInManifest;
                    }
                }
            }

            _log.LogInformation("Completing parsing value files defined in application source ...");

            // start updating files
            var manifestsChanged = false;
            DeploymentUpdate deploymentUpdate = null;

            while ((deploymentUpdate = await _deploymentService.GetNextPendingDeploymentUpdate(application)) != null)
            {
                _log.LogInformation("Executing pending deployment update ...");

                await _deploymentService.ChangeDeploymentUpdateStatus(
                    deploymentUpdate,
                    DeploymentUpdateStatus.UpdatingManifests
                    );

                manifestsChanged = await UpdateDeploymentManifests(gitRepository,
                                                                   deploymentUpdate,
                                                                   imageToFilenameMap,
                                                                   imageToTagInManifest.TryGetAndReturn(deploymentUpdate.Image) ?? "n/a",
                                                                   applicationSourcePath,
                                                                   yamlUtilities,
                                                                   relativePath);

                if (manifestsChanged)
                {
                    imageToTagInManifest[deploymentUpdate.Image] = deploymentUpdate.TargetTag;
                }

                await _deploymentService.FinishDeploymentUpdate(
                    deploymentUpdate,
                    manifestsChanged?DeploymentUpdateStatus.Complete : DeploymentUpdateStatus.Failed
                    );
            }

            if (!manifestsChanged)
            {
                // we don't have a deployment, ensure application manifest is up to date with latest image tags
                foreach (var keyValuePair in imageToTagInManifest)
                {
                    _applicationService.SetCurrentImageTag(application, keyValuePair.Key, keyValuePair.Value);
                }
            }

            return(manifestsChanged);
        }
        public void VmssNetworkInterfaceApiTest()
        {
            var handler1 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };
            var handler2 = new RecordedDelegatingHandler {
                StatusCodeToReturn = HttpStatusCode.OK
            };

            using (MockContext context = MockContext.Start(this.GetType()))
            {
                var resourcesClient         = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
                var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);

                var    location          = NetworkManagementTestUtilities.GetResourceLocation(resourcesClient, "Microsoft.Compute/virtualMachineScaleSets");
                string resourceGroupName = TestUtilities.GenerateName();
                string deploymentName    = TestUtilities.GenerateName("vmss");
                resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
                                                              new ResourceGroup
                {
                    Location = location
                });

                DeploymentUpdate.CreateVmss(resourcesClient, resourceGroupName, deploymentName);

                string virtualMachineScaleSetName = "vmssip";
                var    vmssListAllPageResult      = networkManagementClient.PublicIPAddresses.ListVirtualMachineScaleSetPublicIPAddresses(resourceGroupName, virtualMachineScaleSetName);
                var    vmssListAllResult          = vmssListAllPageResult.ToList();
                var    firstResult = vmssListAllResult.First();

                Assert.NotNull(vmssListAllResult);
                Assert.Equal("Succeeded", firstResult.ProvisioningState);
                Assert.NotNull(firstResult.ResourceGuid);

                var idItem       = firstResult.Id;
                var vmIndex      = GetNameById(idItem, "virtualMachines");
                var nicName      = GetNameById(idItem, "networkInterfaces");
                var ipConfigName = GetNameById(idItem, "ipConfigurations");
                var ipName       = GetNameById(idItem, "publicIPAddresses");

                // Verify that NICs contain refernce to publicip, nsg and dns settings
                var listNicPerVmss = networkManagementClient.NetworkInterfaces.ListVirtualMachineScaleSetNetworkInterfaces(resourceGroupName, virtualMachineScaleSetName).ToList();
                Assert.NotNull(listNicPerVmss);

                foreach (var nic in listNicPerVmss)
                {
                    this.VerifyVmssNicProperties(nic);
                }

                // Verify nics on a vm level
                var listNicPerVm = networkManagementClient.NetworkInterfaces.ListVirtualMachineScaleSetVMNetworkInterfaces(resourceGroupName, virtualMachineScaleSetName, vmIndex).ToList();
                Assert.NotNull(listNicPerVm);
                Assert.Single(listNicPerVm);

                foreach (var nic in listNicPerVm)
                {
                    this.VerifyVmssNicProperties(nic);
                }

                // Verify getting individual nic
                var getNic = networkManagementClient.NetworkInterfaces.GetVirtualMachineScaleSetNetworkInterface(resourceGroupName, virtualMachineScaleSetName, vmIndex, nicName);
                Assert.NotNull(getNic);
                this.VerifyVmssNicProperties(getNic);

                resourcesClient.ResourceGroups.Delete(resourceGroupName);
            }
        }