public async Task <IMessageHandle> SendDeploymentUpdateNotification(
string channel,
DeploymentUpdate deploymentUpdate,
DeploymentUpdateStatus status
)
{
if (channel == null)
{
throw new ArgumentNullException(nameof(channel));
}
if (deploymentUpdate == null)
{
throw new ArgumentNullException(nameof(deploymentUpdate));
}
// var channelsResponse = await _actualClient.APIRequestWithTokenAsync<ConversationsListResponse>(
// new Tuple<string, string>("exclude_archived", "true"),
// new Tuple<string, string>("types", "public_channel,private_channel")
// );
//
// var channelMetadata = channelsResponse.channels.FirstOrDefault(c =>
// c.name.Equals(channel, StringComparison.OrdinalIgnoreCase)
// );
//
// if (channelMetadata == null)
// {
// throw new Exception($"Could not find channel with name {channel}");
// }
return(await PostMessageAsync(
channel,
BuildDeploymentUpdateMessage(deploymentUpdate, status)
));
}
public async Task AddDeploymentUpdate(Application application, Image image, string newTag)
{
var currentTags = _applicationService.GetCurrentImageTags(application);
if (!currentTags.TryGetValue(image, out var currentTag))
{
currentTag = "<new image>";
}
var deploymentUpdate = new DeploymentUpdate(application, image, currentTag, newTag);
if (!_deploymentUpdates.TryAdd(deploymentUpdate, DeploymentUpdateStatus.Pending))
{
_log.LogInformation(
"Image tag update operation already in queue for '{Repository}' with {Tag} for application {Application} with new tag {NewTag}",
image.Repository,
currentTags[image],
application.Name,
newTag
);
return;
}
_log.LogInformation(
"Adding image tag update operation for '{Repository}' with {Tag} for application {Application} with new tag {NewTag}",
image.Repository,
currentTags[image],
application.Name,
newTag
);
var queue = _pendingDeploymentUpdates.GetOrAdd(application, key => new ConcurrentQueue <DeploymentUpdate>());
queue.Enqueue(deploymentUpdate);
var channel = application.Notifications.Channels.FirstOrDefault();
if (channel != null)
{
_log.LogInformation(
"Sending notification about image tag update operation for '{Repository}' with {Tag} for application {Application} with new tag {NewTag}",
image.Repository,
currentTags[image],
application.Name,
newTag
);
try
{
var handle = await _slackClient.SendDeploymentUpdateNotification(channel, deploymentUpdate, DeploymentUpdateStatus.Pending);
_notificationHandles.TryAdd(deploymentUpdate, handle);
}
catch (Exception e)
{
_log.LogError(e, "Failed to send deployment update notification to slack");
}
}
}
public async Task FinishDeploymentUpdate(
DeploymentUpdate deploymentUpdate,
DeploymentUpdateStatus finalStatus
)
{
await ChangeDeploymentUpdateStatus(deploymentUpdate, finalStatus);
_applicationService.SetCurrentImageTag(deploymentUpdate.Application, deploymentUpdate.Image, deploymentUpdate.TargetTag);
}
public void VmssPublicIpAddressApiTest()
{
var handler1 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
var handler2 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
using (MockContext context = MockContext.Start(this.GetType()))
{
var resourcesClient = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);
var location = NetworkManagementTestUtilities.GetResourceLocation(resourcesClient, "Microsoft.Compute/virtualMachineScaleSets");
string resourceGroupName = TestUtilities.GenerateName();
string deploymentName = TestUtilities.GenerateName("vmss");
resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
new ResourceGroup
{
Location = location
});
DeploymentUpdate.CreateVmss(resourcesClient, resourceGroupName, deploymentName);
string virtualMachineScaleSetName = "vmssip";
var vmssListAllPageResult = networkManagementClient.PublicIPAddresses.ListVirtualMachineScaleSetPublicIPAddresses(resourceGroupName, virtualMachineScaleSetName);
var vmssListAllResult = vmssListAllPageResult.ToList();
var firstResult = vmssListAllResult.First();
Assert.NotNull(vmssListAllResult);
Assert.Equal("Succeeded", firstResult.ProvisioningState);
Assert.NotNull(firstResult.ResourceGuid);
var idItem = firstResult.Id;
var vmIndex = GetNameById(idItem, "virtualMachines");
var nicName = GetNameById(idItem, "networkInterfaces");
var ipConfigName = GetNameById(idItem, "ipConfigurations");
var ipName = GetNameById(idItem, "publicIPAddresses");
var vmssListPageResult = networkManagementClient.PublicIPAddresses.ListVirtualMachineScaleSetVMPublicIPAddresses(
resourceGroupName, virtualMachineScaleSetName, vmIndex, nicName, ipConfigName);
var vmssListResult = vmssListPageResult.ToList();
Assert.Single(vmssListResult);
var vmssGetResult = networkManagementClient.PublicIPAddresses.GetVirtualMachineScaleSetPublicIPAddress(
resourceGroupName, virtualMachineScaleSetName, vmIndex, nicName, ipConfigName, ipName);
Assert.NotNull(vmssGetResult);
Assert.Equal("Succeeded", vmssGetResult.ProvisioningState);
Assert.NotNull(vmssGetResult.ResourceGuid);
resourcesClient.ResourceGroups.Delete(resourceGroupName);
}
}
public async Task StartDeploymentUpdateJob(DeploymentUpdate deploymentUpdate)
{
var jobkey = new JobKey($"gitwatch-{deploymentUpdate.Application.Name}", "gitrepowatcher");
var data = new JobDataMap
{
["DeploymentUpdate"] = deploymentUpdate
};
await _scheduler.TriggerJob(jobkey, data, CancellationToken.None);
}
private SlackMessage BuildDeploymentUpdateMessage(DeploymentUpdate deploymentUpdate, DeploymentUpdateStatus status)
{
return(new SlackMessage(
$"A new image of *{deploymentUpdate.Image.Repository}* was detected (tag *{deploymentUpdate.TargetTag}*).",
new IBlock[]
{
new SectionBlock()
{
text = new Text()
{
type = "mrkdwn",
text =
$"A new image of *{deploymentUpdate.Image.Repository}* was detected (tag *{deploymentUpdate.TargetTag}*)."
}
},
new DividerBlock(),
new SectionBlock()
{
fields = new Text[]
{
new Text()
{
text = $"*From*\n{deploymentUpdate.CurrentTag}",
type = "mrkdwn"
},
new Text()
{
text = $"*To*\n{deploymentUpdate.TargetTag}",
type = "mrkdwn"
},
}
},
new SectionBlock()
{
fields = new Text[]
{
new Text()
{
text = $"*Application*\n{deploymentUpdate.Application}",
type = "mrkdwn"
},
new Text()
{
text = $"*Status*\n{status}",
type = "mrkdwn"
},
}
}
}
));
}
public async Task ChangeDeploymentUpdateStatus(DeploymentUpdate deploymentUpdate, DeploymentUpdateStatus status)
{
_deploymentUpdates[deploymentUpdate] = status;
if (_notificationHandles.TryGetValue(deploymentUpdate, out var handle))
{
try
{
_log.LogInformation("Submitting {@DeploymentUpdate} notification change to slack {@MessageHandle}. ", deploymentUpdate, handle);
var newHandle = await _slackClient.UpdateDeploymentUpdateNotification(handle, deploymentUpdate, status);
_notificationHandles.TryUpdate(deploymentUpdate, newHandle, handle);
}
catch (Exception e)
{
_log.LogError(e, "Failed to submit {@DeploymentUpdate} notification {@MessageHandle}", deploymentUpdate, handle);
}
}
}
public async Task <IMessageHandle> UpdateDeploymentUpdateNotification(
IMessageHandle handle,
DeploymentUpdate deploymentUpdate,
DeploymentUpdateStatus status
)
{
if (handle == null)
{
throw new ArgumentNullException(nameof(handle));
}
if (deploymentUpdate == null)
{
throw new ArgumentNullException(nameof(deploymentUpdate));
}
return(await UpdateMessageAsync(
handle as SingleMessageHandle,
BuildDeploymentUpdateMessage(deploymentUpdate, status)
));
}
public void DeleteConnectionMonitorTest()
{
var handler1 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
var handler2 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
var handler3 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
using (MockContext context = MockContext.Start(this.GetType().FullName))
{
var resourcesClient = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);
var computeManagementClient = NetworkManagementTestUtilities.GetComputeManagementClientWithHandler(context, handler3);
string location = "centraluseuap";
string resourceGroupName = TestUtilities.GenerateName();
resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
new ResourceGroup
{
Location = location
});
string virtualMachineName = TestUtilities.GenerateName();
string networkInterfaceName = TestUtilities.GenerateName();
string networkSecurityGroupName = virtualMachineName + "-nsg";
//Deploy VM with a template
DeploymentUpdate.CreateVm(
resourcesClient: resourcesClient,
resourceGroupName: resourceGroupName,
location: location,
virtualMachineName: virtualMachineName,
storageAccountName: TestUtilities.GenerateName(),
networkInterfaceName: networkInterfaceName,
networkSecurityGroupName: networkSecurityGroupName,
diagnosticsStorageAccountName: TestUtilities.GenerateName(),
deploymentName: TestUtilities.GenerateName()
);
var getVm = computeManagementClient.VirtualMachines.Get(resourceGroupName, virtualMachineName);
//Deploy networkWatcherAgent on VM
VirtualMachineExtension parameters = new VirtualMachineExtension
{
Publisher = "Microsoft.Azure.NetworkWatcher",
TypeHandlerVersion = "1.4",
VirtualMachineExtensionType = "NetworkWatcherAgentWindows",
Location = location
};
var addExtension = computeManagementClient.VirtualMachineExtensions.CreateOrUpdate(resourceGroupName, getVm.Name, "NetworkWatcherAgent", parameters);
string networkWatcherName = TestUtilities.GenerateName();
NetworkWatcher properties = new NetworkWatcher
{
Location = location
};
//Create network Watcher
var createNetworkWatcher = networkManagementClient.NetworkWatchers.CreateOrUpdate(resourceGroupName, networkWatcherName, properties);
string connectionMonitorName1 = TestUtilities.GenerateName();
string connectionMonitorName2 = TestUtilities.GenerateName();
ConnectionMonitor cm = new ConnectionMonitor
{
Location = location,
Source = new ConnectionMonitorSource
{
ResourceId = getVm.Id
},
Destination = new ConnectionMonitorDestination
{
Address = "bing.com",
Port = 80
},
MonitoringIntervalInSeconds = 30,
AutoStart = false
};
var connectionMonitor1 = networkManagementClient.ConnectionMonitors.CreateOrUpdate(resourceGroupName, networkWatcherName, connectionMonitorName1, cm);
var connectionMonitor2 = networkManagementClient.ConnectionMonitors.CreateOrUpdate(resourceGroupName, networkWatcherName, connectionMonitorName2, cm);
var getConnectionMonitors1 = networkManagementClient.ConnectionMonitors.List(resourceGroupName, networkWatcherName);
Assert.Equal(2, getConnectionMonitors1.Count());
networkManagementClient.ConnectionMonitors.Delete(resourceGroupName, networkWatcherName, connectionMonitorName2);
var getConnectionMonitors2 = networkManagementClient.ConnectionMonitors.List(resourceGroupName, networkWatcherName);
Assert.Single(getConnectionMonitors2);
}
}
public void CheckConnectivityVmToInternetTest()
{
var handler1 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
var handler2 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
var handler3 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
using (MockContext context = MockContext.Start(this.GetType()))
{
var resourcesClient = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);
var computeManagementClient = NetworkManagementTestUtilities.GetComputeManagementClientWithHandler(context, handler3);
string location = "westcentralus";
string resourceGroupName = TestUtilities.GenerateName();
resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
new ResourceGroup
{
Location = location
});
string virtualMachineName = TestUtilities.GenerateName();
string networkInterfaceName = TestUtilities.GenerateName();
string networkSecurityGroupName = virtualMachineName + "-nsg";
//Deploy VM with a template
DeploymentUpdate.CreateVm(
resourcesClient: resourcesClient,
resourceGroupName: resourceGroupName,
location: location,
virtualMachineName: virtualMachineName,
storageAccountName: TestUtilities.GenerateName(),
networkInterfaceName: networkInterfaceName,
networkSecurityGroupName: networkSecurityGroupName,
diagnosticsStorageAccountName: TestUtilities.GenerateName(),
deploymentName: TestUtilities.GenerateName()
);
var getVm = computeManagementClient.VirtualMachines.Get(resourceGroupName, virtualMachineName);
//Deploy networkWatcherAgent on VM
VirtualMachineExtension parameters = new VirtualMachineExtension();
parameters.Publisher = "Microsoft.Azure.NetworkWatcher.Edp";
parameters.TypeHandlerVersion = "1.4";
parameters.VirtualMachineExtensionType = "NetworkWatcherAgentWindows";
parameters.Location = location;
var addExtension = computeManagementClient.VirtualMachineExtensions.CreateOrUpdate(resourceGroupName, getVm.Name, "NetworkWatcherAgent", parameters);
string networkWatcherName = TestUtilities.GenerateName();
NetworkWatcher properties = new NetworkWatcher();
properties.Location = location;
//Create network Watcher
var createNetworkWatcher = networkManagementClient.NetworkWatchers.CreateOrUpdate(resourceGroupName, networkWatcherName, properties);
ConnectivityParameters connectivityParameters = new ConnectivityParameters();
connectivityParameters.Source = new ConnectivitySource();
connectivityParameters.Source.ResourceId = getVm.Id;
connectivityParameters.Destination = new ConnectivityDestination();
connectivityParameters.Destination.Address = "bing.com";
connectivityParameters.Destination.Port = 80;
var connectivityCheck = networkManagementClient.NetworkWatchers.CheckConnectivity(resourceGroupName, networkWatcherName, connectivityParameters);
//Validation
Assert.Equal("Reachable", connectivityCheck.ConnectionStatus);
Assert.Equal(0, connectivityCheck.ProbesFailed);
Assert.Equal("Source", connectivityCheck.Hops.FirstOrDefault().Type);
Assert.Equal("Internet", connectivityCheck.Hops.LastOrDefault().Type);
}
}
private async Task <bool> UpdateDeploymentManifests(
Repository gitRepository,
DeploymentUpdate deploymentUpdate,
Dictionary <Image, string> imageToFilenameMap,
string currentImageTag,
string applicationSourcePath,
YamlUtilities yamlUtilities,
string relativePath)
{
if (!imageToFilenameMap.TryGetValue(deploymentUpdate.Image, out string file))
{
// TODO: warn that we have an image tag update but no corresponding file
_log.LogWarning("Update to {Repository} cannot be applied since there isn't matching file.");
return(false);
}
_log.LogInformation("Upgrading {Repository} to {NewTag} from {Tag}",
deploymentUpdate.Image.Repository,
deploymentUpdate.TargetTag,
currentImageTag
);
var yaml = new YamlStream();
var filePath = Path.Combine(applicationSourcePath, file);
yamlUtilities.ReadYamlStream(yaml, filePath);
var image = deploymentUpdate.Image;
var doCommit = false;
foreach (var doc in yaml.Documents)
{
var tagInManifest = yamlUtilities.ExtractValueFromDoc(image.TagProperty.Path, doc);
if (tagInManifest == null)
{
continue;
}
if (tagInManifest == deploymentUpdate.TargetTag)
{
_log.LogInformation("Tag for {Repository} matches new tag {NewTag}", image.Repository, deploymentUpdate.TargetTag);
continue;
}
_log.LogInformation("Setting current-tag for {Repository} to {Tag}", image.Repository,
deploymentUpdate.TargetTag);
yamlUtilities.SetValueInDoc(image.TagProperty.Path, doc, deploymentUpdate.TargetTag);
doCommit = true;
}
if (!doCommit)
{
return(true);
}
yamlUtilities.WriteYamlStream(yaml, filePath);
_log.LogInformation("Adding {Path} to repository staging", filePath);
var gitFilePath = Path.Combine(relativePath, file);
Commands.Stage(gitRepository, gitFilePath);
gitRepository.Commit(
$"[{deploymentUpdate.Application}] Updated deployment for {image.RepositoryName}; {deploymentUpdate.CurrentTag} to {deploymentUpdate.TargetTag}",
new Signature("deploy-bot", "*****@*****.**", DateTimeOffset.Now),
new Signature("deploy-bot", "*****@*****.**", DateTimeOffset.Now)
);
// add delay instruction due to the deployment update notification delivered too fast
await Task.Delay(500);
return(true);
}
private async Task <bool> SynchronizeHelmApplicationSource(Repository gitRepository,
ApplicationSourceTrackingContext context,
HelmApplicationSource helmApplicationSource)
{
var relativePath = helmApplicationSource.Path;
var applicationSourcePath = Path.Combine(context.GitRepositoryPath, relativePath);
var application = context.Application;
var yamlUtilities = new YamlUtilities();
// build map of images -> yaml file that defines them and image -> current tag
_log.LogInformation("Beginning to parse value files defined in application source ...");
var imageToFilenameMap = new Dictionary <Image, string>();
var imageToTagInManifest = new Dictionary <Image, string>();
foreach (var file in helmApplicationSource.ValuesFiles)
{
var yaml = new YamlStream();
var filePath = Path.Combine(applicationSourcePath, file);
yamlUtilities.ReadYamlStream(yaml, filePath);
foreach (var doc in yaml.Documents)
{
foreach (var image in context.Application.Images)
{
var tagInManifest = yamlUtilities.ExtractValueFromDoc(image.TagProperty.Path, doc);
if (tagInManifest == null)
{
continue;
}
if (imageToFilenameMap.ContainsKey(image))
{
// TODO: handle situation where multiple files define the same image tag (ERROR and warn user)
}
imageToFilenameMap[image] = file;
imageToTagInManifest[image] = tagInManifest;
}
}
}
_log.LogInformation("Completing parsing value files defined in application source ...");
// start updating files
var manifestsChanged = false;
DeploymentUpdate deploymentUpdate = null;
while ((deploymentUpdate = await _deploymentService.GetNextPendingDeploymentUpdate(application)) != null)
{
_log.LogInformation("Executing pending deployment update ...");
await _deploymentService.ChangeDeploymentUpdateStatus(
deploymentUpdate,
DeploymentUpdateStatus.UpdatingManifests
);
manifestsChanged = await UpdateDeploymentManifests(gitRepository,
deploymentUpdate,
imageToFilenameMap,
imageToTagInManifest.TryGetAndReturn(deploymentUpdate.Image) ?? "n/a",
applicationSourcePath,
yamlUtilities,
relativePath);
if (manifestsChanged)
{
imageToTagInManifest[deploymentUpdate.Image] = deploymentUpdate.TargetTag;
}
await _deploymentService.FinishDeploymentUpdate(
deploymentUpdate,
manifestsChanged?DeploymentUpdateStatus.Complete : DeploymentUpdateStatus.Failed
);
}
if (!manifestsChanged)
{
// we don't have a deployment, ensure application manifest is up to date with latest image tags
foreach (var keyValuePair in imageToTagInManifest)
{
_applicationService.SetCurrentImageTag(application, keyValuePair.Key, keyValuePair.Value);
}
}
return(manifestsChanged);
}
public void VmssNetworkInterfaceApiTest()
{
var handler1 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
var handler2 = new RecordedDelegatingHandler {
StatusCodeToReturn = HttpStatusCode.OK
};
using (MockContext context = MockContext.Start(this.GetType()))
{
var resourcesClient = ResourcesManagementTestUtilities.GetResourceManagementClientWithHandler(context, handler1, true);
var networkManagementClient = NetworkManagementTestUtilities.GetNetworkManagementClientWithHandler(context, handler2);
var location = NetworkManagementTestUtilities.GetResourceLocation(resourcesClient, "Microsoft.Compute/virtualMachineScaleSets");
string resourceGroupName = TestUtilities.GenerateName();
string deploymentName = TestUtilities.GenerateName("vmss");
resourcesClient.ResourceGroups.CreateOrUpdate(resourceGroupName,
new ResourceGroup
{
Location = location
});
DeploymentUpdate.CreateVmss(resourcesClient, resourceGroupName, deploymentName);
string virtualMachineScaleSetName = "vmssip";
var vmssListAllPageResult = networkManagementClient.PublicIPAddresses.ListVirtualMachineScaleSetPublicIPAddresses(resourceGroupName, virtualMachineScaleSetName);
var vmssListAllResult = vmssListAllPageResult.ToList();
var firstResult = vmssListAllResult.First();
Assert.NotNull(vmssListAllResult);
Assert.Equal("Succeeded", firstResult.ProvisioningState);
Assert.NotNull(firstResult.ResourceGuid);
var idItem = firstResult.Id;
var vmIndex = GetNameById(idItem, "virtualMachines");
var nicName = GetNameById(idItem, "networkInterfaces");
var ipConfigName = GetNameById(idItem, "ipConfigurations");
var ipName = GetNameById(idItem, "publicIPAddresses");
// Verify that NICs contain refernce to publicip, nsg and dns settings
var listNicPerVmss = networkManagementClient.NetworkInterfaces.ListVirtualMachineScaleSetNetworkInterfaces(resourceGroupName, virtualMachineScaleSetName).ToList();
Assert.NotNull(listNicPerVmss);
foreach (var nic in listNicPerVmss)
{
this.VerifyVmssNicProperties(nic);
}
// Verify nics on a vm level
var listNicPerVm = networkManagementClient.NetworkInterfaces.ListVirtualMachineScaleSetVMNetworkInterfaces(resourceGroupName, virtualMachineScaleSetName, vmIndex).ToList();
Assert.NotNull(listNicPerVm);
Assert.Single(listNicPerVm);
foreach (var nic in listNicPerVm)
{
this.VerifyVmssNicProperties(nic);
}
// Verify getting individual nic
var getNic = networkManagementClient.NetworkInterfaces.GetVirtualMachineScaleSetNetworkInterface(resourceGroupName, virtualMachineScaleSetName, vmIndex, nicName);
Assert.NotNull(getNic);
this.VerifyVmssNicProperties(getNic);
resourcesClient.ResourceGroups.Delete(resourceGroupName);
}
}
