static string GetValidLoggerParamName(string resourceName)
{
string[] loggerNameStrs = resourceName.Split(new char[] { ',' });
string validLoggerName = ParameterNamingHelper.GenerateValidParameterName(loggerNameStrs[loggerNameStrs.Length - 1], ParameterPrefix.LogResourceId);
return(validLoggerName);
}
async Task LoadAllReferencedLoggers(
List <string> apisToExtract,
ExtractorParameters extractorParameters)
{
var serviceDiagnostics = await this.diagnosticClient.GetAllAsync(extractorParameters);
foreach (var serviceDiagnostic in serviceDiagnostics)
{
string loggerId = serviceDiagnostic.Properties.LoggerId;
var serviceDiagnosticsKey = ParameterNamingHelper.GenerateValidParameterName(serviceDiagnostic.Name, ParameterPrefix.Diagnostic);
if (!this.Cache.ServiceLevelDiagnosticLoggerBindings.ContainsKey(serviceDiagnosticsKey))
{
this.Cache.ServiceLevelDiagnosticLoggerBindings.Add(serviceDiagnosticsKey, loggerId);
}
}
if (apisToExtract.IsNullOrEmpty())
{
this.logger.LogWarning("No apis to extract are passed to {0}", nameof(LoggerExtractor));
return;
}
foreach (string curApiName in apisToExtract)
{
var diagnostics = await this.diagnosticClient.GetApiDiagnosticsAsync(curApiName, extractorParameters);
if (diagnostics.IsNullOrEmpty())
{
this.logger.LogWarning("No diagnostics found for '{0}' api", curApiName);
continue;
}
var diagnosticLoggerBindings = new HashSet <DiagnosticLoggerBinding>();
foreach (var diagnostic in diagnostics)
{
diagnosticLoggerBindings.Add(new DiagnosticLoggerBinding
{
DiagnosticName = ParameterNamingHelper.GenerateValidParameterName(diagnostic.Name, ParameterPrefix.Diagnostic),
LoggerId = diagnostic.Properties.LoggerId
});
}
if (!diagnosticLoggerBindings.IsNullOrEmpty())
{
this.Cache.ApiDiagnosticLoggerBindings.Add(
ParameterNamingHelper.GenerateValidParameterName(curApiName, ParameterPrefix.Api),
diagnosticLoggerBindings);
}
}
}
public string GetCachedPolicyContent(PolicyTemplateResource policyTemplateResource, string baseFilesGenerationDirectory)
{
if (policyTemplateResource?.Properties is null)
{
this.logger.LogWarning("Policy was not initialized correctly {0}", policyTemplateResource.Name);
return(string.Empty);
}
// easy code flow, if we already have a policy-content-file-full-path to use as a key for policy content caches
if (!string.IsNullOrEmpty(policyTemplateResource.Properties.PolicyContentFileFullPath))
{
if (this.policyPathToContentCache.ContainsKey(policyTemplateResource.Properties.PolicyContentFileFullPath))
{
return(this.policyPathToContentCache[policyTemplateResource.Properties.PolicyContentFileFullPath]);
}
else
{
this.logger.LogWarning("Policy content at '{0}' was initialized, but wasn't cached properly", policyTemplateResource.Properties.PolicyContentFileFullPath);
}
}
// if no path found already, trying to get one from policy content
// check if this is a file or is it the raw policy content
var policyContent = policyTemplateResource?.Properties?.PolicyContent ?? string.Empty;
if (policyContent.Contains(".xml"))
{
var fileNameSection = policyContent.Split(',')[1];
var policyFileName = ParameterNamingHelper.GetSubstringBetweenTwoCharacters('\'', '\'', fileNameSection);
var policyFileFullPath = Path.Combine(baseFilesGenerationDirectory, PoliciesDirectoryName, policyFileName);
if (File.Exists(policyFileFullPath))
{
var policyContentFromFile = File.ReadAllText(policyFileFullPath);
policyTemplateResource.Properties.PolicyContentFileFullPath = policyFileFullPath;
this.policyPathToContentCache[policyFileFullPath] = policyContentFromFile;
return(policyContentFromFile);
}
}
return(policyContent);
}
async Task <Dictionary <string, object> > GetAllReferencedLoggers(List <string> apisToExtract, ExtractorParameters extractorParameters)
{
var apiLoggerId = new Dictionary <string, object>();
var serviceDiagnostics = await this.apiExtractor.GetServiceDiagnosticsAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup);
JObject oServiceDiagnostics = JObject.Parse(serviceDiagnostics);
var serviceloggerIds = new Dictionary <string, string>();
foreach (var serviceDiagnostic in oServiceDiagnostics["value"])
{
string diagnosticName = ((JValue)serviceDiagnostic["name"]).Value.ToString();
string loggerId = ((JValue)serviceDiagnostic["properties"]["loggerId"]).Value.ToString();
apiLoggerId.Add(ParameterNamingHelper.GenerateValidParameterName(diagnosticName, ParameterPrefix.Diagnostic), loggerId);
}
foreach (string curApiName in apisToExtract)
{
var loggerIds = new Dictionary <string, string>();
var diagnostics = await this.apiExtractor.GetApiDiagnosticsAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup, curApiName);
JObject oDiagnostics = JObject.Parse(diagnostics);
foreach (var diagnostic in oDiagnostics["value"])
{
string diagnosticName = ((JValue)diagnostic["name"]).Value.ToString();
string loggerId = ((JValue)diagnostic["properties"]["loggerId"]).Value.ToString();
loggerIds.Add(ParameterNamingHelper.GenerateValidParameterName(diagnosticName, ParameterPrefix.Diagnostic), loggerId);
}
if (loggerIds.Count != 0)
{
apiLoggerId.Add(ParameterNamingHelper.GenerateValidParameterName(curApiName, ParameterPrefix.Api), loggerIds);
}
}
return(apiLoggerId);
}
/// <summary>
/// Generate the ARM assets for the backend resources
/// </summary>
/// <param name="apimname"></param>
/// <param name="resourceGroup"></param>
/// <param name="singleApiName"></param>
/// <param name="apiTemplateResources"></param>
/// <param name="propertyResources"></param>
/// <param name="policyXMLBaseUrl"></param>
/// <param name="policyXMLSasToken"></param>
/// <param name="extractBackendParameters"></param>
/// <returns>a combination of a Template and the value for the BackendSettings parameter</returns>
public async Task <Tuple <Template, Dictionary <string, BackendApiParameters> > > GenerateBackendsARMTemplateAsync(string apimname, string resourceGroup, string singleApiName, List <TemplateResource> apiTemplateResources, List <TemplateResource> propertyResources, ExtractorParameters extractorParameters)
{
Console.WriteLine("------------------------------------------");
Console.WriteLine("Extracting backends from service");
Template armTemplate = this.templateBuilder.GenerateTemplateWithApimServiceNameProperty().Build();
if (extractorParameters.ParameterizeBackend)
{
TemplateParameterProperties extractBackendParametersProperties = new TemplateParameterProperties()
{
type = "object"
};
armTemplate.Parameters.Add(ParameterNames.BackendSettings, extractBackendParametersProperties);
}
List <TemplateResource> templateResources = new List <TemplateResource>();
// isolate api and operation policy resources in the case of a single api extraction, as they may reference backends
var policyResources = apiTemplateResources.Where(resource => resource.Type == ResourceTypeConstants.APIPolicy || resource.Type == ResourceTypeConstants.APIOperationPolicy || resource.Type == ResourceTypeConstants.ProductPolicy);
var namedValueResources = propertyResources.Where(resource => resource.Type == ResourceTypeConstants.Property);
// pull all backends for service
JObject oBackends = new JObject();
var oBackendParameters = new Dictionary <string, BackendApiParameters>();
int skipNumberOfBackends = 0;
do
{
string backends = await this.GetBackendsAsync(apimname, resourceGroup, skipNumberOfBackends);
oBackends = JObject.Parse(backends);
foreach (var item in oBackends["value"])
{
string backendName = ((JValue)item["name"]).Value.ToString();
string backend = await this.GetBackendDetailsAsync(apimname, resourceGroup, backendName);
// convert returned backend to template resource class
BackendTemplateResource backendTemplateResource = backend.Deserialize <BackendTemplateResource>();
backendTemplateResource.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{backendName}')]";
backendTemplateResource.ApiVersion = GlobalConstants.ApiVersion;
bool includeBackend = false;
////only extract the backend if this is a full extraction, or in the case of a single api, if it is referenced by one of the policies
if (singleApiName == null)
{
// if the user is extracting all apis, extract all the backends
includeBackend = true;
}
else
{
// check extracted policies to see if the backend is referenced.
foreach (PolicyTemplateResource policyTemplateResource in policyResources)
{
string policyContent = PolicyTemplateUtils.GetPolicyContent(extractorParameters, policyTemplateResource);
if (this.DoesPolicyReferenceBackend(policyContent, namedValueResources, backendName, backendTemplateResource))
{
// backend was used in policy, extract it
includeBackend = true;
// dont need to go through all policies if the back end has already been found
break;
}
}
}
if (includeBackend)
{
if (extractorParameters.ParameterizeBackend)
{
var apiToken = new BackendApiParameters();
string validApiParamName = ParameterNamingHelper.GenerateValidParameterName(backendName, ParameterPrefix.Diagnostic).ToLower();
if (!string.IsNullOrEmpty(backendTemplateResource.Properties.resourceId))
{
apiToken.resourceId = backendTemplateResource.Properties.resourceId;
backendTemplateResource.Properties.resourceId = $"[parameters('{ParameterNames.BackendSettings}').{validApiParamName}.resourceId]";
}
if (!string.IsNullOrEmpty(backendTemplateResource.Properties.url))
{
apiToken.url = backendTemplateResource.Properties.url;
backendTemplateResource.Properties.url = $"[parameters('{ParameterNames.BackendSettings}').{validApiParamName}.url]";
}
if (!string.IsNullOrEmpty(backendTemplateResource.Properties.protocol))
{
apiToken.protocol = backendTemplateResource.Properties.protocol;
backendTemplateResource.Properties.protocol = $"[parameters('{ParameterNames.BackendSettings}').{validApiParamName}.protocol]";
}
oBackendParameters.Add(validApiParamName, apiToken);
}
Console.WriteLine("'{0}' Backend found", backendName);
templateResources.Add(backendTemplateResource);
}
}
skipNumberOfBackends += GlobalConstants.NumOfRecords;
}while (oBackends["nextLink"] != null);
armTemplate.Resources = templateResources.ToArray();
return(new Tuple <Template, Dictionary <string, BackendApiParameters> >(armTemplate, oBackendParameters));
}
public async Task <Template <NamedValuesResources> > GenerateNamedValuesTemplateAsync(
string singleApiName,
List <PolicyTemplateResource> apiPolicies,
List <LoggerTemplateResource> loggerResources,
ExtractorParameters extractorParameters,
string baseFilesGenerationDirectory)
{
var namedValuesTemplate = this.templateBuilder
.GenerateTemplateWithApimServiceNameProperty()
.AddParameterizeNamedValueParameters(extractorParameters)
.AddParameterizeNamedValuesKeyVaultSecretParameters(extractorParameters)
.Build <NamedValuesResources>();
if (extractorParameters.NotIncludeNamedValue)
{
this.logger.LogInformation("'{0}' parameter is turned on, so not processing named-values template generation at all.", nameof(extractorParameters.NotIncludeNamedValue));
return(namedValuesTemplate);
}
var namedValues = await this.namedValuesClient.GetAllAsync(extractorParameters);
if (namedValues.IsNullOrEmpty())
{
this.logger.LogWarning("No named values found for apim '{0}'", extractorParameters.SourceApimName);
return(namedValuesTemplate);
}
foreach (var namedValueResource in namedValues)
{
var originalNamedValueName = namedValueResource.Name;
// convert returned named value to template resource class
namedValueResource.OriginalName = originalNamedValueName;
namedValueResource.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{originalNamedValueName}')]";
namedValueResource.Type = ResourceTypeConstants.NamedValues;
namedValueResource.ApiVersion = GlobalConstants.ApiVersion;
namedValueResource.Scale = null;
if (extractorParameters.ParameterizeNamedValue)
{
namedValueResource.Properties.Value = $"[parameters('{ParameterNames.NamedValues}').{ParameterNamingHelper.GenerateValidParameterName(originalNamedValueName, ParameterPrefix.Property)}]";
}
//Hide the value field if it is a keyvault named value
if (namedValueResource.Properties.KeyVault != null)
{
namedValueResource.Properties.Value = null;
}
if (namedValueResource.Properties.KeyVault != null && extractorParameters.ParamNamedValuesKeyVaultSecrets)
{
namedValueResource.Properties.KeyVault.SecretIdentifier = $"[parameters('{ParameterNames.NamedValueKeyVaultSecrets}').{ParameterNamingHelper.GenerateValidParameterName(originalNamedValueName, ParameterPrefix.Property)}]";
}
if (string.IsNullOrEmpty(singleApiName))
{
namedValuesTemplate.TypedResources.NamedValues.Add(namedValueResource);
}
else
{
// if the user is executing a single api, extract all the named values used in the template resources
var foundInPolicy = this.DoesPolicyReferenceNamedValue(
apiPolicies,
originalNamedValueName,
namedValueResource,
baseFilesGenerationDirectory);
var foundInBackEnd = await this.backendExtractor.IsNamedValueUsedInBackends(
singleApiName,
apiPolicies,
originalNamedValueName,
namedValueResource.Properties.DisplayName,
extractorParameters,
baseFilesGenerationDirectory);
var foundInLogger = this.DoesLoggerReferenceNamedValue(
loggerResources,
originalNamedValueName,
namedValueResource);
if (foundInPolicy || foundInBackEnd || foundInLogger)
{
namedValuesTemplate.TypedResources.NamedValues.Add(namedValueResource);
}
}
}
return(namedValuesTemplate);
}
void SetArmTemplateValuesToApiTemplateResource(ApiTemplateResource apiResource, ExtractorParameters extractorParameters)
{
var originalServiceApiName = apiResource.Name;
apiResource.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{originalServiceApiName}')]";
apiResource.ApiVersion = GlobalConstants.ApiVersion;
apiResource.Scale = null;
if (extractorParameters.ParameterizeServiceUrl)
{
apiResource.Properties.ServiceUrl = $"[parameters('{ParameterNames.ServiceUrl}').{ParameterNamingHelper.GenerateValidParameterName(originalServiceApiName, ParameterPrefix.Api)}]";
}
if (apiResource.Properties.ApiVersionSetId != null)
{
apiResource.DependsOn = Array.Empty <string>();
string versionSetName = apiResource.Properties.ApiVersionSetId;
int versionSetPosition = versionSetName.IndexOf("apiVersionSets/");
versionSetName = versionSetName.Substring(versionSetPosition, versionSetName.Length - versionSetPosition);
apiResource.Properties.ApiVersionSetId = $"[concat(resourceId('Microsoft.ApiManagement/service', parameters('{ParameterNames.ApimServiceName}')), '/{versionSetName}')]";
}
else
{
apiResource.DependsOn = Array.Empty <string>();
}
}
public async Task <Template> GenerateNamedValuesTemplateAsync(string singleApiName, List <TemplateResource> apiTemplateResources, ExtractorParameters extractorParameters, IBackendExtractor backendExtractor, List <TemplateResource> loggerTemplateResources)
{
Template armTemplate = this.templateBuilder.GenerateTemplateWithApimServiceNameProperty().Build();
if (extractorParameters.ParameterizeNamedValue)
{
TemplateParameterProperties namedValueParameterProperties = new TemplateParameterProperties()
{
type = "object"
};
armTemplate.Parameters.Add(ParameterNames.NamedValues, namedValueParameterProperties);
}
if (extractorParameters.ParamNamedValuesKeyVaultSecrets)
{
TemplateParameterProperties keyVaultNamedValueParameterProperties = new TemplateParameterProperties()
{
type = "object"
};
armTemplate.Parameters.Add(ParameterNames.NamedValueKeyVaultSecrets, keyVaultNamedValueParameterProperties);
}
if (extractorParameters.NotIncludeNamedValue == true)
{
Console.WriteLine("------------------------------------------");
Console.WriteLine("Skipping extracting named values from service");
return(armTemplate);
}
Console.WriteLine("------------------------------------------");
Console.WriteLine("Extracting named values from service");
List <TemplateResource> templateResources = new List <TemplateResource>();
// pull all named values (properties) for service
string[] properties = await this.GetPropertiesAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup);
// isolate api and operation policy resources in the case of a single api extraction, as they may reference named value
var policyResources = apiTemplateResources.Where(resource => resource.Type == ResourceTypeConstants.APIPolicy || resource.Type == ResourceTypeConstants.APIOperationPolicy || resource.Type == ResourceTypeConstants.ProductPolicy);
foreach (var extractedProperty in properties)
{
JToken oProperty = JObject.Parse(extractedProperty);
string propertyName = ((JValue)oProperty["name"]).Value.ToString();
string fullPropertyResource = await this.GetPropertyDetailsAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup, propertyName);
// convert returned named value to template resource class
PropertyTemplateResource propertyTemplateResource = fullPropertyResource.Deserialize <PropertyTemplateResource>();
propertyTemplateResource.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{propertyName}')]";
propertyTemplateResource.Type = ResourceTypeConstants.Property;
propertyTemplateResource.ApiVersion = GlobalConstants.ApiVersion;
propertyTemplateResource.Scale = null;
if (extractorParameters.ParameterizeNamedValue)
{
propertyTemplateResource.Properties.value = $"[parameters('{ParameterNames.NamedValues}').{ParameterNamingHelper.GenerateValidParameterName(propertyName, ParameterPrefix.Property)}]";
}
//Hide the value field if it is a keyvault named value
if (propertyTemplateResource.Properties.keyVault != null)
{
propertyTemplateResource.Properties.value = null;
}
if (propertyTemplateResource.Properties.keyVault != null && extractorParameters.ParamNamedValuesKeyVaultSecrets)
{
propertyTemplateResource.Properties.keyVault.secretIdentifier = $"[parameters('{ParameterNames.NamedValueKeyVaultSecrets}').{ParameterNamingHelper.GenerateValidParameterName(propertyName, ParameterPrefix.Property)}]";
}
if (singleApiName == null)
{
// if the user is executing a full extraction, extract all the loggers
Console.WriteLine("'{0}' Named value found", propertyName);
templateResources.Add(propertyTemplateResource);
}
else
{
// if the user is executing a single api, extract all the named values used in the template resources
bool foundInPolicy = this.DoesPolicyReferenceNamedValue(extractorParameters, policyResources, propertyName, propertyTemplateResource);
bool foundInBackEnd = await backendExtractor.IsNamedValueUsedInBackends(extractorParameters.SourceApimName, extractorParameters.ResourceGroup, singleApiName, apiTemplateResources, extractorParameters, propertyName, propertyTemplateResource.Properties.displayName);
bool foundInLogger = this.DoesLoggerReferenceNamedValue(loggerTemplateResources, propertyName, propertyTemplateResource);
// check if named value is referenced in a backend
if (foundInPolicy || foundInBackEnd || foundInLogger)
{
// named value was used in policy, extract it
Console.WriteLine("'{0}' Named value found", propertyName);
templateResources.Add(propertyTemplateResource);
}
}
}
armTemplate.Resources = templateResources.ToArray();
return(armTemplate);
}
public async Task <Template <LoggerTemplateResources> > GenerateLoggerTemplateAsync(
List <string> apisToExtract,
List <PolicyTemplateResource> apiPolicies,
ExtractorParameters extractorParameters)
{
var loggerTemplate = this.templateBuilder
.GenerateTemplateWithApimServiceNameProperty()
.Build <LoggerTemplateResources>();
if (extractorParameters.ParameterizeApiLoggerId)
{
await this.LoadAllReferencedLoggers(apisToExtract, extractorParameters);
}
var loggers = await this.loggerClient.GetAllAsync(extractorParameters);
foreach (var logger in loggers)
{
var originalLoggerName = logger.Name;
logger.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{originalLoggerName}')]";
logger.Type = ResourceTypeConstants.Logger;
logger.ApiVersion = GlobalConstants.ApiVersion;
logger.Scale = null;
// single api extraction
if (apisToExtract?.Count != 1)
{
loggerTemplate.TypedResources.Loggers.Add(logger);
}
else
{
// if the user is extracting a single api, extract the loggers referenced by its diagnostics and api policies
var isReferencedInPolicy = apiPolicies?.Any(x => x.Properties.PolicyContent.Contains(originalLoggerName));
bool isReferencedInDiagnostic = false;
var validApiName = ParameterNamingHelper.GenerateValidParameterName(apisToExtract.First(), ParameterPrefix.Api);
if (extractorParameters.ParameterizeApiLoggerId && this.Cache.ApiDiagnosticLoggerBindings.ContainsKey(validApiName))
{
var diagnosticLoggerBindings = this.Cache.ApiDiagnosticLoggerBindings[validApiName];
if (!diagnosticLoggerBindings.IsNullOrEmpty())
{
var validDiagnosticName = ParameterNamingHelper.GenerateValidParameterName(logger.Properties.LoggerType, ParameterPrefix.Diagnostic).ToLower();
if (diagnosticLoggerBindings.Any(x => x.DiagnosticName == validDiagnosticName))
{
isReferencedInDiagnostic = true;
}
}
}
if (isReferencedInPolicy == true || isReferencedInDiagnostic)
{
// logger was used in policy or diagnostic, extract it
loggerTemplate.TypedResources.Loggers.Add(logger);
}
}
}
return(loggerTemplate);
}
public Template CreatePropertyTemplate(CreatorConfig creatorConfig)
{
// create empty template
Template propertyTemplate = this.templateBuilder.GenerateEmptyTemplate().Build();
// add parameters
propertyTemplate.Parameters = new Dictionary <string, TemplateParameterProperties>
{
{ ParameterNames.ApimServiceName, new TemplateParameterProperties()
{
Type = "string"
} }
};
if (creatorConfig.parameterizeNamedValues)
{
if (creatorConfig.namedValues.Any(x => x.Value != null))
{
propertyTemplate.Parameters.Add(ParameterNames.NamedValues, new TemplateParameterProperties {
Type = "object"
});
}
if (creatorConfig.namedValues.Any(x => x.KeyVault != null))
{
propertyTemplate.Parameters.Add(ParameterNames.NamedValueKeyVaultSecrets, new TemplateParameterProperties {
Type = "object"
});
}
}
List <TemplateResource> resources = new List <TemplateResource>();
foreach (PropertyConfig namedValue in creatorConfig.namedValues)
{
string value = namedValue.Value == null ? null
: creatorConfig.parameterizeNamedValues
? $"[parameters('{ParameterNames.NamedValues}').{ParameterNamingHelper.GenerateValidParameterName(namedValue.DisplayName, ParameterPrefix.Property)}]"
: namedValue.Value;
var keyVault = namedValue.KeyVault == null ? null
: creatorConfig.parameterizeNamedValues
? new NamedValueResourceKeyVaultProperties
{
SecretIdentifier = $"[parameters('{ParameterNames.NamedValueKeyVaultSecrets}').{ParameterNamingHelper.GenerateValidParameterName(namedValue.DisplayName, ParameterPrefix.Property)}]"
}
: namedValue.KeyVault;
// create property resource with properties
NamedValueTemplateResource propertyTemplateResource = new NamedValueTemplateResource()
{
Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{namedValue.DisplayName}')]",
Type = ResourceTypeConstants.NamedValues,
ApiVersion = GlobalConstants.ApiVersion,
Properties = new NamedValueProperties()
{
DisplayName = namedValue.DisplayName,
Value = value,
Secret = namedValue.Secret,
Tags = namedValue.Tags,
KeyVault = keyVault
},
DependsOn = new string[] { }
};
resources.Add(propertyTemplateResource);
}
propertyTemplate.Resources = resources.ToArray();
return(propertyTemplate);
}
/// <summary>
/// Generate the ARM assets for the backend resources
/// </summary>
/// <returns>a combination of a Template and the value for the BackendSettings parameter</returns>
public async Task <Template <BackendTemplateResources> > GenerateBackendsTemplateAsync(
string singleApiName,
List <PolicyTemplateResource> apiPolicies,
List <NamedValueTemplateResource> namedValues,
string baseFilesGenerationDirectory,
ExtractorParameters extractorParameters)
{
var backendTemplate = this.templateBuilder
.GenerateTemplateWithApimServiceNameProperty()
.AddParameterizeBackendProperty(extractorParameters)
.Build <BackendTemplateResources>();
var backends = await this.backendClient.GetAllAsync(extractorParameters);
if (backends.IsNullOrEmpty())
{
this.logger.LogWarning("No backends found for apim instance '{0}'", extractorParameters.SourceApimName);
return(backendTemplate);
}
foreach (var backendResource in backends)
{
var originalBackendName = backendResource.Name;
backendResource.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{originalBackendName}')]";
backendResource.Type = ResourceTypeConstants.Backend;
backendResource.ApiVersion = GlobalConstants.ApiVersion;
if (string.IsNullOrEmpty(singleApiName))
{
// if the user is extracting all apis, extract all the backends
backendTemplate.TypedResources.Backends.Add(backendResource);
SaveBackendApiParametersToCache();
}
else
{
if (apiPolicies.IsNullOrEmpty())
{
continue;
}
foreach (var policyTemplateResource in apiPolicies)
{
var policyContent = this.policyExtractor.GetCachedPolicyContent(policyTemplateResource, baseFilesGenerationDirectory);
if (this.DoesPolicyReferenceBackend(policyContent, namedValues, originalBackendName, backendResource))
{
// backend was used in policy, extract it
backendTemplate.TypedResources.Backends.Add(backendResource);
SaveBackendApiParametersToCache();
// don't need to go through all policies if the back end has already been found
break;
}
}
}
void SaveBackendApiParametersToCache()
{
if (!extractorParameters.ParameterizeBackend)
{
this.logger.LogDebug("Parameter '{0}' is false. Skipping storing api-backend mapping in cache", nameof(ExtractorParameters.ParameterizeBackend));
return;
}
var backendApiParameters = new BackendApiParameters();
var backendValidName = ParameterNamingHelper.GenerateValidParameterName(originalBackendName, ParameterPrefix.Diagnostic).ToLower();
if (!string.IsNullOrEmpty(backendResource.Properties.ResourceId))
{
backendApiParameters.ResourceId = backendResource.Properties.ResourceId;
backendResource.Properties.ResourceId = $"[parameters('{ParameterNames.BackendSettings}').{backendValidName}.resourceId]";
}
if (!string.IsNullOrEmpty(backendResource.Properties.Url))
{
backendApiParameters.Url = backendResource.Properties.Url;
backendResource.Properties.Url = $"[parameters('{ParameterNames.BackendSettings}').{backendValidName}.url]";
}
if (!string.IsNullOrEmpty(backendResource.Properties.Protocol))
{
backendApiParameters.Protocol = backendResource.Properties.Protocol;
backendResource.Properties.Protocol = $"[parameters('{ParameterNames.BackendSettings}').{backendValidName}.protocol]";
}
if (!backendTemplate.TypedResources.BackendNameParametersCache.ContainsKey(backendValidName))
{
backendTemplate.TypedResources.BackendNameParametersCache.Add(backendValidName, backendApiParameters);
}
}
}
return(backendTemplate);
}
async Task <ApiTemplateResource> GenerateApiTemplateResourceAsVersioned(string apiName, ExtractorParameters extractorParameters)
{
var apiDetails = await this.apisClient.GetSingleAsync(apiName, extractorParameters);
apiDetails.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{apiName}')]";
apiDetails.ApiVersion = GlobalConstants.ApiVersion;
apiDetails.Scale = null;
apiDetails.Properties.IsCurrent = null;
if (extractorParameters.ParameterizeServiceUrl)
{
apiDetails.Properties.ServiceUrl = $"[parameters('{ParameterNames.ServiceUrl}').{ParameterNamingHelper.GenerateValidParameterName(apiName, ParameterPrefix.Api)}]";
}
if (apiDetails.Properties.ApiVersionSetId != null)
{
apiDetails.DependsOn = Array.Empty <string>();
string versionSetName = apiDetails.Properties.ApiVersionSetId;
int versionSetPosition = versionSetName.IndexOf("apiVersionSets/");
versionSetName = versionSetName.Substring(versionSetPosition, versionSetName.Length - versionSetPosition);
apiDetails.Properties.ApiVersionSetId = $"[concat(resourceId('Microsoft.ApiManagement/service', parameters('{ParameterNames.ApimServiceName}')), '/{versionSetName}')]";
}
else
{
apiDetails.DependsOn = Array.Empty <string>();
}
return(apiDetails);
}
/// <summary>
/// Gets the "All API" level diagnostic resources, these are common to all APIs.
/// </summary>
/// <param name="extractorParameters">The extractor.</param>
/// <returns>a list of DiagnosticTemplateResources</returns>
public async Task <List <DiagnosticTemplateResource> > GetServiceDiagnosticsTemplateResourcesAsync(ExtractorParameters extractorParameters)
{
var serviceDiagnostics = await this.diagnosticClient.GetAllAsync(extractorParameters);
if (serviceDiagnostics.IsNullOrEmpty())
{
this.logger.LogWarning("No diagnostic settings were found...");
}
var templateResources = new List <DiagnosticTemplateResource>();
foreach (var serviceDiagnostic in serviceDiagnostics)
{
this.logger.LogDebug("Found {0} diagnostic setting...", serviceDiagnostic.Name);
var originalDiagnosticName = serviceDiagnostic.Name;
serviceDiagnostic.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{originalDiagnosticName}')]";
serviceDiagnostic.Type = ResourceTypeConstants.APIServiceDiagnostic;
serviceDiagnostic.ApiVersion = GlobalConstants.ApiVersion;
serviceDiagnostic.Scale = null;
serviceDiagnostic.DependsOn = Array.Empty <string>();
if (extractorParameters.ParameterizeApiLoggerId)
{
serviceDiagnostic.Properties.loggerId = $"[parameters('{ParameterNames.ApiLoggerId}').{ParameterNamingHelper.GenerateValidParameterName(originalDiagnosticName, ParameterPrefix.Diagnostic)}]";
}
if (!originalDiagnosticName.Contains("applicationinsights"))
{
// enableHttpCorrelationHeaders only works for application insights, causes errors otherwise
//TODO: Check this settings still valid?
serviceDiagnostic.Properties.enableHttpCorrelationHeaders = null;
}
templateResources.Add(serviceDiagnostic);
}
return(templateResources);
}
public async Task <List <DiagnosticTemplateResource> > GetApiDiagnosticsResourcesAsync(string apiName, ExtractorParameters extractorParameters)
{
var apiDiagnostics = await this.diagnosticClient.GetApiDiagnosticsAsync(apiName, extractorParameters);
if (apiDiagnostics.IsNullOrEmpty())
{
this.logger.LogWarning("No diagnostics found for api {0}", apiName);
return(null);
}
var templateResources = new List <DiagnosticTemplateResource>();
foreach (var apiDiagnostic in apiDiagnostics)
{
var apiDiagnosticOriginalName = apiDiagnostic.Name;
apiDiagnostic.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{apiName}/{apiDiagnosticOriginalName}')]";
apiDiagnostic.Type = ResourceTypeConstants.APIDiagnostic;
apiDiagnostic.ApiVersion = GlobalConstants.ApiVersion;
apiDiagnostic.Scale = null;
apiDiagnostic.DependsOn = new string[] { $"[resourceId('Microsoft.ApiManagement/service/apis', parameters('{ParameterNames.ApimServiceName}'), '{apiName}')]" };
if (extractorParameters.ParameterizeApiLoggerId)
{
apiDiagnostic.Properties.loggerId = $"[parameters('{ParameterNames.ApiLoggerId}').{ParameterNamingHelper.GenerateValidParameterName(apiName, ParameterPrefix.Api)}.{ParameterNamingHelper.GenerateValidParameterName(apiDiagnosticOriginalName, ParameterPrefix.Diagnostic)}]";
}
if (!apiDiagnosticOriginalName.Contains("applicationinsights"))
{
// enableHttpCorrelationHeaders only works for application insights, causes errors otherwise
apiDiagnostic.Properties.enableHttpCorrelationHeaders = null;
}
templateResources.Add(apiDiagnostic);
}
return(templateResources);
}
public async Task <Template> CreateMasterTemplateParameterValues(
List <string> apisToExtract,
LoggersCache loggersCache,
LoggerTemplateResources loggerTemplateResources,
BackendTemplateResources backendResources,
NamedValuesResources namedValuesResources,
ExtractorParameters extractorParameters)
{
// used to create the parameter values for use in parameters file
// create empty template
var parametersTemplate = this.templateBuilder
.GenerateEmptyTemplate()
.Build();
parametersTemplate.Parameters = new();
var parameters = parametersTemplate.Parameters;
parameters.Add(ParameterNames.ApimServiceName, new() { Value = extractorParameters.DestinationApimName });
AddLinkedUrlParameters();
AddPolicyParameters();
AddNamedValuesParameters();
await AddServiceUrlParameterAsync();
void AddLinkedUrlParameters()
{
if (string.IsNullOrEmpty(extractorParameters.LinkedTemplatesBaseUrl))
{
return;
}
parameters.Add(ParameterNames.LinkedTemplatesBaseUrl, new() { Value = extractorParameters.LinkedTemplatesBaseUrl });
if (!string.IsNullOrEmpty(extractorParameters.LinkedTemplatesSasToken))
{
parameters.Add(ParameterNames.LinkedTemplatesSasToken, new() { Value = extractorParameters.LinkedTemplatesSasToken });
}
if (!string.IsNullOrEmpty(extractorParameters.LinkedTemplatesUrlQueryString))
{
parameters.Add(ParameterNames.LinkedTemplatesUrlQueryString, new() { Value = extractorParameters.LinkedTemplatesUrlQueryString });
}
}
void AddPolicyParameters()
{
if (string.IsNullOrEmpty(extractorParameters.PolicyXMLBaseUrl))
{
return;
}
parameters.Add(ParameterNames.PolicyXMLBaseUrl, new() { Value = extractorParameters.PolicyXMLBaseUrl });
if (!string.IsNullOrEmpty(extractorParameters.PolicyXMLSasToken))
{
parameters.Add(ParameterNames.PolicyXMLSasToken, new() { Value = extractorParameters.PolicyXMLSasToken });
}
}
async Task AddServiceUrlParameterAsync()
{
if (!extractorParameters.ParameterizeServiceUrl)
{
return;
}
var serviceUrls = new Dictionary <string, string>();
foreach (var apiName in apisToExtract)
{
var validApiName = ParameterNamingHelper.GenerateValidParameterName(apiName, ParameterPrefix.Api);
string serviceUrl;
if (extractorParameters.ServiceUrlParameters is null)
{
var apiDetails = await this.apisClient.GetSingleAsync(apiName, extractorParameters);
serviceUrl = apiDetails.Properties.ServiceUrl;
}
else
{
serviceUrl = extractorParameters.ServiceUrlParameters.FirstOrDefault(x => x.ApiName.Equals(apiName))?.ServiceUrl;
}
serviceUrls.Add(validApiName, serviceUrl);
}
parameters.Add(ParameterNames.ServiceUrl, new TemplateObjectParameterProperties()
{
Value = serviceUrls
});
}
void AddNamedValuesParameters()
{
if (!extractorParameters.ParameterizeNamedValue &&
!extractorParameters.ParamNamedValuesKeyVaultSecrets)
{
return;
}
var namedValuesParameters = new Dictionary <string, string>();
var keyVaultNamedValues = new Dictionary <string, string>();
foreach (var namedValue in namedValuesResources.NamedValues)
{
if (extractorParameters.ParameterizeNamedValue && namedValue?.Properties.KeyVault == null)
{
var propertyValue = namedValue.Properties.Value;
var validPName = ParameterNamingHelper.GenerateValidParameterName(namedValue.OriginalName, ParameterPrefix.Property);
namedValuesParameters.Add(validPName, propertyValue);
}
if (extractorParameters.ParamNamedValuesKeyVaultSecrets && namedValue?.Properties.KeyVault is not null)
{
var propertyValue = namedValue.Properties.KeyVault.SecretIdentifier;
var validPName = ParameterNamingHelper.GenerateValidParameterName(namedValue.OriginalName, ParameterPrefix.Property);
keyVaultNamedValues.Add(validPName, propertyValue);
}
}
parameters.Add(ParameterNames.NamedValues, new TemplateObjectParameterProperties()
{
Value = namedValuesParameters
});
parameters.Add(ParameterNames.NamedValueKeyVaultSecrets, new TemplateObjectParameterProperties()
{
Value = keyVaultNamedValues
});
}
if (extractorParameters.ParameterizeApiLoggerId)
{
parameters.Add(ParameterNames.ApiLoggerId, new TemplateObjectParameterProperties()
{
Value = loggersCache.CreateResultingMap()
});
}
if (extractorParameters.ParameterizeLogResourceId)
{
parameters.Add(ParameterNames.LoggerResourceId, new TemplateObjectParameterProperties()
{
Value = loggerTemplateResources.LoggerResourceIds
});
}
if (extractorParameters.ParameterizeBackend)
{
parameters.Add(ParameterNames.BackendSettings, new TemplateObjectParameterProperties()
{
Value = backendResources.BackendNameParametersCache
});
}
return(parametersTemplate);
}
public async Task <Template> GenerateLoggerTemplateAsync(ExtractorParameters extractorParameters, string singleApiName, List <TemplateResource> apiTemplateResources, Dictionary <string, object> apiLoggerId)
{
Console.WriteLine("------------------------------------------");
Console.WriteLine("Extracting loggers from service");
Template armTemplate = this.templateBuilder.GenerateTemplateWithApimServiceNameProperty().Build();
if (extractorParameters.ParameterizeLogResourceId)
{
TemplateParameterProperties loggerResourceIdParameterProperties = new TemplateParameterProperties()
{
type = "object"
};
armTemplate.Parameters.Add(ParameterNames.LoggerResourceId, loggerResourceIdParameterProperties);
}
// isolate product api associations in the case of a single api extraction
var policyResources = apiTemplateResources.Where(resource => resource.Type == ResourceTypeConstants.APIPolicy || resource.Type == ResourceTypeConstants.APIOperationPolicy || resource.Type == ResourceTypeConstants.ProductPolicy);
List <TemplateResource> templateResources = new List <TemplateResource>();
// pull all loggers for service
string loggers = await this.GetLoggersAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup);
JObject oLoggers = JObject.Parse(loggers);
foreach (var extractedLogger in oLoggers["value"])
{
string loggerName = ((JValue)extractedLogger["name"]).Value.ToString();
string fullLoggerResource = await this.GetLoggerDetailsAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup, loggerName);
// convert returned logger to template resource class
LoggerTemplateResource loggerResource = fullLoggerResource.Deserialize <LoggerTemplateResource>();
loggerResource.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{loggerName}')]";
loggerResource.Type = ResourceTypeConstants.Logger;
loggerResource.ApiVersion = GlobalConstants.ApiVersion;
loggerResource.Scale = null;
if (singleApiName == null)
{
// if the user is extracting all apis, extract all the loggers
Console.WriteLine("'{0}' Logger found", loggerName);
templateResources.Add(loggerResource);
}
else
{
// if the user is extracting a single api, extract the loggers referenced by its diagnostics and api policies
bool isReferencedInPolicy = false;
bool isReferencedInDiagnostic = false;
foreach (PolicyTemplateResource policyTemplateResource in policyResources)
{
if (policyTemplateResource.Properties.PolicyContent.Contains(loggerName))
{
isReferencedInPolicy = true;
}
}
string validApiName = ParameterNamingHelper.GenerateValidParameterName(singleApiName, ParameterPrefix.Api);
if (extractorParameters.ParameterizeApiLoggerId && apiLoggerId.ContainsKey(validApiName))
{
object diagnosticObj = apiLoggerId[validApiName];
if (diagnosticObj is Dictionary <string, string> )
{
Dictionary <string, string> curDiagnostic = (Dictionary <string, string>)diagnosticObj;
string validDName = ParameterNamingHelper.GenerateValidParameterName(loggerResource.Properties.loggerType, ParameterPrefix.Diagnostic).ToLower();
if (curDiagnostic.ContainsKey(validDName) && curDiagnostic[validDName].Contains(loggerName))
{
isReferencedInDiagnostic = true;
}
}
}
if (isReferencedInPolicy == true || isReferencedInDiagnostic == true)
{
// logger was used in policy or diagnostic, extract it
Console.WriteLine("'{0}' Logger found", loggerName);
templateResources.Add(loggerResource);
}
};
}
armTemplate.Resources = templateResources.ToArray();
return(armTemplate);
}
public async Task <Template> CreateMasterTemplateParameterValues(List <string> apisToExtract, ExtractorParameters extractorParameters,
Dictionary <string, object> apiLoggerId,
Dictionary <string, string> loggerResourceIds,
Dictionary <string, BackendApiParameters> backendParams,
List <TemplateResource> propertyResources)
{
// used to create the parameter values for use in parameters file
// create empty template
Template masterTemplate = this.templateBuilder.GenerateEmptyTemplate().Build();
// add parameters with value property
Dictionary <string, TemplateParameterProperties> parameters = new Dictionary <string, TemplateParameterProperties>();
TemplateParameterProperties apimServiceNameProperties = new TemplateParameterProperties()
{
value = extractorParameters.DestinationApimName
};
parameters.Add(ParameterNames.ApimServiceName, apimServiceNameProperties);
if (extractorParameters.LinkedTemplatesBaseUrl != null)
{
TemplateParameterProperties linkedTemplatesBaseUrlProperties = new TemplateParameterProperties()
{
value = extractorParameters.LinkedTemplatesBaseUrl
};
parameters.Add(ParameterNames.LinkedTemplatesBaseUrl, linkedTemplatesBaseUrlProperties);
// add linkedTemplatesSasToken parameter if provided and if the user has provided a linkedTemplatesBaseUrl
if (extractorParameters.LinkedTemplatesSasToken != null)
{
TemplateParameterProperties linkedTemplatesSasTokenProperties = new TemplateParameterProperties()
{
value = extractorParameters.LinkedTemplatesSasToken
};
parameters.Add(ParameterNames.LinkedTemplatesSasToken, linkedTemplatesSasTokenProperties);
}
// add linkedTemplatesUrlQueryString parameter if provided and if the user has provided a linkedTemplatesBaseUrl
if (extractorParameters.LinkedTemplatesUrlQueryString != null)
{
TemplateParameterProperties linkedTemplatesUrlQueryStringProperties = new TemplateParameterProperties()
{
value = extractorParameters.LinkedTemplatesUrlQueryString
};
parameters.Add(ParameterNames.LinkedTemplatesUrlQueryString, linkedTemplatesUrlQueryStringProperties);
}
}
if (extractorParameters.PolicyXMLBaseUrl != null)
{
TemplateParameterProperties policyTemplateBaseUrlProperties = new TemplateParameterProperties()
{
value = extractorParameters.PolicyXMLBaseUrl
};
parameters.Add(ParameterNames.PolicyXMLBaseUrl, policyTemplateBaseUrlProperties);
// add policyXMLSasToken parameter if provided and if the user has provided a policyXMLBaseUrl
if (extractorParameters.PolicyXMLSasToken != null)
{
TemplateParameterProperties policyTemplateSasTokenProperties = new TemplateParameterProperties()
{
value = extractorParameters.PolicyXMLSasToken
};
parameters.Add(ParameterNames.PolicyXMLSasToken, policyTemplateSasTokenProperties);
}
}
if (extractorParameters.ParameterizeServiceUrl)
{
Dictionary <string, string> serviceUrls = new Dictionary <string, string>();
foreach (string apiName in apisToExtract)
{
string validApiName = ParameterNamingHelper.GenerateValidParameterName(apiName, ParameterPrefix.Api);
string serviceUrl = extractorParameters.ServiceUrlParameters != null?this.GetApiServiceUrlFromParameters(apiName, extractorParameters.ServiceUrlParameters) :
await this.GetAPIServiceUrl(extractorParameters.SourceApimName, extractorParameters.ResourceGroup, apiName);
serviceUrls.Add(validApiName, serviceUrl);
}
TemplateObjectParameterProperties serviceUrlProperties = new TemplateObjectParameterProperties()
{
value = serviceUrls
};
parameters.Add(ParameterNames.ServiceUrl, serviceUrlProperties);
}
if (extractorParameters.ParameterizeNamedValue)
{
Dictionary <string, string> namedValues = new Dictionary <string, string>();
PropertyExtractor pExc = new PropertyExtractor(this.templateBuilder);
string[] properties = await pExc.GetPropertiesAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup);
foreach (var extractedProperty in properties)
{
JToken oProperty = JObject.Parse(extractedProperty);
string propertyName = ((JValue)oProperty["name"]).Value.ToString();
// check if the property has been extracted as it is being used in a policy or backend
if (propertyResources.Count(item => item.Name.Contains(propertyName)) > 0)
{
string fullPropertyResource = await pExc.GetPropertyDetailsAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup, propertyName);
PropertyTemplateResource propertyTemplateResource = fullPropertyResource.Deserialize <PropertyTemplateResource>();
//Only add the property if it is not controlled by keyvault
if (propertyTemplateResource?.Properties.keyVault == null)
{
string propertyValue = propertyTemplateResource.Properties.value;
string validPName = ParameterNamingHelper.GenerateValidParameterName(propertyName, ParameterPrefix.Property);
namedValues.Add(validPName, propertyValue);
}
}
}
TemplateObjectParameterProperties namedValueProperties = new TemplateObjectParameterProperties()
{
value = namedValues
};
parameters.Add(ParameterNames.NamedValues, namedValueProperties);
}
if (extractorParameters.ParamNamedValuesKeyVaultSecrets)
{
Dictionary <string, string> keyVaultNamedValues = new Dictionary <string, string>();
PropertyExtractor pExc = new PropertyExtractor(this.templateBuilder);
string[] properties = await pExc.GetPropertiesAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup);
foreach (var extractedProperty in properties)
{
JToken oProperty = JObject.Parse(extractedProperty);
string propertyName = ((JValue)oProperty["name"]).Value.ToString();
// check if the property has been extracted as it is being used in a policy or backend
if (propertyResources.Count(item => item.Name.Contains(propertyName)) > 0)
{
string fullPropertyResource = await pExc.GetPropertyDetailsAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup, propertyName);
PropertyTemplateResource propertyTemplateResource = fullPropertyResource.Deserialize <PropertyTemplateResource>();
if (propertyTemplateResource?.Properties.keyVault != null)
{
string propertyValue = propertyTemplateResource.Properties.keyVault.secretIdentifier;
string validPName = ParameterNamingHelper.GenerateValidParameterName(propertyName, ParameterPrefix.Property);
keyVaultNamedValues.Add(validPName, propertyValue);
}
}
}
TemplateObjectParameterProperties keyVaultNamedValueProperties = new TemplateObjectParameterProperties()
{
value = keyVaultNamedValues
};
parameters.Add(ParameterNames.NamedValueKeyVaultSecrets, keyVaultNamedValueProperties);
}
if (extractorParameters.ParameterizeApiLoggerId)
{
TemplateObjectParameterProperties loggerIdProperties = new TemplateObjectParameterProperties()
{
value = apiLoggerId
};
parameters.Add(ParameterNames.ApiLoggerId, loggerIdProperties);
}
if (extractorParameters.ParameterizeLogResourceId)
{
TemplateObjectParameterProperties loggerResourceIdProperties = new TemplateObjectParameterProperties()
{
value = loggerResourceIds
};
parameters.Add(ParameterNames.LoggerResourceId, loggerResourceIdProperties);
}
if (extractorParameters.ParameterizeBackend)
{
TemplateObjectParameterProperties backendProperties = new TemplateObjectParameterProperties()
{
value = backendParams
};
parameters.Add(ParameterNames.BackendSettings, backendProperties);
}
masterTemplate.Parameters = parameters;
return(masterTemplate);
}