public async Task RefreshCloudFormationSpecAsync(
string specifcationUrl,
string destinationZipLocation,
string destinationJsonLocation
)
{
Console.WriteLine();
// download json specification
Console.WriteLine($"Fetching specification from {specifcationUrl}");
var response = await new HttpClient().GetAsync(specifcationUrl);
string text;
using (var decompressionStream = new GZipStream(await response.Content.ReadAsStreamAsync(), CompressionMode.Decompress))
using (var decompressedMemoryStream = new MemoryStream()) {
await decompressionStream.CopyToAsync(decompressedMemoryStream);
text = Encoding.UTF8.GetString(decompressedMemoryStream.ToArray());
}
var json = JObject.Parse(text);
// apply patches
var jsonPatchesUris = new[] {
"https://raw.githubusercontent.com/aws-cloudformation/cfn-python-lint/master/src/cfnlint/data/ExtendedSpecs/all/01_spec_patch.json",
// "https://raw.githubusercontent.com/aws-cloudformation/cfn-python-lint/master/src/cfnlint/data/ExtendedSpecs/all/02_parameter_types.json",
// "https://raw.githubusercontent.com/aws-cloudformation/cfn-python-lint/master/src/cfnlint/data/ExtendedSpecs/all/03_value_types.json",
// "https://raw.githubusercontent.com/aws-cloudformation/cfn-python-lint/master/src/cfnlint/data/ExtendedSpecs/all/04_property_values.json",
};
foreach (var jsonPatchUri in jsonPatchesUris)
{
// fetch patch document from URI
var httpResponse = await _httpClient.SendAsync(new HttpRequestMessage {
RequestUri = new Uri(jsonPatchUri),
Method = HttpMethod.Get
});
if (!httpResponse.IsSuccessStatusCode)
{
LogError($"unable to fetch '{jsonPatchUri}'");
continue;
}
var patch = PatchDocument.Parse(await httpResponse.Content.ReadAsStringAsync());
// apply each patch operation individually
JToken token = json;
var patcher = new JsonPatcher();
foreach (var patchOperation in patch.Operations)
{
try {
token = patcher.ApplyOperation(patchOperation, token);
} catch {
LogWarn($"unable to apply patch operation to '{patchOperation.Path}'");
}
}
json = (JObject)token;
}
// strip all "Documentation" fields to reduce document size
Console.WriteLine($"Original size: {text.Length:N0}");
json.SelectTokens("$..UpdateType").ToList().ForEach(property => property.Parent.Remove());
json.SelectTokens("$.PropertyTypes..Documentation").ToList().ForEach(property => property.Parent.Remove());
json.SelectTokens("$.ResourceTypes.*.*..Documentation").ToList().ForEach(property => property.Parent.Remove());
json = OrderFields(json);
text = json.ToString(Formatting.None);
Console.WriteLine($"Stripped size: {text.Length:N0}");
if (destinationJsonLocation != null)
{
Directory.CreateDirectory(Path.GetDirectoryName(destinationJsonLocation));
var cloudformationJson = json.ToString(Formatting.Indented);
if (File.Exists(destinationJsonLocation) && ((await File.ReadAllTextAsync(destinationJsonLocation)).ToMD5Hash() == cloudformationJson.ToMD5Hash()))
{
// not changes, nothing else to do
return;
}
await File.WriteAllTextAsync(destinationJsonLocation, cloudformationJson);
}
// save compressed file
if (destinationZipLocation != null)
{
Directory.CreateDirectory(Path.GetDirectoryName(destinationZipLocation));
using (var fileStream = File.OpenWrite(destinationZipLocation))
using (var compressionStream = new GZipStream(fileStream, CompressionLevel.Optimal))
using (var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(text))) {
await memoryStream.CopyToAsync(compressionStream);
}
var info = new FileInfo(destinationZipLocation);
Console.WriteLine($"Stored compressed spec file {destinationZipLocation}");
Console.WriteLine($"Compressed file size: {info.Length:N0}");
// write timestamp when the spec was updated (helps with determining if the embedded or downloaded spec is newer)
await File.WriteAllTextAsync($"{destinationZipLocation}.timestamp", DateTime.UtcNow.ToString("u"));
}
// local functions
JObject OrderFields(JObject value)
{
var result = new JObject();
foreach (var property in value.Properties().ToList().OrderBy(property => property.Name))
{
result.Add(property.Name, (property.Value is JObject propertyValue)
? OrderFields(propertyValue)
: property.Value
);
}
return(result);
}
}
