public async Task WriteAsync(DataRepositoryManifest manifest)
{
this.logger.LogInformation($"Writing manifest for {this.ManifestFilePath}");
var tmpFile = this.ManifestFilePath + ".tmp";
using var fs = File.Create(tmpFile);
await JsonSerializer.SerializeAsync(fs, manifest);
File.Move(tmpFile, this.ManifestFilePath, overwrite: true);
}
public async Task <DataRepositoryKnownGoods> GetOrUpdateAsync(string ns, DataRepositoryManifest manifest, IReadOnlyList <PodDataRequestInfo> pods, CancellationToken token)
{
var lkg = await this.client.GetEndpointAnnotationsAsync(ns, AutoCraneLastKnownGoodEndpointName, token);
var itemsToAdd = new Dictionary <string, string>();
foreach (var source in manifest.Sources)
{
if (source.Value.Count > 0)
{
bool shouldUpgrade = false;
if (!lkg.ContainsKey(source.Key))
{
// LKG entry does not exist
shouldUpgrade = true;
}
else if (lkg.TryGetValue(source.Key, out var req))
{
var lkgRequest = DataDownloadRequestDetails.FromBase64Json(req);
// if the manifest no longer contains the LKG, we shouldn't send people to download the LKG
shouldUpgrade = !source.Value.Any(sv => sv.ArchiveFilePath == lkgRequest?.Path);
}
if (shouldUpgrade)
{
var mostRecentData = source.Value.ToList().OrderByDescending(k => k.Timestamp).First();
var req = new DataDownloadRequestDetails(mostRecentData.ArchiveFilePath, mostRecentData.Hash);
this.logger.LogInformation($"Setting LKG for {source.Key} to hash={req.Hash} filePath={req.Path}");
itemsToAdd[source.Key] = req.ToBase64String();
}
}
}
foreach (var knownGood in lkg)
{
var dataSource = knownGood.Key;
var currentVersionString = knownGood.Value;
var requestsForThisSource = pods.Select(p => p.Requests.FirstOrDefault(r => r.Key == dataSource).Value)
.Select(r => r == null ? null : DataDownloadRequestDetails.FromBase64Json(r))
.ToList();
var distinctRequestsForThisSource = requestsForThisSource.Select(r => r?.Path).Distinct().ToList();
var firstRequestForThisSource = requestsForThisSource.FirstOrDefault();
if (distinctRequestsForThisSource.Count == 1 && firstRequestForThisSource != null)
{
var everyPodHasThisPath = distinctRequestsForThisSource.First();
var currentVersion = DataDownloadRequestDetails.FromBase64Json(currentVersionString);
if (currentVersion == null)
{
this.logger.LogError($"Couldn't read current version of data {dataSource}, value: {currentVersionString}");
}
else
{
if (currentVersion.Path != everyPodHasThisPath)
{
this.logger.LogInformation($"Upgrading LKG for {dataSource} to hash={firstRequestForThisSource!.Hash} filePath={firstRequestForThisSource!.Path}");
itemsToAdd[dataSource] = firstRequestForThisSource.ToBase64String();
}
}
}
}
if (itemsToAdd.Any())
{
await this.client.PutEndpointAnnotationsAsync(ns, AutoCraneLastKnownGoodEndpointName, itemsToAdd, token);
}
var newDict = new Dictionary <string, string>(lkg);
foreach (var item in itemsToAdd)
{
newDict[item.Key] = item.Value;
}
return(new DataRepositoryKnownGoods(newDict));
}
public async Task <DataRepositoryLatestVersionInfo> GetOrUpdateAsync(string ns, DataRepositoryManifest manifest, CancellationToken token)
{
var currentVer = await this.client.GetEndpointAnnotationsAsync(ns, AutoCraneDataDeployEndpointName, token);
var itemsToAdd = new Dictionary <string, string>();
foreach (var item in manifest.Sources)
{
var recentData = item.Value.ToList().OrderByDescending(k => k.Timestamp);
if (!recentData.Any())
{
this.logger.LogWarning($"Missing recent data for {item.Key}/{item.Value}");
continue;
}
var mostRecentData = recentData.First();
var req = new DataDownloadRequestDetails(mostRecentData.ArchiveFilePath, mostRecentData.Hash);
var latestVersion = req.ToBase64String();
if (currentVer.TryGetValue(item.Key, out var currentVerString))
{
var currentVerJson = DataDownloadRequestDetails.FromBase64Json(currentVerString);
if (currentVerJson?.Path == req.Path)
{
continue;
}
}
this.logger.LogInformation($"Setting Latest for {item.Key} to hash={req.Hash} filePath={req.Path}");
itemsToAdd[item.Key] = latestVersion;
}
if (itemsToAdd.Any())
{
await this.client.PutEndpointAnnotationsAsync(ns, AutoCraneDataDeployEndpointName, itemsToAdd, token);
}
var union = itemsToAdd;
foreach (var item in currentVer)
{
union.TryAdd(item.Key, item.Value);
}
return(new DataRepositoryLatestVersionInfo(union));
}
