private static void AssertSameData(string expected, DataDownloadRequestDetails?actual, string msg)
{
Assert.IsNotNull(actual);
var expectedData = DataDownloadRequestDetails.FromBase64Json(expected);
AssertSameData(expectedData, actual, msg);
}
public async Task <IList <DataDownloadRequest> > GetPodRequestsAsync(PodIdentifier pod)
{
this.logger.LogInformation($"Getting pod info {pod}");
var podInfo = await this.podDataRequestGetter.GetAsync(pod);
var list = new List <DataDownloadRequest>();
if (string.IsNullOrEmpty(podInfo.DropFolder))
{
this.logger.LogError($"{CommonAnnotations.DataStoreLocation} is not set, returning empty list of pod data requests");
return(list);
}
foreach (var repo in podInfo.DataSources)
{
if (podInfo.Requests.TryGetValue(repo, out var request))
{
var details = DataDownloadRequestDetails.FromBase64Json(request);
if (details is null || details.Hash is null || details.Path is null)
{
this.logger.LogError($"Cannot parse pod {podInfo.Id} DataDownloadRequestDetails {request}");
continue;
}
var extractionLocation = Path.Combine(podInfo.DropFolder, details.Path.Replace(Path.PathSeparator, '_'));
list.Add(new DataDownloadRequest(pod, repo, podInfo.DropFolder, extractionLocation, details));
}
else
{
list.Add(new DataDownloadRequest(pod, repo, podInfo.DropFolder, string.Empty, null));
}
}
return(list);
}
public void RequestUpgradeMiddleVersionToLatest()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
});
var middle = new DataDownloadRequestDetails("c", "c");
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "name"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = middle.ToBase64String(),
}),
};
var o = f.Create(kg, lv, pods);
AssertSameData(lv.UpgradeInfo["d"], o.GetDataRequest(pods[0].Id, "d"), "upgrade to latest");
}
public async Task <DataRepositoryLatestVersionInfo> GetOrUpdateAsync(string ns, DataRepositoryManifest manifest, CancellationToken token)
{
var currentVer = await this.client.GetEndpointAnnotationsAsync(ns, AutoCraneDataDeployEndpointName, token);
var itemsToAdd = new Dictionary <string, string>();
foreach (var item in manifest.Sources)
{
var recentData = item.Value.ToList().OrderByDescending(k => k.Timestamp);
if (!recentData.Any())
{
this.logger.LogWarning($"Missing recent data for {item.Key}/{item.Value}");
continue;
}
var mostRecentData = recentData.First();
var req = new DataDownloadRequestDetails(mostRecentData.ArchiveFilePath, mostRecentData.Hash);
var latestVersion = req.ToBase64String();
if (currentVer.TryGetValue(item.Key, out var currentVerString))
{
var currentVerJson = DataDownloadRequestDetails.FromBase64Json(currentVerString);
if (currentVerJson?.Path == req.Path)
{
continue;
}
}
this.logger.LogInformation($"Setting Latest for {item.Key} to hash={req.Hash} filePath={req.Path}");
itemsToAdd[item.Key] = latestVersion;
}
if (itemsToAdd.Any())
{
await this.client.PutEndpointAnnotationsAsync(ns, AutoCraneDataDeployEndpointName, itemsToAdd, token);
}
var union = itemsToAdd;
foreach (var item in currentVer)
{
union.TryAdd(item.Key, item.Value);
}
return(new DataRepositoryLatestVersionInfo(union));
}
private bool PodIsOnVersionForAtLeast(PodDataRequestInfo p, string repoSpec, DataDownloadRequestDetails ver, TimeSpan?timespan)
{
var request = p.Requests.FirstOrDefault(r => r.Key == repoSpec).Value;
if (request is not null)
{
var version = DataDownloadRequestDetails.FromBase64Json(request);
if (version != null && version.Equals(ver))
{
if (!timespan.HasValue)
{
return(true);
}
else
{
return(version.UnixTimestampSeconds < (this.clock.Get() - timespan.Value).ToUnixTimeSeconds());
}
}
}
return(false);
}
public void RequestResetNoTimestampVersionToLKG()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
});
var badReq = new DataDownloadRequestDetails("b", "b")
{
UnixTimestampSeconds = null
};
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "name"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = badReq.ToBase64String(),
}),
};
var o = f.Create(kg, lv, pods);
AssertSameData(kg.KnownGoodVersions["d"], o.GetDataRequest(pods[0].Id, "d"), $"Reset to LKG on bad timestamp");
}
public async Task <DataRepositoryKnownGoods> GetOrUpdateAsync(string ns, DataRepositoryManifest manifest, IReadOnlyList <PodDataRequestInfo> pods, CancellationToken token)
{
var lkg = await this.client.GetEndpointAnnotationsAsync(ns, AutoCraneLastKnownGoodEndpointName, token);
var itemsToAdd = new Dictionary <string, string>();
foreach (var source in manifest.Sources)
{
if (source.Value.Count > 0)
{
bool shouldUpgrade = false;
if (!lkg.ContainsKey(source.Key))
{
// LKG entry does not exist
shouldUpgrade = true;
}
else if (lkg.TryGetValue(source.Key, out var req))
{
var lkgRequest = DataDownloadRequestDetails.FromBase64Json(req);
// if the manifest no longer contains the LKG, we shouldn't send people to download the LKG
shouldUpgrade = !source.Value.Any(sv => sv.ArchiveFilePath == lkgRequest?.Path);
}
if (shouldUpgrade)
{
var mostRecentData = source.Value.ToList().OrderByDescending(k => k.Timestamp).First();
var req = new DataDownloadRequestDetails(mostRecentData.ArchiveFilePath, mostRecentData.Hash);
this.logger.LogInformation($"Setting LKG for {source.Key} to hash={req.Hash} filePath={req.Path}");
itemsToAdd[source.Key] = req.ToBase64String();
}
}
}
foreach (var knownGood in lkg)
{
var dataSource = knownGood.Key;
var currentVersionString = knownGood.Value;
var requestsForThisSource = pods.Select(p => p.Requests.FirstOrDefault(r => r.Key == dataSource).Value)
.Select(r => r == null ? null : DataDownloadRequestDetails.FromBase64Json(r))
.ToList();
var distinctRequestsForThisSource = requestsForThisSource.Select(r => r?.Path).Distinct().ToList();
var firstRequestForThisSource = requestsForThisSource.FirstOrDefault();
if (distinctRequestsForThisSource.Count == 1 && firstRequestForThisSource != null)
{
var everyPodHasThisPath = distinctRequestsForThisSource.First();
var currentVersion = DataDownloadRequestDetails.FromBase64Json(currentVersionString);
if (currentVersion == null)
{
this.logger.LogError($"Couldn't read current version of data {dataSource}, value: {currentVersionString}");
}
else
{
if (currentVersion.Path != everyPodHasThisPath)
{
this.logger.LogInformation($"Upgrading LKG for {dataSource} to hash={firstRequestForThisSource!.Hash} filePath={firstRequestForThisSource!.Path}");
itemsToAdd[dataSource] = firstRequestForThisSource.ToBase64String();
}
}
}
}
if (itemsToAdd.Any())
{
await this.client.PutEndpointAnnotationsAsync(ns, AutoCraneLastKnownGoodEndpointName, itemsToAdd, token);
}
var newDict = new Dictionary <string, string>(lkg);
foreach (var item in itemsToAdd)
{
newDict[item.Key] = item.Value;
}
return(new DataRepositoryKnownGoods(newDict));
}
public DataDownloadRequestDetails?GetDataRequest(PodIdentifier pi, string repoName)
{
var pod = this.pods.FirstOrDefault(p => p.Id == pi);
if (pod == null)
{
this.logger.LogError($"Cannot find pod {pi}, so cannot determine correct data request");
return(null);
}
// there is no existing request
if (!pod.DataSources.Contains(repoName))
{
// we can't even find the data source, bailout--this shouldn't happen
this.logger.LogError($"Pod {pod.Id} has data request {repoName} but could not find data source.");
return(null);
}
// try parsing the LKG and latest values
if (!this.knownGoods.KnownGoodVersions.TryGetValue(repoName, out var repoDetailsKnownGoodVersion))
{
this.logger.LogError($"{pi} {repoName}: LKG missing, cannot set LKG");
return(null);
}
if (!this.latestVersionInfo.UpgradeInfo.TryGetValue(repoName, out var repoDetailsLatestVersion))
{
this.logger.LogError($"{pi} {repoName}: latest version missing");
return(null);
}
var knownGoodVersion = DataDownloadRequestDetails.FromBase64Json(repoDetailsKnownGoodVersion);
if (knownGoodVersion is null)
{
this.logger.LogError($"{pi} Cannot parse known good version of data {repoName}: {repoDetailsKnownGoodVersion}");
return(null);
}
var latestVersion = DataDownloadRequestDetails.FromBase64Json(repoDetailsLatestVersion);
if (latestVersion is null)
{
this.logger.LogError($"{pi} Cannot parse known good version of data {repoName}: {repoDetailsLatestVersion}");
return(null);
}
if (pod.Requests.TryGetValue(repoName, out var existingVersionString))
{
var existingVersion = DataDownloadRequestDetails.FromBase64Json(existingVersionString);
if (existingVersion is null)
{
// if we can't parse the existing version, try setting it to LKG
this.logger.LogError($"{pi} Cannot parse existing version {existingVersionString}, setting to LKG: {knownGoodVersion}");
return(knownGoodVersion);
}
if (existingVersion.UnixTimestampSeconds is null)
{
this.logger.LogError($"{pi} Existing version {existingVersionString}, does not have timestamp, returning LKG: {knownGoodVersion}");
return(knownGoodVersion);
}
if (existingVersion.Equals(latestVersion))
{
this.logger.LogTrace($"{pi} {repoName} is on latest version, doing nothing.");
return(null);
}
var existingVersionTimestamp = DateTimeOffset.FromUnixTimeSeconds(existingVersion.UnixTimestampSeconds.GetValueOrDefault());
if (existingVersionTimestamp > this.clock.Get() - UpgradeProbationTimeSpan)
{
this.logger.LogTrace($"{pi} {repoName} upgraded recently ({existingVersionTimestamp}). skipping");
return(null);
}
// we know we aren't on the latest version, if we aren't on LKG, upgrade to latest
if (!existingVersion.Equals(knownGoodVersion))
{
this.logger.LogInformation($"{pi} {repoName} is on version between LKG and latest, moving to latest");
return(latestVersion);
}
// if FailingLimit% has been on latest version for at least UpgradeProbationTimeSpan, and there are no watchdog failures on dependent users, then
// set LKG to latest, upgrade everyone to latest
var podsUsingThisData = this.podsWithRepo[repoName];
if (this.podsDependingOnRepo.TryGetValue(repoName, out var podsDependingOnThisData))
{
var watchdogFailureCount = podsDependingOnThisData.Where(p => this.watchdogStatusAggregator.Aggregate(p.Annotations) == WatchdogStatus.ErrorLevel).Count();
var pctFailing = (double)watchdogFailureCount / podsDependingOnThisData.Count;
if (pctFailing > UpgradePercent)
{
this.logger.LogInformation($"{pi} {repoName} found watchdog failures on pods {watchdogFailureCount}/{podsDependingOnThisData.Count}, taking no action");
return(null);
}
}
var podsOnLatestVersionForProbationTimeSpanCount = podsUsingThisData.Where(p => this.PodIsOnVersionForAtLeast(p, repoName, latestVersion, UpgradeProbationTimeSpan)).Count();
var podsOnLatestVersionForProbation = (double)podsOnLatestVersionForProbationTimeSpanCount / podsUsingThisData.Count;
if (podsOnLatestVersionForProbation >= UpgradePercent)
{
this.logger.LogInformation($"{pi} {repoName} found pods {podsOnLatestVersionForProbationTimeSpanCount}/{podsUsingThisData.Count} on latest for probation period, upgrading to latest {latestVersion}");
return(latestVersion);
}
// otherwise put FailingLimit% on Latest and the rest on LKG
var podsOnLKG = podsUsingThisData.Where(p => this.PodIsOnVersionForAtLeast(p, repoName, knownGoodVersion, null)).ToList();
// round down or we might never upgrade anyone
// take at least one, but zero if there is only one
// var numberToTake = Math.Max(podsUsingThisData.Count - 1, Math.Min(1, (int)(Math.Floor(1.0 - UpgradePercent) * podsUsingThisData.Count));
var numberToTake = (int)Math.Floor((1.0 - UpgradePercent) * podsUsingThisData.Count);
var shouldNotUpgradeList = podsOnLKG.Take(numberToTake).Select(p => p.Id).ToHashSet();
if (shouldNotUpgradeList.Contains(pod.Id))
{
this.logger.LogInformation($"{pi} {repoName} in do not upgrade list");
return(null);
}
// put on latest version
this.logger.LogInformation($"{pi} {repoName} upgrading to: {latestVersion}");
return(latestVersion);
}
else
{
// doesn't have a request set, so default to LKG
this.logger.LogInformation($"{pi} {repoName} has no request, setting to LKG: {knownGoodVersion}");
return(knownGoodVersion);
}
}
