public void RequestRecentUpgrade()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.UtcNow,
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
});
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "name"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = kg.KnownGoodVersions["d"],
}),
};
var o = f.Create(kg, lv, pods);
Assert.IsNull(o.GetDataRequest(pods[0].Id, "d"), $"Should not give a new request because still in probation");
}
public void RequestNoLkg()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
});
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "name"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = lv.UpgradeInfo["d"],
}),
};
var o = f.Create(kg, lv, pods);
Assert.IsNull(o.GetDataRequest(pods[0].Id, "d"));
}
public void RequestMatchesLastKnownGood()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.FromUnixTimeSeconds(1_000),
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "name"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = kg.KnownGoodVersions["d"],
}),
};
var o = f.Create(kg, lv, pods);
Assert.IsNull(o.GetDataRequest(pods[0].Id, "d"), "Request should return nothing if upgrade would be same as LKG");
}
public void WatchdogsFailingDoNotUpgrade()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
});
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "pod1"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
}),
new PodDataRequestInfo(
new PodIdentifier("ns", "pod2"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
}),
new PodDataRequestInfo(
new PodIdentifier("ns", "pod3"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
}),
new PodDataRequestInfo(
new PodIdentifier("ns", "pod4"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataDependsOn}"] = "d, e, f",
[$"{WatchdogStatus.Prefix}x"] = $"{WatchdogStatus.ErrorLevel}/ohno",
}),
};
var o = f.Create(kg, lv, pods);
Assert.IsNull(o.GetDataRequest(pods[0].Id, "d"), "watchdog failure on pod 4");
Assert.IsNull(o.GetDataRequest(pods[1].Id, "d"), "watchdog failure on pod 4");
Assert.IsNull(o.GetDataRequest(pods[2].Id, "d"), "watchdog failure on pod 4");
}
public void FinishUpgradeToLatest()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
});
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "pod1"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
}),
new PodDataRequestInfo(
new PodIdentifier("ns", "pod2"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
}),
new PodDataRequestInfo(
new PodIdentifier("ns", "pod3"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
}),
};
var o = f.Create(kg, lv, pods);
// upgrade 1 to latest
var pod1 = o.GetDataRequest(pods[0].Id, "d");
var pod2 = o.GetDataRequest(pods[1].Id, "d");
var pod3 = o.GetDataRequest(pods[2].Id, "d");
AssertSameData(lv.UpgradeInfo["d"], pod1, "upgrade to latest");
AssertSameData(lv.UpgradeInfo["d"], pod2, "upgrade to latest");
Assert.IsNull(pod3, "pod already on latest");
}
public DataRepositoryUpgradeOracle(DataRepositoryKnownGoods knownGoods, DataRepositoryLatestVersionInfo latestVersionInfo, IReadOnlyList <PodDataRequestInfo> pods, ILogger <DataRepositoryUpgradeOracle> logger, IClock clock, IWatchdogStatusAggregator watchdogStatusAggregator)
{
this.knownGoods = knownGoods;
this.latestVersionInfo = latestVersionInfo;
this.pods = pods;
this.logger = logger;
this.clock = clock;
this.watchdogStatusAggregator = watchdogStatusAggregator;
// this is likely a list of data deployment daemonsets
this.podsWithRepo = new Dictionary <string, List <PodDataRequestInfo> >();
foreach (var pod in pods)
{
foreach (var repoSpec in pod.DataSources)
{
if (!this.podsWithRepo.TryGetValue(repoSpec, out var list))
{
list = new List <PodDataRequestInfo>();
this.podsWithRepo[repoSpec] = list;
}
list.Add(pod);
}
}
// this is a list of pods mounting volumes to the daemonsets supplying the data
// we look at their watchdogs
this.podsDependingOnRepo = new Dictionary <string, List <PodDataRequestInfo> >();
foreach (var pod in pods)
{
foreach (var repo in pod.DependsOn)
{
if (!this.podsDependingOnRepo.TryGetValue(repo, out var list))
{
list = new List <PodDataRequestInfo>();
this.podsDependingOnRepo[repo] = list;
}
list.Add(pod);
}
}
}
public void RequestResetNoTimestampVersionToLKG()
{
var fakeClock = new ManualClock()
{
Time = DateTimeOffset.UtcNow + TimeSpan.FromDays(1),
};
var f = new DataRepositoryUpgradeOracleFactory(new LoggerFactory(), fakeClock, new WatchdogStatusAggregator());
var kg = new DataRepositoryKnownGoods(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("a", "a").ToBase64String(),
});
var lv = new DataRepositoryLatestVersionInfo(new Dictionary <string, string>()
{
["d"] = new DataDownloadRequestDetails("b", "b").ToBase64String(),
});
var badReq = new DataDownloadRequestDetails("b", "b")
{
UnixTimestampSeconds = null
};
var pods = new List <PodDataRequestInfo>()
{
new PodDataRequestInfo(
new PodIdentifier("ns", "name"),
new Dictionary <string, string>()
{
[$"{CommonAnnotations.DataSources}"] = "d",
[$"{CommonAnnotations.DataRequestPrefix}d"] = badReq.ToBase64String(),
}),
};
var o = f.Create(kg, lv, pods);
AssertSameData(kg.KnownGoodVersions["d"], o.GetDataRequest(pods[0].Id, "d"), $"Reset to LKG on bad timestamp");
}
public IDataRepositoryUpgradeOracle Create(DataRepositoryKnownGoods knownGoods, DataRepositoryLatestVersionInfo latestVersionInfo, IReadOnlyList <PodDataRequestInfo> pods)
{
return(new DataRepositoryUpgradeOracle(knownGoods, latestVersionInfo, pods, this.logger, this.clock, this.watchdogStatusAggregator));
}
