public async Task VerifyStep123And4_MaxPages2(DownloadDepth depth)
{
_dd.Depth = depth;
_maxPages = 2;
CopyFilesToWebRoot(Step1);
CopyFilesToWebRoot(Step2a);
CopyFilesToWebRoot(Step2b);
CopyFilesToWebRoot(Step3);
CopyFilesToWebRoot(Step4);
await ExecuteAsync();
AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json");
AssertDownload(DownloadDepth.CatalogIndex, Step4, "catalog/index.json");
AssertDownload(DownloadDepth.CatalogPage, Step2a, "catalog/page0.json", "catalog/page0-page499/page0.json");
AssertDownload(DownloadDepth.CatalogPage, Step4, "catalog/page1.json", "catalog/page0-page499/page1.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step1, "catalog/2020.10.20.00.00.00/a.1.0.0.json", "catalog/2020/10/20/00/00.00/a.1.0.0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/a.2.0.0.json", "catalog/2020/10/21/00/00.00/a.2.0.0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/b.1.0.0.json", "catalog/2020/10/21/00/00.00/b.1.0.0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step3, "catalog/2020.10.22.00.00.00/b.2.0.0.json", "catalog/2020/10/22/00/00.00/b.2.0.0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step4, "catalog/2020.10.23.00.00.00/c.1.0.0.json", "catalog/2020/10/23/00/00.00/c.1.0.0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step4, "catalog/2020.10.24.00.00.00/c.2.0.0.json", "catalog/2020/10/24/00/00.00/c.2.0.0.json");
AssertRequestCount();
_dd.AssertDownloadCursor("catalog", "\"2020-10-24T00:00:00+00:00\"");
await ExecuteAsync();
AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json");
AssertDownload(DownloadDepth.CatalogIndex, Step4, "catalog/index.json");
AssertDownload(DownloadDepth.CatalogPage, Step4, "catalog/page2.json", "catalog/page0-page499/page2.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step4, "catalog/2020.10.25.00.00.00/c.3.0.0.json", "catalog/2020/10/25/00/00.00/c.3.0.0.json");
AssertRequestCount();
_dd.AssertDownloadCursor("catalog", "\"2020-10-25T00:00:00+00:00\"");
}
public async Task VerifyStep1And2a_ThenStep2bAnd3(DownloadDepth depth)
{
_dd.Depth = depth;
CopyFilesToWebRoot(Step1);
CopyFilesToWebRoot(Step2a);
await ExecuteAsync();
AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json");
AssertDownload(DownloadDepth.CatalogIndex, Step1, "catalog/index.json");
AssertDownload(DownloadDepth.CatalogPage, Step2a, "catalog/page0.json", "catalog/page0-page499/page0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step1, "catalog/2020.10.20.00.00.00/a.1.0.0.json", "catalog/2020/10/20/00/00.00/a.1.0.0.json");
AssertRequestCount();
_dd.AssertDownloadCursor("catalog", "\"2020-10-20T00:00:00+00:00\"");
CopyFilesToWebRoot(Step2b);
CopyFilesToWebRoot(Step3);
await ExecuteAsync();
AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json");
AssertDownload(DownloadDepth.CatalogIndex, Step3, "catalog/index.json");
AssertDownload(DownloadDepth.CatalogPage, Step2a, "catalog/page0.json", "catalog/page0-page499/page0.json");
AssertDownload(DownloadDepth.CatalogPage, Step3, "catalog/page1.json", "catalog/page0-page499/page1.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/a.2.0.0.json", "catalog/2020/10/21/00/00.00/a.2.0.0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/b.1.0.0.json", "catalog/2020/10/21/00/00.00/b.1.0.0.json");
AssertDownload(DownloadDepth.CatalogLeaf, Step3, "catalog/2020.10.22.00.00.00/b.2.0.0.json", "catalog/2020/10/22/00/00.00/b.2.0.0.json");
AssertRequestCount();
_dd.AssertDownloadCursor("catalog", "\"2020-10-22T00:00:00+00:00\"");
}
private void AssertDownload(DownloadDepth depth, string testDataDir, string requestPath, string filePath = null)
{
if (depth <= _dd.Depth)
{
Assert.Contains('/' + requestPath, _paths);
_expectedRequestCount++;
_dd.AssertTestData(testDataDir, requestPath, filePath);
}
}
public static async Task ExecuteAsync(
HttpClient httpClient,
string serviceIndexUrl,
string dataDir,
DownloadDepth depth,
JsonFormatting jsonFormatting,
int?maxPages,
int?maxCommits,
bool formatPaths,
int parallelDownloads,
IDepthLogger logger)
{
var cursorProvider = new CursorFactory(
cursorSuffix: $"download.{depth}",
defaultCursorValue: DateTimeOffset.MinValue,
logger: logger);
var downloader = new Downloader(
httpClient,
new DownloaderConfiguration
{
ServiceIndexUrl = serviceIndexUrl,
DataDirectory = dataDir,
Depth = depth,
JsonFormatting = jsonFormatting,
MaxPages = maxPages,
MaxCommits = maxCommits,
SaveToDisk = true,
FormatPaths = formatPaths,
ParallelDownloads = parallelDownloads,
},
cursorProvider,
NullVisitor.Instance,
logger);
await downloader.DownloadAsync();
}
public DataDirectoryHelper(string dataDir, DownloadDepth depth, string host)
{
_dataDir = dataDir;
_host = host;
Depth = depth;
}
