public async Task VerifyStep123And4_MaxPages2(DownloadDepth depth) { _dd.Depth = depth; _maxPages = 2; CopyFilesToWebRoot(Step1); CopyFilesToWebRoot(Step2a); CopyFilesToWebRoot(Step2b); CopyFilesToWebRoot(Step3); CopyFilesToWebRoot(Step4); await ExecuteAsync(); AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json"); AssertDownload(DownloadDepth.CatalogIndex, Step4, "catalog/index.json"); AssertDownload(DownloadDepth.CatalogPage, Step2a, "catalog/page0.json", "catalog/page0-page499/page0.json"); AssertDownload(DownloadDepth.CatalogPage, Step4, "catalog/page1.json", "catalog/page0-page499/page1.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step1, "catalog/2020.10.20.00.00.00/a.1.0.0.json", "catalog/2020/10/20/00/00.00/a.1.0.0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/a.2.0.0.json", "catalog/2020/10/21/00/00.00/a.2.0.0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/b.1.0.0.json", "catalog/2020/10/21/00/00.00/b.1.0.0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step3, "catalog/2020.10.22.00.00.00/b.2.0.0.json", "catalog/2020/10/22/00/00.00/b.2.0.0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step4, "catalog/2020.10.23.00.00.00/c.1.0.0.json", "catalog/2020/10/23/00/00.00/c.1.0.0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step4, "catalog/2020.10.24.00.00.00/c.2.0.0.json", "catalog/2020/10/24/00/00.00/c.2.0.0.json"); AssertRequestCount(); _dd.AssertDownloadCursor("catalog", "\"2020-10-24T00:00:00+00:00\""); await ExecuteAsync(); AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json"); AssertDownload(DownloadDepth.CatalogIndex, Step4, "catalog/index.json"); AssertDownload(DownloadDepth.CatalogPage, Step4, "catalog/page2.json", "catalog/page0-page499/page2.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step4, "catalog/2020.10.25.00.00.00/c.3.0.0.json", "catalog/2020/10/25/00/00.00/c.3.0.0.json"); AssertRequestCount(); _dd.AssertDownloadCursor("catalog", "\"2020-10-25T00:00:00+00:00\""); }
public async Task VerifyStep1And2a_ThenStep2bAnd3(DownloadDepth depth) { _dd.Depth = depth; CopyFilesToWebRoot(Step1); CopyFilesToWebRoot(Step2a); await ExecuteAsync(); AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json"); AssertDownload(DownloadDepth.CatalogIndex, Step1, "catalog/index.json"); AssertDownload(DownloadDepth.CatalogPage, Step2a, "catalog/page0.json", "catalog/page0-page499/page0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step1, "catalog/2020.10.20.00.00.00/a.1.0.0.json", "catalog/2020/10/20/00/00.00/a.1.0.0.json"); AssertRequestCount(); _dd.AssertDownloadCursor("catalog", "\"2020-10-20T00:00:00+00:00\""); CopyFilesToWebRoot(Step2b); CopyFilesToWebRoot(Step3); await ExecuteAsync(); AssertDownload(DownloadDepth.ServiceIndex, Step1, "index.json"); AssertDownload(DownloadDepth.CatalogIndex, Step3, "catalog/index.json"); AssertDownload(DownloadDepth.CatalogPage, Step2a, "catalog/page0.json", "catalog/page0-page499/page0.json"); AssertDownload(DownloadDepth.CatalogPage, Step3, "catalog/page1.json", "catalog/page0-page499/page1.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/a.2.0.0.json", "catalog/2020/10/21/00/00.00/a.2.0.0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step2a, "catalog/2020.10.21.00.00.00/b.1.0.0.json", "catalog/2020/10/21/00/00.00/b.1.0.0.json"); AssertDownload(DownloadDepth.CatalogLeaf, Step3, "catalog/2020.10.22.00.00.00/b.2.0.0.json", "catalog/2020/10/22/00/00.00/b.2.0.0.json"); AssertRequestCount(); _dd.AssertDownloadCursor("catalog", "\"2020-10-22T00:00:00+00:00\""); }
private void AssertDownload(DownloadDepth depth, string testDataDir, string requestPath, string filePath = null) { if (depth <= _dd.Depth) { Assert.Contains('/' + requestPath, _paths); _expectedRequestCount++; _dd.AssertTestData(testDataDir, requestPath, filePath); } }
public static async Task ExecuteAsync( HttpClient httpClient, string serviceIndexUrl, string dataDir, DownloadDepth depth, JsonFormatting jsonFormatting, int?maxPages, int?maxCommits, bool formatPaths, int parallelDownloads, IDepthLogger logger) { var cursorProvider = new CursorFactory( cursorSuffix: $"download.{depth}", defaultCursorValue: DateTimeOffset.MinValue, logger: logger); var downloader = new Downloader( httpClient, new DownloaderConfiguration { ServiceIndexUrl = serviceIndexUrl, DataDirectory = dataDir, Depth = depth, JsonFormatting = jsonFormatting, MaxPages = maxPages, MaxCommits = maxCommits, SaveToDisk = true, FormatPaths = formatPaths, ParallelDownloads = parallelDownloads, }, cursorProvider, NullVisitor.Instance, logger); await downloader.DownloadAsync(); }
public DataDirectoryHelper(string dataDir, DownloadDepth depth, string host) { _dataDir = dataDir; _host = host; Depth = depth; }