private async Task TraverseLinks(IList <string> links) { var storeTasks = new List <Task>(); var fetchTasks = new List <Task <(byte[] Document, string Path)> >(); foreach (var link in links) { fetchTasks.Add(_dataFetcher.Fetch(link)); _linkTracker.TryAdd(link, 0); } var fetchResults = await Task.WhenAll(fetchTasks); var foundLinks = new List <string>(); foreach (var item in fetchResults.Where(i => i.Path != _invalidResourceTag)) { foundLinks.AddRange(_linkParser.FindLinks(item.Document, item.Path)); storeTasks.Add(_fileHandler.Store(item.Path, item.Document)); } foundLinks = foundLinks.Except(_linkTracker.Keys).ToList(); if (foundLinks.Count > 0) { await TraverseLinks(foundLinks); } await Task.WhenAll(storeTasks); }
public async Task Save_VisitsAllValidLinks_WhenRootDocumentContainsLinks() { _dataFetcher.Fetch(Arg.Is("/")).Returns(Task.FromResult((new byte[0], ""))); _linkParser.FindLinks(Arg.Any <byte[]>(), Arg.Is("")).Returns(new string[] { "blog" }); await _sut.Save(); await _dataFetcher.Received().Fetch(Arg.Is("blog")); }
public Provider FindProviderById(string id) { var dtProvider = _dataFiller.Fetch("FindProviderById", new SqlParameter("@Id", id)); var provider = new Provider(); if (dtProvider.Rows.Count == 0) { return(null); } var rProvider = dtProvider.Rows[0] as DataRow; provider.Id = rProvider["Id"].ToString(); provider.ClientId = rProvider["ClientId"].ToString(); provider.ClientSecret = rProvider["ClientSecret"].ToString(); return(provider); }