private ScalarSegments CalculateParallelScalars(ScalarPlan plan, Transceiver[] transceivers) { var pm = new ParallelManager <Transceiver>(Mpi).WithTasks(transceivers); GreenScalars gs = null; ScalarSegments localSegments = null; pm.Run(tasks => { using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoACalc)) { gs = _scalarCalc.Calculate(plan, tasks, Profiler); } using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoASegments)) { localSegments = ScalarSegments.AllocateAndConvert(MemoryProvider, plan, gs); } }); using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoACommunicate)) { var allSegments1 = ScalarSegments.AllocateScalarSegments(MemoryProvider, plan, gs, transceivers.Length); var allSegments2 = ScalarSegments.AllocateScalarSegments(MemoryProvider, plan, gs, transceivers.Length); DistributeSegments(plan, pm, gs, localSegments, allSegments1, allSegments2); localSegments.Dispose(); allSegments1.Dispose(); return(allSegments2); } }
/// <summary> /// Enqueue a new Url to the Urls Queue /// Add to the UrlsSeen list /// </summary> /// <returns></returns> protected bool AddProcess(Url url) { string link = url.link; if (UrlsSeen.Any(u => u.link == link)) { _logger.LogInformation("Skipping...URL already queued", link); } else if (_config.UseWhiteList && !IsWhiteListedDomain(url.uri.Authority)) { _logger.LogInformation("URL domain not on whitelist", link); } else if (IsExcludedDomain(link)) { _logger.LogInformation("Skipping...URL domain is excluded", link); } else if (IsExcludedFileType(link)) { _logger.LogInformation("Skipping...file type is excluded", link); } else if (ShouldDownload(link)) { UrlsSeen.Add(url); Download(url); } else if (ShouldContinue(url.depth)) { UrlsSeen.Add(url); ParallelManager.Add(FetchNewPage, url); return(true); } return(false); }
public void Go() { List <string> urls = UseSitemap ? GetUrlsFromSitemap(SitemapURLs) : Urls; foreach (string seed in urls) { AddProcess(seed); } if (urls.Count == 0) { _logger.LogInformation("Need at least one seed URL."); } //wait for all threads ParallelManager.StartWait(); }
static void Main() { Operator A1 = new Operator(); TariffsBase tariffs = new TariffsBase(); tariffs.AddTariff(A1.MakeTariff("Comfort", 12.1)); tariffs.AddTariff(A1.MakeTariff("Lite", 1)); UsersBase usersBase = new UsersBase(); for (int i = 0; i < 10_000_000; i++) { usersBase.AddUser(A1.MakeUser($"{i}", tariffs.Tariffs["Lite"])); } var st = new Stopwatch(); //usersBase.AddUser(A1.MakeUser("Kostya", tariffs.Tariffs["Lite"])); NonParallelManager nonParallelManager = new NonParallelManager(); ParallelManager parallelManager = new ParallelManager(); st.Start(); Printer.Print($"{nonParallelManager.MonthlyIncome(usersBase.Base)}\n"); st.Stop(); Printer.Print($"NonParllel - {st.ElapsedMilliseconds}\n"); st.Reset(); st.Start(); Printer.Print($"{parallelManager.MonthlyIncome(usersBase.Base)}\n"); st.Stop(); Printer.Print($"Parallel - {st.ElapsedMilliseconds}\n"); }
private void DistributeSegments(ScalarPlan plan, ParallelManager <Transceiver> pm, GreenScalars gs, ScalarSegments local, ScalarSegments all, ScalarSegments result) { var starts = pm.GetAllStartIndecies(); var lengths = pm.GetAllLength(); var fullStarts = new int[lengths.Length]; var fullLength = new int[lengths.Length]; var localLength = gs.SingleScalars.Length; int nComp = gs.GetNumberOfAvailableIs(plan); int nRho = local.Radii.Length - 1; var localSize = localLength * nComp * 2 * nRho; for (int i = 0; i < starts.Length; i++) { fullStarts[i] = starts[i] * nComp * 2 * nRho; fullLength[i] = lengths[i] * nComp * 2 * nRho; } Mpi.AllGatherV(local.Ptr, localSize, all.Ptr, fullLength, fullStarts); using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoATrans)) { for (int i = 0; i < starts.Length; i++) { int start = starts[i]; int length = lengths[i]; var tmp = ScalarSegments.ReUseScalarSegments(all.Ptr + fullStarts[i], plan, gs, length); for (int j = 0; j < result.SingleSegment.Length; j++) { var src = tmp.SingleSegment[j]; var dst = result.SingleSegment[j]; if (src.I1A != null) { for (int k = 0; k < length; k++) { dst.I1A[k + start] = src.I1A[k]; } } if (src.I2A != null) { for (int k = 0; k < length; k++) { dst.I2A[k + start] = src.I2A[k]; } } if (src.I3A != null) { for (int k = 0; k < length; k++) { dst.I3A[k + start] = src.I3A[k]; } } if (src.I4A != null) { for (int k = 0; k < length; k++) { dst.I4A[k + start] = src.I4A[k]; } } if (src.I5A != null) { for (int k = 0; k < length; k++) { dst.I5A[k + start] = src.I5A[k]; } } if (src.I1B != null) { for (int k = 0; k < length; k++) { dst.I1B[k + start] = src.I1B[k]; } } if (src.I2B != null) { for (int k = 0; k < length; k++) { dst.I2B[k + start] = src.I2B[k]; } } if (src.I3B != null) { for (int k = 0; k < length; k++) { dst.I3B[k + start] = src.I3B[k]; } } if (src.I4B != null) { for (int k = 0; k < length; k++) { dst.I4B[k + start] = src.I4B[k]; } } if (src.I5B != null) { for (int k = 0; k < length; k++) { dst.I5B[k + start] = src.I5B[k]; } } } } } }
protected void FetchNewPage(Url url) { url.status = UrlStatus.Process; string link = url.uri.AbsoluteUri; if (ShouldReprocess(url)) { //put the url back to the queue ParallelManager.Retry(FetchNewPage, url); _logger.LogInformation("Queuing again this url, {0}", link); return; } using Page page = new Page(_logger, url, Connection.Read(url)); if (page.Source != null && page.Source.Length > 0) { _logger.LogInformation("Page load successful, {0}", link); T?item = default; try { item = Crawl(page); _logger.LogInformation("Finished crawling page. {0}", GetUrlState()); } catch (Exception ex)//parser error { _logger.LogDebug("Parser Error on this url, {0}", ex.Message); if (_config.SaveErrorItens) { FileUrlsLog.Write(page.Url); } if (OnSpiderError != null) { OnSpiderError(); } if (OnUrlProcessError != null) { OnUrlProcessError(url); } } finally { if (item != null) { ExtractData.Add(item); } ParallelManager.IncrementItensProcess(); url.status = UrlStatus.Completed; } } //if the page.source is null or empty this mean connection error else if (_config.TryAgainOnError) { url.status = UrlStatus.Retry; //put the url back to the queue ParallelManager.Retry(FetchNewPage, url); _logger.LogInformation("Queuing again this url, {0}", link); } else { ParallelManager.IncrementItensProcess(); url.status = UrlStatus.Completed; } }