private ScalarSegments CalculateParallelScalars(ScalarPlan plan, Transceiver[] transceivers)
{
var pm = new ParallelManager <Transceiver>(Mpi).WithTasks(transceivers);
GreenScalars gs = null;
ScalarSegments localSegments = null;
pm.Run(tasks =>
{
using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoACalc))
{
gs = _scalarCalc.Calculate(plan, tasks, Profiler);
}
using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoASegments))
{
localSegments = ScalarSegments.AllocateAndConvert(MemoryProvider, plan, gs);
}
});
using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoACommunicate))
{
var allSegments1 = ScalarSegments.AllocateScalarSegments(MemoryProvider, plan, gs, transceivers.Length);
var allSegments2 = ScalarSegments.AllocateScalarSegments(MemoryProvider, plan, gs, transceivers.Length);
DistributeSegments(plan, pm, gs, localSegments, allSegments1, allSegments2);
localSegments.Dispose();
allSegments1.Dispose();
return(allSegments2);
}
}
/// <summary>
/// Enqueue a new Url to the Urls Queue
/// Add to the UrlsSeen list
/// </summary>
/// <returns></returns>
protected bool AddProcess(Url url)
{
string link = url.link;
if (UrlsSeen.Any(u => u.link == link))
{
_logger.LogInformation("Skipping...URL already queued", link);
}
else if (_config.UseWhiteList && !IsWhiteListedDomain(url.uri.Authority))
{
_logger.LogInformation("URL domain not on whitelist", link);
}
else if (IsExcludedDomain(link))
{
_logger.LogInformation("Skipping...URL domain is excluded", link);
}
else if (IsExcludedFileType(link))
{
_logger.LogInformation("Skipping...file type is excluded", link);
}
else if (ShouldDownload(link))
{
UrlsSeen.Add(url);
Download(url);
}
else if (ShouldContinue(url.depth))
{
UrlsSeen.Add(url);
ParallelManager.Add(FetchNewPage, url);
return(true);
}
return(false);
}
public void Go()
{
List <string> urls = UseSitemap ? GetUrlsFromSitemap(SitemapURLs) : Urls;
foreach (string seed in urls)
{
AddProcess(seed);
}
if (urls.Count == 0)
{
_logger.LogInformation("Need at least one seed URL.");
}
//wait for all threads
ParallelManager.StartWait();
}
static void Main()
{
Operator A1 = new Operator();
TariffsBase tariffs = new TariffsBase();
tariffs.AddTariff(A1.MakeTariff("Comfort", 12.1));
tariffs.AddTariff(A1.MakeTariff("Lite", 1));
UsersBase usersBase = new UsersBase();
for (int i = 0; i < 10_000_000; i++)
{
usersBase.AddUser(A1.MakeUser($"{i}", tariffs.Tariffs["Lite"]));
}
var st = new Stopwatch();
//usersBase.AddUser(A1.MakeUser("Kostya", tariffs.Tariffs["Lite"]));
NonParallelManager nonParallelManager = new NonParallelManager();
ParallelManager parallelManager = new ParallelManager();
st.Start();
Printer.Print($"{nonParallelManager.MonthlyIncome(usersBase.Base)}\n");
st.Stop();
Printer.Print($"NonParllel - {st.ElapsedMilliseconds}\n");
st.Reset();
st.Start();
Printer.Print($"{parallelManager.MonthlyIncome(usersBase.Base)}\n");
st.Stop();
Printer.Print($"Parallel - {st.ElapsedMilliseconds}\n");
}
private void DistributeSegments(ScalarPlan plan, ParallelManager <Transceiver> pm, GreenScalars gs, ScalarSegments local, ScalarSegments all, ScalarSegments result)
{
var starts = pm.GetAllStartIndecies();
var lengths = pm.GetAllLength();
var fullStarts = new int[lengths.Length];
var fullLength = new int[lengths.Length];
var localLength = gs.SingleScalars.Length;
int nComp = gs.GetNumberOfAvailableIs(plan);
int nRho = local.Radii.Length - 1;
var localSize = localLength * nComp * 2 * nRho;
for (int i = 0; i < starts.Length; i++)
{
fullStarts[i] = starts[i] * nComp * 2 * nRho;
fullLength[i] = lengths[i] * nComp * 2 * nRho;
}
Mpi.AllGatherV(local.Ptr, localSize, all.Ptr, fullLength, fullStarts);
using (Profiler?.StartAuto(ProfilerEvent.GreenScalarAtoATrans))
{
for (int i = 0; i < starts.Length; i++)
{
int start = starts[i];
int length = lengths[i];
var tmp = ScalarSegments.ReUseScalarSegments(all.Ptr + fullStarts[i], plan, gs, length);
for (int j = 0; j < result.SingleSegment.Length; j++)
{
var src = tmp.SingleSegment[j];
var dst = result.SingleSegment[j];
if (src.I1A != null)
{
for (int k = 0; k < length; k++)
{
dst.I1A[k + start] = src.I1A[k];
}
}
if (src.I2A != null)
{
for (int k = 0; k < length; k++)
{
dst.I2A[k + start] = src.I2A[k];
}
}
if (src.I3A != null)
{
for (int k = 0; k < length; k++)
{
dst.I3A[k + start] = src.I3A[k];
}
}
if (src.I4A != null)
{
for (int k = 0; k < length; k++)
{
dst.I4A[k + start] = src.I4A[k];
}
}
if (src.I5A != null)
{
for (int k = 0; k < length; k++)
{
dst.I5A[k + start] = src.I5A[k];
}
}
if (src.I1B != null)
{
for (int k = 0; k < length; k++)
{
dst.I1B[k + start] = src.I1B[k];
}
}
if (src.I2B != null)
{
for (int k = 0; k < length; k++)
{
dst.I2B[k + start] = src.I2B[k];
}
}
if (src.I3B != null)
{
for (int k = 0; k < length; k++)
{
dst.I3B[k + start] = src.I3B[k];
}
}
if (src.I4B != null)
{
for (int k = 0; k < length; k++)
{
dst.I4B[k + start] = src.I4B[k];
}
}
if (src.I5B != null)
{
for (int k = 0; k < length; k++)
{
dst.I5B[k + start] = src.I5B[k];
}
}
}
}
}
}
protected void FetchNewPage(Url url)
{
url.status = UrlStatus.Process;
string link = url.uri.AbsoluteUri;
if (ShouldReprocess(url))
{
//put the url back to the queue
ParallelManager.Retry(FetchNewPage, url);
_logger.LogInformation("Queuing again this url, {0}", link);
return;
}
using Page page = new Page(_logger, url, Connection.Read(url));
if (page.Source != null && page.Source.Length > 0)
{
_logger.LogInformation("Page load successful, {0}", link);
T?item = default;
try
{
item = Crawl(page);
_logger.LogInformation("Finished crawling page. {0}", GetUrlState());
}
catch (Exception ex)//parser error
{
_logger.LogDebug("Parser Error on this url, {0}", ex.Message);
if (_config.SaveErrorItens)
{
FileUrlsLog.Write(page.Url);
}
if (OnSpiderError != null)
{
OnSpiderError();
}
if (OnUrlProcessError != null)
{
OnUrlProcessError(url);
}
}
finally
{
if (item != null)
{
ExtractData.Add(item);
}
ParallelManager.IncrementItensProcess();
url.status = UrlStatus.Completed;
}
}
//if the page.source is null or empty this mean connection error
else if (_config.TryAgainOnError)
{
url.status = UrlStatus.Retry;
//put the url back to the queue
ParallelManager.Retry(FetchNewPage, url);
_logger.LogInformation("Queuing again this url, {0}", link);
}
else
{
ParallelManager.IncrementItensProcess();
url.status = UrlStatus.Completed;
}
}
