private System.Collections.Generic.List <string> DeepSearchSitemapParallel(SiteMape sm, System.Collections.Generic.List <string> externalListLinks, System.Boolean flag = false, System.Int32 QUANTITY = 250)
{
//local list of links
System.Collections.Generic.List <string> innerList = new System.Collections.Generic.List <string>();
if (externalListLinks.Count > QUANTITY)
{
QueryLimiter(sm, externalListLinks.ToList <string>(), QUANTITY, innerList);
}
else
{
ExecuteLimitedQuerys(sm, externalListLinks, innerList);
}
listVisitedLinks.AddRange(externalListLinks);
listVisitedLinks = listVisitedLinks.Distinct().ToList <string>();
//delete rows that listVisitedLinks has
innerList = innerList.Except(listVisitedLinks).ToList <string>();
if (flag == false && innerList.Count > 0)
{
innerList.AddRange(DeepSearchSitemapParallel(sm, innerList, flag, QUANTITY).Except(innerList));
innerList = innerList.Except(listVisitedLinks).ToList <string>();
}
return(innerList);
}
private void ExecuteLimitedQuerys(SiteMape sm, System.Collections.Generic.List <string> externalListLinks, System.Collections.Generic.List <string> innerList)
{
//CreateTask(externalListLinks);
SaveListPageThreadPool(externalListLinks, sm);
System.Threading.Tasks.Parallel.For(0, externalListLinks.Count, (INDEX) =>
{
string address = externalListLinks[INDEX];
System.Collections.Generic.List <string> temp = Researching(address);
lock (objForLock)
{
innerList.AddRange(temp.Except(innerList));
}
});
}
private void SaveListPageThreadPool(List <string> list, SiteMape sm)
{
if (list.Count == 0)
{
return;
}
int count = 0;
System.Threading.Tasks.Parallel.ForEach(list, (page) => {
List <TimeSpan> lts = new List <TimeSpan>();
object objLock = new object();
System.Threading.Tasks.Parallel.For(0, this._amountOfTests, (x) =>
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Reset();
sw.Start();
RequestByUrl(page);
sw.Stop();
lock (objLock)
{
lts.Add(sw.Elapsed);
}
});
lock (this._threadLock)
{
count++;
this._db.SitePages.Add(new SitePage {
SiteMape = sm, NameSitePage = page, TimeMax = lts.Max(), TimeMin = lts.Min(), TimeAverage = new TimeSpan((long)lts.Average(x => x.Ticks)), Number = _counterPages++
});
if (_counterSave <= count)
{
this._db.SaveChanges();
count = 0;
}
}
});
lock (this._threadLock)
{
this._db.SaveChanges();
}
}
private void CreateSitemapeParallel(SiteAddress sa)
{
listVisitedLinks = new System.Collections.Generic.List <string>();
targetSiteObj = new TargetSITE(sa.UrlAddress);
SiteMape sm = new SiteMape {
SiteAddress = sa, NameSateMape = "AUTO_GENERATED"
};
this._db.SiteMapes.Add(sm);
this._db.SaveChanges();
DeepSearchSitemapParallel(sm, Researching(targetSiteObj.WholeName));
listVisitedLinks = null;
}
private int SearchSitePage(List <string> listSiteMap, SiteAddress sa)
{
List <TimeSpan> lts = new List <TimeSpan>();
List <string> l_s_m;//list site mape
string str = "";
object o = new object();
int countFoundPages = 0;
foreach (string xmlPage in listSiteMap)
{
System.Threading.Tasks.Parallel.For(0, this._amountOfTests, (x) =>
{
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
sw.Reset();
sw.Start();
str = RequestByUrl(xmlPage);
sw.Stop();
lock (o)
{
lts.Add(sw.Elapsed);
}
});
l_s_m = SearchPattern(str, _httpwebPage, 1);
if (l_s_m.Count == 0)
{
continue;
}
countFoundPages++;
SiteMape sm = new SiteMape {
SiteAddress = sa, NameSateMape = xmlPage, TimeMax = lts.Max(), TimeMin = lts.Min(), TimeAverage = new TimeSpan((long)lts.Average(x => x.Ticks))
};
this._db.SiteMapes.Add(sm);
this._db.SaveChanges();
SaveListPageThreadPool(l_s_m, sm);
}
return(countFoundPages);
}
private void QueryLimiter(SiteMape sm, System.Collections.Generic.List <string> externalListLinks, System.Int32 QUANTITY, System.Collections.Generic.List <string> innerList)
{
int amount = QUANTITY;
do
{
innerList.AddRange(DeepSearchSitemapParallel(sm, externalListLinks.GetRange(0, amount), flag: true));
innerList = innerList.Except(listVisitedLinks).ToList <string>();
externalListLinks = externalListLinks.Except(listVisitedLinks).ToList <string>();
if (amount > externalListLinks.Count)
{
amount = externalListLinks.Count;
if (amount <= 0)
{
break;
}
}
} while (true);
}
