public void TablesUniquelyNamedOnlyWithinThread()
{
var uniqueIntegerList = new System.Collections.Concurrent.ConcurrentBag <int>();
var method = new ThreadStart(() =>
{
Table tbl1 = new Table();
Table tbl2 = new Table();
// Store these values for later comparison
uniqueIntegerList.Add(tbl1.UniqueInteger);
uniqueIntegerList.Add(tbl2.UniqueInteger);
// Ensure that within a thread we have unique integers
Assert.AreEqual(tbl1.UniqueInteger + 1, tbl2.UniqueInteger);
});
var thread1 = new CrossThreadTestRunner(method);
var thread2 = new CrossThreadTestRunner(method);
thread1.Start();
thread2.Start();
thread1.Join();
thread2.Join();
// There should in total be 4 tables, but only two distinct identifiers.
Assert.AreEqual(4, uniqueIntegerList.Count);
Assert.AreEqual(2, uniqueIntegerList.Distinct().Count());
}
//全ピクセルを詰め込んでから重複を取り除いたリストをカウントする
//遅い、メモリを大量に消費する
private int Count8Concurrent32bppとビットシフト(byte[] pixels)
{
var concurrent = new System.Collections.Concurrent.ConcurrentBag <uint>();
Parallel.For(0, pixels.Length / 4, i =>
{
concurrent.Add((uint)(pixels[i * 4] | (pixels[i * 4 + 1] << 8) | (pixels[i * 4 + 2] << 16) | (pixels[i * 4 + 3] << 24)));
});
return(concurrent.Distinct().ToArray().Length);
}
public override void Run(bool runChildren)
{
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink urlListCreate");
urlList = new System.Collections.Concurrent.ConcurrentBag<string>();
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink autoNextLink.Run");
autoNextLink.Run();
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink autoNextLink.Run End");
var page = this.GetUpperRawler<Page>();
if (page != null)
{
if (maxCount > count)
{
var url = page.GetCurrentUrl();
Uri url_uri = new Uri(url);
if (urlList.Any())
{
var test = urlList.Distinct().Where(n => new Uri(n).Host == url_uri.Host && urlHash.Contains(n) == false).ToList();
var nextUrl = urlList.Distinct().Where(n => new Uri(n).Host == url_uri.Host && urlHash.Contains(n) == false)
.Select(n => new { url = n, Distance = Rawler.NPL.LevenshteinDistance.Compute(url, n) })
.OrderBy(n => n.Distance);
if (nextUrl.Any())
{
page.PushUrl(nextUrl.First().url);
urlHash.Add(nextUrl.First().url);
count++;
}
urlHash.Add(url);
}
else
{
ReportManage.ErrReport(this, "NextLinkの取得がありません");
}
}
}
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink End");
base.Run(runChildren);
}
public override void Run(bool runChildren)
{
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink urlListCreate");
urlList = new System.Collections.Concurrent.ConcurrentBag <string>();
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink autoNextLink.Run");
autoNextLink.Run();
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink autoNextLink.Run End");
var page = this.GetUpperRawler <Page>();
if (page != null)
{
if (maxCount > count)
{
var url = page.GetCurrentUrl();
Uri url_uri = new Uri(url);
if (urlList.Any())
{
var test = urlList.Distinct().Where(n => new Uri(n).Host == url_uri.Host && urlHash.Contains(n) == false).ToList();
var nextUrl = urlList.Distinct().Where(n => new Uri(n).Host == url_uri.Host && urlHash.Contains(n) == false)
.Select(n => new { url = n, Distance = Rawler.NPL.LevenshteinDistance.Compute(url, n) })
.OrderBy(n => n.Distance);
if (nextUrl.Any())
{
page.PushUrl(nextUrl.First().url);
urlHash.Add(nextUrl.First().url);
count++;
}
urlHash.Add(url);
}
else
{
ReportManage.ErrReport(this, "NextLinkの取得がありません");
}
}
}
RawlerLib.Timer.StopWatch.Write("RawlerAutoNextLink End");
base.Run(runChildren);
}
/// <inheritdoc />
public async Task ScanTimePageForNewPlayersAsync(DateTime?stopAt)
{
var(validPlayers, bannedPlayers) = await GetPlayersAsync().ConfigureAwait(false);
// Takes GoldenEye as default date (older than Perfect Dark)
var allDatesToLoop = (stopAt ?? Game.GoldenEye.GetEliteFirstDate()).LoopBetweenDates(DateStep.Month).Reverse().ToList();
var playersToCreate = new System.Collections.Concurrent.ConcurrentBag <string>();
const int parallel = 8;
for (var i = 0; i < allDatesToLoop.Count; i += parallel)
{
await Task.WhenAll(allDatesToLoop.Skip(i).Take(parallel).Select(async loopDate =>
{
var results = await _siteParser
.ExtractTimeEntriesAsync(loopDate.Year, loopDate.Month, false)
.ConfigureAwait(false);
foreach (var entry in results)
{
if (!bannedPlayers.Any(p => p.UrlName.Equals(entry.PlayerUrlName, StringComparison.InvariantCultureIgnoreCase)) &&
!validPlayers.Any(p => p.UrlName.Equals(entry.PlayerUrlName, StringComparison.InvariantCultureIgnoreCase)))
{
playersToCreate.Add(entry.PlayerUrlName);
}
}
})).ConfigureAwait(false);
}
foreach (var pName in playersToCreate.Distinct())
{
await _writeRepository
.InsertPlayerAsync(pName, Player.DefaultPlayerHexColor)
.ConfigureAwait(false);
}
}
public async Task <IReadOnlyCollection <DownloadEntry> > GetallEntriesAsync()
{
ReportInformation("Loading BoerseUser");
BoerseUser boerseUser;
if (!TryLoadingBoerseUser(out boerseUser))
{
ReportInformation("Boerse-User is not configured! Please switch the the 'Config User' View.");
return(Enumerable.Empty <DownloadEntry>().ToList().AsReadOnly());
}
ReportInformation("Loading Configurations...");
var concurrentDownloadEntriesFromWebSite = new System.Collections.Concurrent.ConcurrentBag <DownloadEntry>();
List <DownloadEntry> allDownloadEntries = new List <DownloadEntry>();
_processingTimer.StartShowingProcess();
await Task.Run(() =>
{
var allDownloadContexts = _downloadContextConfigurationLogic.LoadAll();
Parallel.ForEach(allDownloadContexts, context =>
{
var downloadEntriesFromBoerseWebsite = DownloadEntries(context, boerseUser);
foreach (var de in downloadEntriesFromBoerseWebsite)
{
concurrentDownloadEntriesFromWebSite.Add(de);
}
});
allDownloadEntries = concurrentDownloadEntriesFromWebSite.Distinct(_downloadEntryEqualityComparer).ToList();
_processingTimer.WrapUpAndFinishProcess();
});
return(allDownloadEntries.AsReadOnly());
}
public void TablesUniquelyNamedOnlyWithinThread()
{
var uniqueIntegerList = new System.Collections.Concurrent.ConcurrentBag<int>();
var method = new ThreadStart(() =>
{
Table tbl1 = new Table();
Table tbl2 = new Table();
// Store these values for later comparison
uniqueIntegerList.Add(tbl1.UniqueInteger);
uniqueIntegerList.Add(tbl2.UniqueInteger);
// Ensure that within a thread we have unique integers
Assert.AreEqual(tbl1.UniqueInteger + 1, tbl2.UniqueInteger);
});
var thread1 = new CrossThreadTestRunner(method);
var thread2 = new CrossThreadTestRunner(method);
thread1.Start();
thread2.Start();
thread1.Join();
thread2.Join();
// There should in total be 4 tables, but only two distinct identifiers.
Assert.AreEqual(4, uniqueIntegerList.Count);
Assert.AreEqual(2, uniqueIntegerList.Distinct().Count());
}