public OneXBetParserHostedService(ILogService logger, INatsConnectionFactory connectionFactory,
IOptions <ParsingSettings> options, ThreadProvider threadProvider)
: base(logger, connectionFactory, threadProvider, By.Id("games_content"),
options.Value, options.Value.OneXBet)
{
_document = new HtmlDocument();
}
private List <CSVFile> ReadCSVFiles(GenerationOrLoad generationOrLoad, [NotNull] DirectoryInfo di)
{
Info("Reading " + di.FullName);
string filepattern = "*." + generationOrLoad + ".csv";
var csvfiles = di.GetFiles(filepattern);
if (csvfiles.Length == 0)
{
throw new FlaException("No exported files found in: " + di.FullName + " for file pattern " + filepattern);
}
var csvs = new List <CSVFile>();
var activeFiles = new List <CSVFile>();
foreach (var info in csvfiles)
{
var cs = new CSVFile(generationOrLoad, info.Name);
csvs.Add(cs);
cs.MyThread = ThreadProvider.Get().MakeThreadAndStart(() => { cs.Read(info); }, "CSVProfileVisualizer");
activeFiles.Add(cs);
WaitForJoining(activeFiles, 8);
}
WaitForJoining(activeFiles, 0);
if (csvs.Count != csvfiles.Length)
{
throw new FlaException("Missing files");
}
Info("Read " + csvs.Count + " for " + generationOrLoad);
return(csvs);
}
public ThreadClass([NotNull] string name, [NotNull] Action func, [NotNull] ILogger logger) : base(logger, Stage.Plotting, name)
{
Func = func;
var t = ThreadProvider.Get().MakeThreadAndStart(SaveExecute, Name, true);
Thread = t;
}
public Logger([CanBeNull] ITestOutputHelper unittestoutput, [NotNull] RunningConfig config)
{
config.CheckInitalisation();
_unittestoutput = unittestoutput;
_config = config;
_myThread = ThreadProvider.Get().MakeThreadAndStart(SafeRun, "Logger");
SLogger.Init(this);
}
protected BetsParserHostedService(ILogService logger, INatsConnectionFactory connectionFactory,
ThreadProvider threadProvider, By waitBy, ParsingSettings parsingSettings,
ParsingBookmakerSettings bkSettings)
{
_connectionFactory = connectionFactory;
_waitBy = waitBy;
ParsingSettings = parsingSettings;
_settings = bkSettings;
LogService = logger;
ThreadProvider = threadProvider;
}
protected override Task <List <BkGame> > FetchGames()
{
return(Task.Run(() =>
{
try
{
var webDriver = WebDriver.GetPage();
var jsResult =
webDriver.ExecuteJavaScript <IReadOnlyCollection <object> >(Js, GetSportTypeArg(), _groups);
var games = jsResult
.OfType <Dictionary <string, object> >()
.ToArray();
var foundGames = new List <BkGame>();
var waiters = GetWaiters(games.Length);
if (games.Length <= 0)
{
return foundGames;
}
for (var i = 0; i < games.Length; i++)
{
var node = games[i];
var waiter = waiters[i];
ThreadProvider.Run(() =>
{
var parsedGames = ParseGames(node);
if (parsedGames.Any())
{
lock (foundGames)
{
foundGames.AddRange(parsedGames);
}
}
}, () => waiter.Set());
}
WaitHandle.WaitAll(waiters, TimeSpan.FromMinutes(1));
return foundGames;
}
catch (Exception ex)
{
LogError(ex, "Parsing exception");
return new List <BkGame>();
}
}));
}
public ThreadClass([NotNull] string name, [NotNull] Action func, [NotNull] string dstFullFileName, [NotNull] ILogger logger) : base(
logger,
Stage.Plotting,
name)
{
if (File.Exists(dstFullFileName))
{
File.Delete(dstFullFileName);
}
Func = func;
var t = ThreadProvider.Get().MakeThreadAndStart(SaveExecute, name, true);
Thread = t;
DstFullFileName = dstFullFileName;
}
protected override Task <List <BkGame> > FetchGames()
{
return(Task.Run(() =>
{
var webDriver = WebDriver.GetPage();
var html = webDriver.ExecuteJavaScript <string>(
"return document.getElementById('games_content').innerHTML");
_document.LoadHtml(html);
var htmlNodes = _document.DocumentNode
.SelectNodes("//div[@data-name='dashboard-champ-content']")
.Where(node => _groups.Contains(node.SelectSingleNode("div/div[@class='c-events__name']").InnerText))
.ToArray();
var foundGames = new List <BkGame>();
var waiters = GetWaiters(htmlNodes.Length);
if (htmlNodes.Length <= 0)
{
return foundGames;
}
for (var i = 0; i < htmlNodes.Length; i++)
{
var node = htmlNodes[i];
var waiter = waiters[i];
ThreadProvider.Run(() =>
{
var games = ParseGames(node);
if (games.Any())
{
lock (foundGames)
{
foundGames.AddRange(games);
}
}
}, () => waiter.Set());
}
WaitHandle.WaitAll(waiters, 60000);
return foundGames;
}));
}
internal ThreadProviderFactory(IConfigService configService)
{
ThreadProvider threadProvider = configService.Get <ThreadProvider>("Threading", "Provider", ThreadProvider.ThreadPool);
switch (threadProvider)
{
case ThreadProvider.ThreadPool:
this.provider = ThreadPoolThreadProvider.Instance;
break;
case ThreadProvider.UserThread:
this.provider = UserThreadProvider.Instance;
break;
default:
case ThreadProvider.None:
throw new ArgumentOutOfRangeException("Threading.Provider", "Invalid Threading Provider configured");
}
}
// ReSharper disable once NotNullMemberIsNotInitialized
public ProsumerComponentResultArchiver(Stage myStage, [NotNull] ScenarioSliceParameters parameters,
HouseProcessor.ProcessingMode processingMode,
[NotNull] ServiceRepository services)
{
_myStage = myStage;
_services = services;
if (processingMode == HouseProcessor.ProcessingMode.Collecting)
{
_generationSa = SaveableEntry <Prosumer> .GetSaveableEntry(_services.SqlConnectionPreparer.GetDatabaseConnection(myStage, parameters, DatabaseCode.HouseProfiles),
SaveableEntryTableType.HouseGeneration,
_services.Logger);
_generationSa.MakeCleanTableForListOfFields(false);
_loadSa = SaveableEntry <Prosumer> .GetSaveableEntry(_services.SqlConnectionPreparer.GetDatabaseConnection(myStage, parameters, DatabaseCode.HouseProfiles),
SaveableEntryTableType.HouseLoad,
_services.Logger);
_loadSa.MakeCleanTableForListOfFields(false);
}
_myThread = ThreadProvider.Get().MakeThreadAndStart(SafeRun, "Archiver");
}
public void ThreadDemo(ICLIFlags toolFlags)
{
string basePath;
if (toolFlags is ExtractFlags flags)
{
basePath = flags.OutputPath;
}
else
{
throw new Exception("no output path");
}
const string container = "DebugThreadDemo";
string path = Path.Combine(basePath, container);
/*ThreadProvider provider = new ThreadProvider();
*
* for (int i = 0; i < 8; i++) {
* provider.AddTask(new FakeTaskA());
* }
*
* for (int i = 0; i < 8; i++) {
* provider.AddTask(new FakeTaskB());
* }
*
* provider.Run(4);
* provider.Wait();*/
ThreadProvider provider = new ThreadProvider();
const int max = 2000;
int i = 0;
HashSet <WorkTask> tasks = new HashSet <WorkTask>();
foreach (ulong key in TrackedFiles[0x4])
{
ulong dataKey = (key & 0xFFFFFFFFUL) | 0x100000000UL | 0x0320000000000000UL;
if (Files.ContainsKey(dataKey))
{
tasks.Add(new ConvertTextureTaskWithData(path, key, dataKey));
tasks.Add(new WriteOWMatTaskFake(path, i));
}
else
{
tasks.Add(new ConvertTextureTask(path, key));
tasks.Add(new WriteOWMatTaskFake(path, i));
}
if (i == max)
{
break;
}
i++;
}
foreach (WorkTask workTask in tasks)
{
provider.AddTask(workTask);
}
// without threads
// {
// foreach (WorkTask workTask in tasks) {
// workTask.DynamicallyAssigned = true;
// workTask.CASCStreams = new ConcurrentDictionary<ulong, Stream>();
// foreach (ulong requiredCASCFile in workTask.RequiredCASCFiles) {
// Stream stream = provider.GetStream(requiredCASCFile);
// workTask.CASCStreams[requiredCASCFile] = stream;
// }
// workTask.Run(null);
// }
// }
// return;
provider.Run(5);
provider.Wait();
}
#pragma warning disable xUnit1013 // Public method should be marked as test
public void RunBasedOnSettings([NotNull] RunningConfig settings, [NotNull] Logger logger)
#pragma warning restore xUnit1013 // Public method should be marked as test
{
Stopwatch sw = new Stopwatch();
sw.Start();
var container = CreateBuilderContainer(_output, logger, settings);
Dictionary <string, double> secondsPerStep = new Dictionary <string, double>();
using (var scope = container.BeginLifetimeScope()) {
var allItems = container.Resolve <IEnumerable <IBasicRunner> >().ToList();
logger.Info("#The following scenarios are going to be processed.", Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
if (settings.Slices == null)
{
throw new FlaException("slices was not initalized");
}
foreach (ScenarioSliceParameters scenarioSliceParameters in settings.Slices)
{
logger.Info("\t" + scenarioSliceParameters, Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
}
logger.Info("#The following stages are going to be processed.", Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
foreach (Stage stage in settings.StagesToExecute)
{
logger.Info("\t" + stage, Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
}
for (int stageIdx = 0; stageIdx < settings.StagesToExecute.Count; stageIdx++)
{
Stage currentStage = settings.StagesToExecute[stageIdx];
logger.Info("## Starting Stage " + currentStage, Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
var steps = allItems.Where(x => x.MyStage == currentStage).ToList();
steps.Sort((x, y) => x.SequenceNumber.CompareTo(y.SequenceNumber));
logger.Info("### The following steps in this stage are going to be processed.",
Stage.Preparation,
nameof(MainBurgdorfStatisticsCreator));
foreach (var step in steps)
{
logger.Info("\t" + step.SequenceNumber + " - " + step.Name, Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
}
CheckAndValidateSteps(steps, logger, currentStage, out var stepType);
switch (stepType)
{
case Steptype.Global: {
ProcessGlobalSteps(steps, secondsPerStep);
logger.SaveToDatabase(Constants.PresentSlice);
break;
}
case Steptype.SliceProcessors: {
ProcessSliceSteps(steps, logger, settings, secondsPerStep, currentStage);
break;
}
case Steptype.Scenario: {
List <RunnableForScenarioWithBenchmark> scenarioSteps = steps.Select(x => (RunnableForScenarioWithBenchmark)x).ToList();
var scenariosToProcess = settings.Slices.Select(x => x.DstScenario).Distinct().ToList();
if (scenariosToProcess.Contains(Scenario.Present()) && currentStage == Stage.ScenarioCreation)
{
scenariosToProcess.Remove(Scenario.Present());
}
foreach (var scenario in scenariosToProcess)
{
logger.Info("Processing " + scenario, Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
var slicesWithScenario = settings.Slices.Where(x => x.DstScenario == scenario).ToList();
foreach (var step in scenarioSteps)
{
Stopwatch stopwatch = Stopwatch.StartNew();
step.Run(settings.Slices.ToList());
logger.SaveToDatabase(slicesWithScenario.Last());
var stepname = currentStage + "-" + step.Name;
stopwatch.Stop();
if (!secondsPerStep.ContainsKey(stepname))
{
secondsPerStep.Add(stepname, 0);
}
secondsPerStep[stepname] += stopwatch.ElapsedMilliseconds / 1000.0;
}
logger.Info("Finished processing " + scenario, Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
}
}
break;
case Steptype.AllScenarios: {
List <RunnableForAllScenarioWithBenchmark> crossScenarioSteps =
steps.Select(x => (RunnableForAllScenarioWithBenchmark)x).ToList();
AnalysisRepository analysisRepository = new AnalysisRepository(settings);
foreach (var step in crossScenarioSteps)
{
Stopwatch stopwatch = Stopwatch.StartNew();
step.Run(settings.Slices.ToList(), analysisRepository);
logger.SaveToDatabase(Constants.PresentSlice);
var stepname = currentStage + "-" + step.Name;
stopwatch.Stop();
if (!secondsPerStep.ContainsKey(stepname))
{
secondsPerStep.Add(stepname, 0);
}
secondsPerStep[stepname] += stopwatch.ElapsedMilliseconds / 1000.0;
}
}
break;
default:
throw new Exception("unknown steptype");
}
logger.Info("Finished Stage " + currentStage + ", waiting for the charts to finish.",
Stage.Preparation,
nameof(MainBurgdorfStatisticsCreator));
var plotMaker = scope.Resolve <PlotMaker>();
plotMaker.Finish();
logger.Info("Finished Stage " + currentStage, Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
}
logger.SaveToDatabase(Constants.PresentSlice);
if (settings.MakeStageEntries)
{
PngSvgConverter psc = new PngSvgConverter(logger, Stage.Preparation);
psc.ConvertAllSVG(settings);
}
sw.Stop();
logger.Info("Finished everything after " + sw.Elapsed.TotalMinutes + " minutes",
Stage.Preparation,
nameof(MainBurgdorfStatisticsCreator));
foreach (var pair in secondsPerStep)
{
logger.Info(pair.Key + "\t" + pair.Value.ToString("F2"), Stage.Preparation, "Main");
}
logger.Info("Listing open threads before closing the logger", Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
ThreadProvider.Get().DisplayRunningThreads();
logger.Info("Listing open threads after closing the logger", Stage.Preparation, nameof(MainBurgdorfStatisticsCreator));
logger.FinishSavingEverything();
ThreadProvider.Get().DisplayRunningThreads();
}
container.Dispose();
}
private void InitializeProviders()
{
listThreads.BeginUpdate();
_threadP = new ThreadProvider(_pid);
Program.SecondaryProviderThread.Add(_threadP);
_threadP.Updated += this._threadP_Updated;
listThreads.Provider = _threadP;
listModules.BeginUpdate();
_moduleP = new ModuleProvider(_pid);
Program.SecondaryProviderThread.Add(_moduleP);
_moduleP.Updated += this._moduleP_Updated;
listModules.Provider = _moduleP;
listMemory.BeginUpdate();
_memoryP = new MemoryProvider(_pid);
Program.SecondaryProviderThread.Add(_memoryP);
_memoryP.IgnoreFreeRegions = true;
_memoryP.Updated += this._memoryP_Updated;
listMemory.Provider = _memoryP;
listHandles.BeginUpdate();
_handleP = new HandleProvider(_pid);
Program.SecondaryProviderThread.Add(_handleP);
_handleP.HideHandlesWithNoName = Settings.Instance.HideHandlesWithNoName;
_handleP.Updated += this._handleP_Updated;
listHandles.Provider = _handleP;
this.InitializeShortcuts();
}
private void InitializeProviders()
{
listThreads.BeginUpdate();
_threadP = new ThreadProvider(_pid);
Program.SecondarySharedThreadProvider.Add(_threadP);
_threadP.Interval = Properties.Settings.Default.RefreshInterval;
_threadP.Updated += new ThreadProvider.ProviderUpdateOnce(_threadP_Updated);
listThreads.Provider = _threadP;
listModules.BeginUpdate();
_moduleP = new ModuleProvider(_pid);
Program.SecondarySharedThreadProvider.Add(_moduleP);
_moduleP.Interval = Properties.Settings.Default.RefreshInterval;
_moduleP.Updated += new ModuleProvider.ProviderUpdateOnce(_moduleP_Updated);
listModules.Provider = _moduleP;
listMemory.BeginUpdate();
_memoryP = new MemoryProvider(_pid);
Program.SecondarySharedThreadProvider.Add(_memoryP);
_memoryP.IgnoreFreeRegions = true;
_memoryP.Interval = Properties.Settings.Default.RefreshInterval;
_memoryP.Updated += new MemoryProvider.ProviderUpdateOnce(_memoryP_Updated);
listMemory.Provider = _memoryP;
listHandles.BeginUpdate();
_handleP = new HandleProvider(_pid);
Program.SecondarySharedThreadProvider.Add(_handleP);
_handleP.HideHandlesWithNoName = Properties.Settings.Default.HideHandlesWithNoName;
_handleP.Interval = Properties.Settings.Default.RefreshInterval;
_handleP.Updated += new HandleProvider.ProviderUpdateOnce(_handleP_Updated);
listHandles.Provider = _handleP;
listThreads.List.SetTheme("explorer");
listModules.List.SetTheme("explorer");
listMemory.List.SetTheme("explorer");
listHandles.List.SetTheme("explorer");
this.InitializeShortcuts();
}
public WinlineParserHostedService(ILogService logger, INatsConnectionFactory connectionFactory,
IOptions <ParsingSettings> options, ThreadProvider threadProvider)
: base(logger, connectionFactory, threadProvider, By.Id("sticky-header-bottom-sticky-wrapper"),
options.Value, options.Value.Winline)
{
}
public ThreadProviderTests()
{
target = new ThreadProvider();
}
