//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testStatisticsRetriedFailedJobs() public virtual void testStatisticsRetriedFailedJobs() { // given Batch batch = helper.createMigrationBatchWithSize(3); // when helper.completeSeedJobs(batch); helper.failExecutionJobs(batch, 3); // then BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().singleResult(); assertEquals(3, batchStatistics.TotalJobs); assertEquals(3, batchStatistics.JobsCreated); assertEquals(3, batchStatistics.RemainingJobs); assertEquals(0, batchStatistics.CompletedJobs); assertEquals(3, batchStatistics.FailedJobs); // when helper.setRetries(batch, 3, 1); helper.completeJobs(batch, 3); // then batchStatistics = managementService.createBatchStatisticsQuery().singleResult(); assertEquals(3, batchStatistics.TotalJobs); assertEquals(3, batchStatistics.JobsCreated); assertEquals(0, batchStatistics.RemainingJobs); assertEquals(3, batchStatistics.CompletedJobs); assertEquals(0, batchStatistics.FailedJobs); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testBatchStatisticsQueryFilterWithoutTenantId() public virtual void testBatchStatisticsQueryFilterWithoutTenantId() { // when BatchStatistics returnedBatch = managementService.createBatchStatisticsQuery().withoutTenantId().singleResult(); // then Assert.assertNotNull(returnedBatch); Assert.assertEquals(sharedBatch.Id, returnedBatch.Id); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testBatchStatisticsQueryFilterByTenant() public virtual void testBatchStatisticsQueryFilterByTenant() { // when BatchStatistics returnedBatch = managementService.createBatchStatisticsQuery().tenantIdIn(TENANT_ONE).singleResult(); // then Assert.assertNotNull(returnedBatch); Assert.assertEquals(tenant1Batch.Id, returnedBatch.Id); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testQueryById() public virtual void testQueryById() { // given helper.createMigrationBatchWithSize(1); Batch batch = helper.createMigrationBatchWithSize(1); // when BatchStatistics statistics = managementService.createBatchStatisticsQuery().batchId(batch.Id).singleResult(); // then assertEquals(batch.Id, statistics.Id); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testStatisticsSuspend() public virtual void testStatisticsSuspend() { // given Batch batch = helper.migrateProcessInstancesAsync(1); // when managementService.suspendBatchById(batch.Id); // then BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().batchId(batch.Id).singleResult(); assertTrue(batchStatistics.Suspended); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testStatisticsNoExecutionJobsGenerated() public virtual void testStatisticsNoExecutionJobsGenerated() { // given helper.createMigrationBatchWithSize(3); // when BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().singleResult(); // then assertEquals(3, batchStatistics.TotalJobs); assertEquals(0, batchStatistics.JobsCreated); assertEquals(3, batchStatistics.RemainingJobs); assertEquals(0, batchStatistics.CompletedJobs); assertEquals(0, batchStatistics.FailedJobs); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testStatisticsWithDeletedJobs() public virtual void testStatisticsWithDeletedJobs() { // given Batch batch = helper.createMigrationBatchWithSize(13); // when helper.executeJob(helper.getSeedJob(batch)); deleteMigrationJobs(batch); // then BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().singleResult(); assertEquals(13, batchStatistics.TotalJobs); assertEquals(10, batchStatistics.JobsCreated); assertEquals(3, batchStatistics.RemainingJobs); assertEquals(10, batchStatistics.CompletedJobs); assertEquals(0, batchStatistics.FailedJobs); }
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes: //ORIGINAL LINE: @Test public void testBatchStatisticsAndCreateUserId() public virtual void testBatchStatisticsAndCreateUserId() { // given ProcessInstance pi = createMigrationPlan(); // when authRule.createGrantAuthorization(Resources.BATCH, "*", "userId", Permissions.CREATE); authRule.createGrantAuthorization(Resources.PROCESS_DEFINITION, "*", "userId", Permissions.MIGRATE_INSTANCE); authRule.enableAuthorization("userId"); batch3 = engineRule.RuntimeService.newMigration(migrationPlan).processInstanceIds(Arrays.asList(pi.Id)).executeAsync(); authRule.disableAuthorization(); // then BatchStatistics batchStatistics = engineRule.ManagementService.createBatchStatisticsQuery().batchId(batch3.Id).singleResult(); assertEquals("userId", batchStatistics.CreateUserId); }
public void ExecuteBatch <T>(IList <T> src, Action <T> action, string description = null, bool allowPartialBatchResumption = false, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1) { if (src.Count == 0) { return; } if (allowPartialBatchResumption && src.Count == 1) { var threadTask = new ThreadTask { Action = () => { action(src[0]); }, BatchStats = new BatchStatistics { Total = 1, Completed = 0 }, EarlyBreak = false, Description = new OperationDescription { From = 1, To = 1, Total = 1, PlainText = description } }; object _; try { threadTask.Action(); threadTask.BatchStats.Completed++; _runningTasks.TryAdd(threadTask, null); } catch (Exception e) { logger.ErrorException( string.Format( "Error occured while executing RavenThreadPool task; ThreadPool name :{0} ; Task queued at: {1} ; Task Description: {2} ", Name, threadTask.QueuedAt, threadTask.Description), e); } finally { _runningTasks.TryRemove(threadTask, out _); } return; } var now = DateTime.UtcNow; CountdownEvent lastEvent; var itemsCount = 0; var batch = new BatchStatistics { Total = src.Count, Completed = 0 }; if (_concurrentEvents.TryDequeue(out lastEvent) == false) { lastEvent = new CountdownEvent(src.Count); } else { lastEvent.Reset(src.Count); } for (; itemsCount < src.Count && _ct.IsCancellationRequested == false; itemsCount++) { var copy = itemsCount; var threadTask = new ThreadTask { Action = () => { try { action(src[copy]); } finally { lastEvent.Signal(); } }, Description = new OperationDescription { Type = OperationDescription.OpeartionType.Atomic, PlainText = description, From = itemsCount + 1, To = itemsCount + 1, Total = src.Count }, BatchStats = batch, EarlyBreak = allowPartialBatchResumption, QueuedAt = now, DoneEvent = lastEvent }; _tasks.Add(threadTask, _ct); } if (!allowPartialBatchResumption) { WaitForBatchToCompletion(lastEvent); return; } WaitForBatchAllowingPartialBatchResumption(lastEvent, batch, completedMultiplier, freeThreadsMultiplier, maxWaitMultiplier); }
public BatchStatistics CalculateBatchStatistics(BindingList<BatchIntervalMarked> batchIntervals, CaptureState captureState, BatchType batchType) { decimal batchIntervalsMean = 0; decimal batchIntervalsTrimmedMean = 0; // Trim zero packets from the batch interval BindingList<BatchIntervalMarked> batchIntervalsTrimmed = new BindingList<BatchIntervalMarked>(); foreach (BatchIntervalMarked bim in batchIntervals) { if (bim.PacketCount > AnalysisConfiguration.HistogramBinSize) { batchIntervalsTrimmed.Add(bim); } } // Calculate statistics for the batch BatchStatistics bs = new BatchStatistics(); if (AnalysisConfiguration.TrimSmallPackets) { BaseStatistics stats = new BaseStatistics(batchIntervalsTrimmed); bs.IntervalCountTrimmed = stats.Count; bs.PacketCountMaximum = stats.Maximum; bs.PacketCountMinimum = stats.Minimum; bs.PacketCountMean = stats.Mean; bs.PacketCountStandardDeviation = stats.StdDev; // Calculate both means for updating the capture batch intervals batchIntervalsTrimmedMean = stats.Mean; batchIntervalsMean = Convert.ToDecimal((from t in batchIntervals select t.PacketCount).Average()); // Get both counts for the batch bs.IntervalCountTrimmed = stats.Count < 0 ? 0 : stats.Count; bs.IntervalCount = batchIntervals.Count < 0 ? 0 : batchIntervals.Count; } else { BaseStatistics stats = new BaseStatistics(batchIntervals); bs.IntervalCount = stats.Count; bs.PacketCountMaximum = stats.Maximum; bs.PacketCountMinimum = stats.Minimum; bs.PacketCountMean = stats.Mean; bs.PacketCountStandardDeviation = stats.StdDev; // Calculate both means for updating the capture batch intervals batchIntervalsMean = bs.PacketCountMean; batchIntervalsTrimmedMean = Convert.ToDecimal((from t in batchIntervalsTrimmed select t.PacketCount).Average()); // Get both counts for the batch bs.IntervalCount = stats.Count < 0 ? 0 : stats.Count; bs.IntervalCountTrimmed = batchIntervalsTrimmed.Count < 0 ? 0 : batchIntervalsTrimmed.Count; } // Update the batch mean - only for single batches, not cumulative batches CurrentCaptureFile captureFile = new CurrentCaptureFile(); ProcessCapturePackets pcp = new ProcessCapturePackets(); captureFile = pcp.GetCurrentCaptureFile(_CaptureFileName); int captureBatchId = captureFile.CaptureBatchId; if (batchType == BatchType.Single && captureBatchId != 0) { try { //ProcessCapturePackets pcp = new ProcessCapturePackets(); //if (!pcp.UpdateBatchMean(Convert.ToInt32(captureBatchId), bs.PacketCountMean)) if (!pcp.UpdateBatchMean(Convert.ToInt32(captureBatchId), batchIntervalsMean, batchIntervalsTrimmedMean)) { throw new Exception("Error updating batch mean for CaptureBatchId " + captureBatchId); } } catch (Exception ex) { throw new Exception("Error updating batch mean for CaptureBatchId " + captureBatchId + ": " + ex.Message); } } // Save the statistics to the database for display on the Analysis tab (save to DisplayStatistics table) SaveDisplayStatistics(bs, captureBatchId, captureState, batchType, true); return bs; }
public void SaveDisplayStatistics(BatchStatistics batchStatistics, int captureBatchId, CaptureState captureState, BatchType batchType, bool saveData) { // Save the statistics to the database for display on the Analysis tab (save to DisplayStatistics table) // Replace cumulative data with new data DisplayStatisticsData dsd = new DisplayStatisticsData(); switch (captureState) { case CaptureState.Marked: switch (batchType) { case BatchType.Single: //dsd.DeleteSingleMarkedDisplayStatitics(); dsd.InsertSingleMarkedDisplayStatitics(batchStatistics); dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Single, true); break; case BatchType.Cumulative: dsd.DeleteCumulativeMarkedDisplayStatitics(); dsd.InsertCumulativeMarkedDisplayStatitics(batchStatistics); dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Cumulative, true); break; } break; case CaptureState.Unmarked: switch (batchType) { case BatchType.Single: //dsd.DeleteSingleUnmarkedDisplayStatitics(); dsd.InsertSingleUnarkedDisplayStatitics(batchStatistics); dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Single, true); break; case BatchType.Cumulative: dsd.DeleteCumulativeUnmarkedDisplayStatitics(); dsd.InsertCumulativeUnarkedDisplayStatitics(batchStatistics); dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Cumulative, true); break; } break; } }
public bool CalculateSingleBatchStatistics() { bool result = false; // Calculate single batch statistics //// Get the last marked and unmarked batches and add them to the graph //int lastBatchId = 0; ProcessCapturePackets pcp = new ProcessCapturePackets(); //lastBatchId = pcp.GetLastCaptureBatchId(); CurrentCaptureFile captureFile = new CurrentCaptureFile(); CaptureFileData cfd = new CaptureFileData(); //captureFile = cfd.GetLastCaptureBatchRecord(); captureFile = cfd.GetCurrentCaptureFile(_CaptureFileName); // Set the global variable _CaptureState = captureFile.CaptureState; BindingList<BatchIntervalMarked> batchIntervals = new BindingList<BatchIntervalMarked>(); // Calculate probabilities batchIntervals = pcp.GetMarkedBatchIntervals(captureFile.CaptureBatchId); int histogramBinSize = Convert.ToInt32(_HistogramBinSize); SortedDictionary<int, decimal> probabilities = new CalculateProbability(batchIntervals).GetProbabilityByPacketRange(_TrimZeroPacketIntervals, histogramBinSize); BatchStatistics markedSingleStats = new BatchStatistics(); BatchStatistics unmarkedSingleStats = new BatchStatistics(); // Add the results to the DisplayStatistics table DisplayStatisticsData dsd = new DisplayStatisticsData(); if(captureFile.CaptureState == CaptureState.Marked) { markedSingleStats = CalculateBatchStatistics(batchIntervals, CaptureState.Marked, BatchType.Single); } else { unmarkedSingleStats = CalculateBatchStatistics(batchIntervals, CaptureState.Unmarked, BatchType.Single); } return result; }
public bool CalculateCumulativeBatchStatistics() { bool result = false; // Get the cumulative interval counts ProcessCapturePackets pcp = new ProcessCapturePackets(); BindingList<CumulativeInterval> cumulativeIntervals = new BindingList<CumulativeInterval>(); cumulativeIntervals = pcp.GetCumulativeIntervals(); // Get the batch intervals BindingList<BatchIntervalMarked> unmarkedBatchIntervals = new BindingList<BatchIntervalMarked>(); BindingList<BatchIntervalMarked> markedBatchIntervals = new BindingList<BatchIntervalMarked>(); foreach (CumulativeInterval ci in cumulativeIntervals) { if (ci.Marked) { BatchIntervalMarked bim = new BatchIntervalMarked(); bim.BatchIntervalId = 0; bim.CaptureBatchId = 0; bim.IntervalNumber = ci.CumulativeIntervalNumber; bim.Marked = CaptureState.Marked; bim.PacketCount = ci.PacketCount; markedBatchIntervals.Add(bim); } else { BatchIntervalMarked bim = new BatchIntervalMarked(); bim.BatchIntervalId = 0; bim.CaptureBatchId = 0; bim.IntervalNumber = ci.CumulativeIntervalNumber; bim.Marked = CaptureState.Unmarked; bim.PacketCount = ci.PacketCount; unmarkedBatchIntervals.Add(bim); } } BatchStatistics markedCumulativeStats = new BatchStatistics(); BatchStatistics unmarkedCumulativeStats = new BatchStatistics(); //decimal markedMeanOfMeans = 0; //decimal markedStdDevMeanOfMeans = 0; //decimal unmarkedMeanOfMeans = 0; //decimal unmarkedStdDevMeanOfMeans = 0; /******************************************************************************************** * * Note: must have at least two samples to calculate mean of means and mean of means std dev... * * ******************************************************************************************/ /* TO DO: - need to retrieve std dev value for batch for cumulative stats when only one batch * has been processed. * - Have to calculate CumulativeStats first, then update MeanOfMeans and StdDev */ if (_CaptureState == CaptureState.Marked && markedBatchIntervals.Count > 0) { markedCumulativeStats = CalculateBatchStatistics(markedBatchIntervals, CaptureState.Marked, BatchType.Cumulative); } if (_CaptureState == CaptureState.Unmarked && unmarkedBatchIntervals.Count > 0) { unmarkedCumulativeStats = CalculateBatchStatistics(unmarkedBatchIntervals, CaptureState.Unmarked, BatchType.Cumulative); } if (pcp.GetMeanCount() > 1) { if (_CaptureState == CaptureState.Marked && markedBatchIntervals.Count > 0) { //markedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false); markedCumulativeStats.MeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false); //markedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false); markedCumulativeStats.MeanOfMeansStandardDeviation = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false); SaveDisplayStatistics(markedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Marked, BatchType.Cumulative, true); } if (_CaptureState == CaptureState.Unmarked && unmarkedBatchIntervals.Count > 0) { //unmarkedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false); unmarkedCumulativeStats.MeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false); //unmarkedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false); unmarkedCumulativeStats.MeanOfMeansStandardDeviation = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false); SaveDisplayStatistics(unmarkedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Unmarked, BatchType.Cumulative, true); } } else { // Only one batch - use the mean and standard deviation from the first batch if (_CaptureState == CaptureState.Marked && markedBatchIntervals.Count > 0) { //markedMeanOfMeans = pcp.GetMean(CaptureState.Marked, _TrimZeroPacketIntervals); //markedMeanOfMeans = markedCumulativeStats.PacketCountMean; markedCumulativeStats.MeanOfMeans = markedCumulativeStats.PacketCountMean; //markedStdDevMeanOfMeans = markedCumulativeStats.PacketCountStandardDeviation; markedCumulativeStats.MeanOfMeansStandardDeviation = markedCumulativeStats.PacketCountStandardDeviation; SaveDisplayStatistics(markedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Marked, BatchType.Cumulative, true); } if (_CaptureState == CaptureState.Unmarked && unmarkedBatchIntervals.Count > 0) { //unmarkedMeanOfMeans = pcp.GetMean(CaptureState.Unmarked, _TrimZeroPacketIntervals); //unmarkedMeanOfMeans = unmarkedCumulativeStats.PacketCountMean; unmarkedCumulativeStats.MeanOfMeans = unmarkedCumulativeStats.PacketCountMean; //unmarkedStdDevMeanOfMeans = unmarkedCumulativeStats.PacketCountStandardDeviation; unmarkedCumulativeStats.MeanOfMeansStandardDeviation = unmarkedCumulativeStats.PacketCountStandardDeviation; SaveDisplayStatistics(unmarkedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Unmarked, BatchType.Cumulative, true); } } // public bool GetHypothesisTestResult() // NOTE: use _HypothesisTest variable to determine which test result to return //if (markedBatchIntervals.Count > 0 && unmarkedBatchIntervals.Count > 0) //{ // // Cumulative variance column //} //// Update the K-S statistics object - for display //_KsStatistics.MarkedMean = markedMeanOfMeans; //_KsStatistics.MarkedStdDev = markedStdDevMeanOfMeans; ////_KsStatistics.MarkedMean = markedCumulativeStats.PacketCountMean; ////_KsStatistics.MarkedStdDev = markedCumulativeStats.PacketCountStandardDeviation; //_KsStatistics.MarkedIntervalCount = TrimIntervalsCheckBox.Checked == true ? markedCumulativeStats.IntervalCountTrimmed : markedCumulativeStats.IntervalCount; //_KsStatistics.UnmarkedMean = unmarkedMeanOfMeans; //_KsStatistics.UnmarkedStdDev = unmarkedStdDevMeanOfMeans; ////_KsStatistics.UnmarkedMean = unmarkedCumulativeStats.PacketCountMean; ////_KsStatistics.UnmarkedStdDev = unmarkedCumulativeStats.PacketCountStandardDeviation; //_KsStatistics.UnmarkedIntervalCount = TrimIntervalsCheckBox.Checked == true ? unmarkedCumulativeStats.IntervalCountTrimmed : unmarkedCumulativeStats.IntervalCount; return result; }
private void RefreshSingleBatchStatistics() { // Get the last marked and unmarked batches BindingList<CurrentCaptureFile> lastBatchIds = new BindingList<CurrentCaptureFile>(); ProcessCapturePackets pcp = new ProcessCapturePackets(); lastBatchIds = pcp.GetLastCaptureBatchIds(); // Get the batch intervals BindingList<BatchIntervalMarked> unmarkedBatchIntervals = new BindingList<BatchIntervalMarked>(); BindingList<BatchIntervalMarked> markedBatchIntervals = new BindingList<BatchIntervalMarked>(); foreach (CurrentCaptureFile file in lastBatchIds) { if (file.CaptureState == CaptureState.Marked) { markedBatchIntervals = pcp.GetMarkedBatchIntervals(file.CaptureBatchId); } else if(file.CaptureState == CaptureState.Unmarked) { unmarkedBatchIntervals = pcp.GetMarkedBatchIntervals(file.CaptureBatchId); } else { MessageBox.Show("Error retrieving batch intervals: capture state is unknown!", "GetMarkedBatchIntervals by CaptureBatchId", MessageBoxButtons.OK, MessageBoxIcon.Error); } } BatchStatistics markedSingleStats = new BatchStatistics(); BatchStatistics unmarkedSingleStats = new BatchStatistics(); // Get this data from DisplayStatistics table; except on refresh... //AnalysisEngine ae = new AnalysisEngine(); //markedSingleStats = ae.GetBatchStatistics(markedBatchIntervals, CaptureState.Marked, BatchType.Single); //unmarkedSingleStats = ae.GetBatchStatistics(unmarkedBatchIntervals, CaptureState.Unmarked, BatchType.Single); // Load up the table // Single unmarked column int row = 0; _AnalysisDataGridView.Rows[row++].Cells[1].Value = unmarkedSingleStats.IntervalCount; _AnalysisDataGridView.Rows[row++].Cells[1].Value = TrimIntervals == true ? unmarkedSingleStats.IntervalCountTrimmed.ToString() : "N/A"; _AnalysisDataGridView.Rows[row++].Cells[1].Value = string.Format("{0:N2}", unmarkedSingleStats.PacketCountMean); _AnalysisDataGridView.Rows[row++].Cells[1].Value = string.Format("{0:N2}", unmarkedSingleStats.PacketCountStandardDeviation); _AnalysisDataGridView.Rows[row++].Cells[1].Value = unmarkedSingleStats.PacketCountMinimum; _AnalysisDataGridView.Rows[row++].Cells[1].Value = unmarkedSingleStats.PacketCountMaximum; _AnalysisDataGridView.Rows[row++].Cells[1].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[1].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[1].Value = "N/A"; // Single marked column row = 0; _AnalysisDataGridView.Rows[row++].Cells[2].Value = markedSingleStats.IntervalCount; _AnalysisDataGridView.Rows[row++].Cells[2].Value = TrimIntervals == true ? markedSingleStats.IntervalCountTrimmed.ToString() : "N/A"; _AnalysisDataGridView.Rows[row++].Cells[2].Value = string.Format("{0:N2}", markedSingleStats.PacketCountMean); _AnalysisDataGridView.Rows[row++].Cells[2].Value = string.Format("{0:N2}", markedSingleStats.PacketCountStandardDeviation); _AnalysisDataGridView.Rows[row++].Cells[2].Value = markedSingleStats.PacketCountMinimum; _AnalysisDataGridView.Rows[row++].Cells[2].Value = markedSingleStats.PacketCountMaximum; _AnalysisDataGridView.Rows[row++].Cells[2].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[2].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[2].Value = "N/A"; // Single variance column row = 0; _AnalysisDataGridView.Rows[row++].Cells[3].Value = unmarkedSingleStats.IntervalCount - markedSingleStats.IntervalCount; _AnalysisDataGridView.Rows[row++].Cells[3].Value = TrimIntervals == true ? (unmarkedSingleStats.IntervalCountTrimmed - markedSingleStats.IntervalCountTrimmed).ToString() : "N/A"; _AnalysisDataGridView.Rows[row++].Cells[3].Value = string.Format("{0:N2}", (unmarkedSingleStats.PacketCountMean - markedSingleStats.PacketCountMean)); _AnalysisDataGridView.Rows[row++].Cells[3].Value = string.Format("{0:N2}",(unmarkedSingleStats.PacketCountStandardDeviation - markedSingleStats.PacketCountStandardDeviation)); _AnalysisDataGridView.Rows[row++].Cells[3].Value = unmarkedSingleStats.PacketCountMinimum - markedSingleStats.PacketCountMinimum; _AnalysisDataGridView.Rows[row++].Cells[3].Value = unmarkedSingleStats.PacketCountMaximum - markedSingleStats.PacketCountMaximum; _AnalysisDataGridView.Rows[row++].Cells[3].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[3].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[3].Value = "N/A"; }
private bool WaitForBatchAllowingPartialBatchResumption(CountdownEvent completionEvent, BatchStatistics batch, List <ThreadTask> childTasks, DocumentDatabase database = null) { Interlocked.Increment(ref _hasPartialBatchResumption); var cancellationToken = database?.WorkContext.CancellationToken ?? _ct; try { var waitHandles = new[] { completionEvent.WaitHandle, _threadHasNoWorkToDo }; var sp = Stopwatch.StartNew(); var batchRanToCompletion = false; var lastThreadIndexChecked = 0; while (cancellationToken.IsCancellationRequested == false) { // First, try to find work to do among child tasks, instead of just waiting ThreadTask busyWaitWorkToDo = null; for (; lastThreadIndexChecked < childTasks.Count && completionEvent.IsSet == false && _ct.IsCancellationRequested == false; lastThreadIndexChecked++) { var task = childTasks[lastThreadIndexChecked]; if (Interlocked.CompareExchange(ref task.Proccessing, 1, 0) == 1) { continue; } busyWaitWorkToDo = task; break; } int returnedWaitHandleIndex; // Run found work or just wait for all work to finish (returnedWaitHandleIndex=0) or to be notified about free threads in the system (returnedWaitHandleIndex=1) // After which we'll decide whether we should early exit, or to wait some more if (busyWaitWorkToDo != null) { RunThreadTask(busyWaitWorkToDo); returnedWaitHandleIndex = WaitHandle.WaitAny(waitHandles, 0); } else { returnedWaitHandleIndex = WaitHandle.WaitAny(waitHandles); } if (returnedWaitHandleIndex == 0) { break; } // we won't consider breaking early if we haven't completed at least half the work if (Thread.VolatileRead(ref batch.Completed) < batch.Total / 2) { continue; } var currentFreeThreads = _freedThreadsValue.Values.Count(isFree => isFree); // we will break early only if there are more then half free threads if (currentFreeThreads > _currentWorkingThreadsAmount / 2) { break; } } // after we've decided to quit early, we will wait some more time, allowing a normal wait. // we decide how much we will wait by choosing the biggest among the next: // 1) half the time we've waited on the current batch // 2) a waiting time factor that increase for every second left early batch and decreases for every leave early batch that completed before leaving early var elapsedMilliseconds = (int)sp.ElapsedMilliseconds; if (completionEvent.Wait(Math.Max(elapsedMilliseconds / 2, Thread.VolatileRead(ref _partialMaxWait)), cancellationToken)) { _concurrentEvents.Enqueue(completionEvent); } // updating the waiting time factor if (batch.Completed != batch.Total) { if (_partialMaxWaitChangeFlag > 0) { Interlocked.Exchange(ref _partialMaxWait, Math.Min(2500, (int)(Thread.VolatileRead(ref _partialMaxWait) * 1.25))); } } else if (_partialMaxWaitChangeFlag < 0) { batchRanToCompletion = true; Interlocked.Exchange(ref _partialMaxWait, Math.Max(Thread.VolatileRead(ref _partialMaxWait) / 2, 10)); } Interlocked.Exchange(ref _partialMaxWaitChangeFlag, Thread.VolatileRead(ref _partialMaxWaitChangeFlag) * -1); // completionEvent is explicitly left to the finalizer // we expect this to be rare, and it isn't worth the trouble of trying // to manage this if (logger.IsDebugEnabled) { logger.Debug($"Raven Thread Pool ended batch for database {database?.Name}. Done {batch.Completed} items out of {batch.Total}. Batch {(batchRanToCompletion ? "was not" : "was")} left early"); } return(batchRanToCompletion); } finally { Interlocked.Decrement(ref _hasPartialBatchResumption); } }
public bool ExecuteBatch <T>(IList <T> src, Action <T> action, DocumentDatabase database = null, string description = null, bool allowPartialBatchResumption = false, Action runAfterCompletion = null) { //, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1 switch (src.Count) { case 0: return(true); case 1: //if we have only one source to go through, //we should execute it in the current thread, without using RTP threads ExecuteSingleBatchSynchronously(src, action, description, database); runAfterCompletion?.Invoke(); return(true); } var now = DateTime.UtcNow; CountdownEvent countdownEvent; var itemsCount = 0; var batch = new BatchStatistics { Total = src.Count, Completed = 0 }; if (_concurrentEvents.TryDequeue(out countdownEvent) == false) { countdownEvent = new CountdownEvent(src.Count); } else { countdownEvent.Reset(src.Count); } var exceptions = new ConcurrentSet <Exception>(); var currentBatchTasks = new List <ThreadTask>(src.Count); for (; itemsCount < src.Count && _ct.IsCancellationRequested == false; itemsCount++) { var copy = itemsCount; var threadTask = new ThreadTask { Action = () => { try { database?.WorkContext.CancellationToken.ThrowIfCancellationRequested(); action(src[copy]); } catch (Exception e) { exceptions.Add(e); throw; } finally { countdownEvent.Signal(); } }, Description = new OperationDescription { Type = OperationDescription.OperationType.Atomic, PlainText = description, From = itemsCount + 1, To = itemsCount + 1, Total = src.Count }, Database = database, BatchStats = batch, EarlyBreak = allowPartialBatchResumption, QueuedAt = now, DoneEvent = countdownEvent, Proccessing = 0, RunAfterCompletion = runAfterCompletion }; _tasks.Add(threadTask); currentBatchTasks.Add(threadTask); } var ranToCompletion = false; if (allowPartialBatchResumption == false) { WaitForBatchToCompletion(countdownEvent, currentBatchTasks, database?.WorkContext.CancellationToken); ranToCompletion = true; } else { ranToCompletion = WaitForBatchAllowingPartialBatchResumption(countdownEvent, batch, currentBatchTasks, database); } switch (exceptions.Count) { case 0: return(ranToCompletion); case 1: ExceptionDispatchInfo.Capture(exceptions.First()).Throw(); return(ranToCompletion); // won't happen default: throw new AggregateException(exceptions); } }
public void ExecuteBatch <T>(IList <T> src, Action <T> action, string description = null, bool allowPartialBatchResumption = false, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1) { switch (src.Count) { case 0: return; case 1: //if we have only one source to go through, //we should execute it in the current thread, without using RTP threads ExecuteSingleBatchSynchronously(src, action, description); return; } var now = DateTime.UtcNow; CountdownEvent lastEvent; var itemsCount = 0; var batch = new BatchStatistics { Total = src.Count, Completed = 0 }; if (_concurrentEvents.TryDequeue(out lastEvent) == false) { lastEvent = new CountdownEvent(src.Count); } else { lastEvent.Reset(src.Count); } var exceptions = new ConcurrentSet <Exception>(); for (; itemsCount < src.Count && _ct.IsCancellationRequested == false; itemsCount++) { var copy = itemsCount; var threadTask = new ThreadTask { Action = () => { try { action(src[copy]); } catch (Exception e) { exceptions.Add(e); throw; } finally { lastEvent.Signal(); } }, Description = new OperationDescription { Type = OperationDescription.OperationType.Atomic, PlainText = description, From = itemsCount + 1, To = itemsCount + 1, Total = src.Count }, BatchStats = batch, EarlyBreak = allowPartialBatchResumption, QueuedAt = now, DoneEvent = lastEvent }; _tasks.Add(threadTask); } if (allowPartialBatchResumption == false) { WaitForBatchToCompletion(lastEvent); } else { WaitForBatchAllowingPartialBatchResumption(lastEvent, batch, completedMultiplier, freeThreadsMultiplier, maxWaitMultiplier); } switch (exceptions.Count) { case 0: return; case 1: ExceptionDispatchInfo.Capture(exceptions.First()).Throw(); break; default: throw new AggregateException(exceptions); } }
public void InsertSingleUnarkedDisplayStatitics(BatchStatistics batchStatistics) { using (var context = new PacketAnalysisEntity()) { context.DisplayStatisticsInsert( batchStatistics.IntervalCount, batchStatistics.IntervalCountTrimmed, batchStatistics.PacketCountMean, batchStatistics.PacketCountStandardDeviation, batchStatistics.PacketCountMinimum, batchStatistics.PacketCountMaximum, batchStatistics.MeanOfMeans, batchStatistics.MeanOfMeansStandardDeviation, false, // Marked Convert.ToInt32(BatchType.Single) ); context.SaveChanges(); } }
private void WaitForBatchAllowingPartialBatchResumption(CountdownEvent completionEvent, BatchStatistics batch, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1) { Interlocked.Increment(ref _hasPartialBatchResumption); try { var waitHandles = new[] { completionEvent.WaitHandle, _threadHasNoWorkToDo }; var sp = Stopwatch.StartNew(); var batchLeftEarly = false; while (_ct.IsCancellationRequested == false) { var i = WaitHandle.WaitAny(waitHandles); if (i == 0) { break; } if (Thread.VolatileRead(ref batch.Completed) < batch.Total / completedMultiplier) { continue; } var currentFreeThreads = _freedThreadsValue.Values.Count(isFree => isFree); if (currentFreeThreads > _currentWorkingThreadsAmount / freeThreadsMultiplier) { break; } } var elapsedMilliseconds = (int)sp.ElapsedMilliseconds; if (completionEvent.Wait(Math.Max(elapsedMilliseconds / 2, Thread.VolatileRead(ref _partialMaxWait)) / maxWaitMultiplier, _ct)) { _concurrentEvents.Enqueue(completionEvent); } if (batch.Completed != batch.Total) { batchLeftEarly = true; if (_partialMaxWaitChangeFlag > 0) { Interlocked.Exchange(ref _partialMaxWait, Math.Min(2500, (int)(Thread.VolatileRead(ref _partialMaxWait) * 1.25))); } } else if (_partialMaxWaitChangeFlag < 0) { Interlocked.Exchange(ref _partialMaxWait, Math.Max(Thread.VolatileRead(ref _partialMaxWait) / 2, 10)); } Interlocked.Exchange(ref _partialMaxWaitChangeFlag, Thread.VolatileRead(ref _partialMaxWaitChangeFlag) * -1); // completionEvent is explicitly left to the finalizer // we expect this to be rare, and it isn't worth the trouble of trying // to manage this if (logger.IsDebugEnabled) { logger.Info("Raven Thread Pool named {0} ended batch. Done {1} items out of {2}. Batch {3} left early", Name, batch.Completed, batch.Total, batchLeftEarly ? "was" : "was not"); } } finally { Interlocked.Decrement(ref _hasPartialBatchResumption); } }
private void RefreshCumulativeBatchStatistics() { // Get the cumulative interval counts ProcessCapturePackets pcp = new ProcessCapturePackets(); BindingList<CumulativeInterval> cumulativeIntervals = new BindingList<CumulativeInterval>(); cumulativeIntervals = pcp.GetCumulativeIntervals(); // Get the batch intervals BindingList<BatchIntervalMarked> unmarkedBatchIntervals = new BindingList<BatchIntervalMarked>(); BindingList<BatchIntervalMarked> markedBatchIntervals = new BindingList<BatchIntervalMarked>(); foreach (CumulativeInterval ci in cumulativeIntervals) { if (ci.Marked) { BatchIntervalMarked bim = new BatchIntervalMarked(); bim.BatchIntervalId = 0; bim.CaptureBatchId = 0; bim.IntervalNumber = ci.CumulativeIntervalNumber; bim.Marked = CaptureState.Marked; bim.PacketCount = ci.PacketCount; markedBatchIntervals.Add(bim); } else { BatchIntervalMarked bim = new BatchIntervalMarked(); bim.BatchIntervalId = 0; bim.CaptureBatchId = 0; bim.IntervalNumber = ci.CumulativeIntervalNumber; bim.Marked = CaptureState.Unmarked; bim.PacketCount = ci.PacketCount; unmarkedBatchIntervals.Add(bim); } } BatchStatistics markedCumulativeStats = new BatchStatistics(); BatchStatistics unmarkedCumulativeStats = new BatchStatistics(); decimal markedMeanOfMeans = 0; decimal markedStdDevMeanOfMeans = 0; decimal unmarkedMeanOfMeans = 0; decimal unmarkedStdDevMeanOfMeans = 0; AnalysisEngine ae = new AnalysisEngine(); if(markedBatchIntervals.Count > 0) { markedCumulativeStats = ae.CalculateBatchStatistics(markedBatchIntervals,CaptureState.Marked, BatchType.Cumulative); markedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Marked, TrimIntervals ? true : false); markedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Marked, TrimIntervals ? true : false); // Load up the table // Cumulative marked column int row = 0; _AnalysisDataGridView.Rows[row++].Cells[5].Value = markedCumulativeStats.IntervalCount; _AnalysisDataGridView.Rows[row++].Cells[5].Value = TrimIntervals == true ? markedCumulativeStats.IntervalCountTrimmed.ToString() : "N/A"; _AnalysisDataGridView.Rows[row++].Cells[5].Value = string.Format("{0:N2}", markedCumulativeStats.PacketCountMean); _AnalysisDataGridView.Rows[row++].Cells[5].Value = string.Format("{0:N2}", markedCumulativeStats.PacketCountStandardDeviation); _AnalysisDataGridView.Rows[row++].Cells[5].Value = markedCumulativeStats.PacketCountMinimum; _AnalysisDataGridView.Rows[row++].Cells[5].Value = markedCumulativeStats.PacketCountMaximum; _AnalysisDataGridView.Rows[row++].Cells[5].Value = string.Format("{0:N2}", markedMeanOfMeans); _AnalysisDataGridView.Rows[row++].Cells[5].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[5].Value = "N/A"; } if (unmarkedBatchIntervals.Count > 0) { unmarkedCumulativeStats = ae.CalculateBatchStatistics(unmarkedBatchIntervals, CaptureState.Marked, BatchType.Cumulative); unmarkedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Unmarked, TrimIntervals ? true : false); unmarkedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Unmarked, TrimIntervals ? true : false); // Load up the table // Cumulative unmarked column int row = 0; _AnalysisDataGridView.Rows[row++].Cells[4].Value = unmarkedCumulativeStats.IntervalCount; _AnalysisDataGridView.Rows[row++].Cells[4].Value = TrimIntervals == true ? unmarkedCumulativeStats.IntervalCountTrimmed.ToString() : "N/A"; _AnalysisDataGridView.Rows[row++].Cells[4].Value = string.Format("{0:N2}", unmarkedCumulativeStats.PacketCountMean); _AnalysisDataGridView.Rows[row++].Cells[4].Value = string.Format("{0:N2}", unmarkedCumulativeStats.PacketCountStandardDeviation); _AnalysisDataGridView.Rows[row++].Cells[4].Value = unmarkedCumulativeStats.PacketCountMinimum; _AnalysisDataGridView.Rows[row++].Cells[4].Value = unmarkedCumulativeStats.PacketCountMaximum; _AnalysisDataGridView.Rows[row++].Cells[4].Value = string.Format("{0:N2}", unmarkedMeanOfMeans); _AnalysisDataGridView.Rows[row++].Cells[4].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[4].Value = "N/A"; } if (markedBatchIntervals.Count > 0 && unmarkedBatchIntervals.Count > 0) { // Cumulative variance column int row = 0; _AnalysisDataGridView.Rows[row++].Cells[6].Value = unmarkedCumulativeStats.IntervalCount - markedCumulativeStats.IntervalCount; _AnalysisDataGridView.Rows[row++].Cells[6].Value = TrimIntervals == true ? (unmarkedCumulativeStats.IntervalCountTrimmed - markedCumulativeStats.IntervalCountTrimmed).ToString() : "N/A"; _AnalysisDataGridView.Rows[row++].Cells[6].Value = string.Format("{0:N2}", (unmarkedCumulativeStats.PacketCountMean - markedCumulativeStats.PacketCountMean)); _AnalysisDataGridView.Rows[row++].Cells[6].Value = string.Format("{0:N2}", (unmarkedCumulativeStats.PacketCountStandardDeviation - markedCumulativeStats.PacketCountStandardDeviation)); _AnalysisDataGridView.Rows[row++].Cells[6].Value = unmarkedCumulativeStats.PacketCountMinimum - markedCumulativeStats.PacketCountMinimum; _AnalysisDataGridView.Rows[row++].Cells[6].Value = unmarkedCumulativeStats.PacketCountMaximum - markedCumulativeStats.PacketCountMaximum; _AnalysisDataGridView.Rows[row++].Cells[6].Value = string.Format("{0:N2}", (unmarkedMeanOfMeans - markedMeanOfMeans)); _AnalysisDataGridView.Rows[row++].Cells[6].Value = "N/A"; _AnalysisDataGridView.Rows[row++].Cells[6].Value = "N/A"; } // Update the K-S statistics object _KsStatistics.MarkedMean = markedMeanOfMeans; _KsStatistics.MarkedStdDev = markedStdDevMeanOfMeans; //_KsStatistics.MarkedMean = markedCumulativeStats.PacketCountMean; //_KsStatistics.MarkedStdDev = markedCumulativeStats.PacketCountStandardDeviation; _KsStatistics.MarkedIntervalCount = TrimIntervals == true ? markedCumulativeStats.IntervalCountTrimmed : markedCumulativeStats.IntervalCount; _KsStatistics.UnmarkedMean = unmarkedMeanOfMeans; _KsStatistics.UnmarkedStdDev = unmarkedStdDevMeanOfMeans; //_KsStatistics.UnmarkedMean = unmarkedCumulativeStats.PacketCountMean; //_KsStatistics.UnmarkedStdDev = unmarkedCumulativeStats.PacketCountStandardDeviation; _KsStatistics.UnmarkedIntervalCount = TrimIntervals == true ? unmarkedCumulativeStats.IntervalCountTrimmed : unmarkedCumulativeStats.IntervalCount; }