//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testStatisticsRetriedFailedJobs()
public virtual void testStatisticsRetriedFailedJobs()
{
// given
Batch batch = helper.createMigrationBatchWithSize(3);
// when
helper.completeSeedJobs(batch);
helper.failExecutionJobs(batch, 3);
// then
BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().singleResult();
assertEquals(3, batchStatistics.TotalJobs);
assertEquals(3, batchStatistics.JobsCreated);
assertEquals(3, batchStatistics.RemainingJobs);
assertEquals(0, batchStatistics.CompletedJobs);
assertEquals(3, batchStatistics.FailedJobs);
// when
helper.setRetries(batch, 3, 1);
helper.completeJobs(batch, 3);
// then
batchStatistics = managementService.createBatchStatisticsQuery().singleResult();
assertEquals(3, batchStatistics.TotalJobs);
assertEquals(3, batchStatistics.JobsCreated);
assertEquals(0, batchStatistics.RemainingJobs);
assertEquals(3, batchStatistics.CompletedJobs);
assertEquals(0, batchStatistics.FailedJobs);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testBatchStatisticsQueryFilterWithoutTenantId()
public virtual void testBatchStatisticsQueryFilterWithoutTenantId()
{
// when
BatchStatistics returnedBatch = managementService.createBatchStatisticsQuery().withoutTenantId().singleResult();
// then
Assert.assertNotNull(returnedBatch);
Assert.assertEquals(sharedBatch.Id, returnedBatch.Id);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testBatchStatisticsQueryFilterByTenant()
public virtual void testBatchStatisticsQueryFilterByTenant()
{
// when
BatchStatistics returnedBatch = managementService.createBatchStatisticsQuery().tenantIdIn(TENANT_ONE).singleResult();
// then
Assert.assertNotNull(returnedBatch);
Assert.assertEquals(tenant1Batch.Id, returnedBatch.Id);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testQueryById()
public virtual void testQueryById()
{
// given
helper.createMigrationBatchWithSize(1);
Batch batch = helper.createMigrationBatchWithSize(1);
// when
BatchStatistics statistics = managementService.createBatchStatisticsQuery().batchId(batch.Id).singleResult();
// then
assertEquals(batch.Id, statistics.Id);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testStatisticsSuspend()
public virtual void testStatisticsSuspend()
{
// given
Batch batch = helper.migrateProcessInstancesAsync(1);
// when
managementService.suspendBatchById(batch.Id);
// then
BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().batchId(batch.Id).singleResult();
assertTrue(batchStatistics.Suspended);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testStatisticsNoExecutionJobsGenerated()
public virtual void testStatisticsNoExecutionJobsGenerated()
{
// given
helper.createMigrationBatchWithSize(3);
// when
BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().singleResult();
// then
assertEquals(3, batchStatistics.TotalJobs);
assertEquals(0, batchStatistics.JobsCreated);
assertEquals(3, batchStatistics.RemainingJobs);
assertEquals(0, batchStatistics.CompletedJobs);
assertEquals(0, batchStatistics.FailedJobs);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testStatisticsWithDeletedJobs()
public virtual void testStatisticsWithDeletedJobs()
{
// given
Batch batch = helper.createMigrationBatchWithSize(13);
// when
helper.executeJob(helper.getSeedJob(batch));
deleteMigrationJobs(batch);
// then
BatchStatistics batchStatistics = managementService.createBatchStatisticsQuery().singleResult();
assertEquals(13, batchStatistics.TotalJobs);
assertEquals(10, batchStatistics.JobsCreated);
assertEquals(3, batchStatistics.RemainingJobs);
assertEquals(10, batchStatistics.CompletedJobs);
assertEquals(0, batchStatistics.FailedJobs);
}
//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
//ORIGINAL LINE: @Test public void testBatchStatisticsAndCreateUserId()
public virtual void testBatchStatisticsAndCreateUserId()
{
// given
ProcessInstance pi = createMigrationPlan();
// when
authRule.createGrantAuthorization(Resources.BATCH, "*", "userId", Permissions.CREATE);
authRule.createGrantAuthorization(Resources.PROCESS_DEFINITION, "*", "userId", Permissions.MIGRATE_INSTANCE);
authRule.enableAuthorization("userId");
batch3 = engineRule.RuntimeService.newMigration(migrationPlan).processInstanceIds(Arrays.asList(pi.Id)).executeAsync();
authRule.disableAuthorization();
// then
BatchStatistics batchStatistics = engineRule.ManagementService.createBatchStatisticsQuery().batchId(batch3.Id).singleResult();
assertEquals("userId", batchStatistics.CreateUserId);
}
public void ExecuteBatch <T>(IList <T> src, Action <T> action, string description = null,
bool allowPartialBatchResumption = false, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1)
{
if (src.Count == 0)
{
return;
}
if (allowPartialBatchResumption && src.Count == 1)
{
var threadTask = new ThreadTask
{
Action = () => { action(src[0]); },
BatchStats = new BatchStatistics
{
Total = 1,
Completed = 0
},
EarlyBreak = false,
Description = new OperationDescription
{
From = 1,
To = 1,
Total = 1,
PlainText = description
}
};
object _;
try
{
threadTask.Action();
threadTask.BatchStats.Completed++;
_runningTasks.TryAdd(threadTask, null);
}
catch (Exception e)
{
logger.ErrorException(
string.Format(
"Error occured while executing RavenThreadPool task; ThreadPool name :{0} ; Task queued at: {1} ; Task Description: {2} ",
Name, threadTask.QueuedAt, threadTask.Description), e);
}
finally
{
_runningTasks.TryRemove(threadTask, out _);
}
return;
}
var now = DateTime.UtcNow;
CountdownEvent lastEvent;
var itemsCount = 0;
var batch = new BatchStatistics
{
Total = src.Count,
Completed = 0
};
if (_concurrentEvents.TryDequeue(out lastEvent) == false)
{
lastEvent = new CountdownEvent(src.Count);
}
else
{
lastEvent.Reset(src.Count);
}
for (; itemsCount < src.Count && _ct.IsCancellationRequested == false; itemsCount++)
{
var copy = itemsCount;
var threadTask = new ThreadTask
{
Action = () =>
{
try
{
action(src[copy]);
}
finally
{
lastEvent.Signal();
}
},
Description = new OperationDescription
{
Type = OperationDescription.OpeartionType.Atomic,
PlainText = description,
From = itemsCount + 1,
To = itemsCount + 1,
Total = src.Count
},
BatchStats = batch,
EarlyBreak = allowPartialBatchResumption,
QueuedAt = now,
DoneEvent = lastEvent
};
_tasks.Add(threadTask, _ct);
}
if (!allowPartialBatchResumption)
{
WaitForBatchToCompletion(lastEvent);
return;
}
WaitForBatchAllowingPartialBatchResumption(lastEvent, batch, completedMultiplier, freeThreadsMultiplier, maxWaitMultiplier);
}
public BatchStatistics CalculateBatchStatistics(BindingList<BatchIntervalMarked> batchIntervals, CaptureState captureState, BatchType batchType)
{
decimal batchIntervalsMean = 0;
decimal batchIntervalsTrimmedMean = 0;
// Trim zero packets from the batch interval
BindingList<BatchIntervalMarked> batchIntervalsTrimmed = new BindingList<BatchIntervalMarked>();
foreach (BatchIntervalMarked bim in batchIntervals)
{
if (bim.PacketCount > AnalysisConfiguration.HistogramBinSize)
{
batchIntervalsTrimmed.Add(bim);
}
}
// Calculate statistics for the batch
BatchStatistics bs = new BatchStatistics();
if (AnalysisConfiguration.TrimSmallPackets)
{
BaseStatistics stats = new BaseStatistics(batchIntervalsTrimmed);
bs.IntervalCountTrimmed = stats.Count;
bs.PacketCountMaximum = stats.Maximum;
bs.PacketCountMinimum = stats.Minimum;
bs.PacketCountMean = stats.Mean;
bs.PacketCountStandardDeviation = stats.StdDev;
// Calculate both means for updating the capture batch intervals
batchIntervalsTrimmedMean = stats.Mean;
batchIntervalsMean = Convert.ToDecimal((from t in batchIntervals select t.PacketCount).Average());
// Get both counts for the batch
bs.IntervalCountTrimmed = stats.Count < 0 ? 0 : stats.Count;
bs.IntervalCount = batchIntervals.Count < 0 ? 0 : batchIntervals.Count;
}
else
{
BaseStatistics stats = new BaseStatistics(batchIntervals);
bs.IntervalCount = stats.Count;
bs.PacketCountMaximum = stats.Maximum;
bs.PacketCountMinimum = stats.Minimum;
bs.PacketCountMean = stats.Mean;
bs.PacketCountStandardDeviation = stats.StdDev;
// Calculate both means for updating the capture batch intervals
batchIntervalsMean = bs.PacketCountMean;
batchIntervalsTrimmedMean = Convert.ToDecimal((from t in batchIntervalsTrimmed select t.PacketCount).Average());
// Get both counts for the batch
bs.IntervalCount = stats.Count < 0 ? 0 : stats.Count;
bs.IntervalCountTrimmed = batchIntervalsTrimmed.Count < 0 ? 0 : batchIntervalsTrimmed.Count;
}
// Update the batch mean - only for single batches, not cumulative batches
CurrentCaptureFile captureFile = new CurrentCaptureFile();
ProcessCapturePackets pcp = new ProcessCapturePackets();
captureFile = pcp.GetCurrentCaptureFile(_CaptureFileName);
int captureBatchId = captureFile.CaptureBatchId;
if (batchType == BatchType.Single && captureBatchId != 0)
{
try
{
//ProcessCapturePackets pcp = new ProcessCapturePackets();
//if (!pcp.UpdateBatchMean(Convert.ToInt32(captureBatchId), bs.PacketCountMean))
if (!pcp.UpdateBatchMean(Convert.ToInt32(captureBatchId), batchIntervalsMean, batchIntervalsTrimmedMean))
{
throw new Exception("Error updating batch mean for CaptureBatchId " + captureBatchId);
}
}
catch (Exception ex)
{
throw new Exception("Error updating batch mean for CaptureBatchId " + captureBatchId + ": " + ex.Message);
}
}
// Save the statistics to the database for display on the Analysis tab (save to DisplayStatistics table)
SaveDisplayStatistics(bs, captureBatchId, captureState, batchType, true);
return bs;
}
public void SaveDisplayStatistics(BatchStatistics batchStatistics, int captureBatchId, CaptureState captureState, BatchType batchType, bool saveData)
{
// Save the statistics to the database for display on the Analysis tab (save to DisplayStatistics table)
// Replace cumulative data with new data
DisplayStatisticsData dsd = new DisplayStatisticsData();
switch (captureState)
{
case CaptureState.Marked:
switch (batchType)
{
case BatchType.Single:
//dsd.DeleteSingleMarkedDisplayStatitics();
dsd.InsertSingleMarkedDisplayStatitics(batchStatistics);
dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Single, true);
break;
case BatchType.Cumulative:
dsd.DeleteCumulativeMarkedDisplayStatitics();
dsd.InsertCumulativeMarkedDisplayStatitics(batchStatistics);
dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Cumulative, true);
break;
}
break;
case CaptureState.Unmarked:
switch (batchType)
{
case BatchType.Single:
//dsd.DeleteSingleUnmarkedDisplayStatitics();
dsd.InsertSingleUnarkedDisplayStatitics(batchStatistics);
dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Single, true);
break;
case BatchType.Cumulative:
dsd.DeleteCumulativeUnmarkedDisplayStatitics();
dsd.InsertCumulativeUnarkedDisplayStatitics(batchStatistics);
dsd.UpdateDisplayStatsSavedFlag(captureBatchId, BatchType.Cumulative, true);
break;
}
break;
}
}
public bool CalculateSingleBatchStatistics()
{
bool result = false;
// Calculate single batch statistics
//// Get the last marked and unmarked batches and add them to the graph
//int lastBatchId = 0;
ProcessCapturePackets pcp = new ProcessCapturePackets();
//lastBatchId = pcp.GetLastCaptureBatchId();
CurrentCaptureFile captureFile = new CurrentCaptureFile();
CaptureFileData cfd = new CaptureFileData();
//captureFile = cfd.GetLastCaptureBatchRecord();
captureFile = cfd.GetCurrentCaptureFile(_CaptureFileName);
// Set the global variable
_CaptureState = captureFile.CaptureState;
BindingList<BatchIntervalMarked> batchIntervals = new BindingList<BatchIntervalMarked>();
// Calculate probabilities
batchIntervals = pcp.GetMarkedBatchIntervals(captureFile.CaptureBatchId);
int histogramBinSize = Convert.ToInt32(_HistogramBinSize);
SortedDictionary<int, decimal> probabilities = new CalculateProbability(batchIntervals).GetProbabilityByPacketRange(_TrimZeroPacketIntervals, histogramBinSize);
BatchStatistics markedSingleStats = new BatchStatistics();
BatchStatistics unmarkedSingleStats = new BatchStatistics();
// Add the results to the DisplayStatistics table
DisplayStatisticsData dsd = new DisplayStatisticsData();
if(captureFile.CaptureState == CaptureState.Marked)
{
markedSingleStats = CalculateBatchStatistics(batchIntervals, CaptureState.Marked, BatchType.Single);
}
else
{
unmarkedSingleStats = CalculateBatchStatistics(batchIntervals, CaptureState.Unmarked, BatchType.Single);
}
return result;
}
public bool CalculateCumulativeBatchStatistics()
{
bool result = false;
// Get the cumulative interval counts
ProcessCapturePackets pcp = new ProcessCapturePackets();
BindingList<CumulativeInterval> cumulativeIntervals = new BindingList<CumulativeInterval>();
cumulativeIntervals = pcp.GetCumulativeIntervals();
// Get the batch intervals
BindingList<BatchIntervalMarked> unmarkedBatchIntervals = new BindingList<BatchIntervalMarked>();
BindingList<BatchIntervalMarked> markedBatchIntervals = new BindingList<BatchIntervalMarked>();
foreach (CumulativeInterval ci in cumulativeIntervals)
{
if (ci.Marked)
{
BatchIntervalMarked bim = new BatchIntervalMarked();
bim.BatchIntervalId = 0;
bim.CaptureBatchId = 0;
bim.IntervalNumber = ci.CumulativeIntervalNumber;
bim.Marked = CaptureState.Marked;
bim.PacketCount = ci.PacketCount;
markedBatchIntervals.Add(bim);
}
else
{
BatchIntervalMarked bim = new BatchIntervalMarked();
bim.BatchIntervalId = 0;
bim.CaptureBatchId = 0;
bim.IntervalNumber = ci.CumulativeIntervalNumber;
bim.Marked = CaptureState.Unmarked;
bim.PacketCount = ci.PacketCount;
unmarkedBatchIntervals.Add(bim);
}
}
BatchStatistics markedCumulativeStats = new BatchStatistics();
BatchStatistics unmarkedCumulativeStats = new BatchStatistics();
//decimal markedMeanOfMeans = 0;
//decimal markedStdDevMeanOfMeans = 0;
//decimal unmarkedMeanOfMeans = 0;
//decimal unmarkedStdDevMeanOfMeans = 0;
/********************************************************************************************
*
* Note: must have at least two samples to calculate mean of means and mean of means std dev...
*
* ******************************************************************************************/
/* TO DO: - need to retrieve std dev value for batch for cumulative stats when only one batch
* has been processed.
* - Have to calculate CumulativeStats first, then update MeanOfMeans and StdDev
*/
if (_CaptureState == CaptureState.Marked && markedBatchIntervals.Count > 0)
{
markedCumulativeStats = CalculateBatchStatistics(markedBatchIntervals, CaptureState.Marked, BatchType.Cumulative);
}
if (_CaptureState == CaptureState.Unmarked && unmarkedBatchIntervals.Count > 0)
{
unmarkedCumulativeStats = CalculateBatchStatistics(unmarkedBatchIntervals, CaptureState.Unmarked, BatchType.Cumulative);
}
if (pcp.GetMeanCount() > 1)
{
if (_CaptureState == CaptureState.Marked && markedBatchIntervals.Count > 0)
{
//markedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false);
markedCumulativeStats.MeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false);
//markedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false);
markedCumulativeStats.MeanOfMeansStandardDeviation = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Marked, AnalysisConfiguration.TrimSmallPackets ? true : false);
SaveDisplayStatistics(markedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Marked, BatchType.Cumulative, true);
}
if (_CaptureState == CaptureState.Unmarked && unmarkedBatchIntervals.Count > 0)
{
//unmarkedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false);
unmarkedCumulativeStats.MeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false);
//unmarkedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false);
unmarkedCumulativeStats.MeanOfMeansStandardDeviation = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Unmarked, AnalysisConfiguration.TrimSmallPackets ? true : false);
SaveDisplayStatistics(unmarkedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Unmarked, BatchType.Cumulative, true);
}
}
else
{
// Only one batch - use the mean and standard deviation from the first batch
if (_CaptureState == CaptureState.Marked && markedBatchIntervals.Count > 0)
{
//markedMeanOfMeans = pcp.GetMean(CaptureState.Marked, _TrimZeroPacketIntervals);
//markedMeanOfMeans = markedCumulativeStats.PacketCountMean;
markedCumulativeStats.MeanOfMeans = markedCumulativeStats.PacketCountMean;
//markedStdDevMeanOfMeans = markedCumulativeStats.PacketCountStandardDeviation;
markedCumulativeStats.MeanOfMeansStandardDeviation = markedCumulativeStats.PacketCountStandardDeviation;
SaveDisplayStatistics(markedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Marked, BatchType.Cumulative, true);
}
if (_CaptureState == CaptureState.Unmarked && unmarkedBatchIntervals.Count > 0)
{
//unmarkedMeanOfMeans = pcp.GetMean(CaptureState.Unmarked, _TrimZeroPacketIntervals);
//unmarkedMeanOfMeans = unmarkedCumulativeStats.PacketCountMean;
unmarkedCumulativeStats.MeanOfMeans = unmarkedCumulativeStats.PacketCountMean;
//unmarkedStdDevMeanOfMeans = unmarkedCumulativeStats.PacketCountStandardDeviation;
unmarkedCumulativeStats.MeanOfMeansStandardDeviation = unmarkedCumulativeStats.PacketCountStandardDeviation;
SaveDisplayStatistics(unmarkedCumulativeStats, pcp.GetCaptureBatchId(_CaptureFileName), CaptureState.Unmarked, BatchType.Cumulative, true);
}
}
// public bool GetHypothesisTestResult()
// NOTE: use _HypothesisTest variable to determine which test result to return
//if (markedBatchIntervals.Count > 0 && unmarkedBatchIntervals.Count > 0)
//{
// // Cumulative variance column
//}
//// Update the K-S statistics object - for display
//_KsStatistics.MarkedMean = markedMeanOfMeans;
//_KsStatistics.MarkedStdDev = markedStdDevMeanOfMeans;
////_KsStatistics.MarkedMean = markedCumulativeStats.PacketCountMean;
////_KsStatistics.MarkedStdDev = markedCumulativeStats.PacketCountStandardDeviation;
//_KsStatistics.MarkedIntervalCount = TrimIntervalsCheckBox.Checked == true ? markedCumulativeStats.IntervalCountTrimmed : markedCumulativeStats.IntervalCount;
//_KsStatistics.UnmarkedMean = unmarkedMeanOfMeans;
//_KsStatistics.UnmarkedStdDev = unmarkedStdDevMeanOfMeans;
////_KsStatistics.UnmarkedMean = unmarkedCumulativeStats.PacketCountMean;
////_KsStatistics.UnmarkedStdDev = unmarkedCumulativeStats.PacketCountStandardDeviation;
//_KsStatistics.UnmarkedIntervalCount = TrimIntervalsCheckBox.Checked == true ? unmarkedCumulativeStats.IntervalCountTrimmed : unmarkedCumulativeStats.IntervalCount;
return result;
}
private void RefreshSingleBatchStatistics()
{
// Get the last marked and unmarked batches
BindingList<CurrentCaptureFile> lastBatchIds = new BindingList<CurrentCaptureFile>();
ProcessCapturePackets pcp = new ProcessCapturePackets();
lastBatchIds = pcp.GetLastCaptureBatchIds();
// Get the batch intervals
BindingList<BatchIntervalMarked> unmarkedBatchIntervals = new BindingList<BatchIntervalMarked>();
BindingList<BatchIntervalMarked> markedBatchIntervals = new BindingList<BatchIntervalMarked>();
foreach (CurrentCaptureFile file in lastBatchIds)
{
if (file.CaptureState == CaptureState.Marked)
{
markedBatchIntervals = pcp.GetMarkedBatchIntervals(file.CaptureBatchId);
}
else if(file.CaptureState == CaptureState.Unmarked)
{
unmarkedBatchIntervals = pcp.GetMarkedBatchIntervals(file.CaptureBatchId);
}
else
{
MessageBox.Show("Error retrieving batch intervals: capture state is unknown!", "GetMarkedBatchIntervals by CaptureBatchId", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
BatchStatistics markedSingleStats = new BatchStatistics();
BatchStatistics unmarkedSingleStats = new BatchStatistics();
// Get this data from DisplayStatistics table; except on refresh...
//AnalysisEngine ae = new AnalysisEngine();
//markedSingleStats = ae.GetBatchStatistics(markedBatchIntervals, CaptureState.Marked, BatchType.Single);
//unmarkedSingleStats = ae.GetBatchStatistics(unmarkedBatchIntervals, CaptureState.Unmarked, BatchType.Single);
// Load up the table
// Single unmarked column
int row = 0;
_AnalysisDataGridView.Rows[row++].Cells[1].Value = unmarkedSingleStats.IntervalCount;
_AnalysisDataGridView.Rows[row++].Cells[1].Value = TrimIntervals == true ? unmarkedSingleStats.IntervalCountTrimmed.ToString() : "N/A";
_AnalysisDataGridView.Rows[row++].Cells[1].Value = string.Format("{0:N2}", unmarkedSingleStats.PacketCountMean);
_AnalysisDataGridView.Rows[row++].Cells[1].Value = string.Format("{0:N2}", unmarkedSingleStats.PacketCountStandardDeviation);
_AnalysisDataGridView.Rows[row++].Cells[1].Value = unmarkedSingleStats.PacketCountMinimum;
_AnalysisDataGridView.Rows[row++].Cells[1].Value = unmarkedSingleStats.PacketCountMaximum;
_AnalysisDataGridView.Rows[row++].Cells[1].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[1].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[1].Value = "N/A";
// Single marked column
row = 0;
_AnalysisDataGridView.Rows[row++].Cells[2].Value = markedSingleStats.IntervalCount;
_AnalysisDataGridView.Rows[row++].Cells[2].Value = TrimIntervals == true ? markedSingleStats.IntervalCountTrimmed.ToString() : "N/A";
_AnalysisDataGridView.Rows[row++].Cells[2].Value = string.Format("{0:N2}", markedSingleStats.PacketCountMean);
_AnalysisDataGridView.Rows[row++].Cells[2].Value = string.Format("{0:N2}", markedSingleStats.PacketCountStandardDeviation);
_AnalysisDataGridView.Rows[row++].Cells[2].Value = markedSingleStats.PacketCountMinimum;
_AnalysisDataGridView.Rows[row++].Cells[2].Value = markedSingleStats.PacketCountMaximum;
_AnalysisDataGridView.Rows[row++].Cells[2].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[2].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[2].Value = "N/A";
// Single variance column
row = 0;
_AnalysisDataGridView.Rows[row++].Cells[3].Value = unmarkedSingleStats.IntervalCount - markedSingleStats.IntervalCount;
_AnalysisDataGridView.Rows[row++].Cells[3].Value = TrimIntervals == true ? (unmarkedSingleStats.IntervalCountTrimmed - markedSingleStats.IntervalCountTrimmed).ToString() : "N/A";
_AnalysisDataGridView.Rows[row++].Cells[3].Value = string.Format("{0:N2}", (unmarkedSingleStats.PacketCountMean - markedSingleStats.PacketCountMean));
_AnalysisDataGridView.Rows[row++].Cells[3].Value = string.Format("{0:N2}",(unmarkedSingleStats.PacketCountStandardDeviation - markedSingleStats.PacketCountStandardDeviation));
_AnalysisDataGridView.Rows[row++].Cells[3].Value = unmarkedSingleStats.PacketCountMinimum - markedSingleStats.PacketCountMinimum;
_AnalysisDataGridView.Rows[row++].Cells[3].Value = unmarkedSingleStats.PacketCountMaximum - markedSingleStats.PacketCountMaximum;
_AnalysisDataGridView.Rows[row++].Cells[3].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[3].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[3].Value = "N/A";
}
private bool WaitForBatchAllowingPartialBatchResumption(CountdownEvent completionEvent, BatchStatistics batch, List <ThreadTask> childTasks, DocumentDatabase database = null)
{
Interlocked.Increment(ref _hasPartialBatchResumption);
var cancellationToken = database?.WorkContext.CancellationToken ?? _ct;
try
{
var waitHandles = new[] { completionEvent.WaitHandle, _threadHasNoWorkToDo };
var sp = Stopwatch.StartNew();
var batchRanToCompletion = false;
var lastThreadIndexChecked = 0;
while (cancellationToken.IsCancellationRequested == false)
{
// First, try to find work to do among child tasks, instead of just waiting
ThreadTask busyWaitWorkToDo = null;
for (; lastThreadIndexChecked < childTasks.Count && completionEvent.IsSet == false && _ct.IsCancellationRequested == false; lastThreadIndexChecked++)
{
var task = childTasks[lastThreadIndexChecked];
if (Interlocked.CompareExchange(ref task.Proccessing, 1, 0) == 1)
{
continue;
}
busyWaitWorkToDo = task;
break;
}
int returnedWaitHandleIndex;
// Run found work or just wait for all work to finish (returnedWaitHandleIndex=0) or to be notified about free threads in the system (returnedWaitHandleIndex=1)
// After which we'll decide whether we should early exit, or to wait some more
if (busyWaitWorkToDo != null)
{
RunThreadTask(busyWaitWorkToDo);
returnedWaitHandleIndex = WaitHandle.WaitAny(waitHandles, 0);
}
else
{
returnedWaitHandleIndex = WaitHandle.WaitAny(waitHandles);
}
if (returnedWaitHandleIndex == 0)
{
break;
}
// we won't consider breaking early if we haven't completed at least half the work
if (Thread.VolatileRead(ref batch.Completed) < batch.Total / 2)
{
continue;
}
var currentFreeThreads = _freedThreadsValue.Values.Count(isFree => isFree);
// we will break early only if there are more then half free threads
if (currentFreeThreads > _currentWorkingThreadsAmount / 2)
{
break;
}
}
// after we've decided to quit early, we will wait some more time, allowing a normal wait.
// we decide how much we will wait by choosing the biggest among the next:
// 1) half the time we've waited on the current batch
// 2) a waiting time factor that increase for every second left early batch and decreases for every leave early batch that completed before leaving early
var elapsedMilliseconds = (int)sp.ElapsedMilliseconds;
if (completionEvent.Wait(Math.Max(elapsedMilliseconds / 2, Thread.VolatileRead(ref _partialMaxWait)), cancellationToken))
{
_concurrentEvents.Enqueue(completionEvent);
}
// updating the waiting time factor
if (batch.Completed != batch.Total)
{
if (_partialMaxWaitChangeFlag > 0)
{
Interlocked.Exchange(ref _partialMaxWait, Math.Min(2500, (int)(Thread.VolatileRead(ref _partialMaxWait) * 1.25)));
}
}
else if (_partialMaxWaitChangeFlag < 0)
{
batchRanToCompletion = true;
Interlocked.Exchange(ref _partialMaxWait, Math.Max(Thread.VolatileRead(ref _partialMaxWait) / 2, 10));
}
Interlocked.Exchange(ref _partialMaxWaitChangeFlag, Thread.VolatileRead(ref _partialMaxWaitChangeFlag) * -1);
// completionEvent is explicitly left to the finalizer
// we expect this to be rare, and it isn't worth the trouble of trying
// to manage this
if (logger.IsDebugEnabled)
{
logger.Debug($"Raven Thread Pool ended batch for database {database?.Name}. Done {batch.Completed} items out of {batch.Total}. Batch {(batchRanToCompletion ? "was not" : "was")} left early");
}
return(batchRanToCompletion);
}
finally
{
Interlocked.Decrement(ref _hasPartialBatchResumption);
}
}
public bool ExecuteBatch <T>(IList <T> src, Action <T> action, DocumentDatabase database = null, string description = null,
bool allowPartialBatchResumption = false, Action runAfterCompletion = null)
{
//, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1
switch (src.Count)
{
case 0:
return(true);
case 1:
//if we have only one source to go through,
//we should execute it in the current thread, without using RTP threads
ExecuteSingleBatchSynchronously(src, action, description, database);
runAfterCompletion?.Invoke();
return(true);
}
var now = DateTime.UtcNow;
CountdownEvent countdownEvent;
var itemsCount = 0;
var batch = new BatchStatistics
{
Total = src.Count,
Completed = 0
};
if (_concurrentEvents.TryDequeue(out countdownEvent) == false)
{
countdownEvent = new CountdownEvent(src.Count);
}
else
{
countdownEvent.Reset(src.Count);
}
var exceptions = new ConcurrentSet <Exception>();
var currentBatchTasks = new List <ThreadTask>(src.Count);
for (; itemsCount < src.Count && _ct.IsCancellationRequested == false; itemsCount++)
{
var copy = itemsCount;
var threadTask = new ThreadTask
{
Action = () =>
{
try
{
database?.WorkContext.CancellationToken.ThrowIfCancellationRequested();
action(src[copy]);
}
catch (Exception e)
{
exceptions.Add(e);
throw;
}
finally
{
countdownEvent.Signal();
}
},
Description = new OperationDescription
{
Type = OperationDescription.OperationType.Atomic,
PlainText = description,
From = itemsCount + 1,
To = itemsCount + 1,
Total = src.Count
},
Database = database,
BatchStats = batch,
EarlyBreak = allowPartialBatchResumption,
QueuedAt = now,
DoneEvent = countdownEvent,
Proccessing = 0,
RunAfterCompletion = runAfterCompletion
};
_tasks.Add(threadTask);
currentBatchTasks.Add(threadTask);
}
var ranToCompletion = false;
if (allowPartialBatchResumption == false)
{
WaitForBatchToCompletion(countdownEvent, currentBatchTasks, database?.WorkContext.CancellationToken);
ranToCompletion = true;
}
else
{
ranToCompletion = WaitForBatchAllowingPartialBatchResumption(countdownEvent, batch, currentBatchTasks, database);
}
switch (exceptions.Count)
{
case 0:
return(ranToCompletion);
case 1:
ExceptionDispatchInfo.Capture(exceptions.First()).Throw();
return(ranToCompletion); // won't happen
default:
throw new AggregateException(exceptions);
}
}
public void ExecuteBatch <T>(IList <T> src, Action <T> action, string description = null,
bool allowPartialBatchResumption = false, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1)
{
switch (src.Count)
{
case 0:
return;
case 1:
//if we have only one source to go through,
//we should execute it in the current thread, without using RTP threads
ExecuteSingleBatchSynchronously(src, action, description);
return;
}
var now = DateTime.UtcNow;
CountdownEvent lastEvent;
var itemsCount = 0;
var batch = new BatchStatistics
{
Total = src.Count,
Completed = 0
};
if (_concurrentEvents.TryDequeue(out lastEvent) == false)
{
lastEvent = new CountdownEvent(src.Count);
}
else
{
lastEvent.Reset(src.Count);
}
var exceptions = new ConcurrentSet <Exception>();
for (; itemsCount < src.Count && _ct.IsCancellationRequested == false; itemsCount++)
{
var copy = itemsCount;
var threadTask = new ThreadTask
{
Action = () =>
{
try
{
action(src[copy]);
}
catch (Exception e)
{
exceptions.Add(e);
throw;
}
finally
{
lastEvent.Signal();
}
},
Description = new OperationDescription
{
Type = OperationDescription.OperationType.Atomic,
PlainText = description,
From = itemsCount + 1,
To = itemsCount + 1,
Total = src.Count
},
BatchStats = batch,
EarlyBreak = allowPartialBatchResumption,
QueuedAt = now,
DoneEvent = lastEvent
};
_tasks.Add(threadTask);
}
if (allowPartialBatchResumption == false)
{
WaitForBatchToCompletion(lastEvent);
}
else
{
WaitForBatchAllowingPartialBatchResumption(lastEvent, batch, completedMultiplier, freeThreadsMultiplier, maxWaitMultiplier);
}
switch (exceptions.Count)
{
case 0:
return;
case 1:
ExceptionDispatchInfo.Capture(exceptions.First()).Throw();
break;
default:
throw new AggregateException(exceptions);
}
}
public void InsertSingleUnarkedDisplayStatitics(BatchStatistics batchStatistics)
{
using (var context = new PacketAnalysisEntity())
{
context.DisplayStatisticsInsert(
batchStatistics.IntervalCount,
batchStatistics.IntervalCountTrimmed,
batchStatistics.PacketCountMean,
batchStatistics.PacketCountStandardDeviation,
batchStatistics.PacketCountMinimum,
batchStatistics.PacketCountMaximum,
batchStatistics.MeanOfMeans,
batchStatistics.MeanOfMeansStandardDeviation,
false, // Marked
Convert.ToInt32(BatchType.Single)
);
context.SaveChanges();
}
}
private void WaitForBatchAllowingPartialBatchResumption(CountdownEvent completionEvent, BatchStatistics batch, int completedMultiplier = 2, int freeThreadsMultiplier = 2, int maxWaitMultiplier = 1)
{
Interlocked.Increment(ref _hasPartialBatchResumption);
try
{
var waitHandles = new[] { completionEvent.WaitHandle, _threadHasNoWorkToDo };
var sp = Stopwatch.StartNew();
var batchLeftEarly = false;
while (_ct.IsCancellationRequested == false)
{
var i = WaitHandle.WaitAny(waitHandles);
if (i == 0)
{
break;
}
if (Thread.VolatileRead(ref batch.Completed) < batch.Total / completedMultiplier)
{
continue;
}
var currentFreeThreads = _freedThreadsValue.Values.Count(isFree => isFree);
if (currentFreeThreads > _currentWorkingThreadsAmount / freeThreadsMultiplier)
{
break;
}
}
var elapsedMilliseconds = (int)sp.ElapsedMilliseconds;
if (completionEvent.Wait(Math.Max(elapsedMilliseconds / 2, Thread.VolatileRead(ref _partialMaxWait)) / maxWaitMultiplier, _ct))
{
_concurrentEvents.Enqueue(completionEvent);
}
if (batch.Completed != batch.Total)
{
batchLeftEarly = true;
if (_partialMaxWaitChangeFlag > 0)
{
Interlocked.Exchange(ref _partialMaxWait, Math.Min(2500, (int)(Thread.VolatileRead(ref _partialMaxWait) * 1.25)));
}
}
else if (_partialMaxWaitChangeFlag < 0)
{
Interlocked.Exchange(ref _partialMaxWait, Math.Max(Thread.VolatileRead(ref _partialMaxWait) / 2, 10));
}
Interlocked.Exchange(ref _partialMaxWaitChangeFlag, Thread.VolatileRead(ref _partialMaxWaitChangeFlag) * -1);
// completionEvent is explicitly left to the finalizer
// we expect this to be rare, and it isn't worth the trouble of trying
// to manage this
if (logger.IsDebugEnabled)
{
logger.Info("Raven Thread Pool named {0} ended batch. Done {1} items out of {2}. Batch {3} left early",
Name, batch.Completed, batch.Total, batchLeftEarly ? "was" : "was not");
}
}
finally
{
Interlocked.Decrement(ref _hasPartialBatchResumption);
}
}
private void RefreshCumulativeBatchStatistics()
{
// Get the cumulative interval counts
ProcessCapturePackets pcp = new ProcessCapturePackets();
BindingList<CumulativeInterval> cumulativeIntervals = new BindingList<CumulativeInterval>();
cumulativeIntervals = pcp.GetCumulativeIntervals();
// Get the batch intervals
BindingList<BatchIntervalMarked> unmarkedBatchIntervals = new BindingList<BatchIntervalMarked>();
BindingList<BatchIntervalMarked> markedBatchIntervals = new BindingList<BatchIntervalMarked>();
foreach (CumulativeInterval ci in cumulativeIntervals)
{
if (ci.Marked)
{
BatchIntervalMarked bim = new BatchIntervalMarked();
bim.BatchIntervalId = 0;
bim.CaptureBatchId = 0;
bim.IntervalNumber = ci.CumulativeIntervalNumber;
bim.Marked = CaptureState.Marked;
bim.PacketCount = ci.PacketCount;
markedBatchIntervals.Add(bim);
}
else
{
BatchIntervalMarked bim = new BatchIntervalMarked();
bim.BatchIntervalId = 0;
bim.CaptureBatchId = 0;
bim.IntervalNumber = ci.CumulativeIntervalNumber;
bim.Marked = CaptureState.Unmarked;
bim.PacketCount = ci.PacketCount;
unmarkedBatchIntervals.Add(bim);
}
}
BatchStatistics markedCumulativeStats = new BatchStatistics();
BatchStatistics unmarkedCumulativeStats = new BatchStatistics();
decimal markedMeanOfMeans = 0;
decimal markedStdDevMeanOfMeans = 0;
decimal unmarkedMeanOfMeans = 0;
decimal unmarkedStdDevMeanOfMeans = 0;
AnalysisEngine ae = new AnalysisEngine();
if(markedBatchIntervals.Count > 0)
{
markedCumulativeStats = ae.CalculateBatchStatistics(markedBatchIntervals,CaptureState.Marked, BatchType.Cumulative);
markedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Marked, TrimIntervals ? true : false);
markedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Marked, TrimIntervals ? true : false);
// Load up the table
// Cumulative marked column
int row = 0;
_AnalysisDataGridView.Rows[row++].Cells[5].Value = markedCumulativeStats.IntervalCount;
_AnalysisDataGridView.Rows[row++].Cells[5].Value = TrimIntervals == true ? markedCumulativeStats.IntervalCountTrimmed.ToString() : "N/A";
_AnalysisDataGridView.Rows[row++].Cells[5].Value = string.Format("{0:N2}", markedCumulativeStats.PacketCountMean);
_AnalysisDataGridView.Rows[row++].Cells[5].Value = string.Format("{0:N2}", markedCumulativeStats.PacketCountStandardDeviation);
_AnalysisDataGridView.Rows[row++].Cells[5].Value = markedCumulativeStats.PacketCountMinimum;
_AnalysisDataGridView.Rows[row++].Cells[5].Value = markedCumulativeStats.PacketCountMaximum;
_AnalysisDataGridView.Rows[row++].Cells[5].Value = string.Format("{0:N2}", markedMeanOfMeans);
_AnalysisDataGridView.Rows[row++].Cells[5].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[5].Value = "N/A";
}
if (unmarkedBatchIntervals.Count > 0)
{
unmarkedCumulativeStats = ae.CalculateBatchStatistics(unmarkedBatchIntervals, CaptureState.Marked, BatchType.Cumulative);
unmarkedMeanOfMeans = pcp.CalculateMeanOfMeans(CaptureState.Unmarked, TrimIntervals ? true : false);
unmarkedStdDevMeanOfMeans = pcp.CalculateStdDevForMeanOfMeans(CaptureState.Unmarked, TrimIntervals ? true : false);
// Load up the table
// Cumulative unmarked column
int row = 0;
_AnalysisDataGridView.Rows[row++].Cells[4].Value = unmarkedCumulativeStats.IntervalCount;
_AnalysisDataGridView.Rows[row++].Cells[4].Value = TrimIntervals == true ? unmarkedCumulativeStats.IntervalCountTrimmed.ToString() : "N/A";
_AnalysisDataGridView.Rows[row++].Cells[4].Value = string.Format("{0:N2}", unmarkedCumulativeStats.PacketCountMean);
_AnalysisDataGridView.Rows[row++].Cells[4].Value = string.Format("{0:N2}", unmarkedCumulativeStats.PacketCountStandardDeviation);
_AnalysisDataGridView.Rows[row++].Cells[4].Value = unmarkedCumulativeStats.PacketCountMinimum;
_AnalysisDataGridView.Rows[row++].Cells[4].Value = unmarkedCumulativeStats.PacketCountMaximum;
_AnalysisDataGridView.Rows[row++].Cells[4].Value = string.Format("{0:N2}", unmarkedMeanOfMeans);
_AnalysisDataGridView.Rows[row++].Cells[4].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[4].Value = "N/A";
}
if (markedBatchIntervals.Count > 0 && unmarkedBatchIntervals.Count > 0)
{
// Cumulative variance column
int row = 0;
_AnalysisDataGridView.Rows[row++].Cells[6].Value = unmarkedCumulativeStats.IntervalCount - markedCumulativeStats.IntervalCount;
_AnalysisDataGridView.Rows[row++].Cells[6].Value = TrimIntervals == true ? (unmarkedCumulativeStats.IntervalCountTrimmed - markedCumulativeStats.IntervalCountTrimmed).ToString() : "N/A";
_AnalysisDataGridView.Rows[row++].Cells[6].Value = string.Format("{0:N2}", (unmarkedCumulativeStats.PacketCountMean - markedCumulativeStats.PacketCountMean));
_AnalysisDataGridView.Rows[row++].Cells[6].Value = string.Format("{0:N2}", (unmarkedCumulativeStats.PacketCountStandardDeviation - markedCumulativeStats.PacketCountStandardDeviation));
_AnalysisDataGridView.Rows[row++].Cells[6].Value = unmarkedCumulativeStats.PacketCountMinimum - markedCumulativeStats.PacketCountMinimum;
_AnalysisDataGridView.Rows[row++].Cells[6].Value = unmarkedCumulativeStats.PacketCountMaximum - markedCumulativeStats.PacketCountMaximum;
_AnalysisDataGridView.Rows[row++].Cells[6].Value = string.Format("{0:N2}", (unmarkedMeanOfMeans - markedMeanOfMeans));
_AnalysisDataGridView.Rows[row++].Cells[6].Value = "N/A";
_AnalysisDataGridView.Rows[row++].Cells[6].Value = "N/A";
}
// Update the K-S statistics object
_KsStatistics.MarkedMean = markedMeanOfMeans;
_KsStatistics.MarkedStdDev = markedStdDevMeanOfMeans;
//_KsStatistics.MarkedMean = markedCumulativeStats.PacketCountMean;
//_KsStatistics.MarkedStdDev = markedCumulativeStats.PacketCountStandardDeviation;
_KsStatistics.MarkedIntervalCount = TrimIntervals == true ? markedCumulativeStats.IntervalCountTrimmed : markedCumulativeStats.IntervalCount;
_KsStatistics.UnmarkedMean = unmarkedMeanOfMeans;
_KsStatistics.UnmarkedStdDev = unmarkedStdDevMeanOfMeans;
//_KsStatistics.UnmarkedMean = unmarkedCumulativeStats.PacketCountMean;
//_KsStatistics.UnmarkedStdDev = unmarkedCumulativeStats.PacketCountStandardDeviation;
_KsStatistics.UnmarkedIntervalCount = TrimIntervals == true ? unmarkedCumulativeStats.IntervalCountTrimmed : unmarkedCumulativeStats.IntervalCount;
}
