public SlurpItemEventArgs(SlurpItem item, bool willStart) : this(item) { this.willStart = willStart; }
protected override void Process() { // Schedule(); var baseTime = DateTime.Today.AddHours(DateTime.Now.Hour).AddMinutes(DateTime.Now.Minute / 15); Dictionary <int, double> averageExecutionTimes = new Dictionary <int, double>(); Dictionary <int, int> itemsToProcessPerSource = new Dictionary <int, int>(); int batchTime = 5; // minutes int maxDegree = 1; //per site int defaultItemTime = 10; //seconds var datetimeValue = DateTime.Now.AddDays(-2); using (var unit = GetUnitOfWork()) { var slurpQueueList = (from l in unit.Scope.Repository <SlurpQueue>().GetAll() where l.IsCompleted && l.CompletionTime.Value >= datetimeValue select l).ToList(); averageExecutionTimes = (from l in slurpQueueList group l by l.ProductCompareSourceID into grouped select new { ID = grouped.Key, Duration = grouped.Average(x => (x.CompletionTime.Value - x.StartTime.Value).TotalMilliseconds / 1000.0) }).ToDictionary(x => x.ID, y => y.Duration); itemsToProcessPerSource = (from l in unit.Scope.Repository <SlurpQueue>().GetAll() where !l.IsCompleted group l by l.ProductCompareSourceID into grouped select new { ID = grouped.Key, Count = grouped.Count() }).ToDictionary(x => x.ID, y => y.Count); } var remainingHoursToday = 24 - baseTime.Hour; var batchesRemainingToday = (remainingHoursToday * 60) / batchTime; //Random r = new Random(DateTime.Now.Millisecond); var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = maxDegree }; log.DebugFormat("Start Execution"); using (var unit = GetUnitOfWork()) { //var options = new DataLoadOptions(); //options.LoadWith<SlurpQueue>(x => x.Product); //options.LoadWith<SlurpQueue>(x => x.ProductCompareSource); //ctx.LoadOptions = options; var slurpQueueRepo = unit.Scope.Repository <SlurpQueue>().Include(c => c.Product, c => c.ProductCompareSource); var sources = unit.Scope.Repository <ProductCompareSource>().GetAll(x => x.IsActive == true && x.ProductCompareSourceType != null).ToDictionary(x => x.ProductCompareSourceID, y => y); var recordsPerSource = (from q in slurpQueueRepo.GetAll(l => !l.IsCompleted && l.ProductCompareSource.IsActive == true && l.ProductCompareSource.ProductCompareSourceType != null) group q by q.ProductCompareSourceID into grouped select grouped).ToList(); var siteHandles = new ManualResetEvent[recordsPerSource.Count()]; for (int idx = 0; idx < recordsPerSource.Count; idx++) { var source = recordsPerSource[idx]; var compareSource = sources[source.Key]; log.DebugFormat("Processing {0}", compareSource.Source); siteHandles[idx] = new ManualResetEvent(false); List <SlurpItem> sourceQueue = new List <SlurpItem>(); // calculate items to schedule if (!averageExecutionTimes.ContainsKey(source.Key)) { averageExecutionTimes[source.Key] = defaultItemTime; } var maxItemsToQueue = (int)Math.Floor((60.0 / averageExecutionTimes[source.Key]) * batchTime); int itemsInQueue = source.Count(); var itemsToQueue = (int)(Math.Floor(Math.Max(itemsInQueue / (double)batchesRemainingToday, 0)) + 1); itemsToQueue = Math.Min(itemsToQueue, maxItemsToQueue); log.DebugFormat("Queueing {0} items of {2} in queue (Maximum : {1})", itemsToQueue, maxItemsToQueue, itemsInQueue); var startTime = DateTime.Now; var sleepTimes = GetSleepTimes(batchTime * 60, itemsToQueue, false); //var delayIdeal = ((batchTime * 60.0) - (averageExecutionTimes[source.Key] * itemsToQueue)) / itemsToQueue; int stIndex = -1; foreach (var product in source.OrderBy(x => x.CreationTime).Take(itemsToQueue)) { var sleepTime = sleepTimes[++stIndex]; SlurpItem item = new SlurpItem(Type.GetType(compareSource.ProductCompareSourceType), product.QueueID, sleepTime); sourceQueue.Add(item); } var currentHandle = siteHandles[idx]; Action wrappedAction = () => { try { Parallel.ForEach(sourceQueue, parallelOptions, x => { SlurpResult result = x.Process(); lock (_locker) { //var avg = (from l in slurpQueueRepo.GetAll(c => c.IsCompleted && c.StartTime >= startTime) // select (l.CompletionTime.Value - l.StartTime.Value).TotalMilliseconds / 1000.0).ToList().Average(); //log.InfoFormat("Average execution rate for {1} : {0:f2}s / product", avg, result.SiteName); } }); } finally { currentHandle.Set(); } }; ThreadPool.QueueUserWorkItem(x => wrappedAction()); } if (siteHandles != null && siteHandles.Length > 0) { WaitHandle.WaitAll(siteHandles); } //ctx.SubmitChanges(); } }
public SlurpItemEventArgs(SlurpItem item) { this.item = item; }