private void WriteLog(SlurpResult result) { var fn = result.ManufacturerId.Trim().Replace("/", "_").Replace("\\", "_"); string fileName = String.Format(@"c:\slurp\{1}-{0}.txt", fn, result.SiteName); using (FileStream fs = new FileStream(fileName, FileMode.OpenOrCreate)) { ser.Serialize(fs, result); } }
protected override void Process() { // Schedule(); var baseTime = DateTime.Today.AddHours(DateTime.Now.Hour).AddMinutes(DateTime.Now.Minute / 15); Dictionary <int, double> averageExecutionTimes = new Dictionary <int, double>(); Dictionary <int, int> itemsToProcessPerSource = new Dictionary <int, int>(); int batchTime = 5; // minutes int maxDegree = 1; //per site int defaultItemTime = 10; //seconds var datetimeValue = DateTime.Now.AddDays(-2); using (var unit = GetUnitOfWork()) { var slurpQueueList = (from l in unit.Scope.Repository <SlurpQueue>().GetAll() where l.IsCompleted && l.CompletionTime.Value >= datetimeValue select l).ToList(); averageExecutionTimes = (from l in slurpQueueList group l by l.ProductCompareSourceID into grouped select new { ID = grouped.Key, Duration = grouped.Average(x => (x.CompletionTime.Value - x.StartTime.Value).TotalMilliseconds / 1000.0) }).ToDictionary(x => x.ID, y => y.Duration); itemsToProcessPerSource = (from l in unit.Scope.Repository <SlurpQueue>().GetAll() where !l.IsCompleted group l by l.ProductCompareSourceID into grouped select new { ID = grouped.Key, Count = grouped.Count() }).ToDictionary(x => x.ID, y => y.Count); } var remainingHoursToday = 24 - baseTime.Hour; var batchesRemainingToday = (remainingHoursToday * 60) / batchTime; //Random r = new Random(DateTime.Now.Millisecond); var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = maxDegree }; log.DebugFormat("Start Execution"); using (var unit = GetUnitOfWork()) { //var options = new DataLoadOptions(); //options.LoadWith<SlurpQueue>(x => x.Product); //options.LoadWith<SlurpQueue>(x => x.ProductCompareSource); //ctx.LoadOptions = options; var slurpQueueRepo = unit.Scope.Repository <SlurpQueue>().Include(c => c.Product, c => c.ProductCompareSource); var sources = unit.Scope.Repository <ProductCompareSource>().GetAll(x => x.IsActive == true && x.ProductCompareSourceType != null).ToDictionary(x => x.ProductCompareSourceID, y => y); var recordsPerSource = (from q in slurpQueueRepo.GetAll(l => !l.IsCompleted && l.ProductCompareSource.IsActive == true && l.ProductCompareSource.ProductCompareSourceType != null) group q by q.ProductCompareSourceID into grouped select grouped).ToList(); var siteHandles = new ManualResetEvent[recordsPerSource.Count()]; for (int idx = 0; idx < recordsPerSource.Count; idx++) { var source = recordsPerSource[idx]; var compareSource = sources[source.Key]; log.DebugFormat("Processing {0}", compareSource.Source); siteHandles[idx] = new ManualResetEvent(false); List <SlurpItem> sourceQueue = new List <SlurpItem>(); // calculate items to schedule if (!averageExecutionTimes.ContainsKey(source.Key)) { averageExecutionTimes[source.Key] = defaultItemTime; } var maxItemsToQueue = (int)Math.Floor((60.0 / averageExecutionTimes[source.Key]) * batchTime); int itemsInQueue = source.Count(); var itemsToQueue = (int)(Math.Floor(Math.Max(itemsInQueue / (double)batchesRemainingToday, 0)) + 1); itemsToQueue = Math.Min(itemsToQueue, maxItemsToQueue); log.DebugFormat("Queueing {0} items of {2} in queue (Maximum : {1})", itemsToQueue, maxItemsToQueue, itemsInQueue); var startTime = DateTime.Now; var sleepTimes = GetSleepTimes(batchTime * 60, itemsToQueue, false); //var delayIdeal = ((batchTime * 60.0) - (averageExecutionTimes[source.Key] * itemsToQueue)) / itemsToQueue; int stIndex = -1; foreach (var product in source.OrderBy(x => x.CreationTime).Take(itemsToQueue)) { var sleepTime = sleepTimes[++stIndex]; SlurpItem item = new SlurpItem(Type.GetType(compareSource.ProductCompareSourceType), product.QueueID, sleepTime); sourceQueue.Add(item); } var currentHandle = siteHandles[idx]; Action wrappedAction = () => { try { Parallel.ForEach(sourceQueue, parallelOptions, x => { SlurpResult result = x.Process(); lock (_locker) { //var avg = (from l in slurpQueueRepo.GetAll(c => c.IsCompleted && c.StartTime >= startTime) // select (l.CompletionTime.Value - l.StartTime.Value).TotalMilliseconds / 1000.0).ToList().Average(); //log.InfoFormat("Average execution rate for {1} : {0:f2}s / product", avg, result.SiteName); } }); } finally { currentHandle.Set(); } }; ThreadPool.QueueUserWorkItem(x => wrappedAction()); } if (siteHandles != null && siteHandles.Length > 0) { WaitHandle.WaitAll(siteHandles); } //ctx.SubmitChanges(); } }
public SlurpResult Process() { log.InfoFormat("Start task for {0}", PluginType.Name); using (var unit = ServiceLocator.Current.GetInstance <IUnitOfWork>()) { var QueueItem = unit.Scope.Repository <SlurpQueue>().GetSingle(x => x.QueueID == QueueID); QueueItem.StartTime = DateTime.Now; ProductID = QueueItem.ProductID; SlurpResult result = new SlurpResult(QueueItem.Product.VendorItemNumber); try { var plugin = (ISlurpSite)Activator.CreateInstance(PluginType); result = plugin.Process(QueueItem.Product.VendorItemNumber); result.SiteName = PluginType.Name; #region Update Results if (result.ProductStatus == ProductStatus.Valid) { var mappingRepo = unit.Scope.Repository <ProductCompetitorMapping>(); var priceRepo = unit.Scope.Repository <ProductCompetitorPrice>(); var mappings = mappingRepo.GetAll(x => x.ProductCompareSourceID == QueueItem.ProductCompareSourceID).ToList(); var prices = priceRepo.GetAll(x => x.ProductCompetitorMapping.ProductCompareSourceID == QueueItem.ProductCompareSourceID).ToList(); foreach (var shop in result.Shops) { var mapping = mappings.SingleOrDefault(c => c.Competitor == shop.Name.Trim()); if (mapping == null) { //new mapping = new ProductCompetitorMapping() { Competitor = shop.Name.Trim(), ProductCompareSourceID = QueueItem.ProductCompareSourceID, ProductCompetitorID = null }; mappingRepo.Add(mapping); // context.SubmitChanges(); mappings.Add(mapping); } var newPrice = shop.TotalPrice ?? 0; var stockInfo = shop.Delivery.ToString(); //price insert ProductCompetitorPrice price = prices.SingleOrDefault(p => p.ProductCompetitorMappingID == mapping.ProductCompetitorMappingID && p.ProductID == QueueItem.ProductID); if (price == null) { price = new ProductCompetitorPrice() { ProductCompetitorMapping = mapping, ProductID = QueueItem.ProductID, Price = newPrice, Stock = stockInfo, CreationTime = DateTime.Now, LastImport = DateTime.Now.AddDays(-30) }; priceRepo.Add(price); } else { //ledger //check for diff if (price.Price != newPrice || price.Stock != stockInfo) { ProductCompetitorLedger ledger = new ProductCompetitorLedger() { Price = price.Price, Stock = price.Stock, ProductCompetitorPriceID = price.ProductCompetitorPriceID, CreationTime = DateTime.Now }; price.Price = newPrice; price.Stock = shop.Delivery.ToString(); unit.Scope.Repository <ProductCompetitorLedger>().Add(ledger); } } } CompleteItem(unit, QueueItem); //unit.Save(); } else { CompleteItem(unit, QueueItem); } #endregion log.DebugFormat("Sleeping for {0} seconds.", SleepTime); Thread.Sleep(SleepTime * 1000); } catch (Exception ex) { log.Error("Error during processing", ex); result = new SlurpResult(QueueItem.Product.VendorItemNumber); result.SiteName = PluginType.Name; CompleteItem(unit, QueueItem); } finally { WriteLog(result); } return(result); } }