private static void HandleMarketOrderBulk(object sender, MsgHandlerEventArgs args) { var logger = CreateLogger <Program>(); var message = args.Message; try { using var context = new ConfiguredContext(); var marketOrderUpdates = JsonConvert.DeserializeObject <List <MarketOrderDB> >(Encoding.UTF8.GetString(message.Data)); // The database calls the unique Albion ID AlbionId // However the data client sends this as just Id // Transfer Id data to AlbionId to match database naming foreach (var marketOrderUpdate in marketOrderUpdates) { marketOrderUpdate.AlbionId = marketOrderUpdate.Id; marketOrderUpdate.Id = 0; } // AlbionId is always unique var dbOrders = context.MarketOrders .Where(x => marketOrderUpdates.Select(y => y.AlbionId).Contains(x.AlbionId)) .ToDictionary(x => x.AlbionId); foreach (var marketOrderUpdate in marketOrderUpdates) { dbOrders.TryGetValue(marketOrderUpdate.AlbionId, out var dbOrder); if (dbOrder != null) { dbOrder.UnitPriceSilver = marketOrderUpdate.UnitPriceSilver; dbOrder.UpdatedAt = DateTime.UtcNow; dbOrder.Amount = marketOrderUpdate.Amount; dbOrder.LocationId = marketOrderUpdate.LocationId; dbOrder.DeletedAt = null; context.MarketOrders.Update(dbOrder); } else { marketOrderUpdate.InitialAmount = marketOrderUpdate.Amount; marketOrderUpdate.CreatedAt = DateTime.UtcNow; marketOrderUpdate.UpdatedAt = DateTime.UtcNow; if (marketOrderUpdate.Expires > DateTime.UtcNow.AddYears(1)) { marketOrderUpdate.Expires = DateTime.UtcNow.AddDays(7); } context.MarketOrders.Add(marketOrderUpdate); } updatedOrderCounter++; } context.SaveChanges(); logger.LogInformation(GetLogMessage(marketOrderUpdates.Count, "Market Orders", updatedOrderCounter)); } catch (Exception ex) { logger.LogError(ex, "Error handling market order"); } }
private void OnExecute() { Console.CancelKeyPress += (sender, args) => { quitEvent.Set(); args.Cancel = true; }; var logger = CreateLogger <Program>(); if (Debug) { logger.LogInformation("Debugging enabled"); } using (var context = new ConfiguredContext()) { if (context.Database.EnsureCreated()) { logger.LogInformation("Database Created"); context.SaveChanges(); } else { logger.LogInformation("Database Exists"); } } logger.LogInformation($"Nats URL: {NatsUrl}"); logger.LogInformation($"NATS Connected, ID: {NatsConnection.ConnectedId}"); var incomingMarketOrders = NatsConnection.SubscribeAsync(marketOrdersDedupedBulk); var incomingMarketHistories = NatsConnection.SubscribeAsync(marketHistoriesDeduped); var incomingGoldData = NatsConnection.SubscribeAsync(goldDataDeduped); incomingMarketOrders.MessageHandler += HandleMarketOrderBulk; incomingMarketHistories.MessageHandler += HandleMarketHistory; incomingGoldData.MessageHandler += HandleGoldData; incomingMarketOrders.Start(); logger.LogInformation("Listening for Market Order Data"); incomingMarketHistories.Start(); logger.LogInformation("Listening for Market History Data"); incomingGoldData.Start(); logger.LogInformation("Listening for Gold Data"); logger.LogInformation($"Checking Every {ExpireCheckMinutes} Minutes for expired orders."); logger.LogInformation($"Deleting orders after {MaxAgeHours} hours"); expirationTimer.Change(TimeSpan.Zero, TimeSpan.FromMinutes(ExpireCheckMinutes)); quitEvent.WaitOne(); NatsConnection.Close(); }
private static void HandleGoldData(object sender, MsgHandlerEventArgs args) { var logger = CreateLogger <Program>(); var message = args.Message; try { var upload = JsonConvert.DeserializeObject <GoldPriceUpload>(Encoding.UTF8.GetString(message.Data)); if (upload.Prices.Length != upload.Timestamps.Length) { throw new Exception("Different list lengths"); } using var context = new ConfiguredContext(); for (var i = 0; i < upload.Prices.Length; i++) { var price = upload.Prices[i]; var timestamp = new DateTime(upload.Timestamps[i], DateTimeKind.Utc); var dbGold = context.GoldPrices.FirstOrDefault(x => x.Timestamp == timestamp); if (dbGold != null) { if (dbGold.Price != price) { dbGold.Price = price; context.GoldPrices.Update(dbGold); } } else { var goldPrice = new GoldPrice() { Price = price, Timestamp = timestamp }; context.GoldPrices.Add(goldPrice); } updatedGoldCounter++; } context.SaveChanges(); logger.LogInformation(GetLogMessage(upload.Prices.Length, "Gold Records", updatedGoldCounter)); } catch (Exception ex) { logger.LogError(ex, "Error handling gold data"); } }
private static void ExpirationTask(object state) { try { var start = DateTime.Now; File.AppendAllText("last-run.txt", start.ToString("F") + Environment.NewLine); var logger = CreateLogger <Program>(); logger.LogInformation("Expiration Task Starting"); using var context = new ConfiguredContext(); const int batchSize = 1000; var sleepTime = TimeSpan.FromSeconds(10); // Using int.MaxValue allows us to use the rows affected from the previous delete to determine if we should keep deleting // We shouldn't keep deleting if the last delete did not delete a full batch var lastDeletedOrderCount = int.MaxValue; var lastDeletedHistoryCount = int.MaxValue; var totalCount = 0; var changesLeft = true; while (changesLeft) { changesLeft = false; // Delete old market orders if (lastDeletedOrderCount >= batchSize) { lastDeletedOrderCount = context.Database.ExecuteSqlRaw(@$ "DELETE FROM market_orders WHERE deleted_at IS NULL AND ( expires < UTC_TIMESTAMP() OR updated_at < DATE_ADD(UTC_TIMESTAMP(), INTERVAL -{MaxAgeHours} HOUR) ) LIMIT {batchSize}"); totalCount += lastDeletedOrderCount; logger.LogInformation($"Expired {lastDeletedOrderCount} Market Orders. Total Order/History Expiration: {totalCount}"); Thread.Sleep(sleepTime); } // Delete old hourly history records if (lastDeletedHistoryCount >= batchSize) { lastDeletedHistoryCount = context.Database.ExecuteSqlRaw(@$ "DELETE FROM market_history WHERE aggregation = 1 AND timestamp < DATE_ADD(UTC_TIMESTAMP(), INTERVAL -7 DAY) LIMIT {batchSize}"); totalCount += lastDeletedHistoryCount; logger.LogInformation($"Expired {lastDeletedHistoryCount} Market Histories. Total Order/History Expiration: {totalCount}"); Thread.Sleep(sleepTime); } // Keep expiring when we are expiring large numbers at a time // Stop expiring when at less than a full batch if (lastDeletedOrderCount >= batchSize || lastDeletedHistoryCount >= batchSize) { changesLeft = true; } // We have been deleting for too long, kill this thread if ((DateTime.Now - start).TotalMinutes > ExpireCheckMinutes * 0.75) { logger.LogWarning("Killing Long Running Expiration Task"); changesLeft = false; } } logger.LogInformation($"Total Order/History Expirations: {totalCount} Time Spent: {DateTime.Now - start}"); } catch (Exception ex) { File.AppendAllText("last-run.txt", DateTime.Now.ToString("F") + Environment.NewLine + ex.ToString() + Environment.NewLine); } }
private static void HandleMarketHistory(object sender, MsgHandlerEventArgs args) { var logger = CreateLogger <Program>(); var message = args.Message; try { using var context = new ConfiguredContext(); var upload = JsonConvert.DeserializeObject <MarketHistoriesUpload>(Encoding.UTF8.GetString(message.Data)); var aggregationType = TimeAggregation.QuarterDay; if (upload.Timescale == Timescale.Day) { aggregationType = TimeAggregation.Hourly; } // Do not use the last history timestamp because it is a partial period // It is not guaranteed to be updated so it can appear that the count in the period was way lower var marketHistoryUpdates = upload.MarketHistories.OrderBy(x => x.Timestamp).SkipLast(1); var dbHistories = context.MarketHistories.Where(x => x.ItemTypeId == upload.AlbionIdString && x.Location == upload.LocationId && x.QualityLevel == upload.QualityLevel && marketHistoryUpdates.Select(x => new DateTime((long)x.Timestamp)).Contains(x.Timestamp) && x.AggregationType == aggregationType) .ToDictionary(x => x.Timestamp); foreach (var marketHistoryUpdate in marketHistoryUpdates) { var historyDate = new DateTime((long)marketHistoryUpdate.Timestamp); dbHistories.TryGetValue(historyDate, out var dbHistory); if (dbHistory == null) { dbHistory = new MarketHistoryDB { AggregationType = aggregationType, ItemTypeId = upload.AlbionIdString, Location = upload.LocationId, QualityLevel = upload.QualityLevel, ItemAmount = marketHistoryUpdate.ItemAmount, SilverAmount = marketHistoryUpdate.SilverAmount, Timestamp = historyDate }; context.MarketHistories.Add(dbHistory); } else { dbHistory.ItemAmount = marketHistoryUpdate.ItemAmount; dbHistory.SilverAmount = marketHistoryUpdate.SilverAmount; context.MarketHistories.Update(dbHistory); } updatedHistoryCounter++; } context.SaveChanges(); logger.LogInformation(GetLogMessage(marketHistoryUpdates.Count(), "Market Histories", updatedHistoryCounter)); } catch (Exception ex) { logger.LogError(ex, "Error handling market history"); } }