internal static async Task NumberUsers()
{
Task bidNumberTask = null;
using (var context = new HypixelContext())
{
for (int i = 0; i < 3; i++)
{
var doublePlayersId = await context.Players.GroupBy(p => p.Id).Where(p => p.Count() > 1).Select(p => p.Key).FirstOrDefaultAsync();
if (doublePlayersId == 0)
{
break;
}
await ResetDoublePlayers(context, doublePlayersId);
}
await context.SaveChangesAsync();
var unindexedPlayers = await context.Players.Where(p => p.Id == 0).Take(2000).ToListAsync();
if (unindexedPlayers.Any())
{
Console.Write($" numbering: {unindexedPlayers.Count()} ");
foreach (var player in unindexedPlayers)
{
player.Id = System.Threading.Interlocked.Increment(ref Indexer.highestPlayerId);
context.Players.Update(player);
}
// save all the ids
await context.SaveChangesAsync();
}
if (unindexedPlayers.Count < 2000)
{
// all players in the db have an id now
bidNumberTask = Task.Run(NumberBids);
await NumberAuctions(context);
await context.SaveChangesAsync();
}
// temp migration
foreach (var item in context.Auctions.Where(a => a.UId == 0)
.OrderByDescending(a => a.Id).Take(5000))
{
item.UId = AuctionService.Instance.GetId(item.Uuid);
context.Update(item);
}
await context.SaveChangesAsync();
}
if (bidNumberTask != null)
{
await bidNumberTask;
}
// give the db a moment to store everything
await Task.Delay(2000);
}
private static async Task ToDb(List <SaveAuction> auctions)
{
auctions = auctions.Distinct(new AuctionComparer()).ToList();
lock (nameof(highestPlayerId))
{
if (highestPlayerId == 1)
{
LoadFromDB();
}
}
using (var context = new HypixelContext())
{
Dictionary <string, SaveAuction> inDb = await GetExistingAuctions(auctions, context);
var comparer = new BidComparer();
foreach (var auction in auctions)
{
ProcessAuction(context, inDb, comparer, auction);
}
//Program.AddPlayers (context, playerIds);
await context.SaveChangesAsync();
context.Dispose();
}
}
private static async Task <int> DoBatch(IEnumerable <AveragePrice> items, int count, HypixelContext context)
{
var lookup = items.Select(p => p.Date).ToList();
var exising = context.Prices.Where(p => lookup.Any(l => l == p.Date)).ToList();
Console.WriteLine($"loaded a total of {exising.Count} prices to check against");
foreach (var item in items)
{
if (context.Prices.Any(p => p.ItemId == item.ItemId && p.Date == item.Date))
{
continue;
}
item.Id = 0;
context.Prices.Add(item);
count++;
}
await context.SaveChangesAsync();
if (context.Items.Any() && context.Players.Count() > 20_000)
{
Program.Migrated = true;
}
return(count);
}
private static async void NumberBids()
{
using (var context = new HypixelContext())
{
try
{
var bidsWithoutSellerId = await context.Bids.Where(a => a.BidderId == 0).Take(batchSize).ToListAsync();
foreach (var bid in bidsWithoutSellerId)
{
bid.BidderId = GetOrCreatePlayerId(context, bid.Bidder);
if (bid.BidderId == 0)
{
// his player has not yet received his number
continue;
}
context.Bids.Update(bid);
}
await context.SaveChangesAsync();
}
catch (Exception e)
{
Console.WriteLine($"Ran into error on numbering bids {e.Message} {e.StackTrace}");
}
}
}
public override async Task Execute(MessageData data)
{
//data.Data = CacheService.Unzip(data.GetAs<byte[]>());
var items = data.GetAs <List <DBItem> >();
using (var context = new HypixelContext())
{
foreach (var item in items)
{
if (context.Items.Any(p => p.Tag == item.Tag || p.Id == item.Id))
{
continue;
}
context.Items.Add(item);
}
var affected = await context.SaveChangesAsync();
Console.WriteLine($"Synced items {affected}");
if (context.Players.Count() > 20_000)
{
Program.Migrated = true;
}
}
}
internal async Task Send(int userId, string title, string text, string url, string icon, object data = null)
{
var not = new Notification(title, text, url, icon, null, data);
if (!doubleChecker.HasNeverBeenSeen(userId, not))
{
return;
}
try
{
using (var context = new HypixelContext())
{
var devices = context.Users.Where(u => u.Id == userId).SelectMany(u => u.Devices);
foreach (var item in devices)
{
Console.WriteLine("sending to " + item.UserId);
var success = await TryNotifyAsync(item.Token, not);
if (success)
{
return;
}
dev.Logger.Instance.Error("Sending pushnotification failed to");
dev.Logger.Instance.Error(JsonConvert.SerializeObject(item));
context.Remove(item);
}
await context.SaveChangesAsync();
}
}
catch (Exception)
{
dev.Logger.Instance.Error($"Could not send {not.body} to {userId}");
}
}
private static async Task BackfillAuctions(HypixelContext context, DateTime skyblockStart, int itemId)
{
DateTime start = skyblockStart;
var updateDaySize = 7;
var end = skyblockStart;
while (
end < DateTime.Now - TimeSpan.FromDays(updateDaySize))
{
start = end;
end = start + TimeSpan.FromDays(updateDaySize);
if (context.Prices.Where(p => p.ItemId == itemId && p.Date < end.AddMinutes(1) && p.Date > start.AddMinutes(-1)).Any())
{
continue;
}
var select = AuctionSelect(start, end, context, itemId);
var result = await AvgFromAuctions(itemId, select);
await context.Prices.AddRangeAsync(result);
await context.SaveChangesAsync();
if (result.Count() != 0)
{
Console.Write($"SIP {itemId} ");
}
await Task.Delay(100);
}
}
public async Task <int> Unsubscribe(int userId, string topic, SubscribeItem.SubType type)
{
using (var context = new HypixelContext())
{
var subs = context.SubscribeItem.Where(s => s.UserId == userId && s.TopicId == topic && s.Type == type).FirstOrDefault();
if (subs != null)
{
context.SubscribeItem.Remove(subs);
}
if (type.HasFlag(SubscribeItem.SubType.PRICE_HIGHER_THAN) || type.HasFlag(SubscribeItem.SubType.PRICE_LOWER_THAN))
{
RemoveSubscriptionFromCache(userId, topic, type, PriceUpdate);
}
if (type.HasFlag(SubscribeItem.SubType.SOLD))
{
RemoveSubscriptionFromCache(userId, topic, type, Sold);
}
if (type.HasFlag(SubscribeItem.SubType.OUTBID))
{
RemoveSubscriptionFromCache(userId, topic, type, outbid);
}
if (type.HasFlag(SubscribeItem.SubType.AUCTION))
{
RemoveSubscriptionFromCache(userId, topic, type, AuctionSub);
}
return(await context.SaveChangesAsync());
}
}
private async Task AddOccurences(HypixelContext context)
{
foreach (var itemId in ItemDetails.Instance.TagLookup.Values)
{
var sample = await context.Auctions
.Where(a => a.ItemId == itemId)
.OrderByDescending(a => a.Id)
.Take(20)
.Select(a => a.ItemName)
.ToListAsync();
var names = context.AltItemNames.Where(n => n.DBItemId == itemId);
foreach (var item in names)
{
var occured = sample.Count(s => s == item.Name);
if (occured == 0)
{
continue;
}
item.OccuredTimes += occured;
context.Update(item);
}
await context.SaveChangesAsync();
}
await Task.Delay(TimeSpan.FromSeconds(1));
}
public static async Task <int> UpdateFlaggedNames()
{
var updated = 0;
var targetAmount = 100;
using (var context = new HypixelContext())
{
var players = context.Players.Where(p => p.ChangedFlag && p.Id > 0)
.OrderBy(p => p.UpdatedAt)
.Take(targetAmount).ToList();
foreach (var player in players)
{
player.Name = await Program.GetPlayerNameFromUuid(player.UuId);
player.ChangedFlag = false;
player.UpdatedAt = DateTime.Now;
context.Players.Update(player);
}
updated = await context.SaveChangesAsync();
}
LastUpdate = DateTime.Now;
updateCount++;
return(updated);
}
public override async Task Execute(MessageData data)
{
var response = data.GetAs <PlayerSyncCommand.PlayerSyncData>();
var players = response.Players;
var ids = players.Select(p => p.UuId);
if (players.Count == 0)
{
return; // done
}
int count = 0;
using (var context = new HypixelContext())
{
var existingPlayers = context.Players.Where(p => ids.Contains(p.UuId));
var existing = existingPlayers.ToDictionary(p => p.UuId);
foreach (var player in players)
{
if (existing.ContainsKey(player.UuId))
{
var existingPlayer = existingPlayers.Where(p => p.UuId == player.UuId).FirstOrDefault();
if (existingPlayer.Name == null)
{
existingPlayer.Name = player.Name;
context.Update(existingPlayer);
}
continue;
}
context.Players.Add(player);
count++;
if (count % 1000 == 0)
{
await context.SaveChangesAsync();
}
}
try
{
await context.SaveChangesAsync();
}
catch (Exception e)
{
dev.Logger.Instance.Error(e, "playerSync");
}
count = context.Players.Count();
}
await data.SendBack(data.Create("playerSync", response.Offset));
}
private async Task FillYesterDayForever()
{
while (true)
{
try
{
var start = (DateTime.Now - TimeSpan.FromDays(1)).Date;
var end = start + TimeSpan.FromDays(1);
using (var context = new HypixelContext())
{
context.Database.SetCommandTimeout(3600);
var idOfLava = ItemDetails.Instance.GetItemIdForName("ENCHANTED_LAVA_BUCKET");
if (!context.Prices.Where(p => p.Date >= start && p.Date <= end && p.ItemId == idOfLava).Any())
{
await context.Prices.AddRangeAsync(await AvgBazzarHistory(start, end));
}
await context.SaveChangesAsync();
foreach (var itemId in ItemDetails.Instance.TagLookup.Values)
{
if (context.Prices.Where(p => p.Date >= start && p.Date <= end && p.ItemId == itemId).Any())
{
continue;
}
var select = AuctionSelect(start, end, context, itemId);
var result = await AvgFromAuctions(itemId, select);
await context.Prices.AddRangeAsync(result);
await context.SaveChangesAsync();
}
}
// wait for tomorrow (only when no exception)
await Task.Delay(DateTime.Now.Date + TimeSpan.FromDays(1.0001) - DateTime.Now);
}
catch (Exception e)
{
Logger.Instance.Error($"Daily prices failed: {e.Message} \n {e.StackTrace}");
await Task.Delay(TimeSpan.FromMinutes(2));
}
}
}
public async Task SaveHits()
{
using (var context = new HypixelContext())
{
//if (updateCount % 12 == 5)
// PartialUpdateCache(context);
ItemDetails.Instance.SaveHits(context);
PlayerSearch.Instance.SaveHits(context);
await context.SaveChangesAsync();
}
updateCount++;
}
private async Task UpdateItem(DBItem existingItem, DBItem newItem)
{
await Task.Delay(5000);
Console.WriteLine("updating item");
using (var context = new HypixelContext())
{
newItem.Id = existingItem.Id;
context.Items.Update(newItem);
await context.SaveChangesAsync();
}
}
static async Task FlagOldest()
{
// this is a workaround, because the "updatedat" field is only updated when there is a change
using (var context = new HypixelContext())
{
var players = context.Players.Where(p => p.Id > 0)
.OrderBy(p => p.UpdatedAt).Take(50);
foreach (var p in players)
{
p.ChangedFlag = true;
context.Players.Update(p);
}
await context.SaveChangesAsync();
}
}
public async override Task Execute(MessageData data)
{
using (var context = new HypixelContext())
{
var name = data.GetAs <string>();
var device = data.User.Devices.Where(d => d.Name == name).FirstOrDefault();
if (device != null)
{
context.Remove(device);
}
await context.SaveChangesAsync();
await data.Ok();
}
}
private async Task FulfillOrder(Stripe.Checkout.Session session)
{
Console.WriteLine("Furfilling order");
var googleId = Int32.Parse(session.ClientReferenceId);
var id = session.CustomerId;
//var email = session.CustomerEmail;
var days = Int32.Parse(session.Metadata["days"]);
Console.WriteLine("STRIPE");
using (var context = new HypixelContext())
{
var user = await context.Users.Where(u => u.Id == googleId).FirstAsync();
UserService.Instance.SavePurchase(user, days, session.Id);
await context.SaveChangesAsync();
Console.WriteLine("order completed");
}
}
public async Task DoBatch()
{
await Task.Delay(TimeSpan.FromMinutes(5));
var batchSize = 2000;
using (var context = new HypixelContext())
{
var select = context.Auctions
.Where(a => a.Id < minId)
.OrderByDescending(a => a.Id)
.Include(a => a.NBTLookup)
.Include(a => a.NbtData)
.Take(batchSize);
foreach (var auction in select)
{
if (auction.NBTLookup != null && auction.NBTLookup.Count > 0)
{
continue;
}
try
{
auction.NBTLookup = NBT.CreateLookup(auction.NbtData);
context.Update(auction);
}
catch (Exception e)
{
Console.WriteLine($"could not generate nbtlookup for {auction.Uuid} {e.Message} \n{e.StackTrace} \n {e.InnerException?.Message} {e.InnerException.StackTrace}");
}
}
int updated = await context.SaveChangesAsync();
Console.WriteLine($"updated nbt lookup for {updated} auctions, highest: {minId}");
minId -= batchSize;
}
}
private static async Task ResetDoublePlayers(HypixelContext context, int doublePlayersId)
{
if (doublePlayersId % 3 == 0)
{
Console.WriteLine($"Found Double player id: {doublePlayersId}, renumbering, highestId: {Indexer.highestPlayerId}");
}
foreach (var item in context.Players.Where(p => p.Id == doublePlayersId))
{
item.Id = 0;
context.Update(item);
}
foreach (var item in context.Auctions.Where(p => p.SellerId == doublePlayersId))
{
item.SellerId = 0;
context.Update(item);
}
foreach (var item in context.Bids.Where(p => p.BidderId == doublePlayersId))
{
item.BidderId = 0;
context.Update(item);
}
await context.SaveChangesAsync();
}
public async Task BackfillPrices()
{
Console.WriteLine("starting to backfill item prices :)");
using (var context = new HypixelContext())
{
context.Database.SetCommandTimeout(3600);
// bazzar
DateTime start = new DateTime();
var idOfLava = ItemDetails.Instance.GetItemIdForName("ENCHANTED_LAVA_BUCKET");
var end = new DateTime(2020, 5, 1);
while (
end < DateTime.Now - TimeSpan.FromDays(1))
{
start = end;
var size = 1d;
// a lot of data in between these
if (start >= new DateTime(2020, 05, 25) && start < new DateTime(2020, 6, 1))
{
size = 0.25;
}
end = start + TimeSpan.FromDays(size);
// only requery if lava has no price
if (context.Prices.Where(p => p.Date >= start - TimeSpan.FromSeconds(1) && p.Date <= end && p.ItemId == idOfLava).Any())
{
continue;
}
try
{
var data = await AvgBazzarHistory(start, end);
if (data.Count() == 0)
{
continue;
}
await context.Prices.AddRangeAsync(data);
await context.SaveChangesAsync();
if (data.Count() != 0)
{
Console.WriteLine("Saved bazaar prices for day " + start);
}
}
catch (Exception e)
{
Console.WriteLine($"Backfill failed :( for day {start} \n{e.Message}\n {e.InnerException?.Message} {e.StackTrace}");
}
await Task.Delay(1000);
}
var skyblockStart = new DateTime(2019, 5, 1);
foreach (var itemId in ItemDetails.Instance.TagLookup.Values.ToList())
{
await BackfillAuctions(context, skyblockStart, itemId);
}
}
Console.WriteLine("## Backfill completed :)");
await FillYesterDayForever();
}