public static void GrabAuctions(HypixelApi hypixelApi)
{
var expired = hypixelApi.getAuctionsEnded();
var auctions = expired.Auctions.Select(item =>
{
var a = new SaveAuction()
{
Uuid = item.Uuid,
AuctioneerId = item.Seller,
Bids = new List <SaveBids>()
{
new SaveBids()
{
Amount = item.Price,
Bidder = item.Buyer,
Timestamp = item.TimeStamp,
ProfileId = "unknown"
}
},
HighestBidAmount = item.Price,
Bin = item.BuyItemNow,
End = DateTime.Now,
UId = AuctionService.Instance.GetId(item.Uuid)
};
NBT.FillDetails(a, item.ItemBytes);
return(a);
}).ToList();
SoldLastMin = auctions;
Indexer.AddToQueue(auctions);
Task.Run(() =>
{
foreach (var item in auctions)
{
SubscribeEngine.Instance.BinSold(item);
Flipper.FlipperEngine.Instance.AuctionSold(item);
}
}).ConfigureAwait(false);
Console.WriteLine($"Updated {expired.Auctions.Count} bin sells eg {expired.Auctions.First().Uuid}");
}
// builds the index for all auctions in the last hour
Task <int> Save(GetAuctionPage res, DateTime lastUpdate, ConcurrentDictionary <string, bool> activeUuids)
{
int count = 0;
var processed = res.Auctions.Where(item =>
{
activeUuids[item.Uuid] = true;
// nothing changed if the last bid is older than the last update
return(!(item.Bids.Count > 0 && item.Bids[item.Bids.Count - 1].Timestamp < lastUpdate ||
item.Bids.Count == 0 && item.Start < lastUpdate));
})
.Select(a =>
{
if (Program.Migrated)
{
ItemDetails.Instance.AddOrIgnoreDetails(a);
}
count++;
var auction = new SaveAuction(a);
return(auction);
}).ToList();
if (DateTime.Now.Minute % 30 == 7)
{
foreach (var a in res.Auctions)
{
var auction = new SaveAuction(a);
AuctionCount.AddOrUpdate(auction.Tag, k =>
{
return(DetermineWorth(0, auction));
}, (k, c) =>
{
return(DetermineWorth(c, auction));
});
}
}
var ended = res.Auctions.Where(a => a.End < DateTime.Now).Select(a => new SaveAuction(a));
/* var variableHereToRemoveWarning = taskFactory.StartNew(async () =>
* {
* await Task.Delay(TimeSpan.FromSeconds(20));
* await ItemPrices.Instance.AddEndedAuctions(ended);
* });*/
if (Program.FullServerMode)
{
Indexer.AddToQueue(processed);
}
else
{
FileController.SaveAs($"apull/{DateTime.Now.Ticks}", processed);
}
var started = processed.Where(a => a.Start > lastUpdate).ToList();
// do not slow down the update
var min = DateTime.Now - TimeSpan.FromMinutes(15);
Flipper.FlipperEngine.Instance.NewAuctions(started.Where(a => a.Start > min));
foreach (var auction in started)
{
SubscribeEngine.Instance.NewAuction(auction);
}
return(Task.FromResult(count));
}
async Task <DateTime> RunUpdate(DateTime updateStartTime)
{
/* Task.Run(()
* => BinUpdater.GrabAuctions(hypixel)
* );*/
BinUpdater.GrabAuctions(hypixel);
long max = 1;
var lastUpdate = lastUpdateDone; // new DateTime (1970, 1, 1);
//if (FileController.Exists ("lastUpdate"))
// lastUpdate = FileController.LoadAs<DateTime> ("lastUpdate").ToLocalTime ();
var lastUpdateStart = new DateTime(0);
if (FileController.Exists("lastUpdateStart"))
{
lastUpdateStart = FileController.LoadAs <DateTime>("lastUpdateStart").ToLocalTime();
}
if (!minimumOutput)
{
Console.WriteLine($"{lastUpdateStart > lastUpdate} {DateTime.Now - lastUpdateStart}");
}
FileController.SaveAs("lastUpdateStart", DateTime.Now);
TimeSpan timeEst = new TimeSpan(0, 1, 1);
Console.WriteLine($"Updating Data {DateTime.Now}");
// add extra miniute to start to catch lost auctions
lastUpdate = updateStartTime - new TimeSpan(0, 1, 0);
DateTime lastHypixelCache = lastUpdate;
var tasks = new List <Task>();
int sum = 0;
int doneCont = 0;
object sumloc = new object();
var firstPage = await hypixel?.GetAuctionPageAsync(0);
max = firstPage.TotalPages;
if (firstPage.LastUpdated == updateStartTime)
{
// wait for the server cache to refresh
await Task.Delay(5000);
return(updateStartTime);
}
OnNewUpdateStart?.Invoke();
var cancelToken = new CancellationToken();
AuctionCount = new ConcurrentDictionary <string, int>();
var activeUuids = new ConcurrentDictionary <string, bool>();
for (int i = 0; i < max; i++)
{
var index = i;
await Task.Delay(200);
tasks.Add(taskFactory.StartNew(async() =>
{
try
{
var res = index != 0 ? await hypixel?.GetAuctionPageAsync(index) : firstPage;
if (res == null)
{
return;
}
max = res.TotalPages;
if (index == 0)
{
lastHypixelCache = res.LastUpdated;
// correct update time
Console.WriteLine($"Updating difference {lastUpdate} {res.LastUpdated}\n");
}
var val = await Save(res, lastUpdate, activeUuids);
lock (sumloc)
{
sum += val;
// process done
doneCont++;
}
PrintUpdateEstimate(index, doneCont, sum, updateStartTime, max);
}
catch (Exception e)
{
try // again
{
var res = await hypixel?.GetAuctionPageAsync(index);
var val = await Save(res, lastUpdate, activeUuids);
}
catch (System.Exception)
{
Logger.Instance.Error($"Single page ({index}) could not be loaded twice because of {e.Message} {e.StackTrace} {e.InnerException?.Message}");
}
}
}, cancelToken).Unwrap());
PrintUpdateEstimate(i, doneCont, sum, updateStartTime, max);
// try to stay under 600MB
if (System.GC.GetTotalMemory(false) > 500000000)
{
Console.Write("\t mem: " + System.GC.GetTotalMemory(false));
System.GC.Collect();
}
//await Task.Delay(100);
}
foreach (var item in tasks)
{
//Console.Write($"\r {index++}/{updateEstimation} \t({index}) {timeEst:mm\\:ss}");
if (item != null)
{
await item;
}
PrintUpdateEstimate(max, doneCont, sum, updateStartTime, max);
}
if (AuctionCount.Count > 2)
{
LastAuctionCount = AuctionCount;
}
//BinUpdateSold(currentUpdateBins);
var lastUuids = ActiveAuctions;
ActiveAuctions = activeUuids;
var canceledTask = Task.Run(() =>
{
foreach (var item in ActiveAuctions.Keys)
{
lastUuids.TryRemove(item, out bool val);
}
foreach (var item in BinUpdater.SoldLastMin)
{
lastUuids.TryRemove(item.Uuid, out bool val);
}
Console.WriteLine($"canceled last min: {lastUuids.Count} {lastUuids.FirstOrDefault().Key}");
Indexer.AddToQueue(lastUuids.Select(id => new SaveAuction(id.Key)));
foreach (var item in lastUuids)
{
Flipper.FlipperEngine.Instance.AuctionInactive(item.Key);
}
}).ConfigureAwait(false);
if (sum > 10)
{
LastPull = DateTime.Now;
}
Console.WriteLine($"Updated {sum} auctions {doneCont} pages");
UpdateSize = sum;
OnNewUpdateEnd?.Invoke();
return(lastHypixelCache);
}