// builds the index for all auctions in the last hour
Task <int> Save(GetAuctionPage res, DateTime lastUpdate, ConcurrentDictionary <string, bool> activeUuids)
{
int count = 0;
var processed = res.Auctions.Where(item =>
{
activeUuids[item.Uuid] = true;
// nothing changed if the last bid is older than the last update
return(!(item.Bids.Count > 0 && item.Bids[item.Bids.Count - 1].Timestamp < lastUpdate ||
item.Bids.Count == 0 && item.Start < lastUpdate));
})
.Select(a =>
{
if (Program.Migrated)
{
ItemDetails.Instance.AddOrIgnoreDetails(a);
}
count++;
var auction = new SaveAuction(a);
return(auction);
}).ToList();
if (DateTime.Now.Minute % 30 == 7)
{
foreach (var a in res.Auctions)
{
var auction = new SaveAuction(a);
AuctionCount.AddOrUpdate(auction.Tag, k =>
{
return(DetermineWorth(0, auction));
}, (k, c) =>
{
return(DetermineWorth(c, auction));
});
}
}
var ended = res.Auctions.Where(a => a.End < DateTime.Now).Select(a => new SaveAuction(a));
/* var variableHereToRemoveWarning = taskFactory.StartNew(async () =>
* {
* await Task.Delay(TimeSpan.FromSeconds(20));
* await ItemPrices.Instance.AddEndedAuctions(ended);
* });*/
if (Program.FullServerMode)
{
Indexer.AddToQueue(processed);
}
else
{
FileController.SaveAs($"apull/{DateTime.Now.Ticks}", processed);
}
var started = processed.Where(a => a.Start > lastUpdate).ToList();
// do not slow down the update
var min = DateTime.Now - TimeSpan.FromMinutes(15);
Flipper.FlipperEngine.Instance.NewAuctions(started.Where(a => a.Start > min));
foreach (var auction in started)
{
SubscribeEngine.Instance.NewAuction(auction);
}
return(Task.FromResult(count));
}
async Task <DateTime> RunUpdate(DateTime updateStartTime)
{
/* Task.Run(()
* => BinUpdater.GrabAuctions(hypixel)
* );*/
BinUpdater.GrabAuctions(hypixel);
long max = 1;
var lastUpdate = lastUpdateDone; // new DateTime (1970, 1, 1);
//if (FileController.Exists ("lastUpdate"))
// lastUpdate = FileController.LoadAs<DateTime> ("lastUpdate").ToLocalTime ();
var lastUpdateStart = new DateTime(0);
if (FileController.Exists("lastUpdateStart"))
{
lastUpdateStart = FileController.LoadAs <DateTime>("lastUpdateStart").ToLocalTime();
}
if (!minimumOutput)
{
Console.WriteLine($"{lastUpdateStart > lastUpdate} {DateTime.Now - lastUpdateStart}");
}
FileController.SaveAs("lastUpdateStart", DateTime.Now);
TimeSpan timeEst = new TimeSpan(0, 1, 1);
Console.WriteLine($"Updating Data {DateTime.Now}");
// add extra miniute to start to catch lost auctions
lastUpdate = updateStartTime - new TimeSpan(0, 1, 0);
DateTime lastHypixelCache = lastUpdate;
var tasks = new List <Task>();
int sum = 0;
int doneCont = 0;
object sumloc = new object();
var firstPage = await hypixel?.GetAuctionPageAsync(0);
max = firstPage.TotalPages;
if (firstPage.LastUpdated == updateStartTime)
{
// wait for the server cache to refresh
await Task.Delay(5000);
return(updateStartTime);
}
OnNewUpdateStart?.Invoke();
var cancelToken = new CancellationToken();
AuctionCount = new ConcurrentDictionary <string, int>();
var activeUuids = new ConcurrentDictionary <string, bool>();
for (int i = 0; i < max; i++)
{
var index = i;
await Task.Delay(200);
tasks.Add(taskFactory.StartNew(async() =>
{
try
{
var res = index != 0 ? await hypixel?.GetAuctionPageAsync(index) : firstPage;
if (res == null)
{
return;
}
max = res.TotalPages;
if (index == 0)
{
lastHypixelCache = res.LastUpdated;
// correct update time
Console.WriteLine($"Updating difference {lastUpdate} {res.LastUpdated}\n");
}
var val = await Save(res, lastUpdate, activeUuids);
lock (sumloc)
{
sum += val;
// process done
doneCont++;
}
PrintUpdateEstimate(index, doneCont, sum, updateStartTime, max);
}
catch (Exception e)
{
try // again
{
var res = await hypixel?.GetAuctionPageAsync(index);
var val = await Save(res, lastUpdate, activeUuids);
}
catch (System.Exception)
{
Logger.Instance.Error($"Single page ({index}) could not be loaded twice because of {e.Message} {e.StackTrace} {e.InnerException?.Message}");
}
}
}, cancelToken).Unwrap());
PrintUpdateEstimate(i, doneCont, sum, updateStartTime, max);
// try to stay under 600MB
if (System.GC.GetTotalMemory(false) > 500000000)
{
Console.Write("\t mem: " + System.GC.GetTotalMemory(false));
System.GC.Collect();
}
//await Task.Delay(100);
}
foreach (var item in tasks)
{
//Console.Write($"\r {index++}/{updateEstimation} \t({index}) {timeEst:mm\\:ss}");
if (item != null)
{
await item;
}
PrintUpdateEstimate(max, doneCont, sum, updateStartTime, max);
}
if (AuctionCount.Count > 2)
{
LastAuctionCount = AuctionCount;
}
//BinUpdateSold(currentUpdateBins);
var lastUuids = ActiveAuctions;
ActiveAuctions = activeUuids;
var canceledTask = Task.Run(() =>
{
foreach (var item in ActiveAuctions.Keys)
{
lastUuids.TryRemove(item, out bool val);
}
foreach (var item in BinUpdater.SoldLastMin)
{
lastUuids.TryRemove(item.Uuid, out bool val);
}
Console.WriteLine($"canceled last min: {lastUuids.Count} {lastUuids.FirstOrDefault().Key}");
Indexer.AddToQueue(lastUuids.Select(id => new SaveAuction(id.Key)));
foreach (var item in lastUuids)
{
Flipper.FlipperEngine.Instance.AuctionInactive(item.Key);
}
}).ConfigureAwait(false);
if (sum > 10)
{
LastPull = DateTime.Now;
}
Console.WriteLine($"Updated {sum} auctions {doneCont} pages");
UpdateSize = sum;
OnNewUpdateEnd?.Invoke();
return(lastHypixelCache);
}
private static void FullServer()
{
Console.WriteLine($"\n - Starting FullServer {version} - \n");
Console.Write("Key: " + apiKey);
FullServerMode = true;
Indexer.MiniumOutput();
server = new Server();
Task.Run(() => server.Start()).ConfigureAwait(false);
Task.Run(() => CreateHost(new string[0])).ConfigureAwait(false);
var mode = SimplerConfig.Config.Instance["MODE"];
LightClient = mode == "light";
if (LightClient)
{
ItemDetails.Instance.LoadLookup();
RunIsolatedForever(async() =>
{
await Task.Delay(TimeSpan.FromMinutes(1));
await SearchService.Instance.SaveHits();
}, "saving hits failed");
System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
}
updater = new Updater(apiKey);
updater.UpdateForEver();
// bring the db up to date
GetDBToDesiredState();
ItemDetails.Instance.LoadFromDB();
SubscribeEngine.Instance.LoadFromDb();
var redisInit = MakeSureRedisIsInitialized();
Console.WriteLine("booting db dependend stuff");
var bazzar = new BazaarUpdater();
bazzar.UpdateForEver(apiKey);
RunIndexer();
Flipper.FlipperEngine.diabled = FileController.Exists("blockFlipper");
for (int i = 0; i < 4; i++)
{
RunIsolatedForever(Flipper.FlipperEngine.Instance.ProcessPotentialFlipps, $"flipper worker {i} got error");
}
NameUpdater.Run();
SearchService.Instance.RunForEver();
CacheService.Instance.RunForEver();
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromMinutes(3));
await ItemPrices.Instance.BackfillPrices();
}).ConfigureAwait(false);;
onStop += () =>
{
StopServices(updater, server, bazzar);
};
try
{
CleanDB();
}
catch (Exception e)
{
Console.WriteLine($"Cleaning failed {e.Message}");
}
redisInit.GetAwaiter().GetResult();
System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
}
static void Main(string[] args)
{
StripeKey = SimplerConfig.Config.Instance["stripeKey"];
StripeSigningSecret = SimplerConfig.Config.Instance["stripeSecret"];
StripeConfiguration.ApiKey = Program.StripeKey;
Console.CancelKeyPress += delegate
{
Console.WriteLine("\nAbording");
onStop?.Invoke();
var cacheCount = StorageManager.CacheItems;
StorageManager.Stop();
Indexer.Stop();
var t = StorageManager.Save();
Console.WriteLine("Saving");
t.Wait();
Console.WriteLine($"Saved {cacheCount}");
};
if (args.Length > 0)
{
FileController.dataPaht = args[0];
Directory.CreateDirectory(FileController.dataPaht);
Directory.CreateDirectory(FileController.dataPaht + "/users");
Directory.CreateDirectory(FileController.dataPaht + "/auctions");
if (args.Length > 1)
{
runSubProgram(args[1][0]);
return;
}
}
displayMode = true;
while (true)
{
//try {
Console.WriteLine("1) List Auctions");
Console.WriteLine("2) List Bids");
Console.WriteLine("3) Display");
Console.WriteLine("4) List Won Bids");
Console.WriteLine("5) Search For auction");
Console.WriteLine("6) Avherage selling price in the last 2 weeks");
Console.WriteLine("9) End");
var res = Console.ReadKey();
if (runSubProgram(res.KeyChar))
{
return;
}
//} catch(Exception e)
//{
// Console.WriteLine("Error Occured: "+ e.Message);
// throw e;
//}
}
}
/// <summary>
/// returns true if application should be closed
/// </summary>
/// <param name="mode"></param>
/// <returns></returns>
static bool runSubProgram(char mode)
{
switch (mode)
{
case 't':
// test
//NotificationService.Instance.NotifyAsync("dPRj0dnG2NcY_kMTdNbpjz:APA91bHJINgv1SjuUlv-sGM21wLlHX5ISC5nYgl8DKP2r0fm273Cs0ujcESW6NR1RyGvFDtTBdQLK0SSq5TY_guLgc57VylKk8AAnH_xKq3zDIrdA1F6UhJNTu-Q0wNDKKIIQkYoVcyj","test","click me","https://sky.coflnet.com").Wait();
SetGoogleIdCommand.ValidateToken("eyJhbGciOiJSUzI1NiIsImtpZCI6IjI1MmZjYjk3ZGY1YjZiNGY2ZDFhODg1ZjFlNjNkYzRhOWNkMjMwYzUiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJhY2NvdW50cy5nb29nbGUuY29tIiwiYXpwIjoiNTcwMzAyODkwNzYwLW5sa2dkOTliNzFxNGQ2MWFtNGxwcWRoZW4xcGVuZGR0LmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29tIiwiYXVkIjoiNTcwMzAyODkwNzYwLW5sa2dkOTliNzFxNGQ2MWFtNGxwcWRoZW4xcGVuZGR0LmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29tIiwic3ViIjoiMTAxOTkzNTcwNzI0MDg4NDMyMjk4IiwiZW1haWwiOiJ0by5jb2ZsbmV0QGdtYWlsLmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJhdF9oYXNoIjoiYWdLN21RM2YySFZQclZNQ3l1UVVmdyIsIm5hbWUiOiJFa3dhdiBDb2ZsbmV0IiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hLS9BT2gxNEdobEx6TjV5U1o3VDZWYnpYRnFhUlR4c3dNRXJLaW1VQk1uem41Nz1zOTYtYyIsImdpdmVuX25hbWUiOiJFa3dhdiIsImZhbWlseV9uYW1lIjoiQ29mbG5ldCIsImxvY2FsZSI6ImRlIiwiaWF0IjoxNjEwMjk4MTE5LCJleHAiOjE2MTAzMDE3MTksImp0aSI6ImIzMWYzODUwNDMwYjNhOWMxNTQ5YTRjMDFiNTFiNTBlZjBhZTkwYTAifQ.cvsqp0GaYca---qkBAm-nS3QI-x_ZTGkzZh7sk-SsYctubikHqJz9VpafY_ih88ouOFTg_CWHKPMvS9dTrR8T4W_iY65cYp2hxsc-iMignDBgxbP6KlUCm3MvpRTHTdLAtL3Eq4JeXAL6_BN21AetRMaOhsWMgvz6yprhTkirOgFSuDt386Q8NXr19csjDhAW6bb2bRwEYJp4ZlBXD77zfzP_kZaF2y671M_lZUXnrqKrDqF7sFL2Jx4r6htKV_e86IuKhx0N1ttNTuEOeqccIZHdRQasivVO9Nq0twjhFIWn-5-azkPyz0VstxzIuYc7mTi2LSVjF4QDl-aLiOlPQ");
break;
case 'b':
//var key = System.Text.Encoding.UTF8.GetString (FileController.ReadAllBytes ("apiKey")).Trim ();
BazaarUpdater.NewUpdate(apiKey).Wait();
break;
case 'f':
FullServer();
break;
case 's':
var server = new Server();
server.Start().Wait();
break;
case 'u':
var updater = new Updater(apiKey);
updater.Update().Wait();
break;
case '7':
displayMode = false;
StorageManager.Migrate();
var auction = StorageManager.GetOrCreateAuction("00e45a19c27848829612be8edf53bd71");
Console.WriteLine(auction.ItemName);
//Console.WriteLine(ItemReferences.RemoveReforges("Itchy Bat man"));
break;
case 'i':
Flipper.FlipperEngine.Instance.Test();
break;
case 'p':
Indexer.LastHourIndex().Wait();
//StorageManager.Migrate();
break;
case 'n':
Console.WriteLine(NBT.Pretty("H4sIAAAAAAAAAHVUS28jRRAux8mu4ywbxIkLohdtRCKvkxnb49cByXGcxIJ1IvJYblbPTHmmyUyP6elxyJEbJ84cFgkJpEgcOSMh5afkhyCqZ5woHLjMo/qrr756dRVgHUqiCgClFVgRfqlegrVhkkldqkJZ82AdVlF64RJRjhYRlHMkVKAEGxfSVcivuBthqQzrx8LHw4gHKcH/qcJzX6TziN8QyVeJwgpZP4FP7247R8gVO/PI1md3t75j9+jV3bYbVmsnB5xphTLQoTn2araTH9eand2es0MsNkGOkUcFgNcalsOOT+kTt2stiz53CgfH7u62zdEOfE4uBzhDmWLh07UKfMNaghtWY5fCdwk4lhqjSASU+hJtN42G3vYEPZVo4bGa/ejYbNm7Tq6LgvQon7vb6J1I/SRml2+M1wHOdcgoJyqPYuPxGF6R9RC5Dun/kFMoGbBvcuyRSq4JfAnm55RioadFIpdMX+O32QIl10gIgK272/ZhFkXsDDXbT2SW9tloNhOeQKmZVlxIYobPKKXRAtUNUbgmj86Qa+4lsZuyCBcYpW9MuI4ORcq4ihPF5gI9ZAERpKxGBWhswQt6ESjVXKe7FNywHikuNfn4vjAqecRcowJT5vIU/RxD4m+STLEUI8oFfeZnMkCyehFP01fE9Jpk7aNKUV0R3hS8QzG9hnX/yw/sYRRgSCjTdFRPQNy2tswndV8h9QsVz8vFpf9A0yKa31kxLqZi7lse4BMK17Hu3//EnvYcqIsdOjnn8qki3iSqH9lyjMiWh1HoZx5l7POYiMExheRXKJlb1DvXN6OaIvfCZWGpICzkC2T4XSbmTMxgj0DGyhWyUGjjTNPBUh4jixOZapqUa0EdkmZsbQte0su0A71E+tSQLyjWQHnhf8rTc+5//o0NFREOQ06ZPdhtOvi1ODgodAN8fHdL4qPR6XjIDi4mR6OTCds/OTk/K8Oal0SJgr//+rMCqxPSZCrkPC7DwOdzLSid/SShcaCpvH//x/89oQqbo+9pOAeaNsLNNKZl+DBM9HSe0GwlU8/cP6SnWnmIu2H37L7d7fbtTrcClTjxxUyggopcKijDR8upmgqN8TSfaqJYq8B6okQg5DkP4PnF5MvJybtJJb/AXg4OBqfn48vRNE+yCi/MTUfzHNPykKQPfLO207RYWyIrl2Ezi7SIaf2m1/mCmxBknRWrPJ0Vq2y0l6GqHpe1gD0L8t0ufqrzx902BrpOV7OMVL3mLbvddjtOHTvYrrfsZrfOXR/rnPdmlt12vRl3yZ3LhYimZtXIfYPSJF1IuxnPYbO713D2GhZr922bDd4CrMCzxy7Dvy+PLTX1BQAA"));
//Console.WriteLine (JsonConvert.SerializeObject (.Instance.Items.Where (item => item.Value.AltNames != null && item.Value.AltNames.Count > 3 && !item.Key.Contains("DRAGON")).Select((item)=>new P(item.Value))));
break;
case 'g':
var ds = new DataSyncer();
ds.Sync("e5bac11a8cc04ca4bae539aed6500823");
break;
case 'm':
Migrator.Migrate();
break;
default:
return(true);
}
return(false);
}