Ejemplo n.º 1
0
        // builds the index for all auctions in the last hour

        Task <int> Save(GetAuctionPage res, DateTime lastUpdate, ConcurrentDictionary <string, bool> activeUuids)
        {
            int count = 0;

            var processed = res.Auctions.Where(item =>
            {
                activeUuids[item.Uuid] = true;
                // nothing changed if the last bid is older than the last update
                return(!(item.Bids.Count > 0 && item.Bids[item.Bids.Count - 1].Timestamp < lastUpdate ||
                         item.Bids.Count == 0 && item.Start < lastUpdate));
            })
                            .Select(a =>
            {
                if (Program.Migrated)
                {
                    ItemDetails.Instance.AddOrIgnoreDetails(a);
                }
                count++;
                var auction = new SaveAuction(a);
                return(auction);
            }).ToList();


            if (DateTime.Now.Minute % 30 == 7)
            {
                foreach (var a in res.Auctions)
                {
                    var auction = new SaveAuction(a);
                    AuctionCount.AddOrUpdate(auction.Tag, k =>
                    {
                        return(DetermineWorth(0, auction));
                    }, (k, c) =>
                    {
                        return(DetermineWorth(c, auction));
                    });
                }
            }

            var ended = res.Auctions.Where(a => a.End < DateTime.Now).Select(a => new SaveAuction(a));

            /* var variableHereToRemoveWarning = taskFactory.StartNew(async () =>
             * {
             *   await Task.Delay(TimeSpan.FromSeconds(20));
             *   await ItemPrices.Instance.AddEndedAuctions(ended);
             * });*/


            if (Program.FullServerMode)
            {
                Indexer.AddToQueue(processed);
            }
            else
            {
                FileController.SaveAs($"apull/{DateTime.Now.Ticks}", processed);
            }


            var started = processed.Where(a => a.Start > lastUpdate).ToList();

            // do not slow down the update
            var min = DateTime.Now - TimeSpan.FromMinutes(15);

            Flipper.FlipperEngine.Instance.NewAuctions(started.Where(a => a.Start > min));
            foreach (var auction in started)
            {
                SubscribeEngine.Instance.NewAuction(auction);
            }

            return(Task.FromResult(count));
        }
Ejemplo n.º 2
0
        async Task <DateTime> RunUpdate(DateTime updateStartTime)
        {
            /* Task.Run(()
             *    => BinUpdater.GrabAuctions(hypixel)
             * );*/
            BinUpdater.GrabAuctions(hypixel);
            long max        = 1;
            var  lastUpdate = lastUpdateDone; // new DateTime (1970, 1, 1);
            //if (FileController.Exists ("lastUpdate"))
            //    lastUpdate = FileController.LoadAs<DateTime> ("lastUpdate").ToLocalTime ();

            var lastUpdateStart = new DateTime(0);

            if (FileController.Exists("lastUpdateStart"))
            {
                lastUpdateStart = FileController.LoadAs <DateTime>("lastUpdateStart").ToLocalTime();
            }

            if (!minimumOutput)
            {
                Console.WriteLine($"{lastUpdateStart > lastUpdate} {DateTime.Now - lastUpdateStart}");
            }
            FileController.SaveAs("lastUpdateStart", DateTime.Now);

            TimeSpan timeEst = new TimeSpan(0, 1, 1);

            Console.WriteLine($"Updating Data {DateTime.Now}");

            // add extra miniute to start to catch lost auctions
            lastUpdate = updateStartTime - new TimeSpan(0, 1, 0);
            DateTime lastHypixelCache = lastUpdate;

            var    tasks     = new List <Task>();
            int    sum       = 0;
            int    doneCont  = 0;
            object sumloc    = new object();
            var    firstPage = await hypixel?.GetAuctionPageAsync(0);

            max = firstPage.TotalPages;
            if (firstPage.LastUpdated == updateStartTime)
            {
                // wait for the server cache to refresh
                await Task.Delay(5000);

                return(updateStartTime);
            }
            OnNewUpdateStart?.Invoke();

            var cancelToken = new CancellationToken();

            AuctionCount = new ConcurrentDictionary <string, int>();

            var activeUuids = new ConcurrentDictionary <string, bool>();

            for (int i = 0; i < max; i++)
            {
                var index = i;
                await Task.Delay(200);

                tasks.Add(taskFactory.StartNew(async() =>
                {
                    try
                    {
                        var res = index != 0 ? await hypixel?.GetAuctionPageAsync(index) : firstPage;
                        if (res == null)
                        {
                            return;
                        }

                        max = res.TotalPages;

                        if (index == 0)
                        {
                            lastHypixelCache = res.LastUpdated;
                            // correct update time
                            Console.WriteLine($"Updating difference {lastUpdate} {res.LastUpdated}\n");
                        }

                        var val = await Save(res, lastUpdate, activeUuids);
                        lock (sumloc)
                        {
                            sum += val;
                            // process done
                            doneCont++;
                        }
                        PrintUpdateEstimate(index, doneCont, sum, updateStartTime, max);
                    }
                    catch (Exception e)
                    {
                        try // again
                        {
                            var res = await hypixel?.GetAuctionPageAsync(index);
                            var val = await Save(res, lastUpdate, activeUuids);
                        }
                        catch (System.Exception)
                        {
                            Logger.Instance.Error($"Single page ({index}) could not be loaded twice because of {e.Message} {e.StackTrace} {e.InnerException?.Message}");
                        }
                    }
                }, cancelToken).Unwrap());
                PrintUpdateEstimate(i, doneCont, sum, updateStartTime, max);

                // try to stay under 600MB
                if (System.GC.GetTotalMemory(false) > 500000000)
                {
                    Console.Write("\t mem: " + System.GC.GetTotalMemory(false));
                    System.GC.Collect();
                }
                //await Task.Delay(100);
            }

            foreach (var item in tasks)
            {
                //Console.Write($"\r {index++}/{updateEstimation} \t({index}) {timeEst:mm\\:ss}");
                if (item != null)
                {
                    await item;
                }
                PrintUpdateEstimate(max, doneCont, sum, updateStartTime, max);
            }

            if (AuctionCount.Count > 2)
            {
                LastAuctionCount = AuctionCount;
            }

            //BinUpdateSold(currentUpdateBins);
            var lastUuids = ActiveAuctions;

            ActiveAuctions = activeUuids;
            var canceledTask = Task.Run(() =>
            {
                foreach (var item in ActiveAuctions.Keys)
                {
                    lastUuids.TryRemove(item, out bool val);
                }

                foreach (var item in BinUpdater.SoldLastMin)
                {
                    lastUuids.TryRemove(item.Uuid, out bool val);
                }
                Console.WriteLine($"canceled last min: {lastUuids.Count} {lastUuids.FirstOrDefault().Key}");
                Indexer.AddToQueue(lastUuids.Select(id => new SaveAuction(id.Key)));
                foreach (var item in lastUuids)
                {
                    Flipper.FlipperEngine.Instance.AuctionInactive(item.Key);
                }
            }).ConfigureAwait(false);

            if (sum > 10)
            {
                LastPull = DateTime.Now;
            }

            Console.WriteLine($"Updated {sum} auctions {doneCont} pages");
            UpdateSize = sum;

            OnNewUpdateEnd?.Invoke();

            return(lastHypixelCache);
        }
Ejemplo n.º 3
0
        private static void FullServer()
        {
            Console.WriteLine($"\n - Starting FullServer {version} - \n");
            Console.Write("Key: " + apiKey);
            FullServerMode = true;
            Indexer.MiniumOutput();

            server = new Server();
            Task.Run(() => server.Start()).ConfigureAwait(false);
            Task.Run(() => CreateHost(new string[0])).ConfigureAwait(false);

            var mode = SimplerConfig.Config.Instance["MODE"];

            LightClient = mode == "light";
            if (LightClient)
            {
                ItemDetails.Instance.LoadLookup();
                RunIsolatedForever(async() =>
                {
                    await Task.Delay(TimeSpan.FromMinutes(1));
                    await SearchService.Instance.SaveHits();
                }, "saving hits failed");
                System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
            }

            updater = new Updater(apiKey);
            updater.UpdateForEver();

            // bring the db up to date
            GetDBToDesiredState();
            ItemDetails.Instance.LoadFromDB();
            SubscribeEngine.Instance.LoadFromDb();
            var redisInit = MakeSureRedisIsInitialized();

            Console.WriteLine("booting db dependend stuff");

            var bazzar = new BazaarUpdater();

            bazzar.UpdateForEver(apiKey);
            RunIndexer();

            Flipper.FlipperEngine.diabled = FileController.Exists("blockFlipper");
            for (int i = 0; i < 4; i++)
            {
                RunIsolatedForever(Flipper.FlipperEngine.Instance.ProcessPotentialFlipps, $"flipper worker {i} got error");
            }

            NameUpdater.Run();
            SearchService.Instance.RunForEver();
            CacheService.Instance.RunForEver();
            Task.Run(async() =>
            {
                await Task.Delay(TimeSpan.FromMinutes(3));
                await ItemPrices.Instance.BackfillPrices();
            }).ConfigureAwait(false);;


            onStop += () =>
            {
                StopServices(updater, server, bazzar);
            };
            try
            {
                CleanDB();
            }
            catch (Exception e)
            {
                Console.WriteLine($"Cleaning failed {e.Message}");
            }

            redisInit.GetAwaiter().GetResult();

            System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
        }
Ejemplo n.º 4
0
        static void Main(string[] args)
        {
            StripeKey                  = SimplerConfig.Config.Instance["stripeKey"];
            StripeSigningSecret        = SimplerConfig.Config.Instance["stripeSecret"];
            StripeConfiguration.ApiKey = Program.StripeKey;

            Console.CancelKeyPress += delegate
            {
                Console.WriteLine("\nAbording");
                onStop?.Invoke();

                var cacheCount = StorageManager.CacheItems;
                StorageManager.Stop();
                Indexer.Stop();

                var t = StorageManager.Save();
                Console.WriteLine("Saving");
                t.Wait();
                Console.WriteLine($"Saved {cacheCount}");
            };

            if (args.Length > 0)
            {
                FileController.dataPaht = args[0];
                Directory.CreateDirectory(FileController.dataPaht);
                Directory.CreateDirectory(FileController.dataPaht + "/users");
                Directory.CreateDirectory(FileController.dataPaht + "/auctions");

                if (args.Length > 1)
                {
                    runSubProgram(args[1][0]);
                    return;
                }
            }

            displayMode = true;

            while (true)
            {
                //try {

                Console.WriteLine("1) List Auctions");
                Console.WriteLine("2) List Bids");
                Console.WriteLine("3) Display");
                Console.WriteLine("4) List Won Bids");
                Console.WriteLine("5) Search For auction");
                Console.WriteLine("6) Avherage selling price in the last 2 weeks");
                Console.WriteLine("9) End");

                var res = Console.ReadKey();
                if (runSubProgram(res.KeyChar))
                {
                    return;
                }

                //} catch(Exception e)
                //{
                //    Console.WriteLine("Error Occured: "+ e.Message);
                //    throw e;
                //}
            }
        }
Ejemplo n.º 5
0
        /// <summary>
        /// returns true if application should be closed
        /// </summary>
        /// <param name="mode"></param>
        /// <returns></returns>
        static bool runSubProgram(char mode)
        {
            switch (mode)
            {
            case 't':
                // test
                //NotificationService.Instance.NotifyAsync("dPRj0dnG2NcY_kMTdNbpjz:APA91bHJINgv1SjuUlv-sGM21wLlHX5ISC5nYgl8DKP2r0fm273Cs0ujcESW6NR1RyGvFDtTBdQLK0SSq5TY_guLgc57VylKk8AAnH_xKq3zDIrdA1F6UhJNTu-Q0wNDKKIIQkYoVcyj","test","click me","https://sky.coflnet.com").Wait();
                SetGoogleIdCommand.ValidateToken("eyJhbGciOiJSUzI1NiIsImtpZCI6IjI1MmZjYjk3ZGY1YjZiNGY2ZDFhODg1ZjFlNjNkYzRhOWNkMjMwYzUiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJhY2NvdW50cy5nb29nbGUuY29tIiwiYXpwIjoiNTcwMzAyODkwNzYwLW5sa2dkOTliNzFxNGQ2MWFtNGxwcWRoZW4xcGVuZGR0LmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29tIiwiYXVkIjoiNTcwMzAyODkwNzYwLW5sa2dkOTliNzFxNGQ2MWFtNGxwcWRoZW4xcGVuZGR0LmFwcHMuZ29vZ2xldXNlcmNvbnRlbnQuY29tIiwic3ViIjoiMTAxOTkzNTcwNzI0MDg4NDMyMjk4IiwiZW1haWwiOiJ0by5jb2ZsbmV0QGdtYWlsLmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJhdF9oYXNoIjoiYWdLN21RM2YySFZQclZNQ3l1UVVmdyIsIm5hbWUiOiJFa3dhdiBDb2ZsbmV0IiwicGljdHVyZSI6Imh0dHBzOi8vbGgzLmdvb2dsZXVzZXJjb250ZW50LmNvbS9hLS9BT2gxNEdobEx6TjV5U1o3VDZWYnpYRnFhUlR4c3dNRXJLaW1VQk1uem41Nz1zOTYtYyIsImdpdmVuX25hbWUiOiJFa3dhdiIsImZhbWlseV9uYW1lIjoiQ29mbG5ldCIsImxvY2FsZSI6ImRlIiwiaWF0IjoxNjEwMjk4MTE5LCJleHAiOjE2MTAzMDE3MTksImp0aSI6ImIzMWYzODUwNDMwYjNhOWMxNTQ5YTRjMDFiNTFiNTBlZjBhZTkwYTAifQ.cvsqp0GaYca---qkBAm-nS3QI-x_ZTGkzZh7sk-SsYctubikHqJz9VpafY_ih88ouOFTg_CWHKPMvS9dTrR8T4W_iY65cYp2hxsc-iMignDBgxbP6KlUCm3MvpRTHTdLAtL3Eq4JeXAL6_BN21AetRMaOhsWMgvz6yprhTkirOgFSuDt386Q8NXr19csjDhAW6bb2bRwEYJp4ZlBXD77zfzP_kZaF2y671M_lZUXnrqKrDqF7sFL2Jx4r6htKV_e86IuKhx0N1ttNTuEOeqccIZHdRQasivVO9Nq0twjhFIWn-5-azkPyz0VstxzIuYc7mTi2LSVjF4QDl-aLiOlPQ");
                break;

            case 'b':
                //var key = System.Text.Encoding.UTF8.GetString (FileController.ReadAllBytes ("apiKey")).Trim ();
                BazaarUpdater.NewUpdate(apiKey).Wait();
                break;

            case 'f':
                FullServer();
                break;

            case 's':
                var server = new Server();
                server.Start().Wait();
                break;

            case 'u':
                var updater = new Updater(apiKey);
                updater.Update().Wait();
                break;

            case '7':
                displayMode = false;
                StorageManager.Migrate();
                var auction = StorageManager.GetOrCreateAuction("00e45a19c27848829612be8edf53bd71");
                Console.WriteLine(auction.ItemName);
                //Console.WriteLine(ItemReferences.RemoveReforges("Itchy Bat man"));
                break;

            case 'i':
                Flipper.FlipperEngine.Instance.Test();
                break;

            case 'p':
                Indexer.LastHourIndex().Wait();
                //StorageManager.Migrate();
                break;

            case 'n':
                Console.WriteLine(NBT.Pretty("H4sIAAAAAAAAAHVUS28jRRAux8mu4ywbxIkLohdtRCKvkxnb49cByXGcxIJ1IvJYblbPTHmmyUyP6elxyJEbJ84cFgkJpEgcOSMh5afkhyCqZ5woHLjMo/qrr756dRVgHUqiCgClFVgRfqlegrVhkkldqkJZ82AdVlF64RJRjhYRlHMkVKAEGxfSVcivuBthqQzrx8LHw4gHKcH/qcJzX6TziN8QyVeJwgpZP4FP7247R8gVO/PI1md3t75j9+jV3bYbVmsnB5xphTLQoTn2araTH9eand2es0MsNkGOkUcFgNcalsOOT+kTt2stiz53CgfH7u62zdEOfE4uBzhDmWLh07UKfMNaghtWY5fCdwk4lhqjSASU+hJtN42G3vYEPZVo4bGa/ejYbNm7Tq6LgvQon7vb6J1I/SRml2+M1wHOdcgoJyqPYuPxGF6R9RC5Dun/kFMoGbBvcuyRSq4JfAnm55RioadFIpdMX+O32QIl10gIgK272/ZhFkXsDDXbT2SW9tloNhOeQKmZVlxIYobPKKXRAtUNUbgmj86Qa+4lsZuyCBcYpW9MuI4ORcq4ihPF5gI9ZAERpKxGBWhswQt6ESjVXKe7FNywHikuNfn4vjAqecRcowJT5vIU/RxD4m+STLEUI8oFfeZnMkCyehFP01fE9Jpk7aNKUV0R3hS8QzG9hnX/yw/sYRRgSCjTdFRPQNy2tswndV8h9QsVz8vFpf9A0yKa31kxLqZi7lse4BMK17Hu3//EnvYcqIsdOjnn8qki3iSqH9lyjMiWh1HoZx5l7POYiMExheRXKJlb1DvXN6OaIvfCZWGpICzkC2T4XSbmTMxgj0DGyhWyUGjjTNPBUh4jixOZapqUa0EdkmZsbQte0su0A71E+tSQLyjWQHnhf8rTc+5//o0NFREOQ06ZPdhtOvi1ODgodAN8fHdL4qPR6XjIDi4mR6OTCds/OTk/K8Oal0SJgr//+rMCqxPSZCrkPC7DwOdzLSid/SShcaCpvH//x/89oQqbo+9pOAeaNsLNNKZl+DBM9HSe0GwlU8/cP6SnWnmIu2H37L7d7fbtTrcClTjxxUyggopcKijDR8upmgqN8TSfaqJYq8B6okQg5DkP4PnF5MvJybtJJb/AXg4OBqfn48vRNE+yCi/MTUfzHNPykKQPfLO207RYWyIrl2Ezi7SIaf2m1/mCmxBknRWrPJ0Vq2y0l6GqHpe1gD0L8t0ufqrzx902BrpOV7OMVL3mLbvddjtOHTvYrrfsZrfOXR/rnPdmlt12vRl3yZ3LhYimZtXIfYPSJF1IuxnPYbO713D2GhZr922bDd4CrMCzxy7Dvy+PLTX1BQAA"));
                //Console.WriteLine (JsonConvert.SerializeObject (.Instance.Items.Where (item => item.Value.AltNames != null && item.Value.AltNames.Count > 3 && !item.Key.Contains("DRAGON")).Select((item)=>new P(item.Value))));
                break;

            case 'g':
                var ds = new DataSyncer();
                ds.Sync("e5bac11a8cc04ca4bae539aed6500823");
                break;

            case 'm':
                Migrator.Migrate();
                break;

            default:
                return(true);
            }
            return(false);
        }