Example #1
0
        public static void Main(string[] args)
        {
            var appStarted = Stats.Watch.Elapsed;

            Log.Info("Version 1.0rc");
            Garbage.Collect0(); // just to reference Garbage static constructor

            CultureInfo.DefaultThreadCurrentCulture = CultureInfo.GetCultureInfo("ru-RU");

            // path to data
            var path = "/tmp";

            if (args.Length > 0)
            {
                path = args[0];
            }

            // create the storage and load initial dataset
            var store = new Storage();

            Loader.LoadFromZip(path, store);
            Garbage.CollectAll(true);

            // prepare the storage and clean up memory before the start
            store.Prepare(true);
            Garbage.CollectAll(true);

            // create the router and the house keeper
            var router = new Router(store);

            // warmup the solver
            if (store.IsRatingRun == 1)
            {
                var warmupDeadline = new TimeSpan(0, 3, 0);
                var warmer         = new Warmup(router, store);
                Log.Info("Warm up the processors before simulation start");
                warmer.RunGet(Stats.Watch.Elapsed.Add(new System.TimeSpan(0, 0, 5)));
                Log.Info("Warm up completed");
                Garbage.CollectAll();
            }

            // start the Epoll server (only on Linux)
            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
            {
                new EpollServer(router).Run();
                Console.ReadKey();
            }

            // local run
            if (Directory.Exists(path + "/ammo"))
            {
                // local simulation
                new Tester(store, router).TestAll(path);
            }

            Console.WriteLine("Press any key to exit");
            Console.ReadKey();
        }
Example #2
0
        public TimeSpan TestAll(string path)
        {
            bool verify = false;

            Console.WriteLine("Test Stage 1");
            var elapsed1 = TestPhase(1, path, verify);

            Console.WriteLine("Test Stage 1 completed in {0}", elapsed1);
            GetGroup.CachedResults.Clear();
            GetFilter.CachedResults.Clear();
            Garbage.CollectAll();

            Console.WriteLine("Test Stage 2");
            var MAX_PHASE2 = verify? 1: 20;
            var elapsed2   = new TimeSpan();

            for (int i = 1; i <= MAX_PHASE2; i++)
            {
                elapsed2 += TestPhase(2, path, verify);
            }
            elapsed2 /= MAX_PHASE2;
            Console.WriteLine("Test Stage 2 completed in {0}", elapsed2);
            Garbage.CollectAll();
            store.Prepare(false);

            var elapsed3   = new TimeSpan();
            var MAX_PHASE3 = 1;

            for (int i = 1; i <= MAX_PHASE3; i++)
            {
                Garbage.CollectAll();
                Console.WriteLine("Test Stage 3");
                var elapsed = TestPhase(3, path, verify);
                Console.WriteLine("Test Stage 3 completed in {0}", elapsed);
                elapsed3 += elapsed;
            }
            elapsed3 /= MAX_PHASE3;

            Stats.DisplayContextTime();
            Console.WriteLine("====");
            var totalTime = elapsed1 + elapsed2 + elapsed3;

            Console.WriteLine("Total time elapsed: {0}", totalTime);
            Log.Info("Memory footprint: {0}", GC.GetTotalMemory(false));
            return(totalTime);
        }
Example #3
0
        public void Tick()
        {
            int useTimeout = 0; // using timeout is dangerous with cached requests

            switch (Phase)
            {
            case EPhase.P01:
                if (getCounter > 0)
                {
                    Phase = EPhase.P1;
                    OnPhaseChanged(Phase);
                    Log.Info("Phase 1 has started");
                    lastCall = Stats.Watch.Elapsed;
                }
                break;

            case EPhase.P1:
                if (getCounter >= phase1Gets || useTimeout > 0 && (Stats.Watch.Elapsed - lastCall) > TimeSpan.FromMilliseconds(useTimeout))
                {
                    Phase = EPhase.P12;
                    OnPhaseChanged(Phase);
                    Log.Info("Phase 1 has ended");
                    GetGroup.CachedResults.Clear();
                    GetFilter.CachedResults.Clear();
                    Garbage.CollectAll();
                }
                if (postCounter > 0 /*how did we miss it?*/)
                {
                    GetGroup.CachedResults.Clear();
                    GetFilter.CachedResults.Clear();
                    Phase = EPhase.P2;
                    OnPhaseChanged(Phase);
                    Log.Info("Phase 2 has started unexpectedly");
                }
                break;

            case EPhase.P12:
                if (postCounter > 0)
                {
                    Phase = EPhase.P2;
                    OnPhaseChanged(Phase);
                    Log.Info("Phase 2 has started");
                    lastCall = Stats.Watch.Elapsed;
                }
                break;

            case EPhase.P2:
                if (postCounter >= phase2Posts || useTimeout > 0 && (Stats.Watch.Elapsed - lastCall) > TimeSpan.FromMilliseconds(useTimeout))
                {
                    Phase = EPhase.P23;
                    OnPhaseChanged(Phase);
                    Log.Info("Phase 2 has ended");
                    GetGroup.CachedResults.Clear();
                    GetFilter.CachedResults.Clear();
                    Garbage.CollectAll();
                    store.Prepare(false);
                    Garbage.CollectAll();
                }
                break;

            case EPhase.P23:
                if (getCounter > phase1Gets)
                {
                    Phase = EPhase.P3;
                    OnPhaseChanged(Phase);
                    Log.Info("Phase 3 has started");
                    lastCall = Stats.Watch.Elapsed;
                }
                break;

            case EPhase.P3:
                if (getCounter >= phase1Gets + phase3Gets || useTimeout > 0 && (Stats.Watch.Elapsed - lastCall) > TimeSpan.FromMilliseconds(useTimeout))
                {
                    Phase = EPhase.PEND;
                    OnPhaseChanged(Phase);
                    Log.Info("Phase 3 has ended");
                    Stats.DisplayContextTime();
                    Mapper.DisplayStats();
                    EpollServer.DisplayServerStats();
                }
                break;
            }

            // 5 sec report
            var now = Stats.Watch.Elapsed;

            if ((now - lastReport).TotalMilliseconds > 4000)
            {
                lastReport = now;
                if (Phase != EPhase.P01)
                {
                    Console.WriteLine("{0} M{1} S{2}/{3}", lastReport.ToString(@"mm\:ss"),
                                      GC.GetTotalMemory(false) / 1024 / 1024, EpollHandler.OpenedSockets, EpollHandler.ClosedSockets);
                }
            }
        }
Example #4
0
        // zip loader, called from Main() before the web server starts
        public static void LoadFromZip(string path, Storage store)
        {
            // load options
            var optionsFile = path + "/data/options.txt";

            if (File.Exists(optionsFile))
            {
                Log.Info("Opening " + optionsFile);
                using (StreamReader r = new StreamReader(File.OpenRead(optionsFile)))
                {
                    store.Now         = int.Parse(r.ReadLine());
                    store.IsRatingRun = int.Parse(r.ReadLine());
                }
            }
            else
            {
                Log.Error(optionsFile + " is not found");
            }

            // load zipped data
            var zipFile = path + "/data/data.zip";

            if (File.Exists(zipFile))
            {
                Log.Info("Opening " + zipFile);
                var totalAccounts = 0;
                var errorAccounts = 0;
                var fileBuffer    = new byte[20000000];
                using (var file = File.OpenRead(zipFile))
                    using (var zip = new ZipArchive(file, ZipArchiveMode.Read))
                    {
                        foreach (var entry in zip.Entries)
                        {
                            if (entry.Name == "options.txt")
                            {
                                Log.Info("Loading options.txt");
                                using (StreamReader r = new StreamReader(entry.Open()))
                                {
                                    store.Now         = int.Parse(r.ReadLine());
                                    store.IsRatingRun = int.Parse(r.ReadLine());
                                }
                            }
                            else
                            {
                                //Log.Info("Loading " + entry.Name);
                                using (var stream = entry.Open())
                                {
                                    Console.Write('.');
                                    var ms = new MemoryStream(fileBuffer);
                                    stream.CopyTo(ms);

                                    JsonReader reader = new JsonReader(fileBuffer);
                                    if (reader.ReadIsNull() || !reader.ReadIsBeginObject())
                                    {
                                        throw new Exception("Could not read the init json");
                                    }

                                    if (reader.ReadPropertyName() != "accounts")
                                    {
                                        throw new Exception("Unexpected object in init json");
                                    }

                                    // read array members
                                    if (!reader.ReadIsBeginArray())
                                    {
                                        throw new Exception("Array of accounts not found");
                                    }

                                    var fileParseStart = Stats.Watch.Elapsed;
                                    var dto            = DtoAccount.Obtain();
                                    while (true)
                                    {
                                        if (DtoAccount.Parse(ref reader, dto, store))
                                        {
                                            dto.flags = DtoFlags.Init;
                                            // now, add to the storage the account with internal ids/likes
                                            store.InitNewAccout(dto);

                                            totalAccounts++;
                                            dto.Reset();
                                        }
                                        else
                                        {
                                            break;
                                        }
                                        if (!reader.ReadIsValueSeparator())
                                        {
                                            break;
                                        }
                                    }
                                    DtoAccount.Release(dto);
                                    var fileParseEnd = Stats.Watch.Elapsed;
                                    if ((fileParseEnd - fileParseStart).TotalSeconds > 1)
                                    {
                                        Garbage.CollectAll();
                                    }

                                    /*
                                     * else
                                     *  Garbage.Collect0();*/
                                }
                            }
                        }
                    }
                Log.Info("Total accounts loaded: " + totalAccounts + ", " + errorAccounts + " errors found");
                fileBuffer = null;
            }
            else
            {
                Log.Error(zipFile + " not found");
            }
        }