/// <summary>
        /// Creates a timeframe during during which the performance of the current process is optimized.
        /// </summary>
        /// <remarks>
        /// Why use SustainedLowLatency instead of LowLatency: http://www.infoq.com/news/2012/03/Net-403.
        /// </remarks>
        public void Optimize()
        {
            IsOptimized = true;

            OldGarbageCollectorLatencyMode = GCSettings.LatencyMode;

            GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency;
        }
        /// <summary>
        /// Enters a section during which the performance of the current process is optimized.
        /// </summary>
        /// <remarks>
        /// Why use SustainedLowLatency instead of LowLatency: http://www.infoq.com/news/2012/03/Net-403.
        /// </remarks>
        public void EnterOptimizedSection()
        {
            // Flag the section as activated
            IsSectionActivated = true;

            // Store the current latency mode of the Garbage Collector
            OldGcLatencyMode = GCSettings.LatencyMode;

            // Set the latency to the lowest value
            GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency;
        }
        static int Main(string[] args)
        {
            if (args.Length > 0)
                iterations = Int64.Parse(args[0]);

            if (iterations == -1)
                runForever = true;
            if (runForever)
                Console.WriteLine("Run until fail");
            else
                Console.WriteLine("Run {0} iterations", iterations);

            initialMode = GCSettings.LatencyMode;
            Console.WriteLine("Initial mode is: " + initialMode);
            GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency;

            Thread t1 = new Thread(SetSLL);
            Thread t2 = new Thread(Allocate);
            int numThreads = 100;
            Thread[] threadArr = new Thread[numThreads];
            for (int i = 0; i < numThreads; i++)
            {
                threadArr[i] = new Thread(AllocateTempObjects);
                threadArr[i].Start();
            }
           
            t1.Start();
            t2.Start();
            
            t1.Join();
            t2.Join();
            for (int i = 0; i < numThreads; i++)
            {
                threadArr[i].Join();
            }

            if (failed)
            {
                Console.WriteLine("Test failed");
                return 1;
            }
            Console.WriteLine("Test passed");
            return 100;

        }
Esempio n. 4
0
 public static void LatencyRoundtrips_LowLatency(GCLatencyMode value) => LatencyRoundtrips(value);
Esempio n. 5
0
 public static void LatencyRoundtrips(GCLatencyMode value)
 {
     GCLatencyMode orig = GCSettings.LatencyMode;
     try
     {
         GCSettings.LatencyMode = value;
         Assert.Equal(value, GCSettings.LatencyMode);
     }
     finally
     {
         GCSettings.LatencyMode = orig;
         Assert.Equal(orig, GCSettings.LatencyMode);
     }
 }
Esempio n. 6
0
        public static bool RunAllPerfTests(bool enabled, bool resetBaseline, Group group, KeyValuePair <string, bool>[] enables)
        {
            Debug.Assert(enables.Length == CategoryTokens.Length);

            bool   success = false;
            string status  = "SKIPPED";

            Console.WriteLine("Performance Regression Tests - Started");
#if DEBUG
#pragma warning disable CS0162 // complaint about unreachable code
            Console.WriteLine("  DEBUG build - skipping");
            if (false)
#else
            if (enabled)
#endif
            {
                GCLatencyMode        oldGCLatencyMode   = GCSettings.LatencyMode;
                ProcessPriorityClass oldProcessPriority = Process.GetCurrentProcess().PriorityClass;
                ThreadPriority       oldThreadPriority  = Thread.CurrentThread.Priority;
                try
                {
                    GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency;
                    Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.AboveNormal;
                    Thread.CurrentThread.Priority = ThreadPriority.AboveNormal;

                    string baselinePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "TreeLibTest", "perfbaseline.txt");

                    List <Measurement.Result> results = new List <Measurement.Result>();

                    success = true;

                    Dictionary <string, Measurement.Result> baselineResults = new Dictionary <string, Measurement.Result>();
                    if (!resetBaseline)
                    {
                        if (File.Exists(baselinePath))
                        {
                            using (TextReader reader = new StreamReader(baselinePath))
                            {
                                string line    = reader.ReadLine();
                                int    version = Int32.Parse(line);
                                if (version == Measurement.Result.Version)
                                {
                                    reader.ReadLine(); // headers
                                    while ((line = reader.ReadLine()) != null)
                                    {
                                        Measurement.Result result = Measurement.Result.FromString(line);
                                        baselineResults.Add(result.label, result);
                                    }
                                }
                                else
                                {
                                    Console.WriteLine(" ** baseline file has wrong version - not loaded **");
                                }
                            }
                        }
                        else
                        {
                            Console.WriteLine(" ** no baseline file found **");
                        }
                    }

                    foreach (Kind phase in new Kind[] { Kind.Dry, Kind.Real })
                    {
                        results.Clear();

                        TextWriter savedOutput = null;
                        if (phase == Kind.Dry)
                        {
                            savedOutput = Console.Out;
                            Console.SetOut(TextWriter.Null);
                        }

                        if (enables[Array.FindIndex(enables, delegate(KeyValuePair <string, bool> candidate) { return(String.Equals("basic", candidate.Key)); })].Value)
                        {
                            RunTestCategory(phase, group, MapCreators, MapTests, ref success, resetBaseline, baselineResults, results);
                            RunTestCategory(phase, group, RankMapCreators, RankMapTests, ref success, resetBaseline, baselineResults, results);
                            RunTestCategory(phase, group, MultiRankMapCreators, MultiRankMapTests, ref success, resetBaseline, baselineResults, results);
                            RunTestCategory(phase, group, RangeMapCreators, RangeMapTests, ref success, resetBaseline, baselineResults, results);
                            RunTestCategory(phase, group, Range2MapCreators, Range2MapTests, ref success, resetBaseline, baselineResults, results);
                        }

                        if (enables[Array.FindIndex(enables, delegate(KeyValuePair <string, bool> candidate) { return(String.Equals("enum", candidate.Key)); })].Value)
                        {
                            RunTestCategory(phase, group, MapEnumerationCreators, MapEnumerationTests, ref success, resetBaseline, baselineResults, results);
                            RunTestCategory(phase, group, Range2MapEnumerationCreators, Range2MapEnumerationTests, ref success, resetBaseline, baselineResults, results);
                        }

                        if (savedOutput != null)
                        {
                            Console.SetOut(savedOutput);
                        }
                    }

                    if (resetBaseline)
                    {
                        status = "NEW BASELINE";
                        Directory.CreateDirectory(Path.GetDirectoryName(baselinePath));
                        using (TextWriter writer = new StreamWriter(baselinePath))
                        {
                            writer.WriteLine(Measurement.Result.Version);
                            writer.WriteLine(Measurement.Result.Header);
                            for (int i = 0; i < results.Count; i++)
                            {
                                writer.WriteLine(results[i].ToString());
                            }
                        }
                    }
                    else
                    {
                        status = success ? "PASSED" : "FAILED";
                    }
                }
                finally
                {
                    GCSettings.LatencyMode = oldGCLatencyMode;
                    Process.GetCurrentProcess().PriorityClass = oldProcessPriority;
                    Thread.CurrentThread.Priority = oldThreadPriority;
                }
            }

            Program.WritePassFail("Performance Regression Tests - Finished", success ? Program.TestResultCode.Passed : Program.TestResultCode.Failed, status);
            return(success);
        }
Esempio n. 7
0
        private static void RestoreGCLatencyMode(GCLatencyMode originalMode)
        {
            GCSettings.LatencyMode = originalMode;
            s_delay = null;

            // hint to the GC that if it postponed a gen 2 collection, now might be a good time to do it.
            GC.Collect(2, GCCollectionMode.Optimized);
        }
Esempio n. 8
0
 /// <summary>
 /// Creates a time-frame during which the performance of the current process is optimized.
 /// </summary>
 /// <remarks>
 /// Why use SustainedLowLatency instead of LowLatency: http://www.infoq.com/news/2012/03/Net-403.
 /// </remarks>
 public void OptimizeGCLatency()
 {
     IsGCLatencyOptimized            = true;
     _oldGarbageCollectorLatencyMode = GCSettings.LatencyMode;
     GCSettings.LatencyMode          = GCLatencyMode.SustainedLowLatency;
 }
Esempio n. 9
0
 private static void RestoreGCLatencyMode(GCLatencyMode originalMode)
 {
     GCSettings.LatencyMode = originalMode;
     s_delay = null;
 }
Esempio n. 10
0
        static void Main(string[] args)
        {
            var mtreeStopwatch     = new Stopwatch();
            var fastMtreeStopWatch = new Stopwatch();

            var originalTimes = new List <long>();
            var fastTimes     = new List <long>();

            var dataSize     = 100000;
            var testDataSize = 50;
            var range        = 1000000;
            var neighbors    = 10;
            var dimensions   = 1000;

            Console.WriteLine($"{nameof(dataSize)}: {dataSize}");
            Console.WriteLine($"{nameof(dimensions)}: {dimensions}");
            Console.WriteLine($"{nameof(neighbors)}: {neighbors}");
            Console.WriteLine();

            var testData = Supercluster.Tests.Utilities.GenerateDoubles(testDataSize, range, dimensions);

            for (int index = 0; index < testData.Length; index++)
            {
                var treeData = Supercluster.Tests.Utilities.GenerateDoubles(dataSize, range, dimensions);
                var target   = testData[index];


                // 1. Build Trees
                var fastMtree = new FastMTree <double[]> {
                    Capacity = 3, Metric = Metrics.L2Norm_Double
                };
                foreach (var point in treeData)
                {
                    fastMtree.Add(point);
                }


                var mtree = new MTree <double[]> {
                    Capacity = 3, Metric = Metrics.L2Norm_Double
                };
                foreach (var point in treeData)
                {
                    mtree.Add(point);
                }

                GC.Collect();
                GCLatencyMode oldMode = GCSettings.LatencyMode;

                // Make sure we can always go to the catch block,
                // so we can set the latency mode back to `oldMode`
                RuntimeHelpers.PrepareConstrainedRegions();

                try
                {
                    GCSettings.LatencyMode = GCLatencyMode.LowLatency;

                    // Measure Trees
                    mtreeStopwatch.Start();
                    var resultsList = mtree.NearestNeighbors(target, neighbors).ToArray();
                    mtreeStopwatch.Stop();

                    fastMtreeStopWatch.Start();
                    var fastResults = fastMtree.NearestNeighbors(target, neighbors).ToArray();
                    fastMtreeStopWatch.Stop();

                    // Generation 2 garbage collection is now
                    // deferred, except in extremely low-memory situations
                }
                finally
                {
                    // ALWAYS set the latency mode back
                    GCSettings.LatencyMode = oldMode;
                }


                // Print times
                if (index != 0) // We skip first run because of jitting
                {
                    // Record Times
                    originalTimes.Add(mtreeStopwatch.ElapsedTicks);
                    fastTimes.Add(fastMtreeStopWatch.ElapsedTicks);
                    Console.WriteLine(mtreeStopwatch.ElapsedTicks + " " + fastMtreeStopWatch.ElapsedTicks);
                }

                // reset stopwatches
                mtreeStopwatch.Reset();
                fastMtreeStopWatch.Reset();
            }

            Console.WriteLine("Average: " + originalTimes.Average() + " " + fastTimes.Average());
            Console.Read();
        }
Esempio n. 11
0
        /// <summary>
        /// Чтение вперед или назад. Возвращает количество успешно прочитанных секторов.
        /// </summary>
        /// <returns>Возвращает количество успешно прочитанных секторов.</returns>
        public int Read(ScanMode scanMode, bool forward)
        {
            int goodSectors = Params.Image.GoodSectors;

            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
            GCLatencyMode oldGCLatencyMode = GCSettings.LatencyMode;

            try
            {
                GCSettings.LatencyMode = GCLatencyMode.LowLatency;
                int firstTrack   = Params.FirstTrack;
                int prevCylinder = -1;
                int lastTrack    = Params.LastTrack;
                trackTimer.Restart();
                int track = forward ? firstTrack : lastTrack - 1;
                for (int p = firstTrack; p < lastTrack; p++, track = forward ? track + 1 : track - 1)
                {
                    if (Aborted)
                    {
                        goto abort;
                    }
                    if (Params.Side == DiskSide.Side0 && (track % 2 != 0))
                    {
                        continue;
                    }
                    if (Params.Side == DiskSide.Side1 && (track % 2 != 1))
                    {
                        continue;
                    }
                    TrackFormat trackF = Params.Image[track];
                    bool        scan   = ((scanMode == ScanMode.Once) ||
                                          (scanMode == ScanMode.UnscannedOnly && trackF.FormatName == TrackFormatName.Unscanned) ||
                                          scanMode == ScanMode.EachTrackRead) &&
                                         (trackF.MaxGap() > 128 + TrackFormat.MinSectorHeaderSize || trackF.ContainsCalculatedTime());
                    if (!scan && trackF.NotGoodSectors == 0)
                    {
                        continue;
                    }
                    int cylinder = track / 2;
                    if (cylinder != prevCylinder)
                    {
                        Driver.Seek(DriverHandle, track);
                        if (Aborted)
                        {
                            goto abort;
                        }
                        prevCylinder = cylinder;
                    }
                    ReadTrack(track, Params, scanMode);
                }
                return(Params.Image.GoodSectors - goodSectors);

abort:
                Log.Info?.Out("Чтение прервано.");
            }
            finally
            {
                GCSettings.LatencyMode = oldGCLatencyMode;
                Params.Map?.ClearHighlight(MapCell.Processing);
            }
            return(Params.Image.GoodSectors - goodSectors);
        }
Esempio n. 12
0
        /// <summary>
        /// Чтение назад. Возвращает количество успешно прочитанных секторов.
        /// </summary>
        /// <returns>Возвращает количество успешно прочитанных секторов.ы</returns>
        public int ReadBackward()
        {
            int goodSectors = Params.Image.GoodSectors;

            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
            GCLatencyMode oldGCLatencyMode = GCSettings.LatencyMode;

            try
            {
                GCSettings.LatencyMode = GCLatencyMode.LowLatency;
                int firstTrack   = Params.FirstSectorNum / Params.SectorsOnTrack;
                int prevCylinder = -1;
                int lastTrack    = (int)Math.Ceiling((double)Params.LastSectorNum / Params.SectorsOnTrack);
                trackTimer.Restart();
                bool upperHeadScanned = false;
                for (int track = lastTrack - 1; track >= firstTrack; track--)
                {
                    if (Aborted)
                    {
                        goto abort;
                    }
                    if (Params.Side == DiskSide.Side0 && (track % 2 != 0))
                    {
                        continue;
                    }
                    if (Params.Side == DiskSide.Side1 && (track % 2 != 1))
                    {
                        continue;
                    }
                    int diskSectorNum = track * Params.SectorsOnTrack;
                    for (int i = diskSectorNum; i < diskSectorNum + Params.SectorsOnTrack; i++)
                    {
                        if (Params.Image.Sectors[i] != SectorProcessResult.Good)
                        {
                            goto nread;
                        }
                    }
                    continue;
nread:
                    int cylinder = track / 2;
                    if (cylinder != prevCylinder)
                    {
                        if (Aborted)
                        {
                            goto abort;
                        }
                        Driver.Seek(DriverHandle, track);
                        prevCylinder = cylinder;
                    }
                    if (Params.UpperSideHeadAutodetect && !upperHeadScanned && (track & 1) != 0)
                    {
                        if (ScanHeadParameter(ref Params.UpperSideHead, track, Params.CurrentTrackFormat) == 0)
                        {
                            Log.Trace?.Out($"Параметр Head трека {track} определен: {(int)Params.UpperSideHead}");
                            upperHeadScanned = true;
                        }
                        else
                        {
                            Log.Trace?.Out($"Параметр Head трека {track} определить не удалось.");
                        }
                        if (Aborted)
                        {
                            goto abort;
                        }
                    }
                    ReadTrack(track, Params);
                }
                return(Params.Image.GoodSectors - goodSectors);

abort:
                Log.Info?.Out("Чтение прервано.");
            }
            finally
            {
                GCSettings.LatencyMode = oldGCLatencyMode;
                Params.Image.Map?.ClearHighlight(MapCell.Processing);
            }
            return(Params.Image.GoodSectors - goodSectors);
        }
Esempio n. 13
0
 public void SetLatencyMode(GCLatencyMode newMode)
 {
     GCSettings.LatencyMode = newMode;
 }
Esempio n. 14
0
        public static bool Initialize()
        {
            if (Initialized)
            {
                Logs.Instance.Push("App provider already initialized!");
                return(true);
            }

            // Initialize logs instance
            Logs.Instance.Push("App provider initializing...");

            // Check program crashed.
            if (ProgramLock.ProgramCrashed)
            {
                Logs.Instance.Push("Program is terminated abnormally.");
            }

#if DEBUG
            // Check exists instances.
            if (Instance.Count > 1)
            {
                throw new Exception("You must wait for app-provider initialization procedure before using instance-lazy!\n" +
                                    "For more informations, see the development documents.");
            }
#endif

            // GC Setting
            GCLatencyMode oldMode = GCSettings.LatencyMode;
            RuntimeHelpers.PrepareConstrainedRegions();
            GCSettings.LatencyMode = GCLatencyMode.Batch;

            // Extends Connteion Limit
            ServicePointManager.DefaultConnectionLimit = int.MaxValue;

            // Initialize Scheduler
            Scheduler = new NetScheduler(Settings.Instance.Model.ThreadCount);

            // Initialize Postprocessor Scheduler
            PPScheduler = new PostprocessorScheduler(Settings.Instance.Model.PostprocessorThreadCount);

            // Initialize Resource Monitor
            Condition.Instance.Start();

            Logs.Instance.Push("App provider starts.");

            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);

            if (Settings.Instance.Network.UsingFreeProxy && FreeProxy.Instance.IsBuildRequire())
            {
                if (FreeProxy.Instance.IsBuildRequire())
                {
                    Logs.Instance.Push("You must build proxy to use free-proxy. Please run with --build-free-proxy before using free-proxy.");
                }
                else
                {
                    FreeProxy.Instance.Load();
                }
                FreeProxyPass.Init();
            }

            Initialized = true;

            return(true);
        }
        private static void TestSpeed(IWeightedRandomizer <int> randomizer, bool displayText)
        {
            GCLatencyMode oldLatencyMode = GCSettings.LatencyMode;

            try
            {
                //Prevent garbage collection during tests
                GCSettings.LatencyMode = GCLatencyMode.LowLatency;

                const int numIterations = 10000;
                Stopwatch timer         = new Stopwatch();
                if (displayText)
                {
                    Console.WriteLine("Testing {0}", randomizer.GetType().Name);
                    Console.WriteLine("--------------------------------");
                }

                //NextWithReplacement()/1000
                randomizer.Clear();
                timer.Reset();
                timer.Start();
                for (int i = 1; i <= numIterations; i++)
                {
                    randomizer.Add(i, i);
                }
                for (int i = 1; i <= numIterations / 1000; i++)
                {
                    randomizer.NextWithReplacement();
                }
                timer.Stop();
                GC.Collect();
                if (displayText)
                {
                    Console.WriteLine("Add()x{0} + NextWithReplacement()x{1}: {2} ms", numIterations, numIterations / 1000,
                                      timer.ElapsedMilliseconds);
                }

                //NextWithReplacement()
                randomizer.Clear();
                timer.Reset();
                timer.Start();
                for (int i = 1; i <= numIterations; i++)
                {
                    randomizer.Add(i, i);
                }
                for (int i = 1; i <= numIterations; i++)
                {
                    randomizer.NextWithReplacement();
                }
                timer.Stop();
                GC.Collect();
                if (displayText)
                {
                    Console.WriteLine("Add()x{0} + NextWithReplacement()x{1}: {2} ms", numIterations, numIterations,
                                      timer.ElapsedMilliseconds);
                }

                //NextWithReplacement() * 10
                randomizer.Clear();
                timer.Reset();
                timer.Start();
                for (int i = 1; i <= numIterations; i++)
                {
                    randomizer.Add(i, i);
                }
                for (int i = 1; i <= 10 * numIterations; i++)
                {
                    randomizer.NextWithReplacement();
                }
                timer.Stop();
                GC.Collect();
                if (displayText)
                {
                    Console.WriteLine("Add()x{0} + NextWithReplacement()x{1}: {2} ms", numIterations, 10 * numIterations,
                                      timer.ElapsedMilliseconds);
                }

                //NextWithReplacement() (interleaved)
                randomizer.Clear();
                timer.Reset();
                timer.Start();
                for (int i = 1; i <= numIterations; i++)
                {
                    randomizer.Add(i, i);
                    randomizer.NextWithReplacement();
                }
                timer.Stop();
                GC.Collect();
                if (displayText)
                {
                    Console.WriteLine("( Add() + NextWithReplacement() )x{0} (interleaved): {1} ms", numIterations,
                                      timer.ElapsedMilliseconds);
                }

                //NextWithRemoval()
                randomizer.Clear();
                timer.Reset();
                timer.Start();
                for (int i = 1; i <= numIterations; i++)
                {
                    randomizer.Add(i, i);
                }
                for (int i = 1; i <= numIterations; i++)
                {
                    randomizer.NextWithRemoval();
                }
                timer.Stop();
                GC.Collect();
                if (displayText)
                {
                    Console.WriteLine("Add()x{0} + NextWithRemoval()x{1}: {2} ms", numIterations, numIterations,
                                      timer.ElapsedMilliseconds);

                    Console.WriteLine();
                    Console.WriteLine();
                }
            } //end try
            finally
            {
                // ALWAYS set the latency mode back
                GCSettings.LatencyMode = oldLatencyMode;
            }
        }
Esempio n. 16
0
 public bool Pass(GCLatencyMode gcMode, int numCollections)
 {
     return(_numGen1Collections > 0);
 }
Esempio n. 17
0
 private static SetLatencyModeStatus SetGCLatencyMode(GCLatencyMode value) =>
 (SetLatencyModeStatus)RuntimeImports.RhSetGcLatencyMode(value);
Esempio n. 18
0
 public static void LatencyRoundtrips_LowLatency(GCLatencyMode value) => LatencyRoundtrips(value);
Esempio n. 19
0
        static void Main(string[] args)
        {
            GCLatencyMode oldMode = GCSettings.LatencyMode;

            RuntimeHelpers.PrepareConstrainedRegions();
            GCSettings.LatencyMode = GCLatencyMode.Batch;

            ServicePointManager.DefaultConnectionLimit = int.MaxValue;

            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);

            DownloadQueue = new NetQueue(800);

            Console.WriteLine("Robust Arcalive Image Downloader");
            Console.WriteLine("Copyright 2020. ");

            Console.Write("prefix> ");
            string prefix = Console.ReadLine().Trim();

            Console.Write("category> ");
            string category = Console.ReadLine().Trim();

            Console.Write("maxpage> ");
            int page = Convert.ToInt32(Console.ReadLine().Trim());

            var urls = Enumerable.Range(1, page).Select(x => $"{prefix}?category={category}&p={x}").ToList();

            Console.Write("download pages... ");
            List <string> htmls;
            {
                int complete = 0;
                using (var pb = new ExtractingProgressBar())
                {
                    htmls = NetTools.DownloadStrings(urls, "",
                                                     () =>
                    {
                        pb.Report(urls.Count, Interlocked.Increment(ref complete));
                    }).Result;
                }
            }

            var rr       = new Regex(@"<a class=""vrow"" href=""(.*?)""");
            var subPages = new ConcurrentBag <List <string> >();
            var host     = new Uri(prefix).Host;

            Parallel.ForEach(htmls, (html) =>
            {
                subPages.Add(rr.Matches(html).Cast <Match>().Select(x => $"https://{host}{x.Groups[1].Value}").ToList());
            });

            Console.WriteLine($"{subPages.Sum(x=>x.Count):#,#} page loaded");

            Console.Write("download article... ");
            var           subPagesList = subPages.SelectMany(x => x).ToList();
            List <string> subHtmls;

            {
                int complete = 0;
                using (var pb = new ExtractingProgressBar())
                {
                    subHtmls = NetTools.DownloadStrings(subPagesList, "",
                                                        () =>
                    {
                        pb.Report(subPagesList.Count, Interlocked.Increment(ref complete));
                    }).Result;
                }
            }
            Console.WriteLine("complete");

            // url, path
            var images = new ConcurrentBag <List <(string, string)> >();

            Console.Write("extracting contents... ");
            {
                int complete = 0;
                using (var pb = new ExtractingProgressBar())
                {
                    Parallel.ForEach(subHtmls, (html) =>
                    {
                        var imgs = new List <(string, string)>();
                        var doc  = new HtmlDocument();

                        doc.LoadHtml(html);
                        var id   = doc.DocumentNode.SelectSingleNode("//link[@rel='canonical']").GetAttributeValue("href", "").Split('/').Last();
                        var body = doc.DocumentNode.SelectSingleNode("//div[@class='article-body']");

                        var imgnodes = body.SelectNodes(".//img");
                        if (imgnodes != null)
                        {
                            foreach (var img in imgnodes)
                            {
                                var src = img.GetAttributeValue("src", "");
                                imgs.Add(($"https:{src}?type=orig", $"download/[{id}] {imgs.Count:00}." + src.Split('.').Last()));
                            }
                        }

                        var videonodes = body.SelectNodes(".//video");
                        if (videonodes != null)
                        {
                            foreach (var video in body.SelectNodes(".//video"))
                            {
                                var src = video.GetAttributeValue("src", "");
                                if (video.GetAttributeValue("data-orig", "") == "gif")
                                {
                                    imgs.Add(($"https:{src}.gif?type=orig", $"download/[{id}] {imgs.Count:00}." + "gif"));
                                }
                                else
                                {
                                    imgs.Add(($"https:{src}?type=orig", $"download/[{id}] {imgs.Count:00}." + src.Split('.').Last()));
                                }
                            }
                        }

                        images.Add(imgs);
                        pb.Report(subHtmls.Count, Interlocked.Increment(ref complete));
                    });
Esempio n. 20
0
 internal static extern void RhSetGcLatencyMode(GCLatencyMode newLatencyMode);
Esempio n. 21
0
 protected virtual void EnableHighPerformanceSession()
 {
     originalGCMode         = GCSettings.LatencyMode;
     GCSettings.LatencyMode = GCLatencyMode.LowLatency;
 }
 private static extern SetLatencyModeStatus SetGCLatencyMode(GCLatencyMode newLatencyMode);
Esempio n. 23
0
 public bool Pass(GCLatencyMode gcMode, int numCollections)
 {
     return (_numGen1Collections > 0);
 }
Esempio n. 24
0
 internal static extern void RhSetGcLatencyMode(GCLatencyMode newLatencyMode);
Esempio n. 25
0
        static void Main(string[] args)
        {
            Stopwatch tst2 = new Stopwatch();
            Stopwatch tst  = new Stopwatch();

            tst.Start();
            tst2.Start();
            tst.Stop();
            tst.Start();
            tst2.Stop();
            tst.Stop();
            Console.WriteLine(tst2.ElapsedMilliseconds);
            Console.WriteLine(tst.ElapsedMilliseconds);
            tst  = null;
            tst2 = null;
            Console.WriteLine();
            GC.Collect();

            double tmp           = 0;
            long   memoryTracker = 0;

            double[] row = null;
            Console.WriteLine("Jagged");
            Stopwatch     timer   = new Stopwatch();
            GCLatencyMode oldMode = GCSettings.LatencyMode;

            memoryTracker = GC.GetTotalMemory(true);
            RuntimeHelpers.PrepareConstrainedRegions();
            try
            {
                GCSettings.LatencyMode = GCLatencyMode.LowLatency;

                memoryTracker = GC.GetTotalMemory(true);
                long firstSize = 0;
                timer.Start();
                // Initialization
                double[][] jagged = new double[1024][];

                /*timer.Stop();
                 * long firstSize = GC.GetTotalMemory(true) - memoryTracker;
                 * timer.Start();*/
                for (int i = 0; i < 1024; ++i)
                {
                    jagged[i] = new double[1024];
                }
                timer.Stop();
                long secondSize = GC.GetTotalMemory(true) - firstSize;
                Console.WriteLine("\tInitialization: " + timer.ElapsedMilliseconds + " ms");
                timer.Restart();
                // Setting
                for (int i = 0; i < 1024; ++i)
                {
                    for (int j = 0; j < 1024; ++j)
                    {
                        jagged[i][j] = 0;
                    }
                }
                timer.Stop();
                Console.WriteLine("\tSetting: " + timer.ElapsedMilliseconds + " ms");
                long thirdSize = GC.GetTotalMemory(true) - memoryTracker;
                row = new double[1024];
                timer.Restart();
                // Row Setting
                for (int i = 0; i < 1024; ++i)
                {
                    jagged[i] = row;
                }
                timer.Stop();
                Console.WriteLine("\tSetting Row: " + timer.ElapsedMilliseconds + " ms");
                timer.Restart();
                // Getting
                for (int i = 0; i < 1024; ++i)
                {
                    for (int j = 0; j < 1024; ++j)
                    {
                        tmp = jagged[i][j];
                    }
                }
                timer.Stop();
                Console.WriteLine("\tGetting: " + timer.ElapsedMilliseconds + " ms");
                // Getting Rows
                timer.Restart();
                for (int i = 0; i < 1024; ++i)
                {
                    row = jagged[i];
                }
                timer.Stop();
                Console.WriteLine("\tGetting Rows: " + timer.ElapsedMilliseconds + " ms");
                Console.WriteLine("\tInitial Size: " + firstSize + " bytes");
                Console.WriteLine("\tColumns Size: " + secondSize + " bytes");
                Console.WriteLine("\tValues Size: " + thirdSize + " bytes");
                Console.WriteLine();
                jagged = null;
                row    = null;
            }
            finally
            {
                GCSettings.LatencyMode = oldMode;
            }

            GC.Collect();

            RuntimeHelpers.PrepareConstrainedRegions();
            try
            {
                GCSettings.LatencyMode = GCLatencyMode.LowLatency;
                Console.WriteLine("Flattened");
                memoryTracker = GC.GetTotalMemory(true);
                timer.Restart();
                // Initialization
                double[] flattened = new double[1024 * 1024];
                timer.Stop();
                Console.WriteLine("\tInitialization: " + timer.ElapsedMilliseconds + " ms");
                long firstSize = GC.GetTotalMemory(true) - memoryTracker;
                timer.Restart();
                // Setting All
                for (int e = 0; e < 1048576; ++e)
                {
                    flattened[e] = 0;
                }
                timer.Stop();
                Console.WriteLine("\tSetting All: " + timer.ElapsedMilliseconds + " ms");
                long thirdSize = GC.GetTotalMemory(true) - memoryTracker;
                // Setting By Index
                timer.Restart();
                for (int i = 0; i < 1024; ++i)
                {
                    for (int j = 0; j < 1024; ++j)
                    {
                        flattened[i * 1024 + j] = 0;
                    }
                }
                timer.Stop();
                Console.WriteLine("\tSetting Index: " + timer.ElapsedMilliseconds + " ms");
                row = new double[1024];
                timer.Restart();
                // Setting Rows
                for (int i = 0; i < 1024; ++i)
                {
                    int start = i * 1024;
                    int end   = start + 1024;
                    int e     = 0;
                    for (int j = start; j < end; ++j)
                    {
                        flattened[j] = row[e];
                    }
                }
                timer.Stop();
                Console.WriteLine("\tSetting Rows: " + timer.ElapsedMilliseconds + " ms");
                tmp = 0;
                timer.Restart();
                // Getting All
                for (int e = 0; e < 1048576; ++e)
                {
                    tmp = flattened[e];
                }
                timer.Stop();
                Console.WriteLine("\tGetting All: " + timer.ElapsedMilliseconds + " ms");
                timer.Restart();
                // Getting By Index
                for (int i = 0; i < 1024; ++i)
                {
                    for (int j = 0; j < 1024; ++j)
                    {
                        tmp = flattened[i * 1024 + j];
                    }
                }
                timer.Stop();
                Console.WriteLine("\tGetting Index: " + timer.ElapsedMilliseconds + " ms");
                timer.Restart();
                // Getting Rows
                for (int i = 0; i < 1024; ++i)
                {
                    int start = i * 1024;
                    for (int j = 0; j < 1024; ++j)
                    {
                        row[j] = flattened[start + j];
                    }
                }
                timer.Stop();
                Console.WriteLine("\tGetting Rows: " + timer.ElapsedMilliseconds + " ms");
                timer.Restart();
                // Getting Rows 2
                for (int i = 0; i < 1024; ++i)
                {
                    row = Enumerable.Range(i * 1024, 1024).Select(e => flattened[e]).ToArray();
                }
                timer.Stop();
                Console.WriteLine("\tGetting Rows 2: " + timer.ElapsedMilliseconds + " ms");

                Console.WriteLine("\tInitial Size: " + firstSize + " bytes");
                Console.WriteLine("\tValues Size: " + thirdSize + " bytes");
            }
            finally
            {
                GCSettings.LatencyMode = oldMode;
            }
            GC.Collect();
            Console.ReadLine();
        }