/// <summary>
        /// 令牌桶服务
        /// </summary>
        /// <param name="maxTPS">最大每秒处理能力</param>
        /// <param name="limitSize">最大数量</param>
        /// <param name="limitQPH">每小时请求数量</param>
        public TokenBucketLimitingService(int maxTPS, int limitSize, int limitQPH)
        {
            this.limitSize = limitSize;
            this.maxTPS    = maxTPS;

            if (this.limitSize <= 0)
            {
                this.limitSize = 100;
            }
            if (this.maxTPS <= 0)
            {
                this.maxTPS = 1;
            }

            limitedQueue = new LimitedQueue <object>(limitSize);
            for (int i = 0; i < limitSize; i++)
            {
                limitedQueue.Enqueue(new object());
            }
            cancelToken       = new CancellationTokenSource();
            task              = Task.Factory.StartNew(new Action(TokenProcess), cancelToken.Token);
            Node1HTPS         = new System.Collections.Concurrent.ConcurrentDictionary <string, NodeLimit>();
            Node24HTPS        = new System.Collections.Concurrent.ConcurrentDictionary <string, NodeLimit>();
            nodeLimitQPH      = limitQPH;
            H1StartCountTime  = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, DateTime.Now.Hour, 0, 0);
            H24StartCountTime = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0);
        }
        private System.Threading.ManualResetEventSlim slim;//主线程调用后为了判断不会第二次再进入

        public SendTaskManage()
        {
            slim = new System.Threading.ManualResetEventSlim(true);
            //  bRuning = false;
            downloadItems = new ConcurrentDictionary <string, DownloadItem>(System.Environment.ProcessorCount * 2, 500);
            mr            = new ManualResetEvent(true);
        }
Exemple #3
0
        public LockKeyManager(Gateway gateway, IConfiguration configuration, ILogger <LockKeyManager> logger)
        {
            _timeout       = configuration.GetValue <int>("UnLockKeyTimeout");
            _cache         = new System.Collections.Concurrent.ConcurrentDictionary <string, KeyObject>();
            _gateway       = gateway;
            _configuration = configuration;
            _logger        = logger;

            SystemEventCenter.MicroServiceOnffline += SystemEventCenter_MicroServiceOnffline;
            SystemEventCenter.MicroServiceOnline   += SystemEventCenter_MicroServiceOnline;

            new Thread(checkTimeout).Start();
        }
        public static bool TryGetDeviceInfo(string key, out DeviceInfo device)
        {
            _deviceInfos = _impl.GetData("DeviceInfo") as ConcurrentDictionary<string, DeviceInfo>;
            if ( _deviceInfos == null)
            {
                lock (_sync)
                {
                    if( _deviceInfos == null)
                        _deviceInfos = LoadDeviceInfo();
                }
            }

            return _deviceInfos.TryGetValue(key, out device);
        }
        public static bool TryGetDeviceInfo(string key, out DeviceInfo device)
        {
            _deviceInfos = _impl.GetData("DeviceInfo") as ConcurrentDictionary <string, DeviceInfo>;
            if (_deviceInfos == null)
            {
                lock (_sync)
                {
                    if (_deviceInfos == null)
                    {
                        _deviceInfos = LoadDeviceInfo();
                    }
                }
            }

            return(_deviceInfos.TryGetValue(key, out device));
        }
Exemple #6
0
        static MovieManager()
        {
            _movies = new ConcurrentDictionary <string, Movie>();
            var movice = new Movie()
            {
                Name = "The Breaking Bad", Rating = 5, Id = "1"
            };

            _movies[movice.Id] = movice;
            movice             = new Movie()
            {
                Name = "The Avatar", Rating = 5, Id = "2"
            };
            _movies[movice.Id] = movice;
            movice             = new Movie()
            {
                Name = "The Walking Dead", Rating = 4, Id = "3"
            };
            _movies[movice.Id] = movice;
        }
Exemple #7
0
        public static void Start()
        {
            new System.Threading.Thread(() =>
            {
                if (File.Exists(SessionBagFile))
                {
                    _SessionPools = SerializeManager <ConcurrentDictionary <string, SessionIdentity> > .Deserialize(SessionBagFile);
                    if (_SessionPools == null)
                    {
                        _SessionPools = new ConcurrentDictionary <string, SessionIdentity>();
                    }
                }
                runningstate = true;
                int tick     = 0;
                int tick2    = 0;
                while (runningstate)
                {
                    System.Threading.Thread.Sleep(1000);
                    tick  += 1000;
                    tick2 += 1000;
                    if (tick / 1000 >= 5 * 60) //5分钟 写入磁盘一次
                    {
                        SaveSessionToDisk();
                        tick = 0;
                    }

                    //test
                    if (true)
                    {
                        RemoveExpiredSession();
                    }
                    //end test

                    if (tick2 / 1000 >= 3600 * 24) //24个小时清理一次过期session
                    {
                        RemoveExpiredSession();
                        tick2 = 0;
                    }
                }
            }).Start();
        }
Exemple #8
0
        protected void Application_Start()
        {
            AreaRegistration.RegisterAllAreas();
            FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
            RouteConfig.RegisterRoutes(RouteTable.Routes);
            BundleConfig.RegisterBundles(BundleTable.Bundles);

            ServicePointManager.DefaultConnectionLimit = 500;

            if (ConfigurationManager.AppSettings["IsNode"] == "1")
            {
                SearchApi.Init();
            }
            else
            {
                string log_db_path = @"C:\Users\Administrator\Documents\stackoverflow\LOG";
                logdb = new Db(log_db_path);
            }

            ThreadPool.SetMinThreads(500, 500);

            _recent_clean_job = new System.Threading.Thread(f =>
            {
                while (true)
                {
                    try
                    {
                        Thread.Sleep(60000);
                        _recent_data = new System.Collections.Concurrent.ConcurrentDictionary <string, DateTime>();
                    }
                    catch (Exception ex)
                    {
                        LogError("ERROR IN CLEANING THREAD " + DateTime.Now + " \n" + ex.Message + " \n" + ex.StackTrace, ex);
                    }
                }
            });
        }
        /// <summary>
        /// Creates a memoizer with one parameter for the the specified task provider. The task provider is guaranteed to be executed only once per parameter instance.
        /// </summary>
        /// <typeparam name="TResult">The return value type</typeparam>
        /// <typeparam name="TParam"></typeparam>
        /// <param name="func">A function that will call the create the task.</param>
        /// <returns>A function that will return a task </returns>
        public static FuncAsync <TParam, TResult> AsMemoized <TParam, TResult>(this FuncAsync <TParam, TResult> func)
        {
#if HAS_NO_CONCURRENT_DICT
            var values = new SynchronizedDictionary <TParam, FuncAsync <TResult> >();
#else
            var values = new System.Collections.Concurrent.ConcurrentDictionary <TParam, FuncAsync <TResult> >();
#endif
            // It's safe to use default(TParam) as this won't get called anyway if TParam is a value type.
            var nullValue = Funcs.CreateAsyncMemoized(ct => func(ct, default(TParam)));

            return((ct, param) =>
            {
                if (param == null)
                {
                    return nullValue(ct);
                }
                else
                {
                    var memoizedFunc = values.GetOrAdd(param, p => Funcs.CreateAsyncMemoized(c => func(c, p)));

                    return memoizedFunc(ct);
                }
            });
        }
Exemple #10
0
        /// <summary>
        /// Renders the layer
        /// </summary>
        /// <param name="graphics">Graphics object reference</param>
        /// <param name="map">Map which is rendered</param>
        public override void Render(Graphics graphics, MapViewport map)
        {
            if (!map.Size.IsEmpty && map.Size.Width > 0 && map.Size.Height > 0)
            {
                var bmp = new Bitmap(map.Size.Width, map.Size.Height, PixelFormat.Format32bppArgb);

                using (var g = Graphics.FromImage(bmp))
                {
                    g.InterpolationMode = InterpolationMode;
                    g.Transform         = graphics.Transform.Clone();

                    var extent = new Extent(map.Envelope.MinX, map.Envelope.MinY,
                                            map.Envelope.MaxX, map.Envelope.MaxY);
                    var level      = BruTile.Utilities.GetNearestLevel(_source.Schema.Resolutions, Math.Max(map.PixelWidth, map.PixelHeight));
                    var tiles      = new List <TileInfo>(_source.Schema.GetTileInfos(extent, level));
                    var tileWidth  = _source.Schema.GetTileWidth(level);
                    var tileHeight = _source.Schema.GetTileWidth(level);

                    IList <WaitHandle> waitHandles = new List <WaitHandle>();
                    var toRender       = new System.Collections.Concurrent.ConcurrentDictionary <TileIndex, Bitmap>();
                    var takenFromCache = new System.Collections.Concurrent.ConcurrentDictionary <TileIndex, bool>();
                    foreach (TileInfo info in tiles)
                    {
                        var image = _bitmaps.Find(info.Index);
                        if (image != null)
                        {
                            toRender.TryAdd(info.Index, image);
                            takenFromCache.TryAdd(info.Index, true);
                            continue;
                        }
                        if (_fileCache != null && _fileCache.Exists(info.Index))
                        {
                            var tileBitmap = GetImageFromFileCache(info) as Bitmap;
                            _bitmaps.Add(info.Index, tileBitmap);
                            toRender.TryAdd(info.Index, tileBitmap);
                            takenFromCache.TryAdd(info.Index, true);
                            continue;
                        }

                        var waitHandle = new AutoResetEvent(false);
                        waitHandles.Add(waitHandle);
                        ThreadPool.QueueUserWorkItem(GetTileOnThread,
                                                     new object[] { _source, info, toRender, waitHandle, true, takenFromCache });
                    }

                    foreach (var handle in waitHandles)
                    {
                        handle.WaitOne();
                    }

                    using (var ia = new ImageAttributes())
                    {
                        if (!_transparentColor.IsEmpty)
                        {
                            ia.SetColorKey(_transparentColor, _transparentColor);
                        }
#if !PocketPC
                        ia.SetWrapMode(WrapMode.TileFlipXY);
#endif

                        foreach (var info in tiles)
                        {
                            if (!toRender.ContainsKey(info.Index))
                            {
                                continue;
                            }

                            var bitmap = toRender[info.Index];//_bitmaps.Find(info.Index);
                            if (bitmap == null)
                            {
                                continue;
                            }

                            var min = map.WorldToImage(new Coordinate(info.Extent.MinX, info.Extent.MinY));
                            var max = map.WorldToImage(new Coordinate(info.Extent.MaxX, info.Extent.MaxY));

                            min = new PointF((float)Math.Round(min.X), (float)Math.Round(min.Y));
                            max = new PointF((float)Math.Round(max.X), (float)Math.Round(max.Y));

                            try
                            {
                                g.DrawImage(bitmap,
                                            new Rectangle((int)min.X, (int)max.Y, (int)(max.X - min.X),
                                                          (int)(min.Y - max.Y)),
                                            0, 0, tileWidth, tileHeight,
                                            GraphicsUnit.Pixel,
                                            ia);
                            }
                            catch (Exception ee)
                            {
                                Logger.Error(ee.Message);
                            }
                        }
                    }

                    //Add rendered tiles to cache
                    foreach (var kvp in toRender)
                    {
                        if (takenFromCache.ContainsKey(kvp.Key) && !takenFromCache[kvp.Key])
                        {
                            _bitmaps.Add(kvp.Key, kvp.Value);
                        }
                    }

                    graphics.Transform = new Matrix();
                    graphics.DrawImageUnscaled(bmp, 0, 0);
                    graphics.Transform = g.Transform;
                }
            }
        }
Exemple #11
0
        /// <summary>
        /// Renders the layer
        /// </summary>
        /// <param name="graphics">Graphics object reference</param>
        /// <param name="map">Map which is rendered</param>
        public override void Render(Graphics graphics, Map map)
        {
            if (!map.Size.IsEmpty && map.Size.Width > 0 && map.Size.Height > 0)
            {
                var bmp = new Bitmap(map.Size.Width, map.Size.Height, PixelFormat.Format32bppArgb);
                
                using (var g = Graphics.FromImage(bmp))
                {
                    g.InterpolationMode = InterpolationMode;
                    g.Transform = graphics.Transform.Clone();

                    var extent = new Extent(map.Envelope.MinX, map.Envelope.MinY, 
                                            map.Envelope.MaxX, map.Envelope.MaxY);
                    var level = BruTile.Utilities.GetNearestLevel(_source.Schema.Resolutions, map.PixelSize);
                    var tiles = new List<TileInfo>(_source.Schema.GetTileInfos(extent, level));
                    var tileWidth = _source.Schema.GetTileWidth(level);
                    var tileHeight = _source.Schema.GetTileWidth(level);

                    IList<WaitHandle> waitHandles = new List<WaitHandle>();
                    var toRender = new System.Collections.Concurrent.ConcurrentDictionary<TileIndex, Bitmap>();
                    var takenFromCache = new System.Collections.Concurrent.ConcurrentDictionary<TileIndex,bool>();
                    foreach (TileInfo info in tiles)
                    {
                        var image = _bitmaps.Find(info.Index);
                        if (image != null)
                        {
                            toRender.TryAdd(info.Index, image);
                            takenFromCache.TryAdd(info.Index,true);
                            continue;
                        }
                        if (_fileCache != null && _fileCache.Exists(info.Index))
                        {
                            var tileBitmap = GetImageFromFileCache(info) as Bitmap;
                            _bitmaps.Add(info.Index, tileBitmap);
                            toRender.TryAdd(info.Index, tileBitmap);
                            takenFromCache.TryAdd(info.Index, true);
                            continue;
                        }

                        var waitHandle = new AutoResetEvent(false);
                        waitHandles.Add(waitHandle);
                        ThreadPool.QueueUserWorkItem(GetTileOnThread,
                                                     new object[] { _source, info, toRender, waitHandle, true, takenFromCache });
                    }

                    foreach (var handle in waitHandles)
                        handle.WaitOne();

                    using (var ia = new ImageAttributes())
                    {
                        if (!_transparentColor.IsEmpty)
                            ia.SetColorKey(_transparentColor, _transparentColor);
#if !PocketPC
                        ia.SetWrapMode(WrapMode.TileFlipXY);
#endif

                        foreach (var info in tiles)
                        {
                            if (!toRender.ContainsKey(info.Index))
                                continue;

                            var bitmap = toRender[info.Index];//_bitmaps.Find(info.Index);
                            if (bitmap == null) continue;

                            var min = map.WorldToImage(new Coordinate(info.Extent.MinX, info.Extent.MinY));
                            var max = map.WorldToImage(new Coordinate(info.Extent.MaxX, info.Extent.MaxY));

                            min = new PointF((float) Math.Round(min.X), (float) Math.Round(min.Y));
                            max = new PointF((float) Math.Round(max.X), (float) Math.Round(max.Y));

                            try
                            {
                                g.DrawImage(bitmap,
                                            new Rectangle((int) min.X, (int) max.Y, (int) (max.X - min.X),
                                                          (int) (min.Y - max.Y)),
                                            0, 0, tileWidth, tileHeight,
                                            GraphicsUnit.Pixel,
                                            ia);
                            }
                            catch (Exception ee)
                            {
                                Logger.Error(ee.Message);
                            }

                        }
                    }

                    //Add rendered tiles to cache
                    foreach (var kvp in toRender)
                    {
                        if (takenFromCache.ContainsKey(kvp.Key) && !takenFromCache[kvp.Key])
                        {
                            _bitmaps.Add(kvp.Key, kvp.Value);
                        }
                    }

                    graphics.Transform = new Matrix();
                    graphics.DrawImageUnscaled(bmp, 0, 0);
                    graphics.Transform = g.Transform;
                }
            }
        }
        public void CollectIcal(FeedRegistry fr, ZonedEventStore es, bool test)
        {
            this.LoadTags();

            using (ical_ical)
             {
                Dictionary<string, string> feeds = fr.feeds;
                DateTime utc_midnight_in_tz = Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime;

                // enforce the limit. necessary because processing of icalendar sources can involve
                // the unrolling of recurrence, and that can't go on forever
                DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days);

                List<string> feedurls = test ? feeds.Keys.Take(test_feeds).ToList() : feeds.Keys.ToList();

                ParallelOptions options = new ParallelOptions();
                options.MaxDegreeOfParallelism = Convert.ToInt32(this.settings["max_feed_processing_parallelism"]);
                //options.MaxDegreeOfParallelism = 1; // for debugging

                List<string> hubs_to_skip_date_only_recurrence = new List<string>();

                try
                {
                    hubs_to_skip_date_only_recurrence = settings["hubs_to_skip_date_only_recurrence"].Split(',').ToList();
                }
                catch (Exception e)
                {
                    GenUtils.PriorityLogMsg("exception", "hubs_to_skip_date_only_recurrence", e.Message);
                }

                GenUtils.LogMsg("info", id + " loading " + feedurls.Count() + " feeds", null);

                var results_dict = new System.Collections.Concurrent.ConcurrentDictionary<string, string>();

                try
                {
                    //foreach (var feedurl in feedurls)
                    Parallel.ForEach(source: feedurls, parallelOptions: options, body: (feedurl) =>
                    {

                        if (settings["eventful_feeds_enabled"] == "False" && feedurl.StartsWith("http://eventful.com/") )
                            return;

                        if ( settings["eventbrite_feeds_enabled"] == "False" && feedurl.Contains("ics_from_eventbrite") )
                            return;

                        var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;

                        results_dict.TryAdd(feedurl, null);

                        ZonedEventStore es_feed_cache = new ZonedEventStore(calinfo, SourceType.ical);

                        var eids_and_categories = new Dictionary<string, List<string>>(); // for augmenting eventful ics feeds with categories from corresponding atom feeds

                        // http://eventful.com/ical/annarbor/venues/michigan-theater-/V0-001-000827373-5
                        if ( feedurl.StartsWith("http://eventful.com/ical") )
                        {
                            var atom_url = feedurl.Replace("ical","atom");
                            eids_and_categories = Utils.CategoriesFromEventfulAtomFeed(atom_url);
                        }

                        iCalendar ical = new DDay.iCal.iCalendar();
                        string source_name = "source_name";
                        List<DDay.iCal.Event> events_to_include = new List<DDay.iCal.Event>();
                        Dictionary<DDay.iCal.Event, RecurrenceType> event_recurrence_types = new Dictionary<DDay.iCal.Event, RecurrenceType>();
                        List<DDay.iCal.Event> uniques = new List<DDay.iCal.Event>();
                        string feedtext = "";

                        try
                        {
                            source_name = FeedSetup(fr, feeds, feedurl, tid, source_name);
                        }
                        catch (Exception e0)
                        {
                            try
                            {
                                HandleE0(results_dict, feedurl, tid, e0);
                                return;  // http://stackoverflow.com/questions/3765038/is-there-an-equivalent-to-continue-in-a-parallel-foreach
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleE0", e.Message); }
                        }

                        bool changed = false;

                        try
                        {
                            feedtext = GetFeedTextFromFeedUrl(fr, this.calinfo, source_name, feedurl, this.wait_secs, this.max_retries, this.timeout_secs, this.settings, ref changed);
                        }
                        catch (Exception e1)  // exception while loading feed
                        {
                            try
                            {
                                HandleE1(results_dict, feedurl, tid, source_name, e1);
                                return;
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleE1", e.Message); return; }
                        }

                        try
                        {
                            var setting = "use_ics_cached_objects";
                            if (changed == false && Utils.CachedFeedObjExists(id, feedurl) && settings[setting].StartsWith("y") )
                            {
                                GenUtils.LogMsg("info", "CollectIcal: using cached obj for " + feedurl,  null); // the test UnchangedFeedUsesCachedObj checks for this entry
                                AddEventsFromCachedIcalFeedObj(es, feedurl);  // get events for the feedurl from the cached obj
                                TransferCachedResults(this.id, results_dict, feedurl); // and result for this feedurl from the cached results
                                TransferCachedStats(this.id, fr, feedurl); // and stats for this feedurl from cached stats
                                return;  // done for this feedurl
                            }
                            else
                            {
                                ical = ParseTheFeed(feedtext);
                            }
                        }
                        catch (Exception e2)
                        {
                            try
                            {
                                HandleE2(fr, results_dict, feedurl, tid, source_name, e2);
                                return;
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleE2", e.Message); return; }
                        }

                        if (ical == null || ical.Events.Count == 0)
                        {
                            try
                            {
                                HandleNoEvents(results_dict, feedurl, tid, source_name);
                                return;
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleNoEvents", e.Message); return; }
                        }

                        try
                        {
                            var skip_date_only_recurrence = hubs_to_skip_date_only_recurrence.Exists(x => x == this.id);
                            FeedGather(utc_midnight_in_tz, then, tid, ical, source_name, events_to_include, event_recurrence_types, skip_date_only_recurrence);
                        }
                        catch (Exception e3)
                        {
                            try
                            {
                                HandleE3(fr, results_dict, feedurl, tid, source_name, e3);
                                return;
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleE3", e.Message); return; }
                        }

                        try
                        {
                            uniques = Utils.UniqueByTitleAndStart(events_to_include);
                        }
                        catch (Exception e4)
                        {
                            try
                            {
                                HandleE4(fr, results_dict, feedurl, tid, source_name, e4);
                                return;
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleE4", e.Message); return; }
                        }

                        try
                        {
                            foreach (var unique in uniques)
                            {
                                // MaybeAugmentEventfulCategories(unique, eids_and_categories);
                                // sadly eventful categories can be haphazard so this augmentation has to be suppressed for now

                                lock (es)
                                {
                                    AddIcalEvent(unique, fr, es, feedurl, source_name); // add to the all-feeds store
                                }

                                AddIcalEvent(unique, new FeedRegistry(id), es_feed_cache, feedurl, source_name); // and the per-feed cache
                            }
                        }
                        catch (Exception e5)
                        {
                            try
                            {
                                HandleE5(fr, results_dict, feedurl, tid, source_name, e5);
                                return;
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleE5", e.Message); return; }
                        }

                        try
                        {
                            FeedStats(fr, feedurl, event_recurrence_types, uniques);
                        }
                        catch (Exception e6)
                        {
                            try
                            {
                                HandleE6(results_dict, feedurl, tid, source_name, e6);
                                return;
                            }
                            catch (Exception e) { GenUtils.LogMsg("exception", "HandleE6", e.Message); return; }
                        }

                        try
                        {
                            Utils.SaveFeedObjToCache(id, feedurl, es_feed_cache);  // cache the per-feed zoned eventstore
                        }
                        catch (Exception e) { GenUtils.LogMsg("exception", "SaveFeedObjToCache: " + this.id + ", " + feedurl, e.Message); return; }

                        try { results_dict[feedurl] = "ok: " + source_name; }
                        catch (Exception e)
                            {
                            GenUtils.LogMsg("exception", "update results_dict[feedurl]", e.Message);
                            return;
                            }

                });

                }
                catch (AggregateException agg_ex)
                {
                    var inners = agg_ex.InnerExceptions;
                    foreach (var inner in inners)
                        GenUtils.LogMsg("exception", "CollectIcal: parallel inner exception: " + inner.Message, inner.StackTrace);
                }

                var json = JsonConvert.SerializeObject(results_dict);
                json = GenUtils.PrettifyJson(json);
                bs.PutBlob(id, "feed_processing_results.json", json);
                SerializeStatsAndIntermediateOutputs(fr, es, ical_ical, new NonIcalStats(), SourceType.ical);
            }
        }
 public virtual void TestInitialize()
 {
     tagsDict = new ConcurrentDictionary <string, string>();
     helper.SetUp();
 }