/// <summary>
/// 令牌桶服务
/// </summary>
/// <param name="maxTPS">最大每秒处理能力</param>
/// <param name="limitSize">最大数量</param>
/// <param name="limitQPH">每小时请求数量</param>
public TokenBucketLimitingService(int maxTPS, int limitSize, int limitQPH)
{
this.limitSize = limitSize;
this.maxTPS = maxTPS;
if (this.limitSize <= 0)
{
this.limitSize = 100;
}
if (this.maxTPS <= 0)
{
this.maxTPS = 1;
}
limitedQueue = new LimitedQueue <object>(limitSize);
for (int i = 0; i < limitSize; i++)
{
limitedQueue.Enqueue(new object());
}
cancelToken = new CancellationTokenSource();
task = Task.Factory.StartNew(new Action(TokenProcess), cancelToken.Token);
Node1HTPS = new System.Collections.Concurrent.ConcurrentDictionary <string, NodeLimit>();
Node24HTPS = new System.Collections.Concurrent.ConcurrentDictionary <string, NodeLimit>();
nodeLimitQPH = limitQPH;
H1StartCountTime = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, DateTime.Now.Hour, 0, 0);
H24StartCountTime = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0);
}
private System.Threading.ManualResetEventSlim slim;//主线程调用后为了判断不会第二次再进入
public SendTaskManage()
{
slim = new System.Threading.ManualResetEventSlim(true);
// bRuning = false;
downloadItems = new ConcurrentDictionary <string, DownloadItem>(System.Environment.ProcessorCount * 2, 500);
mr = new ManualResetEvent(true);
}
public LockKeyManager(Gateway gateway, IConfiguration configuration, ILogger <LockKeyManager> logger)
{
_timeout = configuration.GetValue <int>("UnLockKeyTimeout");
_cache = new System.Collections.Concurrent.ConcurrentDictionary <string, KeyObject>();
_gateway = gateway;
_configuration = configuration;
_logger = logger;
SystemEventCenter.MicroServiceOnffline += SystemEventCenter_MicroServiceOnffline;
SystemEventCenter.MicroServiceOnline += SystemEventCenter_MicroServiceOnline;
new Thread(checkTimeout).Start();
}
public static bool TryGetDeviceInfo(string key, out DeviceInfo device)
{
_deviceInfos = _impl.GetData("DeviceInfo") as ConcurrentDictionary<string, DeviceInfo>;
if ( _deviceInfos == null)
{
lock (_sync)
{
if( _deviceInfos == null)
_deviceInfos = LoadDeviceInfo();
}
}
return _deviceInfos.TryGetValue(key, out device);
}
public static bool TryGetDeviceInfo(string key, out DeviceInfo device)
{
_deviceInfos = _impl.GetData("DeviceInfo") as ConcurrentDictionary <string, DeviceInfo>;
if (_deviceInfos == null)
{
lock (_sync)
{
if (_deviceInfos == null)
{
_deviceInfos = LoadDeviceInfo();
}
}
}
return(_deviceInfos.TryGetValue(key, out device));
}
static MovieManager()
{
_movies = new ConcurrentDictionary <string, Movie>();
var movice = new Movie()
{
Name = "The Breaking Bad", Rating = 5, Id = "1"
};
_movies[movice.Id] = movice;
movice = new Movie()
{
Name = "The Avatar", Rating = 5, Id = "2"
};
_movies[movice.Id] = movice;
movice = new Movie()
{
Name = "The Walking Dead", Rating = 4, Id = "3"
};
_movies[movice.Id] = movice;
}
public static void Start()
{
new System.Threading.Thread(() =>
{
if (File.Exists(SessionBagFile))
{
_SessionPools = SerializeManager <ConcurrentDictionary <string, SessionIdentity> > .Deserialize(SessionBagFile);
if (_SessionPools == null)
{
_SessionPools = new ConcurrentDictionary <string, SessionIdentity>();
}
}
runningstate = true;
int tick = 0;
int tick2 = 0;
while (runningstate)
{
System.Threading.Thread.Sleep(1000);
tick += 1000;
tick2 += 1000;
if (tick / 1000 >= 5 * 60) //5分钟 写入磁盘一次
{
SaveSessionToDisk();
tick = 0;
}
//test
if (true)
{
RemoveExpiredSession();
}
//end test
if (tick2 / 1000 >= 3600 * 24) //24个小时清理一次过期session
{
RemoveExpiredSession();
tick2 = 0;
}
}
}).Start();
}
protected void Application_Start()
{
AreaRegistration.RegisterAllAreas();
FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
RouteConfig.RegisterRoutes(RouteTable.Routes);
BundleConfig.RegisterBundles(BundleTable.Bundles);
ServicePointManager.DefaultConnectionLimit = 500;
if (ConfigurationManager.AppSettings["IsNode"] == "1")
{
SearchApi.Init();
}
else
{
string log_db_path = @"C:\Users\Administrator\Documents\stackoverflow\LOG";
logdb = new Db(log_db_path);
}
ThreadPool.SetMinThreads(500, 500);
_recent_clean_job = new System.Threading.Thread(f =>
{
while (true)
{
try
{
Thread.Sleep(60000);
_recent_data = new System.Collections.Concurrent.ConcurrentDictionary <string, DateTime>();
}
catch (Exception ex)
{
LogError("ERROR IN CLEANING THREAD " + DateTime.Now + " \n" + ex.Message + " \n" + ex.StackTrace, ex);
}
}
});
}
/// <summary>
/// Creates a memoizer with one parameter for the the specified task provider. The task provider is guaranteed to be executed only once per parameter instance.
/// </summary>
/// <typeparam name="TResult">The return value type</typeparam>
/// <typeparam name="TParam"></typeparam>
/// <param name="func">A function that will call the create the task.</param>
/// <returns>A function that will return a task </returns>
public static FuncAsync <TParam, TResult> AsMemoized <TParam, TResult>(this FuncAsync <TParam, TResult> func)
{
#if HAS_NO_CONCURRENT_DICT
var values = new SynchronizedDictionary <TParam, FuncAsync <TResult> >();
#else
var values = new System.Collections.Concurrent.ConcurrentDictionary <TParam, FuncAsync <TResult> >();
#endif
// It's safe to use default(TParam) as this won't get called anyway if TParam is a value type.
var nullValue = Funcs.CreateAsyncMemoized(ct => func(ct, default(TParam)));
return((ct, param) =>
{
if (param == null)
{
return nullValue(ct);
}
else
{
var memoizedFunc = values.GetOrAdd(param, p => Funcs.CreateAsyncMemoized(c => func(c, p)));
return memoizedFunc(ct);
}
});
}
/// <summary>
/// Renders the layer
/// </summary>
/// <param name="graphics">Graphics object reference</param>
/// <param name="map">Map which is rendered</param>
public override void Render(Graphics graphics, MapViewport map)
{
if (!map.Size.IsEmpty && map.Size.Width > 0 && map.Size.Height > 0)
{
var bmp = new Bitmap(map.Size.Width, map.Size.Height, PixelFormat.Format32bppArgb);
using (var g = Graphics.FromImage(bmp))
{
g.InterpolationMode = InterpolationMode;
g.Transform = graphics.Transform.Clone();
var extent = new Extent(map.Envelope.MinX, map.Envelope.MinY,
map.Envelope.MaxX, map.Envelope.MaxY);
var level = BruTile.Utilities.GetNearestLevel(_source.Schema.Resolutions, Math.Max(map.PixelWidth, map.PixelHeight));
var tiles = new List <TileInfo>(_source.Schema.GetTileInfos(extent, level));
var tileWidth = _source.Schema.GetTileWidth(level);
var tileHeight = _source.Schema.GetTileWidth(level);
IList <WaitHandle> waitHandles = new List <WaitHandle>();
var toRender = new System.Collections.Concurrent.ConcurrentDictionary <TileIndex, Bitmap>();
var takenFromCache = new System.Collections.Concurrent.ConcurrentDictionary <TileIndex, bool>();
foreach (TileInfo info in tiles)
{
var image = _bitmaps.Find(info.Index);
if (image != null)
{
toRender.TryAdd(info.Index, image);
takenFromCache.TryAdd(info.Index, true);
continue;
}
if (_fileCache != null && _fileCache.Exists(info.Index))
{
var tileBitmap = GetImageFromFileCache(info) as Bitmap;
_bitmaps.Add(info.Index, tileBitmap);
toRender.TryAdd(info.Index, tileBitmap);
takenFromCache.TryAdd(info.Index, true);
continue;
}
var waitHandle = new AutoResetEvent(false);
waitHandles.Add(waitHandle);
ThreadPool.QueueUserWorkItem(GetTileOnThread,
new object[] { _source, info, toRender, waitHandle, true, takenFromCache });
}
foreach (var handle in waitHandles)
{
handle.WaitOne();
}
using (var ia = new ImageAttributes())
{
if (!_transparentColor.IsEmpty)
{
ia.SetColorKey(_transparentColor, _transparentColor);
}
#if !PocketPC
ia.SetWrapMode(WrapMode.TileFlipXY);
#endif
foreach (var info in tiles)
{
if (!toRender.ContainsKey(info.Index))
{
continue;
}
var bitmap = toRender[info.Index];//_bitmaps.Find(info.Index);
if (bitmap == null)
{
continue;
}
var min = map.WorldToImage(new Coordinate(info.Extent.MinX, info.Extent.MinY));
var max = map.WorldToImage(new Coordinate(info.Extent.MaxX, info.Extent.MaxY));
min = new PointF((float)Math.Round(min.X), (float)Math.Round(min.Y));
max = new PointF((float)Math.Round(max.X), (float)Math.Round(max.Y));
try
{
g.DrawImage(bitmap,
new Rectangle((int)min.X, (int)max.Y, (int)(max.X - min.X),
(int)(min.Y - max.Y)),
0, 0, tileWidth, tileHeight,
GraphicsUnit.Pixel,
ia);
}
catch (Exception ee)
{
Logger.Error(ee.Message);
}
}
}
//Add rendered tiles to cache
foreach (var kvp in toRender)
{
if (takenFromCache.ContainsKey(kvp.Key) && !takenFromCache[kvp.Key])
{
_bitmaps.Add(kvp.Key, kvp.Value);
}
}
graphics.Transform = new Matrix();
graphics.DrawImageUnscaled(bmp, 0, 0);
graphics.Transform = g.Transform;
}
}
}
/// <summary>
/// Renders the layer
/// </summary>
/// <param name="graphics">Graphics object reference</param>
/// <param name="map">Map which is rendered</param>
public override void Render(Graphics graphics, Map map)
{
if (!map.Size.IsEmpty && map.Size.Width > 0 && map.Size.Height > 0)
{
var bmp = new Bitmap(map.Size.Width, map.Size.Height, PixelFormat.Format32bppArgb);
using (var g = Graphics.FromImage(bmp))
{
g.InterpolationMode = InterpolationMode;
g.Transform = graphics.Transform.Clone();
var extent = new Extent(map.Envelope.MinX, map.Envelope.MinY,
map.Envelope.MaxX, map.Envelope.MaxY);
var level = BruTile.Utilities.GetNearestLevel(_source.Schema.Resolutions, map.PixelSize);
var tiles = new List<TileInfo>(_source.Schema.GetTileInfos(extent, level));
var tileWidth = _source.Schema.GetTileWidth(level);
var tileHeight = _source.Schema.GetTileWidth(level);
IList<WaitHandle> waitHandles = new List<WaitHandle>();
var toRender = new System.Collections.Concurrent.ConcurrentDictionary<TileIndex, Bitmap>();
var takenFromCache = new System.Collections.Concurrent.ConcurrentDictionary<TileIndex,bool>();
foreach (TileInfo info in tiles)
{
var image = _bitmaps.Find(info.Index);
if (image != null)
{
toRender.TryAdd(info.Index, image);
takenFromCache.TryAdd(info.Index,true);
continue;
}
if (_fileCache != null && _fileCache.Exists(info.Index))
{
var tileBitmap = GetImageFromFileCache(info) as Bitmap;
_bitmaps.Add(info.Index, tileBitmap);
toRender.TryAdd(info.Index, tileBitmap);
takenFromCache.TryAdd(info.Index, true);
continue;
}
var waitHandle = new AutoResetEvent(false);
waitHandles.Add(waitHandle);
ThreadPool.QueueUserWorkItem(GetTileOnThread,
new object[] { _source, info, toRender, waitHandle, true, takenFromCache });
}
foreach (var handle in waitHandles)
handle.WaitOne();
using (var ia = new ImageAttributes())
{
if (!_transparentColor.IsEmpty)
ia.SetColorKey(_transparentColor, _transparentColor);
#if !PocketPC
ia.SetWrapMode(WrapMode.TileFlipXY);
#endif
foreach (var info in tiles)
{
if (!toRender.ContainsKey(info.Index))
continue;
var bitmap = toRender[info.Index];//_bitmaps.Find(info.Index);
if (bitmap == null) continue;
var min = map.WorldToImage(new Coordinate(info.Extent.MinX, info.Extent.MinY));
var max = map.WorldToImage(new Coordinate(info.Extent.MaxX, info.Extent.MaxY));
min = new PointF((float) Math.Round(min.X), (float) Math.Round(min.Y));
max = new PointF((float) Math.Round(max.X), (float) Math.Round(max.Y));
try
{
g.DrawImage(bitmap,
new Rectangle((int) min.X, (int) max.Y, (int) (max.X - min.X),
(int) (min.Y - max.Y)),
0, 0, tileWidth, tileHeight,
GraphicsUnit.Pixel,
ia);
}
catch (Exception ee)
{
Logger.Error(ee.Message);
}
}
}
//Add rendered tiles to cache
foreach (var kvp in toRender)
{
if (takenFromCache.ContainsKey(kvp.Key) && !takenFromCache[kvp.Key])
{
_bitmaps.Add(kvp.Key, kvp.Value);
}
}
graphics.Transform = new Matrix();
graphics.DrawImageUnscaled(bmp, 0, 0);
graphics.Transform = g.Transform;
}
}
}
public void CollectIcal(FeedRegistry fr, ZonedEventStore es, bool test)
{
this.LoadTags();
using (ical_ical)
{
Dictionary<string, string> feeds = fr.feeds;
DateTime utc_midnight_in_tz = Utils.MidnightInTz(this.calinfo.tzinfo).UniversalTime;
// enforce the limit. necessary because processing of icalendar sources can involve
// the unrolling of recurrence, and that can't go on forever
DateTime then = utc_midnight_in_tz.AddDays(calinfo.icalendar_horizon_days);
List<string> feedurls = test ? feeds.Keys.Take(test_feeds).ToList() : feeds.Keys.ToList();
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = Convert.ToInt32(this.settings["max_feed_processing_parallelism"]);
//options.MaxDegreeOfParallelism = 1; // for debugging
List<string> hubs_to_skip_date_only_recurrence = new List<string>();
try
{
hubs_to_skip_date_only_recurrence = settings["hubs_to_skip_date_only_recurrence"].Split(',').ToList();
}
catch (Exception e)
{
GenUtils.PriorityLogMsg("exception", "hubs_to_skip_date_only_recurrence", e.Message);
}
GenUtils.LogMsg("info", id + " loading " + feedurls.Count() + " feeds", null);
var results_dict = new System.Collections.Concurrent.ConcurrentDictionary<string, string>();
try
{
//foreach (var feedurl in feedurls)
Parallel.ForEach(source: feedurls, parallelOptions: options, body: (feedurl) =>
{
if (settings["eventful_feeds_enabled"] == "False" && feedurl.StartsWith("http://eventful.com/") )
return;
if ( settings["eventbrite_feeds_enabled"] == "False" && feedurl.Contains("ics_from_eventbrite") )
return;
var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
results_dict.TryAdd(feedurl, null);
ZonedEventStore es_feed_cache = new ZonedEventStore(calinfo, SourceType.ical);
var eids_and_categories = new Dictionary<string, List<string>>(); // for augmenting eventful ics feeds with categories from corresponding atom feeds
// http://eventful.com/ical/annarbor/venues/michigan-theater-/V0-001-000827373-5
if ( feedurl.StartsWith("http://eventful.com/ical") )
{
var atom_url = feedurl.Replace("ical","atom");
eids_and_categories = Utils.CategoriesFromEventfulAtomFeed(atom_url);
}
iCalendar ical = new DDay.iCal.iCalendar();
string source_name = "source_name";
List<DDay.iCal.Event> events_to_include = new List<DDay.iCal.Event>();
Dictionary<DDay.iCal.Event, RecurrenceType> event_recurrence_types = new Dictionary<DDay.iCal.Event, RecurrenceType>();
List<DDay.iCal.Event> uniques = new List<DDay.iCal.Event>();
string feedtext = "";
try
{
source_name = FeedSetup(fr, feeds, feedurl, tid, source_name);
}
catch (Exception e0)
{
try
{
HandleE0(results_dict, feedurl, tid, e0);
return; // http://stackoverflow.com/questions/3765038/is-there-an-equivalent-to-continue-in-a-parallel-foreach
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE0", e.Message); }
}
bool changed = false;
try
{
feedtext = GetFeedTextFromFeedUrl(fr, this.calinfo, source_name, feedurl, this.wait_secs, this.max_retries, this.timeout_secs, this.settings, ref changed);
}
catch (Exception e1) // exception while loading feed
{
try
{
HandleE1(results_dict, feedurl, tid, source_name, e1);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE1", e.Message); return; }
}
try
{
var setting = "use_ics_cached_objects";
if (changed == false && Utils.CachedFeedObjExists(id, feedurl) && settings[setting].StartsWith("y") )
{
GenUtils.LogMsg("info", "CollectIcal: using cached obj for " + feedurl, null); // the test UnchangedFeedUsesCachedObj checks for this entry
AddEventsFromCachedIcalFeedObj(es, feedurl); // get events for the feedurl from the cached obj
TransferCachedResults(this.id, results_dict, feedurl); // and result for this feedurl from the cached results
TransferCachedStats(this.id, fr, feedurl); // and stats for this feedurl from cached stats
return; // done for this feedurl
}
else
{
ical = ParseTheFeed(feedtext);
}
}
catch (Exception e2)
{
try
{
HandleE2(fr, results_dict, feedurl, tid, source_name, e2);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE2", e.Message); return; }
}
if (ical == null || ical.Events.Count == 0)
{
try
{
HandleNoEvents(results_dict, feedurl, tid, source_name);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleNoEvents", e.Message); return; }
}
try
{
var skip_date_only_recurrence = hubs_to_skip_date_only_recurrence.Exists(x => x == this.id);
FeedGather(utc_midnight_in_tz, then, tid, ical, source_name, events_to_include, event_recurrence_types, skip_date_only_recurrence);
}
catch (Exception e3)
{
try
{
HandleE3(fr, results_dict, feedurl, tid, source_name, e3);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE3", e.Message); return; }
}
try
{
uniques = Utils.UniqueByTitleAndStart(events_to_include);
}
catch (Exception e4)
{
try
{
HandleE4(fr, results_dict, feedurl, tid, source_name, e4);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE4", e.Message); return; }
}
try
{
foreach (var unique in uniques)
{
// MaybeAugmentEventfulCategories(unique, eids_and_categories);
// sadly eventful categories can be haphazard so this augmentation has to be suppressed for now
lock (es)
{
AddIcalEvent(unique, fr, es, feedurl, source_name); // add to the all-feeds store
}
AddIcalEvent(unique, new FeedRegistry(id), es_feed_cache, feedurl, source_name); // and the per-feed cache
}
}
catch (Exception e5)
{
try
{
HandleE5(fr, results_dict, feedurl, tid, source_name, e5);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE5", e.Message); return; }
}
try
{
FeedStats(fr, feedurl, event_recurrence_types, uniques);
}
catch (Exception e6)
{
try
{
HandleE6(results_dict, feedurl, tid, source_name, e6);
return;
}
catch (Exception e) { GenUtils.LogMsg("exception", "HandleE6", e.Message); return; }
}
try
{
Utils.SaveFeedObjToCache(id, feedurl, es_feed_cache); // cache the per-feed zoned eventstore
}
catch (Exception e) { GenUtils.LogMsg("exception", "SaveFeedObjToCache: " + this.id + ", " + feedurl, e.Message); return; }
try { results_dict[feedurl] = "ok: " + source_name; }
catch (Exception e)
{
GenUtils.LogMsg("exception", "update results_dict[feedurl]", e.Message);
return;
}
});
}
catch (AggregateException agg_ex)
{
var inners = agg_ex.InnerExceptions;
foreach (var inner in inners)
GenUtils.LogMsg("exception", "CollectIcal: parallel inner exception: " + inner.Message, inner.StackTrace);
}
var json = JsonConvert.SerializeObject(results_dict);
json = GenUtils.PrettifyJson(json);
bs.PutBlob(id, "feed_processing_results.json", json);
SerializeStatsAndIntermediateOutputs(fr, es, ical_ical, new NonIcalStats(), SourceType.ical);
}
}
public virtual void TestInitialize()
{
tagsDict = new ConcurrentDictionary <string, string>();
helper.SetUp();
}
