public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
bool removeZeroGauges,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var items = gauges.ToArray();
var bucket = new GaugesBucket(items, e.Epoch, ns);
if (removeZeroGauges)
{
// Get all zero-value gauges
double placeholder;
var zeroGauges = 0;
for (int index = 0; index < items.Length; index++)
{
if (items[index].Value == 0)
{
gauges.TryRemove(items[index].Key, out placeholder);
zeroGauges += 1;
}
}
if (zeroGauges > 0)
{
log.InfoFormat("Removed {0} empty gauges.", zeroGauges);
}
}
gauges.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary<string, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + ( percentileName ?? ( "p" + percentile ) );
var incoming = new ActionBlock<StatsdMessage>( p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return new DatapointBox(maxItemsPerBucket, latency.ValueMS);
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return bag;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var bucket = new PercentileBucket(latencies.ToArray(),
e.Epoch,
ns,
percentileName,
percentile);
latencies.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<SetsBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<int, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
sets.AddOrUpdate(set.Name,
(key) =>
{
var setDict = new ConcurrentDictionary<int, bool>();
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
},
(key, setDict) =>
{
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var rawData = sets.ToArray();
sets.Clear();
var bucket = new SetsBucket(
rawData.Select(p =>
new KeyValuePair<string, List<KeyValuePair<int, bool>>>(p.Key, p.Value.ToList())
).ToList(),
e.Epoch,
ns);
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
private void refreshWaitingItems()
{
var waitingItemsList = waitingItems.ToArray();
foreach (var item in waitingItemsList)
{
var waitingItemForOneWorkerType = item.Value.ToArray();
foreach (var workItem in waitingItemForOneWorkerType)
{
if ((DateTime.Now - workItem.Value.Item1).TotalSeconds > 60 * 5)
{
getWaitingItems(workItem.Value.Item2.WorkerType).TryRemove(workItem.Key, out Tuple <DateTime, WorkItemInfo> wii);
getQueue(workItem.Value.Item2.WorkerType).Enqueue(workItem.Value.Item2);
}
}
}
}
/// <summary>
/// Disconnect this instance from the server
/// </summary>
public void Disconnect()
{
if (WSConnection != null)
{
WSConnection.Close();
}
WSConnection = null;
var unusedHandlers = responseHandlers.ToArray();
responseHandlers.Clear();
foreach (var cb in unusedHandlers)
{
var tcs = cb.Value;
tcs.TrySetCanceled();
}
}
private static bool RunDictionaryTest_Remove3()
{
TestHarness.TestLog("* RunDictionaryTest_Remove3()");
ConcurrentDictionary <int, int> dict = new ConcurrentDictionary <int, int>();
dict[99] = -99;
ICollection <KeyValuePair <int, int> > col = dict;
// Make sure we cannot "remove" a key/value pair which is not in the dictionary
for (int i = 0; i < 1000; i++)
{
if (i != 99)
{
if (col.Remove(new KeyValuePair <int, int>(i, -99)) || col.Remove(new KeyValuePair <int, int>(99, -i)))
{
TestHarness.TestLog(" > Removed a key/value pair which was not supposed to be in the dictionary.");
return(false);
}
}
}
// Can we remove a key/value pair successfully?
if (!col.Remove(new KeyValuePair <int, int>(99, -99)))
{
TestHarness.TestLog(" > Failed to remove a key/value pair which was supposed to be in the dictionary.");
return(false);
}
// Make sure the key/value pair is gone
if (col.Remove(new KeyValuePair <int, int>(99, -99)))
{
TestHarness.TestLog(" > Removed a key/value pair which was not supposed to be in the dictionary.");
return(false);
}
// And that the dictionary is empty. We will check the count in a few different ways:
if (dict.Count != 0 || dict.ToArray().Count() != 0 || dict.ToList().Count() != 0)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary.");
return(false);
}
return(true);
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <GraphiteLine> target,
string rootNamespace,
bool deleteGaugesOnFlush,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary <string, int>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var bucket = gauges.ToArray();
if (deleteGaugesOnFlush)
{
}
var lines = bucket.Select(q => new GraphiteLine(ns + q.Key, q.Value, e.Epoch)).ToArray();
for (int i = 0; i < lines.Length; i++)
{
target.Post(lines[i]);
}
log.InfoFormat("TimedGaugeAggregatorBlock - Posted {0} lines.", lines.Length);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static bool ParseProtoPackage(string strDir)
{
var Files = FileHelper.GetAllFile(strDir, "*.sdp");
ConcurrentDictionary <string, FileEntity> fileEntitys = new ConcurrentDictionary <string, FileEntity>();
List <Task> tasks = new List <Task>();
foreach (var file in Files)
{
var fileName = file.Replace('\\', '/');
Task task = Task.Factory.StartNew(() =>
{
var entity = ParseFile(fileName);
if (entity == null)
{
throw new Exception("Parse file in file :" + file);
}
fileEntitys.TryAdd(entity.FileName, entity);
});
tasks.Add(task);
}
Task.WaitAll(tasks.ToArray());
foreach (var kv in fileEntitys.ToArray())
{
ProtoPackage.Files.Add(kv.Key, kv.Value);
}
foreach (var file in ProtoPackage.Files)
{
foreach (var ns in file.Value.NameSpaces)
{
foreach (var en in ns.Enums)
{
ProtoPackage.EnumNames.Add(en.Name);
}
foreach (var st in ns.Structs)
{
ProtoPackage.StructNames.Add(st.Name);
}
}
}
return(true);
}
/// <summary>
/// Initialize a new instance of <see cref="Database"/>
/// </summary>
/// <param name="config">The configuration to setup database</param>
public Database(DatabaseConfig config)
{
Config = config;
config.DatabaseConnectionString.EnableBroker();
_messageTransmitter = new MessageTransmitter(this);
_messageTransmitter.MessageRecieved += OnMessageRecieved;
_disposeHelper.Attach(_messageTransmitter);
_disposeHelper.Attach(_tables);
_disposeHelper.Attach(() =>
{
var queries = _customQueries.ToArray();
_customQueries.Clear();
queries.AsParallel().ForAll(q => q.Value.Dispose());
});
}
private void CleanupCache()
{
if ((DateTime.Now - _lastCleanUpTime).TotalMilliseconds > 15)
{
return;
}
_lastCleanUpTime = DateTime.Now;
foreach (var item in _tokenCache.ToArray())
{
var jwtExpValue = long.Parse(item.Value.Claims.FirstOrDefault(x => x.Type == "exp").Value);
var expirationTime = DateTimeOffset.FromUnixTimeSeconds(jwtExpValue).DateTime;
if (DateTime.Now > expirationTime)
{
_tokenCache.TryRemove(item.Key, out _);
}
}
}
public static async Task BattleChecker()
{
foreach (var activeBattle in ActiveBattles.ToArray())
{
var timeactive = (DateTime.UtcNow - activeBattle.Value.StartTime).TotalMinutes;
if (timeactive > 1 && !activeBattle.Value.Activated)
{
await activeBattle.Value.Kill("Took too long to respond");
continue;
}
var lastMessageMinutes = (DateTime.UtcNow - activeBattle.Value.LastMessageTime).TotalMinutes;
if (lastMessageMinutes > 2)
{
await activeBattle.Value.Kill("Battle timed out due to now messages");
}
}
}
private void UpdateIndexMappingFile()
{
if (configuration.RunInMemory)
{
return;
}
var sb = new StringBuilder();
var indexNamesToId = indexNameToId.ToArray();
foreach (var index in indexNamesToId)
{
sb.Append($"{index.Value} - {index.Key}{Environment.NewLine}");
}
File.WriteAllText(Path.Combine(path, "indexes.txt"), sb.ToString());
}
private string Snapshot()
{
KeyValuePair <string, string>[] planArray = _plans.ToArray();
var sb = new StringBuilder();
foreach (var text in planArray.OrderBy(kvp => kvp.Key).Select(kvp => kvp.Value))
{
if (sb.Length > 0)
{
sb.AppendLine();
}
sb.Append(text);
}
return(sb.ToString());
}
private static RoomInfo[] _getRoomInfo()
{
var rooms = Rooms.ToArray();
RoomInfo[] infos = new RoomInfo[rooms.Length];
for (int i = 0; i < rooms.Length; i++)
{
var room = rooms[i];
var info = new RoomInfo();
info.Avatar = room.Value.Avatar;
info.Counts = room.Value.Users.Count;
info.GameBegin = room.Value.GetGameBegin();
info.Name = room.Value.Name;
infos[i] = info;
}
return(infos);
}
/// <summary>
/// Resets all mocks and suits to the clean state and removes all the real instances
/// You could create a new test suit for each test, but using Reset() after (or before) each test
/// is more performant.
/// Becomes meaningful when you have some hundreds of tests
/// </summary>
public virtual void Reset()
{
foreach (var pair in _dependencies.ToArray())
{
if (pair.Value.IsMock)
{
pair.Value.GetMock().Reset();
}
else if (pair.Value.IsSuit)
{
pair.Value.GetSuit().Reset();
}
else
{
_dependencies.TryRemove(pair.Key, out _);
}
}
}
private void MakeExpiredInvisibleMessagesVisible()
{
KeyValuePair <string, MutableStorageQueueMessage>[] invisibleMessagesSnapshot =
_invisibleMessages.ToArray();
IEnumerable <string> expiredInvisibleMessagePopReceipts =
invisibleMessagesSnapshot.Where(
p => p.Value.NextVisibleTime.Value.UtcDateTime < DateTimeOffset.UtcNow).Select(p => p.Key);
foreach (string popReceipt in expiredInvisibleMessagePopReceipts)
{
MutableStorageQueueMessage message;
if (_invisibleMessages.TryRemove(popReceipt, out message))
{
_visibleMessages.Enqueue(message);
}
}
}
private void ReleaseOldestItems()
{
// ToArray reason:
// https://stackoverflow.com/questions/11692389/getting-argument-exception-in-concurrent-dictionary-when-sorting-and-displaying
var items = cache.ToArray().OrderBy(a => a.Value.LastAccess).ToList();
foreach (var item in items)
{
if (currentSize > memorySizeLimit)
{
TryRemove(item.Key, out var removed);
}
else
{
break;
}
}
}
public DataPoint[] FlushCompleteEvents()
{
DataPoint temp;
// Get a snapshot of the complete events, then iterate through and try to remove each
// one, returning only the successfully removed ones. This ensures each data point is
// returned at most once.
var datapoints = completeData.ToArray();
List <DataPoint> removed = new List <DataPoint>();
foreach (var dp in datapoints)
{
if (completeData.TryRemove(dp.Key, out temp))
{
removed.Add(temp);
}
}
return(removed.ToArray());
}
public void FindOptimalBuildings(ConcurrentDictionary <LatLong, int> BuildingStats)
{
Debug.Log("Count: " + BuildingStats.Count);
var BuildingsArray = BuildingStats.ToArray();
BuildingsList = new List <KeyValuePair <LatLong, int> >(BuildingsArray);
//freeing some memory
BuildingsArray = null;
BuildingsList.Sort((x, y) => (y.Value).CompareTo(x.Value)); //Sorting the buildings on frequency basis
foreach (var item in BuildingsList)
{
Debug.Log((item.Key.GetLatitude()).ToString() + " " + (item.Key.GetLongitude()).ToString() + " " + item.Value);
}
}
public void EndSync()
{
if (Logger.IsDebug)
{
Logger.Debug($"EndSync | Channel: {_channelName}, SyncInProgress: {IsSyncInProgress}");
}
if (!IsSyncInProgress)
{
return;
}
try
{
// We can now strip out the ABSENT members, as we have
// received all of the out-of-order sync messages
foreach (var member in members.ToArray())
{
if (member.Value.Action == PresenceAction.Present)
{
members.TryRemove(member.Key, out PresenceMessage _);
}
}
lock (_lock)
{
if (residualMembers != null)
{
// Any members that were present at the start of the sync,
// and have not been seen in sync, can be removed
foreach (var member in residualMembers)
{
members.TryRemove(member, out PresenceMessage _);
}
residualMembers = null;
}
}
}
finally
{
IsSyncInProgress = false;
}
}
public async Task Start(string remoteServerIp, ushort remoteServerPort, ushort localPort, string localIp = null)
{
var clients = new ConcurrentDictionary <IPEndPoint, UdpClient>();
var server = new System.Net.Sockets.UdpClient(AddressFamily.InterNetworkV6);
server.Client.SetSocketOption(SocketOptionLevel.IPv6, SocketOptionName.IPv6Only, false);
IPAddress localIpAddress = string.IsNullOrEmpty(localIp) ? IPAddress.IPv6Any : IPAddress.Parse(localIp);
server.Client.Bind(new IPEndPoint(localIpAddress, localPort));
Console.WriteLine($"UDP proxy started :{localIpAddress}|{localPort} -> {remoteServerIp}|{remoteServerPort}");
var _ = Task.Run(async() =>
{
while (true)
{
await Task.Delay(10000);
foreach (var client in clients.ToArray())
{
if (client.Value.lastActivity + TimeSpan.FromSeconds(60) < DateTime.UtcNow)
{
UdpClient c;
clients.TryRemove(client.Key, out c);
client.Value.Stop();
}
}
}
});
while (true)
{
try
{
var message = await server.ReceiveAsync();
var endpoint = message.RemoteEndPoint;
var client = clients.GetOrAdd(endpoint, ep => new UdpClient(server, endpoint, new IPEndPoint(IPAddress.Parse(remoteServerIp), remoteServerPort)));
await client.SendToServer(message.Buffer);
}
catch (Exception ex)
{
Console.WriteLine($"an exception occurred on recieving a client datagram: {ex}");
}
}
}
public async Task InitializeTimer()
{
try
{
_timer = new Timer(async _ =>
{////await (await Context.User.GetOrCreateDMChannelAsync()).SendMessageAsync("",false,eb);
foreach (var user in _remindDict.ToArray())
{
List <RemindData> itemsToRemove = new List <RemindData>();
foreach (var reminder in user.Value)
{
if (reminder.TimeToRemind.CompareTo(DateTime.UtcNow) <= 0)
{
var userToRemind = _client.GetUser(user.Key);
await(await userToRemind.GetOrCreateDMChannelAsync()).SendMessageAsync($":alarm_clock: **Reminder:** {reminder.message}");
itemsToRemove.Add(reminder);
}
}
foreach (var remove in itemsToRemove)
{
user.Value.Remove(remove);
}
_remindDict.TryUpdate(user.Key, user.Value);
}
ChangeToClosestInterval();
ReminderDB.SaveReminders(_remindDict);
},
null,
TimeSpan.FromSeconds(INITIAL_DELAY), // Time that message should fire after bot has started
TimeSpan.FromSeconds(INITIAL_DELAY)); //time after which message should repeat (timout.infinite for no repeat)
Console.WriteLine("TIMER INITIALIZED");
/*if(str.timeToTriggerAgain.CompareTo(DateTime.UtcNow) >0)
* {
* return;
* }*/
}
catch (Exception e)
{
Console.WriteLine(e);
await SentryService.SendError(e);
}
}
/// <summary>
/// 規制されたアカウントが再度投稿できるようになったかを確認
/// </summary>
private static void UpdateUnderControls(object o)
{
var dicts = underControls.ToArray();
if (dicts.Length == 0)
{
return;
}
foreach (var i in dicts)
{
if (i.Value.Subtract(DateTime.Now).TotalSeconds < 0)
{
// released
underControls.Remove(i.Key);
OnOnUnderControlChanged(new UnderControlEventArgs(i.Key, false));
}
}
}
private void CompactCache()
{
long elapsedMilliseconds = stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
if (elapsedMilliseconds < 1000)
{
// Performance optimization: Do not compact the cache more often than every second.
return;
}
foreach (var entry in musicFilesCache.ToArray())
{
if (!entry.Value.TryGetTarget(out _))
{
TryRemoveFromCache(entry.Key);
}
}
}
public static string ShortenId(string id)
{
int maxPrefix = 0;
foreach (var kv in InnovationIds.ToArray())
{
var otherId = kv.Key;
if (otherId == id)
{
continue;
}
var commonSize = id.TakeWhile((c, i) => c == otherId[i]).Count();
maxPrefix = Math.Max(maxPrefix, commonSize);
}
return(id.Substring(0, Math.Min(maxPrefix + 1, id.Length)));
}
/// <summary>
/// Gets a serializable snapshot of the current call context.
/// </summary>
/// <returns>Array of call context entries</returns>
public static CallContextEntry[] GetSnapshot()
{
var stateSnaphsot = State.ToArray();
var result = new CallContextEntry[stateSnaphsot.Length];
for (int i = 0; i < stateSnaphsot.Length; i++)
{
var entry = stateSnaphsot[i];
result[i] =
new CallContextEntry()
{
Name = entry.Key,
Value = entry.Value.Value
};
}
return(result);
}
private async Task Post(Post post)
{
foreach (var user in _users.ToArray())
{
var response = user.Value;
if (response == null)
{
continue;
}
try
{
await response.WriteAsync(post);
}
catch
{
_users.TryRemove(user.Key, out IServerStreamWriter <Post> r);
}
}
}
static void WebSocketDisposable()
{
foreach (var webSocket in webSocketContexts.ToArray())
{
ThreadPool.QueueUserWorkItem(new WaitCallback(p =>
{
int count = 1;
WebSocketContext webSocketContext = p as WebSocketContext;
webSocketContext.Dispose();
while (!webSocketContexts.TryRemove(webSocketContext.ID, out webSocketContext))
{
if (count++ >= 99)
{
throw new Exception($"尝试从{webSocketContexts}中删除{webSocketContext}失败99次！");
}
}
}), webSocket);
}
}
private static void Run()
{
long ticks;
while (true)
{
ticks = DateTime.Now.Ticks;
foreach (var keyValue in delayedTasks.ToArray())
{
var key = keyValue.Key;
var value = keyValue.Value;
if (value.DelayedTime <= ticks)
{
value.ExecCount += 1;
try
{
Task.Run(value.Callback(value.State, value.ExecCount));
}
catch (Exception ex)
{
throw ex;
}
if (value.NextDelayedTime != null)
{
var timeSpan = value.NextDelayedTime(value.ExecCount);
if (timeSpan != null)
{
value.DelayedTime = ticks + timeSpan.Value.Ticks;
}
else
{
delayedTasks.TryRemove(key, out value);
}
}
else
{
delayedTasks.TryRemove(key, out value);
}
}
}
}
}
/// <inheritdoc />
public async Task ProcessItemsAsync()
{
_logger.LogDebug("ProcessItemsAsync");
// Attempt to process all items in queue.
var currentItems = _itemProcessQueue.ToArray();
foreach (var(key, container) in currentItems)
{
var item = _libraryManager.GetItemById(key);
if (item is null)
{
// Remove item from queue.
_itemProcessQueue.TryRemove(key, out _);
return;
}
_logger.LogDebug("Item {ItemName}", item.Name);
// Metadata not refreshed yet and under retry limit.
if (item.ProviderIds.Keys.Count == 0 && container.RetryCount < MaxRetries)
{
_logger.LogDebug("Requeue {ItemName}, no provider ids", item.Name);
container.RetryCount++;
_itemProcessQueue.AddOrUpdate(key, container, (_, _) => container);
continue;
}
_logger.LogDebug("Notifying for {ItemName}", item.Name);
// Send notification to each configured destination.
var dataObject = DataObjectHelpers
.GetBaseDataObject(_applicationHost, NotificationType.ItemAdded)
.AddBaseItemData(item);
var itemType = item.GetType();
await _webhookSender.SendNotification(NotificationType.ItemAdded, dataObject, itemType)
.ConfigureAwait(false);
// Remove item from queue.
_itemProcessQueue.TryRemove(key, out _);
}
}
/// <summary>
/// Recover space when the MaxCapacity limit is reached
/// </summary>
private void RecoverSpace(RecoveryFinishedCondition finishCondition)
{
// We're finished when the capacity has been reduced to CapacityToTargetWhenRecoveringSpace
bool finished = false;
while (!finished)
{
// If there queue of keys to cleanup is empty, refill it with all of the key's keys
if (_cleanupQueue.Count == 0)
{
_cleanupQueue = new Queue <TKey>(_keyCounts.ToArray().Select(k => k.Key));
}
// Dequeue one key from the clean-up queue (we only run one recovery-task at a time,
// so we don't need task/thread safety here.
TKey key = _cleanupQueue.Dequeue();
// Reduce the count for the current key on the queue and, if the count reaches 0,
// remove that key
uint count;
if (_keyCounts.TryGetValue(key, out count))
{
if (count <= 1)
{
_keyCounts.TryRemove(key, out count);
Interlocked.Add(ref _sumOfAmountsOverAllKeys, -count);
Interlocked.Add(ref _numberOfElements, -1);
}
else
{
_keyCounts.AddOrUpdate(key, k => 0, (k, priorValue) => priorValue - 1);
Interlocked.Add(ref _sumOfAmountsOverAllKeys, -1);
}
}
// We're finished when the capacity has reached the target capacity
finished = finishCondition();
}
// Now that the task is done, clear the task object so that a new task can be started
// if necessary.
lock (_recoveryTaskLock)
{
_recoveryTask = null;
}
}
public void DropAllPendingTileRequests()
{
ThrowExceptionIfDisposed();
// Notes: http://dotspatial.codeplex.com/discussions/473428
threadPool.Cancel(false);
foreach (KeyValuePair <TileIndex, int> request in activeTileRequests.ToArray())
{
if (!openTileRequests.ContainsKey(request.Key))
{
int dummy;
if (!activeTileRequests.TryRemove(request.Key, out dummy))
{
activeTileRequests.TryRemove(request.Key, out dummy);
}
}
}
openTileRequests.Clear();
}
/// <summary>
/// 定时处理无效的缓存 防止泄露
/// </summary>
static MessageManager()
{
Observable.Timer(DateTimeOffset.Now, TimeSpan.FromMinutes(5)).Subscribe(_ =>
{
var tempCache = _cache.ToArray();
var invalidCache = tempCache.Where(item =>
{
if (!item.Value.TryGetTarget(out var target))
{
return(true);
}
return(target == null);
});
foreach (var keyValuePair in invalidCache)
{
_cache.TryRemove(keyValuePair.Key, out var _);
}
});
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var counters = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : (rootNamespace + ".");
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var counter = p as Counter;
counters.AddOrUpdate(counter.Name, counter.Value, (key, oldValue) => oldValue + counter.Value);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (counters.Count == 0)
{
return;
}
var bucket = new CounterBucket(counters.ToArray(), e.Epoch, ns);
counters.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
private static bool RunDictionaryTest_Add1(int cLevel, int initSize, int threads, int addsPerThread)
{
TestHarness.TestLog(
"* RunDictionaryTest_Add1(cLevel={0}, initSize={1}, threads={2}, addsPerThread={3})",
cLevel, initSize, threads, addsPerThread);
IDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1);
int count = threads;
using (ManualResetEvent mre = new ManualResetEvent(false))
{
for (int i = 0; i < threads; i++)
{
int ii = i;
ThreadPool.QueueUserWorkItem(
(o) =>
{
for (int j = 0; j < addsPerThread; j++)
{
dict.Add(j + ii * addsPerThread, -(j + ii * addsPerThread));
}
if (Interlocked.Decrement(ref count) == 0) mre.Set();
});
}
mre.WaitOne();
}
if (dict.Any(pair => pair.Key != -pair.Value))
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
return false;
}
var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray();
var expectKeys = Enumerable.Range(0, threads * addsPerThread);
if (!gotKeys.SequenceEqual(expectKeys))
{
TestHarness.TestLog(" > The set of keys in the dictionary is invalid.");
return false;
}
// Finally, let's verify that the count is reported correctly.
int expectedCount = threads * addsPerThread;
if (dict.Count != expectedCount || dict.ToArray().Length != expectedCount || dict.ToList().Count() != expectedCount)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary.");
return false;
}
return true;
}
public void Given_different_thread_and_same_driver_environments_When_comparing_Then_should_not_be_equal()
{
var sessions = new ConcurrentDictionary<Guid, Session>();
Action action = () =>
{
var session = Threaded<Session>
.With<PhantomJS>();
sessions.TryAdd(Guid.NewGuid(), session);
};
var tasks = Enumerable
.Repeat(0, 2)
.Select(x => Task.Run(action));
Task.WaitAll(tasks.ToArray());
var session1 = sessions.ToArray()[0].Value;
var session2 = sessions.ToArray()[1].Value;
session1.Should().NotBe(session2);
session1.Driver.Should().NotBeNull();
session2.Driver.Should().NotBeNull();
session1.End();
session2.End();
}
private static bool RunDictionaryTest_Remove1(int cLevel, int threads, int removesPerThread)
{
TestHarness.TestLog("* RunDictionaryTest_Remove1(cLevel={0}, threads={1}, removesPerThread={2})", cLevel, threads, removesPerThread);
ConcurrentDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1);
int N = 2 * threads * removesPerThread;
for (int i = 0; i < N; i++) dict[i] = -i;
// The dictionary contains keys [0..N), each key mapped to a value equal to the key.
// Threads will cooperatively remove all even keys.
int running = threads;
bool passed = true;
using (ManualResetEvent mre = new ManualResetEvent(false))
{
for (int i = 0; i < threads; i++)
{
int ii = i;
ThreadPool.QueueUserWorkItem(
(o) =>
{
for (int j = 0; j < removesPerThread; j++)
{
int value;
int key = 2 * (ii + j * threads);
if (!dict.TryRemove(key, out value))
{
TestHarness.TestLog(" > Failed to remove an element, which should be in the dictionary.");
passed = false;
break;
}
if (value != -key)
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
passed = false;
break;
}
}
if (Interlocked.Decrement(ref running) == 0) mre.Set();
});
}
mre.WaitOne();
}
if (!passed) return false;
var res = dict.ToArray();
if (res.Any(pair => pair.Key != -pair.Value))
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
return false;
}
IEnumerable<int> gotKeys = res.Select(pair => pair.Key).OrderBy(i => i);
IEnumerable<int> expectKeys = Enumerable.Range(0, threads * removesPerThread).Select(i => 2 * i + 1);
if (!gotKeys.SequenceEqual(expectKeys))
{
TestHarness.TestLog(" > The set of keys in the dictionary is invalid.");
return false;
}
// Finally, let's verify that the count is reported correctly.
int expectedCount = expectKeys.Count();
if (dict.Count != expectedCount || dict.ToArray().Length != expectedCount || dict.ToList().Count() != expectedCount)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary.");
return false;
}
return true;
}
private static bool RunDictionaryTest_Remove3()
{
TestHarness.TestLog("* RunDictionaryTest_Remove3()");
ConcurrentDictionary<int, int> dict = new ConcurrentDictionary<int, int>();
dict[99] = -99;
ICollection<KeyValuePair<int, int>> col = dict;
// Make sure we cannot "remove" a key/value pair which is not in the dictionary
for (int i = 0; i < 1000; i++)
{
if (i != 99)
{
if (col.Remove(new KeyValuePair<int, int>(i, -99)) || col.Remove(new KeyValuePair<int, int>(99, -i)))
{
TestHarness.TestLog(" > Removed a key/value pair which was not supposed to be in the dictionary.");
return false;
}
}
}
// Can we remove a key/value pair successfully?
if (!col.Remove(new KeyValuePair<int, int>(99, -99)))
{
TestHarness.TestLog(" > Failed to remove a key/value pair which was supposed to be in the dictionary.");
return false;
}
// Make sure the key/value pair is gone
if (col.Remove(new KeyValuePair<int, int>(99, -99)))
{
TestHarness.TestLog(" > Removed a key/value pair which was not supposed to be in the dictionary.");
return false;
}
// And that the dictionary is empty. We will check the count in a few different ways:
if (dict.Count != 0 || dict.ToArray().Count() != 0 || dict.ToList().Count() != 0)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary.");
return false;
}
return true;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<CounterBucket> target,
string rootNamespace,
IIntervalService intervalService,
ITimeWindowService timeWindowService,
ILog log)
{
var windows = new ConcurrentDictionary<string, ConcurrentDictionary<string, double>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var calendargram = p as Calendargram;
var metricName = calendargram.Name + METRIC_IDENTIFIER_SEPARATOR + calendargram.Value;
var period = timeWindowService.GetTimeWindow().GetTimePeriod(calendargram.Period);
windows.AddOrUpdate(period,
(key) =>
{
var window = new ConcurrentDictionary<string, double>();
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
},
(key, window) =>
{
window.AddOrUpdate(metricName, (key2) => 1, (key2, oldValue) => 1);
return window;
}
);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (windows.Count == 0)
{
return;
}
var currentTimeWindow = timeWindowService.GetTimeWindow();
var periodsNotPresent = windows
.ToArray()
.Where(p => !currentTimeWindow.AllPeriods.Contains(p.Key))
.Select(p => p.Key);
CounterBucket bucket;
foreach (var period in periodsNotPresent)
{
ConcurrentDictionary<String, double> window;
if (windows.TryRemove(period, out window))
{
var parts = period.Split(UNDERSCORE);
var qualifier = "." + parts[0] + "." + parts[1];
var metricsAndValues = window.ToArray();
var metrics = new Dictionary<String, double>();
for (int index = 0; index < metricsAndValues.Length; index++)
{
var metricName = metricsAndValues[index].Key.Split(
METRIC_IDENTIFIER_SEPARATOR_SPLITTER,
StringSplitOptions.RemoveEmptyEntries
)[0] + qualifier;
if (metrics.ContainsKey(metricName))
{
metrics[metricName] += 1;
}
else
{
metrics[metricName] = 1;
}
}
var metricList = metrics.Select(metric =>
{
return new KeyValuePair<string, double>(
metric.Key,
metric.Value
);
}).ToArray();
bucket = new CounterBucket(metricList, e.Epoch, ns);
target.Post(bucket);
}
}
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
private static bool RunDictionaryTest(int cLevel, int initSize, int threads, int addsPerThread, TestMethod testMethod)
{
TestHarness.TestLog("* RunDictionaryTest_{0}, Level={1}, initSize={2}, threads={3}, addsPerThread={4})",
PrintTestMethod(testMethod), cLevel, initSize, threads, addsPerThread);
ConcurrentDictionary<int, int> dict = new ConcurrentDictionary<int, int>(cLevel, 1);
int count = threads;
using (ManualResetEvent mre = new ManualResetEvent(false))
{
for (int i = 0; i < threads; i++)
{
int ii = i;
ThreadPool.QueueUserWorkItem(
(o) =>
{
for (int j = 0; j < addsPerThread; j++)
{
//call either of the two overloads of GetOrAdd
if (j + ii % 2 == 0)
{
dict.GetOrAdd(j, -j);
}
else
{
dict.GetOrAdd(j, x => -x);
}
}
if (Interlocked.Decrement(ref count) == 0) mre.Set();
});
}
mre.WaitOne();
}
bool passed = true;
if (dict.Any(pair => pair.Key != -pair.Value))
{
TestHarness.TestLog(" > Invalid value for some key in the dictionary.");
passed = false;
}
var gotKeys = dict.Select(pair => pair.Key).OrderBy(i => i).ToArray();
var expectKeys = Enumerable.Range(0, addsPerThread);
if (!gotKeys.SequenceEqual(expectKeys))
{
TestHarness.TestLog(" > The set of keys in the dictionary is invalid.");
passed = false;
}
// Finally, let's verify that the count is reported correctly.
int expectedCount = addsPerThread;
int count1 = dict.Count, count2 = dict.ToArray().Length,
count3 = dict.ToList().Count;
if (count1 != expectedCount || count2 != expectedCount || count3 != expectedCount)
{
TestHarness.TestLog(" > Incorrect count of elements reported for the dictionary. Expected {0}, Dict.Count {1}, ToArray.Length {2}, ToList.Count {3}",
expectedCount, count1, count2, count3);
passed = false;
}
return passed;
}
/// <summary>
/// Normally this code never runs, it is here just to prevent run-away resource usage.
/// </summary>
private static void TrimActiveActivityStore(ConcurrentDictionary<Guid, int> activities)
{
if (activities.Count > MaxActivityTrackCount)
{
// Remove half of the oldest activity ids.
var keyValues = activities.ToArray();
var tickNow = Environment.TickCount;
// Sort by age, taking into account wrap-around. As long as x and y are within
// 23 days of now then (0x7FFFFFFF & (tickNow - x.Value)) is the delta (even if
// TickCount wraps). I then sort by DESCENDING age. (that is oldest value first)
Array.Sort(keyValues, (x, y) => (0x7FFFFFFF & (tickNow - y.Value)) - (0x7FFFFFFF & (tickNow - x.Value)));
for (int i = 0; i < keyValues.Length / 2; i++)
{
int dummy;
activities.TryRemove(keyValues[i].Key, out dummy);
}
}
}
private void verify(ref ConcurrentDictionary<string, ConcurrentDictionary<string, long>> allresult)
{
// results should be sequential
// ie S1 start S1 end before S1+n start and end
foreach (var queuePair in allresult.ToArray())
{
var qName = queuePair.Key;
var q = queuePair.Value; // value is ConcurrentDictionary<string, long>
var qEntries = q.ToArray();
var startEntries = qEntries.Where( kvp => kvp.Key[0] == 'S' );
// for each start task look for
// a task that started (between this start-end).
// a task that ended (between this start-end).
// must not use Task sequence (as order) because tasks.
// are placed in the actual queue based on priority.
// slow stuff...
foreach (var startEntry in startEntries)
{
var TaskSeq = startEntry.Key.Substring(1, startEntry.Key.Length -1 );
var endEntry=
qEntries.SingleOrDefault(kvp => kvp.Key == string.Concat("E", TaskSeq));
var foundTask = qEntries.Where(
kvp =>
(
// can use one comparison without key
// but kept this way to be able to spit
// out decent fail error if needed.
kvp.Key[0] == 'S' &&
kvp.Value > startEntry.Value &&
kvp.Value < endEntry.Value
||
kvp.Key[0] == 'E' &&
kvp.Value > startEntry.Value &&
kvp.Value < endEntry.Value
)
);
if (0 != foundTask.Count())
Assert.Fail(string.Format("Q:{0} has a task started or ended while Task {1} was running",
qName, TaskSeq));
}
}
}
