private void WatchTimer() { lock (_lockObject) { _cache_Profiles.Update(); _cache_Stores.Update(); _cache_MailMessages.Update(); _cache_ChatMessages.Update(); } }
public CatharsisManager(string configPath, BufferManager bufferManager) { _bufferManager = bufferManager; _settings = new Settings(configPath); _watchTimer = new WatchTimer(this.WatchThread); _updateTimer = new WatchTimer(() => _ipv4ResultMap?.Update()); _updateTimer.Start(new TimeSpan(0, 30, 0)); _ipv4ResultMap = new VolatileHashDictionary <Ipv4, bool>(new TimeSpan(0, 30, 0)); }
private void ConnectionsManagerThread() { var refreshStopwatch = new Stopwatch(); bool refreshThreadRunning = false; var pushBlockDiffusionStopwatch = new Stopwatch(); pushBlockDiffusionStopwatch.Start(); var pushBlockUploadStopwatch = new Stopwatch(); pushBlockUploadStopwatch.Start(); var pushBlockDownloadStopwatch = new Stopwatch(); pushBlockDownloadStopwatch.Start(); var pushMetadataUploadStopwatch = new Stopwatch(); pushMetadataUploadStopwatch.Start(); var pushMetadataDownloadStopwatch = new Stopwatch(); pushMetadataDownloadStopwatch.Start(); var requestCounts = new VolatileHashDictionary<Key, int>(new TimeSpan(0, 10, 0)); for (;;) { Thread.Sleep(1000); if (this.State == ManagerState.Stop) return; var connectionCount = 0; lock (_thisLock) { connectionCount = _connectionManagers.Count; } if (!refreshStopwatch.IsRunning || refreshStopwatch.Elapsed.TotalSeconds >= 60) { refreshStopwatch.Restart(); // トラストにより必要なMetadataを選択し、不要なMetadataを削除する。 Task.Run(() => { if (refreshThreadRunning) return; refreshThreadRunning = true; try { var lockSignatures = this.OnLockSignaturesEvent(); if (lockSignatures == null) return; var lockTags = this.OnLockTagsEvent(); if (lockTags == null) return; _settings.MetadataManager.Refresh(lockSignatures.ToArray(), lockTags.ToArray()); } catch (Exception e) { Log.Error(e); } finally { refreshThreadRunning = false; } }); } // 拡散アップロード if (connectionCount > _diffusionConnectionCountLowerLimit && pushBlockDiffusionStopwatch.Elapsed.TotalSeconds >= 60) { pushBlockDiffusionStopwatch.Restart(); // 拡散アップロードするブロック数を10000以下に抑える。 lock (_thisLock) { lock (_settings.DiffusionBlocksRequest.ThisLock) { if (_settings.DiffusionBlocksRequest.Count > 10000) { foreach (var key in _settings.DiffusionBlocksRequest.Randomize() .Take(_settings.DiffusionBlocksRequest.Count - 10000)) { _settings.DiffusionBlocksRequest.Remove(key); } } } } // 存在しないブロックのKeyをRemoveする。 lock (_thisLock) { lock (_settings.DiffusionBlocksRequest.ThisLock) { foreach (var key in _cacheManager.ExceptFrom(_settings.DiffusionBlocksRequest.ToArray()).ToArray()) { _settings.DiffusionBlocksRequest.Remove(key); } } lock (_settings.UploadBlocksRequest.ThisLock) { foreach (var key in _cacheManager.ExceptFrom(_settings.UploadBlocksRequest.ToArray()).ToArray()) { _settings.UploadBlocksRequest.Remove(key); } } } var baseNode = this.BaseNode; var otherNodes = new List<Node>(); lock (_thisLock) { otherNodes.AddRange(_connectionManagers.Select(n => n.Node)); } var packetManagers = new Dictionary<Node, PacketManager>(); foreach (var node in otherNodes) { packetManagers[node] = _packetControlManager[node]; } var diffusionBlocksList = new HashSet<Key>(); { { var array = _settings.UploadBlocksRequest.ToArray(); _random.Shuffle(array); int count = 1024; for (int i = 0; i < count && i < array.Length; i++) { diffusionBlocksList.Add(array[i]); } } { var array = _settings.DiffusionBlocksRequest.ToArray(); _random.Shuffle(array); int count = 1024; for (int i = 0; i < count && i < array.Length; i++) { diffusionBlocksList.Add(array[i]); } } } { var diffusionBlocksDictionary = new Dictionary<Node, List<Key>>(); foreach (var key in diffusionBlocksList.Randomize()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(key.Hash, baseNode.Id, otherNodes, 2)) { requestNodes.Add(node); } if (requestNodes.Count == 0) { _blockUploadedEventQueue.Enqueue(key); _settings.UploadBlocksRequest.Remove(key); _settings.DiffusionBlocksRequest.Remove(key); continue; } for (int i = 0; i < requestNodes.Count; i++) { List<Key> collection; if (!diffusionBlocksDictionary.TryGetValue(requestNodes[i], out collection)) { collection = new List<Key>(); diffusionBlocksDictionary[requestNodes[i]] = collection; } collection.Add(key); } } catch (Exception e) { Log.Error(e); } } lock (_diffusionBlocksDictionary.ThisLock) { _diffusionBlocksDictionary.Clear(); foreach (var pair in diffusionBlocksDictionary) { var node = pair.Key; var targets = pair.Value; _random.Shuffle(targets); _diffusionBlocksDictionary.Add(node, new Queue<Key>(targets)); } } } } // アップロード if (connectionCount >= _uploadingConnectionCountLowerLimit && pushBlockUploadStopwatch.Elapsed.TotalSeconds >= 60) { pushBlockUploadStopwatch.Restart(); var baseNode = this.BaseNode; var otherNodes = new List<Node>(); lock (_thisLock) { otherNodes.AddRange(_connectionManagers.Select(n => n.Node)); } var packetManagers = new Dictionary<Node, PacketManager>(); foreach (var node in otherNodes) { packetManagers[node] = _packetControlManager[node]; } { var uploadBlocksDictionary = new Dictionary<Node, List<Key>>(); foreach (var pair in packetManagers) { var node = pair.Key; var packetManager = pair.Value; uploadBlocksDictionary.Add(node, _cacheManager.IntersectFrom(packetManager.PullBlocksRequest.ToArray().Randomize()).Take(1024).ToList()); } lock (_uploadBlocksDictionary.ThisLock) { _uploadBlocksDictionary.Clear(); foreach (var pair in uploadBlocksDictionary) { var node = pair.Key; var targets = pair.Value; _uploadBlocksDictionary.Add(node, new Queue<Key>(targets)); } } } } // ダウンロード if (connectionCount >= _downloadingConnectionCountLowerLimit && pushBlockDownloadStopwatch.Elapsed.TotalSeconds >= 60) { pushBlockDownloadStopwatch.Restart(); var baseNode = this.BaseNode; var otherNodes = new List<Node>(); lock (_thisLock) { otherNodes.AddRange(_connectionManagers.Select(n => n.Node)); } var packetManagers = new Dictionary<Node, PacketManager>(); foreach (var node in otherNodes) { packetManagers[node] = _packetControlManager[node]; } var pushBlocksLinkList = new HashSet<Key>(); var pushBlocksRequestList = new HashSet<Key>(); { { var array = _cacheManager.ToArray(); _random.Shuffle(array); int count = _maxBlockLinkCount; pushBlocksLinkList.UnionWith(array.Take(count)); } foreach (var pair in packetManagers) { var node = pair.Key; var packetManager = pair.Value; { var array = packetManager.PullBlocksLink.ToArray(new TimeSpan(0, 10, 0)); _random.Shuffle(array); var count = (int)((_maxBlockLinkCount * ((double)12 / otherNodes.Count)) * this.GetPriority(node)); pushBlocksLinkList.UnionWith(array.Take(count)); } } { var array = _cacheManager.ExceptFrom(_downloadBlocks.ToArray()).ToArray(); _random.Shuffle(array); int count = _maxBlockRequestCount; pushBlocksRequestList.UnionWith(array.Take(count)); } foreach (var pair in packetManagers) { var node = pair.Key; var packetManager = pair.Value; { var array = _cacheManager.ExceptFrom(packetManager.PullBlocksRequest.ToArray(new TimeSpan(0, 10, 0))).ToArray(); _random.Shuffle(array); var count = (int)((_maxBlockRequestCount * ((double)12 / otherNodes.Count)) * this.GetPriority(node)); pushBlocksRequestList.UnionWith(array.Take(count)); } } } { var pushBlocksLinkDictionary = new Dictionary<Node, List<Key>>(); foreach (var key in pushBlocksLinkList.Randomize()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(key.Hash, otherNodes, 2)) { requestNodes.Add(node); } for (int i = 0; i < requestNodes.Count; i++) { List<Key> collection; if (!pushBlocksLinkDictionary.TryGetValue(requestNodes[i], out collection)) { collection = new List<Key>(); pushBlocksLinkDictionary[requestNodes[i]] = collection; } if (collection.Count < _maxBlockLinkCount) { collection.Add(key); } } } catch (Exception e) { Log.Error(e); } } lock (_pushBlocksLinkDictionary.ThisLock) { _pushBlocksLinkDictionary.Clear(); foreach (var pair in pushBlocksLinkDictionary) { var node = pair.Key; var targets = pair.Value; _random.Shuffle(targets); _pushBlocksLinkDictionary.Add(node, targets); } } } { requestCounts.Update(); var pushBlocksRequestDictionary = new Dictionary<Node, List<Key>>(); foreach (var key in pushBlocksRequestList.Randomize()) { int count; { int value; requestCounts.TryGetValue(key, out value); if (value < 6) requestCounts[key] = (value += 2); count = value; } try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(key.Hash, otherNodes, count)) { requestNodes.Add(node); } foreach (var pair in packetManagers) { var node = pair.Key; var packetManager = pair.Value; if (packetManager.PullBlocksLink.Contains(key)) { requestNodes.Add(node); } } for (int i = 0; i < requestNodes.Count; i++) { List<Key> collection; if (!pushBlocksRequestDictionary.TryGetValue(requestNodes[i], out collection)) { collection = new List<Key>(); pushBlocksRequestDictionary[requestNodes[i]] = collection; } if (collection.Count < _maxBlockRequestCount) { collection.Add(key); } } } catch (Exception e) { Log.Error(e); } } lock (_pushBlocksRequestDictionary.ThisLock) { _pushBlocksRequestDictionary.Clear(); foreach (var pair in pushBlocksRequestDictionary) { var node = pair.Key; var targets = pair.Value; _random.Shuffle(targets); _pushBlocksRequestDictionary.Add(node, targets); } } } } // Metadataのアップロード if (connectionCount >= _uploadingConnectionCountLowerLimit && pushMetadataUploadStopwatch.Elapsed.TotalMinutes >= 3) { pushMetadataUploadStopwatch.Restart(); var baseNode = this.BaseNode; var otherNodes = new List<Node>(); lock (_thisLock) { otherNodes.AddRange(_connectionManagers.Select(n => n.Node)); } var packetManagers = new Dictionary<Node, PacketManager>(); foreach (var node in otherNodes) { packetManagers[node] = _packetControlManager[node]; } // Broadcast foreach (var signature in _settings.MetadataManager.GetBroadcastSignatures()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2)) { requestNodes.Add(node); } for (int i = 0; i < requestNodes.Count; i++) { packetManagers[requestNodes[i]].PullBroadcastMetadatasRequest.Add(signature); } } catch (Exception e) { Log.Error(e); } } // Unicast foreach (var signature in _settings.MetadataManager.GetUnicastSignatures()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2)) { requestNodes.Add(node); } for (int i = 0; i < requestNodes.Count; i++) { packetManagers[requestNodes[i]].PullUnicastMetadatasRequest.Add(signature); } } catch (Exception e) { Log.Error(e); } } // Multicast { foreach (var tag in _settings.MetadataManager.GetMulticastTags()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(tag.Id, otherNodes, 2)) { requestNodes.Add(node); } for (int i = 0; i < requestNodes.Count; i++) { packetManagers[requestNodes[i]].PullMulticastMetadatasRequest.Add(tag); } } catch (Exception e) { Log.Error(e); } } } } // Metadataのダウンロード if (connectionCount >= _downloadingConnectionCountLowerLimit && pushMetadataDownloadStopwatch.Elapsed.TotalSeconds >= 60) { pushMetadataDownloadStopwatch.Restart(); var baseNode = this.BaseNode; var otherNodes = new List<Node>(); lock (_thisLock) { otherNodes.AddRange(_connectionManagers.Select(n => n.Node)); } var packetManagers = new Dictionary<Node, PacketManager>(); foreach (var node in otherNodes) { packetManagers[node] = _packetControlManager[node]; } var pushBroadcastMetadatasRequestList = new HashSet<string>(); var pushUnicastMetadatasRequestList = new HashSet<string>(); var pushMulticastMetadatasRequestList = new HashSet<Tag>(); { // Broadcast { { var array = _pushBroadcastMetadatasRequestList.ToArray(); _random.Shuffle(array); int count = _maxMetadataRequestCount; for (int i = 0; count > 0 && i < array.Length; i++) { pushBroadcastMetadatasRequestList.Add(array[i]); count--; } } foreach (var pair in packetManagers) { var node = pair.Key; var packetManager = pair.Value; { var array = packetManager.PullBroadcastMetadatasRequest.ToArray(new TimeSpan(0, 10, 0)); _random.Shuffle(array); int count = _maxMetadataRequestCount; for (int i = 0; count > 0 && i < array.Length; i++) { pushBroadcastMetadatasRequestList.Add(array[i]); count--; } } } } // Unicast { { var array = _pushUnicastMetadatasRequestList.ToArray(); _random.Shuffle(array); int count = _maxMetadataRequestCount; for (int i = 0; count > 0 && i < array.Length; i++) { pushUnicastMetadatasRequestList.Add(array[i]); count--; } } foreach (var pair in packetManagers) { var node = pair.Key; var packetManager = pair.Value; { var array = packetManager.PullUnicastMetadatasRequest.ToArray(new TimeSpan(0, 10, 0)); _random.Shuffle(array); int count = _maxMetadataRequestCount; for (int i = 0; count > 0 && i < array.Length; i++) { pushUnicastMetadatasRequestList.Add(array[i]); count--; } } } } // Multicast { { var array = _pushMulticastMetadatasRequestList.ToArray(); _random.Shuffle(array); int count = _maxMetadataRequestCount; for (int i = 0; count > 0 && i < array.Length; i++) { pushMulticastMetadatasRequestList.Add(array[i]); count--; } } foreach (var pair in packetManagers) { var node = pair.Key; var packetManager = pair.Value; { var array = packetManager.PullMulticastMetadatasRequest.ToArray(new TimeSpan(0, 10, 0)); _random.Shuffle(array); int count = _maxMetadataRequestCount; for (int i = 0; count > 0 && i < array.Length; i++) { pushMulticastMetadatasRequestList.Add(array[i]); count--; } } } } } { // Broadcast { var pushBroadcastMetadatasRequestDictionary = new Dictionary<Node, List<string>>(); foreach (var signature in pushBroadcastMetadatasRequestList.Randomize()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2)) { requestNodes.Add(node); } for (int i = 0; i < requestNodes.Count; i++) { List<string> collection; if (!pushBroadcastMetadatasRequestDictionary.TryGetValue(requestNodes[i], out collection)) { collection = new List<string>(); pushBroadcastMetadatasRequestDictionary[requestNodes[i]] = collection; } if (collection.Count < _maxMetadataRequestCount) { collection.Add(signature); } } } catch (Exception e) { Log.Error(e); } } lock (_pushBroadcastMetadatasRequestDictionary.ThisLock) { _pushBroadcastMetadatasRequestDictionary.Clear(); foreach (var pair in pushBroadcastMetadatasRequestDictionary) { var node = pair.Key; var targets = pair.Value; _random.Shuffle(targets); _pushBroadcastMetadatasRequestDictionary.Add(node, targets); } } } // Unicast { var pushUnicastMetadatasRequestDictionary = new Dictionary<Node, List<string>>(); foreach (var signature in pushUnicastMetadatasRequestList.Randomize()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2)) { requestNodes.Add(node); } for (int i = 0; i < requestNodes.Count; i++) { List<string> collection; if (!pushUnicastMetadatasRequestDictionary.TryGetValue(requestNodes[i], out collection)) { collection = new List<string>(); pushUnicastMetadatasRequestDictionary[requestNodes[i]] = collection; } if (collection.Count < _maxMetadataRequestCount) { collection.Add(signature); } } } catch (Exception e) { Log.Error(e); } } lock (_pushUnicastMetadatasRequestDictionary.ThisLock) { _pushUnicastMetadatasRequestDictionary.Clear(); foreach (var pair in pushUnicastMetadatasRequestDictionary) { var node = pair.Key; var targets = pair.Value; _random.Shuffle(targets); _pushUnicastMetadatasRequestDictionary.Add(node, targets); } } } // Multicast { var pushMulticastMetadatasRequestDictionary = new Dictionary<Node, List<Tag>>(); foreach (var tag in pushMulticastMetadatasRequestList.Randomize()) { try { var requestNodes = new List<Node>(); foreach (var node in Kademlia<Node>.Search(tag.Id, otherNodes, 2)) { requestNodes.Add(node); } for (int i = 0; i < requestNodes.Count; i++) { List<Tag> collection; if (!pushMulticastMetadatasRequestDictionary.TryGetValue(requestNodes[i], out collection)) { collection = new List<Tag>(); pushMulticastMetadatasRequestDictionary[requestNodes[i]] = collection; } if (collection.Count < _maxMetadataRequestCount) { collection.Add(tag); } } } catch (Exception e) { Log.Error(e); } } lock (_pushMulticastMetadatasRequestDictionary.ThisLock) { _pushMulticastMetadatasRequestDictionary.Clear(); foreach (var pair in pushMulticastMetadatasRequestDictionary) { var node = pair.Key; var targets = pair.Value; _random.Shuffle(targets); _pushMulticastMetadatasRequestDictionary.Add(node, targets); } } } } } } }