Esempio n. 1
0
 private void WatchTimer()
 {
     lock (_lockObject)
     {
         _cache_Profiles.Update();
         _cache_Stores.Update();
         _cache_MailMessages.Update();
         _cache_ChatMessages.Update();
     }
 }
Esempio n. 2
0
            public CatharsisManager(string configPath, BufferManager bufferManager)
            {
                _bufferManager = bufferManager;

                _settings = new Settings(configPath);

                _watchTimer  = new WatchTimer(this.WatchThread);
                _updateTimer = new WatchTimer(() => _ipv4ResultMap?.Update());
                _updateTimer.Start(new TimeSpan(0, 30, 0));

                _ipv4ResultMap = new VolatileHashDictionary <Ipv4, bool>(new TimeSpan(0, 30, 0));
            }
        private void ConnectionsManagerThread()
        {
            var refreshStopwatch = new Stopwatch();
            bool refreshThreadRunning = false;

            var pushBlockDiffusionStopwatch = new Stopwatch();
            pushBlockDiffusionStopwatch.Start();
            var pushBlockUploadStopwatch = new Stopwatch();
            pushBlockUploadStopwatch.Start();
            var pushBlockDownloadStopwatch = new Stopwatch();
            pushBlockDownloadStopwatch.Start();

            var pushMetadataUploadStopwatch = new Stopwatch();
            pushMetadataUploadStopwatch.Start();
            var pushMetadataDownloadStopwatch = new Stopwatch();
            pushMetadataDownloadStopwatch.Start();

            var requestCounts = new VolatileHashDictionary<Key, int>(new TimeSpan(0, 10, 0));

            for (;;)
            {
                Thread.Sleep(1000);
                if (this.State == ManagerState.Stop) return;

                var connectionCount = 0;

                lock (_thisLock)
                {
                    connectionCount = _connectionManagers.Count;
                }

                if (!refreshStopwatch.IsRunning || refreshStopwatch.Elapsed.TotalSeconds >= 60)
                {
                    refreshStopwatch.Restart();

                    // トラストにより必要なMetadataを選択し、不要なMetadataを削除する。
                    Task.Run(() =>
                    {
                        if (refreshThreadRunning) return;
                        refreshThreadRunning = true;

                        try
                        {
                            var lockSignatures = this.OnLockSignaturesEvent();
                            if (lockSignatures == null) return;

                            var lockTags = this.OnLockTagsEvent();
                            if (lockTags == null) return;

                            _settings.MetadataManager.Refresh(lockSignatures.ToArray(), lockTags.ToArray());
                        }
                        catch (Exception e)
                        {
                            Log.Error(e);
                        }
                        finally
                        {
                            refreshThreadRunning = false;
                        }
                    });
                }

                // 拡散アップロード
                if (connectionCount > _diffusionConnectionCountLowerLimit
                    && pushBlockDiffusionStopwatch.Elapsed.TotalSeconds >= 60)
                {
                    pushBlockDiffusionStopwatch.Restart();

                    // 拡散アップロードするブロック数を10000以下に抑える。
                    lock (_thisLock)
                    {
                        lock (_settings.DiffusionBlocksRequest.ThisLock)
                        {
                            if (_settings.DiffusionBlocksRequest.Count > 10000)
                            {
                                foreach (var key in _settings.DiffusionBlocksRequest.Randomize()
                                    .Take(_settings.DiffusionBlocksRequest.Count - 10000))
                                {
                                    _settings.DiffusionBlocksRequest.Remove(key);
                                }
                            }
                        }
                    }

                    // 存在しないブロックのKeyをRemoveする。
                    lock (_thisLock)
                    {
                        lock (_settings.DiffusionBlocksRequest.ThisLock)
                        {
                            foreach (var key in _cacheManager.ExceptFrom(_settings.DiffusionBlocksRequest.ToArray()).ToArray())
                            {
                                _settings.DiffusionBlocksRequest.Remove(key);
                            }
                        }

                        lock (_settings.UploadBlocksRequest.ThisLock)
                        {
                            foreach (var key in _cacheManager.ExceptFrom(_settings.UploadBlocksRequest.ToArray()).ToArray())
                            {
                                _settings.UploadBlocksRequest.Remove(key);
                            }
                        }
                    }

                    var baseNode = this.BaseNode;

                    var otherNodes = new List<Node>();

                    lock (_thisLock)
                    {
                        otherNodes.AddRange(_connectionManagers.Select(n => n.Node));
                    }

                    var packetManagers = new Dictionary<Node, PacketManager>();

                    foreach (var node in otherNodes)
                    {
                        packetManagers[node] = _packetControlManager[node];
                    }

                    var diffusionBlocksList = new HashSet<Key>();

                    {
                        {
                            var array = _settings.UploadBlocksRequest.ToArray();
                            _random.Shuffle(array);

                            int count = 1024;

                            for (int i = 0; i < count && i < array.Length; i++)
                            {
                                diffusionBlocksList.Add(array[i]);
                            }
                        }

                        {
                            var array = _settings.DiffusionBlocksRequest.ToArray();
                            _random.Shuffle(array);

                            int count = 1024;

                            for (int i = 0; i < count && i < array.Length; i++)
                            {
                                diffusionBlocksList.Add(array[i]);
                            }
                        }
                    }

                    {
                        var diffusionBlocksDictionary = new Dictionary<Node, List<Key>>();

                        foreach (var key in diffusionBlocksList.Randomize())
                        {
                            try
                            {
                                var requestNodes = new List<Node>();

                                foreach (var node in Kademlia<Node>.Search(key.Hash, baseNode.Id, otherNodes, 2))
                                {
                                    requestNodes.Add(node);
                                }

                                if (requestNodes.Count == 0)
                                {
                                    _blockUploadedEventQueue.Enqueue(key);

                                    _settings.UploadBlocksRequest.Remove(key);
                                    _settings.DiffusionBlocksRequest.Remove(key);

                                    continue;
                                }

                                for (int i = 0; i < requestNodes.Count; i++)
                                {
                                    List<Key> collection;

                                    if (!diffusionBlocksDictionary.TryGetValue(requestNodes[i], out collection))
                                    {
                                        collection = new List<Key>();
                                        diffusionBlocksDictionary[requestNodes[i]] = collection;
                                    }

                                    collection.Add(key);
                                }
                            }
                            catch (Exception e)
                            {
                                Log.Error(e);
                            }
                        }

                        lock (_diffusionBlocksDictionary.ThisLock)
                        {
                            _diffusionBlocksDictionary.Clear();

                            foreach (var pair in diffusionBlocksDictionary)
                            {
                                var node = pair.Key;
                                var targets = pair.Value;

                                _random.Shuffle(targets);

                                _diffusionBlocksDictionary.Add(node, new Queue<Key>(targets));
                            }
                        }
                    }
                }

                // アップロード
                if (connectionCount >= _uploadingConnectionCountLowerLimit
                    && pushBlockUploadStopwatch.Elapsed.TotalSeconds >= 60)
                {
                    pushBlockUploadStopwatch.Restart();

                    var baseNode = this.BaseNode;

                    var otherNodes = new List<Node>();

                    lock (_thisLock)
                    {
                        otherNodes.AddRange(_connectionManagers.Select(n => n.Node));
                    }

                    var packetManagers = new Dictionary<Node, PacketManager>();

                    foreach (var node in otherNodes)
                    {
                        packetManagers[node] = _packetControlManager[node];
                    }

                    {
                        var uploadBlocksDictionary = new Dictionary<Node, List<Key>>();

                        foreach (var pair in packetManagers)
                        {
                            var node = pair.Key;
                            var packetManager = pair.Value;

                            uploadBlocksDictionary.Add(node, _cacheManager.IntersectFrom(packetManager.PullBlocksRequest.ToArray().Randomize()).Take(1024).ToList());
                        }

                        lock (_uploadBlocksDictionary.ThisLock)
                        {
                            _uploadBlocksDictionary.Clear();

                            foreach (var pair in uploadBlocksDictionary)
                            {
                                var node = pair.Key;
                                var targets = pair.Value;

                                _uploadBlocksDictionary.Add(node, new Queue<Key>(targets));
                            }
                        }
                    }
                }

                // ダウンロード
                if (connectionCount >= _downloadingConnectionCountLowerLimit
                    && pushBlockDownloadStopwatch.Elapsed.TotalSeconds >= 60)
                {
                    pushBlockDownloadStopwatch.Restart();

                    var baseNode = this.BaseNode;

                    var otherNodes = new List<Node>();

                    lock (_thisLock)
                    {
                        otherNodes.AddRange(_connectionManagers.Select(n => n.Node));
                    }

                    var packetManagers = new Dictionary<Node, PacketManager>();

                    foreach (var node in otherNodes)
                    {
                        packetManagers[node] = _packetControlManager[node];
                    }

                    var pushBlocksLinkList = new HashSet<Key>();
                    var pushBlocksRequestList = new HashSet<Key>();

                    {
                        {
                            var array = _cacheManager.ToArray();
                            _random.Shuffle(array);

                            int count = _maxBlockLinkCount;

                            pushBlocksLinkList.UnionWith(array.Take(count));
                        }

                        foreach (var pair in packetManagers)
                        {
                            var node = pair.Key;
                            var packetManager = pair.Value;

                            {
                                var array = packetManager.PullBlocksLink.ToArray(new TimeSpan(0, 10, 0));
                                _random.Shuffle(array);

                                var count = (int)((_maxBlockLinkCount * ((double)12 / otherNodes.Count)) * this.GetPriority(node));

                                pushBlocksLinkList.UnionWith(array.Take(count));
                            }
                        }

                        {
                            var array = _cacheManager.ExceptFrom(_downloadBlocks.ToArray()).ToArray();
                            _random.Shuffle(array);

                            int count = _maxBlockRequestCount;

                            pushBlocksRequestList.UnionWith(array.Take(count));
                        }

                        foreach (var pair in packetManagers)
                        {
                            var node = pair.Key;
                            var packetManager = pair.Value;

                            {
                                var array = _cacheManager.ExceptFrom(packetManager.PullBlocksRequest.ToArray(new TimeSpan(0, 10, 0))).ToArray();
                                _random.Shuffle(array);

                                var count = (int)((_maxBlockRequestCount * ((double)12 / otherNodes.Count)) * this.GetPriority(node));

                                pushBlocksRequestList.UnionWith(array.Take(count));
                            }
                        }
                    }

                    {
                        var pushBlocksLinkDictionary = new Dictionary<Node, List<Key>>();

                        foreach (var key in pushBlocksLinkList.Randomize())
                        {
                            try
                            {
                                var requestNodes = new List<Node>();

                                foreach (var node in Kademlia<Node>.Search(key.Hash, otherNodes, 2))
                                {
                                    requestNodes.Add(node);
                                }

                                for (int i = 0; i < requestNodes.Count; i++)
                                {
                                    List<Key> collection;

                                    if (!pushBlocksLinkDictionary.TryGetValue(requestNodes[i], out collection))
                                    {
                                        collection = new List<Key>();
                                        pushBlocksLinkDictionary[requestNodes[i]] = collection;
                                    }

                                    if (collection.Count < _maxBlockLinkCount)
                                    {
                                        collection.Add(key);
                                    }
                                }
                            }
                            catch (Exception e)
                            {
                                Log.Error(e);
                            }
                        }

                        lock (_pushBlocksLinkDictionary.ThisLock)
                        {
                            _pushBlocksLinkDictionary.Clear();

                            foreach (var pair in pushBlocksLinkDictionary)
                            {
                                var node = pair.Key;
                                var targets = pair.Value;

                                _random.Shuffle(targets);

                                _pushBlocksLinkDictionary.Add(node, targets);
                            }
                        }
                    }

                    {
                        requestCounts.Update();

                        var pushBlocksRequestDictionary = new Dictionary<Node, List<Key>>();

                        foreach (var key in pushBlocksRequestList.Randomize())
                        {
                            int count;
                            {
                                int value;
                                requestCounts.TryGetValue(key, out value);

                                if (value < 6) requestCounts[key] = (value += 2);

                                count = value;
                            }

                            try
                            {
                                var requestNodes = new List<Node>();

                                foreach (var node in Kademlia<Node>.Search(key.Hash, otherNodes, count))
                                {
                                    requestNodes.Add(node);
                                }

                                foreach (var pair in packetManagers)
                                {
                                    var node = pair.Key;
                                    var packetManager = pair.Value;

                                    if (packetManager.PullBlocksLink.Contains(key))
                                    {
                                        requestNodes.Add(node);
                                    }
                                }

                                for (int i = 0; i < requestNodes.Count; i++)
                                {
                                    List<Key> collection;

                                    if (!pushBlocksRequestDictionary.TryGetValue(requestNodes[i], out collection))
                                    {
                                        collection = new List<Key>();
                                        pushBlocksRequestDictionary[requestNodes[i]] = collection;
                                    }

                                    if (collection.Count < _maxBlockRequestCount)
                                    {
                                        collection.Add(key);
                                    }
                                }
                            }
                            catch (Exception e)
                            {
                                Log.Error(e);
                            }
                        }

                        lock (_pushBlocksRequestDictionary.ThisLock)
                        {
                            _pushBlocksRequestDictionary.Clear();

                            foreach (var pair in pushBlocksRequestDictionary)
                            {
                                var node = pair.Key;
                                var targets = pair.Value;

                                _random.Shuffle(targets);

                                _pushBlocksRequestDictionary.Add(node, targets);
                            }
                        }
                    }
                }

                // Metadataのアップロード
                if (connectionCount >= _uploadingConnectionCountLowerLimit
                    && pushMetadataUploadStopwatch.Elapsed.TotalMinutes >= 3)
                {
                    pushMetadataUploadStopwatch.Restart();

                    var baseNode = this.BaseNode;

                    var otherNodes = new List<Node>();

                    lock (_thisLock)
                    {
                        otherNodes.AddRange(_connectionManagers.Select(n => n.Node));
                    }

                    var packetManagers = new Dictionary<Node, PacketManager>();

                    foreach (var node in otherNodes)
                    {
                        packetManagers[node] = _packetControlManager[node];
                    }

                    // Broadcast
                    foreach (var signature in _settings.MetadataManager.GetBroadcastSignatures())
                    {
                        try
                        {
                            var requestNodes = new List<Node>();

                            foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2))
                            {
                                requestNodes.Add(node);
                            }

                            for (int i = 0; i < requestNodes.Count; i++)
                            {
                                packetManagers[requestNodes[i]].PullBroadcastMetadatasRequest.Add(signature);
                            }
                        }
                        catch (Exception e)
                        {
                            Log.Error(e);
                        }
                    }

                    // Unicast
                    foreach (var signature in _settings.MetadataManager.GetUnicastSignatures())
                    {
                        try
                        {
                            var requestNodes = new List<Node>();

                            foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2))
                            {
                                requestNodes.Add(node);
                            }

                            for (int i = 0; i < requestNodes.Count; i++)
                            {
                                packetManagers[requestNodes[i]].PullUnicastMetadatasRequest.Add(signature);
                            }
                        }
                        catch (Exception e)
                        {
                            Log.Error(e);
                        }
                    }

                    // Multicast
                    {
                        foreach (var tag in _settings.MetadataManager.GetMulticastTags())
                        {
                            try
                            {
                                var requestNodes = new List<Node>();

                                foreach (var node in Kademlia<Node>.Search(tag.Id, otherNodes, 2))
                                {
                                    requestNodes.Add(node);
                                }

                                for (int i = 0; i < requestNodes.Count; i++)
                                {
                                    packetManagers[requestNodes[i]].PullMulticastMetadatasRequest.Add(tag);
                                }
                            }
                            catch (Exception e)
                            {
                                Log.Error(e);
                            }
                        }
                    }
                }

                // Metadataのダウンロード
                if (connectionCount >= _downloadingConnectionCountLowerLimit
                    && pushMetadataDownloadStopwatch.Elapsed.TotalSeconds >= 60)
                {
                    pushMetadataDownloadStopwatch.Restart();

                    var baseNode = this.BaseNode;

                    var otherNodes = new List<Node>();

                    lock (_thisLock)
                    {
                        otherNodes.AddRange(_connectionManagers.Select(n => n.Node));
                    }

                    var packetManagers = new Dictionary<Node, PacketManager>();

                    foreach (var node in otherNodes)
                    {
                        packetManagers[node] = _packetControlManager[node];
                    }

                    var pushBroadcastMetadatasRequestList = new HashSet<string>();
                    var pushUnicastMetadatasRequestList = new HashSet<string>();
                    var pushMulticastMetadatasRequestList = new HashSet<Tag>();

                    {
                        // Broadcast
                        {
                            {
                                var array = _pushBroadcastMetadatasRequestList.ToArray();
                                _random.Shuffle(array);

                                int count = _maxMetadataRequestCount;

                                for (int i = 0; count > 0 && i < array.Length; i++)
                                {
                                    pushBroadcastMetadatasRequestList.Add(array[i]);

                                    count--;
                                }
                            }

                            foreach (var pair in packetManagers)
                            {
                                var node = pair.Key;
                                var packetManager = pair.Value;

                                {
                                    var array = packetManager.PullBroadcastMetadatasRequest.ToArray(new TimeSpan(0, 10, 0));
                                    _random.Shuffle(array);

                                    int count = _maxMetadataRequestCount;

                                    for (int i = 0; count > 0 && i < array.Length; i++)
                                    {
                                        pushBroadcastMetadatasRequestList.Add(array[i]);

                                        count--;
                                    }
                                }
                            }
                        }

                        // Unicast
                        {
                            {
                                var array = _pushUnicastMetadatasRequestList.ToArray();
                                _random.Shuffle(array);

                                int count = _maxMetadataRequestCount;

                                for (int i = 0; count > 0 && i < array.Length; i++)
                                {
                                    pushUnicastMetadatasRequestList.Add(array[i]);

                                    count--;
                                }
                            }

                            foreach (var pair in packetManagers)
                            {
                                var node = pair.Key;
                                var packetManager = pair.Value;

                                {
                                    var array = packetManager.PullUnicastMetadatasRequest.ToArray(new TimeSpan(0, 10, 0));
                                    _random.Shuffle(array);

                                    int count = _maxMetadataRequestCount;

                                    for (int i = 0; count > 0 && i < array.Length; i++)
                                    {
                                        pushUnicastMetadatasRequestList.Add(array[i]);

                                        count--;
                                    }
                                }
                            }
                        }

                        // Multicast
                        {
                            {
                                var array = _pushMulticastMetadatasRequestList.ToArray();
                                _random.Shuffle(array);

                                int count = _maxMetadataRequestCount;

                                for (int i = 0; count > 0 && i < array.Length; i++)
                                {
                                    pushMulticastMetadatasRequestList.Add(array[i]);

                                    count--;
                                }
                            }

                            foreach (var pair in packetManagers)
                            {
                                var node = pair.Key;
                                var packetManager = pair.Value;

                                {
                                    var array = packetManager.PullMulticastMetadatasRequest.ToArray(new TimeSpan(0, 10, 0));
                                    _random.Shuffle(array);

                                    int count = _maxMetadataRequestCount;

                                    for (int i = 0; count > 0 && i < array.Length; i++)
                                    {
                                        pushMulticastMetadatasRequestList.Add(array[i]);

                                        count--;
                                    }
                                }
                            }
                        }
                    }

                    {
                        // Broadcast
                        {
                            var pushBroadcastMetadatasRequestDictionary = new Dictionary<Node, List<string>>();

                            foreach (var signature in pushBroadcastMetadatasRequestList.Randomize())
                            {
                                try
                                {
                                    var requestNodes = new List<Node>();

                                    foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2))
                                    {
                                        requestNodes.Add(node);
                                    }

                                    for (int i = 0; i < requestNodes.Count; i++)
                                    {
                                        List<string> collection;

                                        if (!pushBroadcastMetadatasRequestDictionary.TryGetValue(requestNodes[i], out collection))
                                        {
                                            collection = new List<string>();
                                            pushBroadcastMetadatasRequestDictionary[requestNodes[i]] = collection;
                                        }

                                        if (collection.Count < _maxMetadataRequestCount)
                                        {
                                            collection.Add(signature);
                                        }
                                    }
                                }
                                catch (Exception e)
                                {
                                    Log.Error(e);
                                }
                            }

                            lock (_pushBroadcastMetadatasRequestDictionary.ThisLock)
                            {
                                _pushBroadcastMetadatasRequestDictionary.Clear();

                                foreach (var pair in pushBroadcastMetadatasRequestDictionary)
                                {
                                    var node = pair.Key;
                                    var targets = pair.Value;

                                    _random.Shuffle(targets);

                                    _pushBroadcastMetadatasRequestDictionary.Add(node, targets);
                                }
                            }
                        }

                        // Unicast
                        {
                            var pushUnicastMetadatasRequestDictionary = new Dictionary<Node, List<string>>();

                            foreach (var signature in pushUnicastMetadatasRequestList.Randomize())
                            {
                                try
                                {
                                    var requestNodes = new List<Node>();

                                    foreach (var node in Kademlia<Node>.Search(Signature.GetHash(signature), otherNodes, 2))
                                    {
                                        requestNodes.Add(node);
                                    }

                                    for (int i = 0; i < requestNodes.Count; i++)
                                    {
                                        List<string> collection;

                                        if (!pushUnicastMetadatasRequestDictionary.TryGetValue(requestNodes[i], out collection))
                                        {
                                            collection = new List<string>();
                                            pushUnicastMetadatasRequestDictionary[requestNodes[i]] = collection;
                                        }

                                        if (collection.Count < _maxMetadataRequestCount)
                                        {
                                            collection.Add(signature);
                                        }
                                    }
                                }
                                catch (Exception e)
                                {
                                    Log.Error(e);
                                }
                            }

                            lock (_pushUnicastMetadatasRequestDictionary.ThisLock)
                            {
                                _pushUnicastMetadatasRequestDictionary.Clear();

                                foreach (var pair in pushUnicastMetadatasRequestDictionary)
                                {
                                    var node = pair.Key;
                                    var targets = pair.Value;

                                    _random.Shuffle(targets);

                                    _pushUnicastMetadatasRequestDictionary.Add(node, targets);
                                }
                            }
                        }

                        // Multicast
                        {
                            var pushMulticastMetadatasRequestDictionary = new Dictionary<Node, List<Tag>>();

                            foreach (var tag in pushMulticastMetadatasRequestList.Randomize())
                            {
                                try
                                {
                                    var requestNodes = new List<Node>();

                                    foreach (var node in Kademlia<Node>.Search(tag.Id, otherNodes, 2))
                                    {
                                        requestNodes.Add(node);
                                    }

                                    for (int i = 0; i < requestNodes.Count; i++)
                                    {
                                        List<Tag> collection;

                                        if (!pushMulticastMetadatasRequestDictionary.TryGetValue(requestNodes[i], out collection))
                                        {
                                            collection = new List<Tag>();
                                            pushMulticastMetadatasRequestDictionary[requestNodes[i]] = collection;
                                        }

                                        if (collection.Count < _maxMetadataRequestCount)
                                        {
                                            collection.Add(tag);
                                        }
                                    }
                                }
                                catch (Exception e)
                                {
                                    Log.Error(e);
                                }
                            }

                            lock (_pushMulticastMetadatasRequestDictionary.ThisLock)
                            {
                                _pushMulticastMetadatasRequestDictionary.Clear();

                                foreach (var pair in pushMulticastMetadatasRequestDictionary)
                                {
                                    var node = pair.Key;
                                    var targets = pair.Value;

                                    _random.Shuffle(targets);

                                    _pushMulticastMetadatasRequestDictionary.Add(node, targets);
                                }
                            }
                        }
                    }
                }
            }
        }