예제 #1
0
        /// <summary>
        /// 添加ACK 信息,供CONSUMER使用
        /// </summary>
        /// <param name="consumerUri"> </param>
        /// <param name="server"></param>
        /// <param name="ack"></param>
        public void AppendAck(string consumerUri,PhysicalServer server,ConsumerAck ack)
        {
            Guard.ArgumentNotNull(server, "server");
            Guard.ArgumentNotNull(ack, "ack");

            Output.AddOrUpdate(server.ServerDomainName,
                               s =>
                                   {
                                       var queue = new ConcurrentQueue<CBox<Tuple<string, ConsumerAck>>>();
                                       queue.Enqueue(new CBox<Tuple<string, ConsumerAck>> { Value = new Tuple<string, ConsumerAck>(consumerUri, ack) });
                                       //定时Ack
                                       _timerScheduler.Register(() =>
                                                                    {
                                                                        var tmp = server;
                                                                        RunTaskAck(tmp);
                                                                    }, s, _ackIntervalTime, false);
                                           //对于新增的SERVER 启动TASK 进行自动ACK
                                       return queue;
                                   },
                               (s, d) =>
                                   {
                                       var tmp = server;
                                       d.Enqueue(new CBox<Tuple<string, ConsumerAck>>
                                                     {Value = new Tuple<string, ConsumerAck>(consumerUri, ack)});

                                       if (d.Count >= Consts.Consumer_DefaultAckQueueBoundary) //Queue 数量大于
                                       {
                                           System.Threading.ThreadPool.QueueUserWorkItem(RunTaskAck, tmp);
                                       }
                                       return d;
                                   });
        }
예제 #2
0
        public PhysicalServer Schedule(string consumer, int receiveTimeout,bool sync)
        {
            const string title = "noserver";
            if (string.IsNullOrEmpty(consumer))
            {
                Logger.Write("consumer is null", LogLevel.Warn, title);
                return null;
            }

            WeightedRandomScheduling<PhysicalServer> wrs;
            if (!_consumerWrsList.TryGetValue(consumer, out wrs))
            {
                Logger.Write("consumer is not exists int WRS LIST", LogLevel.Warn, title, new[] { new KeyValue { Key = "consumer", Value = consumer } });
                return null;
            }

            var server = wrs.Schedule();
            if (server == null)
            {
                Logger.Write("no server when server run schedule", LogLevel.Warn, title, new[] { new KeyValue { Key = "consumer", Value = consumer } });
                return null;
            }
            //if (!server.Mark) return server; //Mark标记为false的不作退避
            string serverName = "";
            DateTime date = DateTime.MaxValue;
            lock (_lockObject)
            {
                Dictionary<string, Tuple<int, int, DateTime>> dictionary;
                if (!_noMessageList.TryGetValue(consumer, out dictionary)) return server;//此CONSUMER没有无消息的SERVER,直接返回
                if (!dictionary.ContainsKey(server.ServerName)) return server; //CONSUMER对应此SERVER, 不存在列表中,直接返回
                if ((DateTime.Now - dictionary[server.ServerName].Item3).TotalMilliseconds > receiveTimeout / 2) return server;//如超500ms,则继续请求

                var tmp = dictionary.ToList();
                if (tmp.Count >= wrs.Servers.Length)
                {
                    PhysicalServer[] servers = new PhysicalServer[wrs.Servers.Length];
                    wrs.Servers.CopyTo(servers, 0);
                    //当都没有数据时,取最长时间未取数据的SERVER
                    foreach (var item in tmp)
                    {
                        var s = servers.FirstOrDefault(x => x.ServerName == item.Key);//如在server中不存在的,不作处理
                        if (s == null) continue;
                        if ( date > item.Value.Item3)
                        {
                            date = item.Value.Item3;
                            serverName = item.Key;
                            server = s;
                        }
                    }
                }
            }
            if (!string.IsNullOrEmpty(serverName))
            {
                if (!sync) Block(consumer, date, receiveTimeout);//异步做休眠
                return server;
            }
            return Schedule(consumer,receiveTimeout,sync);
        }
예제 #3
0
 private void RemoveServer(string consumerUri, PhysicalServer server)
 {
     if (server == null) return;
     //移除无数据SERVER
     Dictionary<string, Tuple<int, int, DateTime>> dictionary;
     if (_noMessageList.TryGetValue(consumerUri, out dictionary))
     {
         lock (_lockObject)
         {
             if (dictionary.ContainsKey(server.ServerName))
             {
                 dictionary.Remove(server.ServerName);
             }
         }
     }
 }
예제 #4
0
 //PULLING请求结束时调用
 public void EndPulling(string consumerUri, PhysicalServer server, bool hasMessages)
 {
     if (server == null) return;
     if (hasMessages)
     {
         RemoveServer(consumerUri, server);//有数据从无数据列表中迁除该SERVER
     }
     else
     {
         AddOrUpdateServer(consumerUri, server.ServerName);
         if (server.Mark == false)
             RemoveMarkServer(consumerUri, server.ServerName);
     }
 }
예제 #5
0
        /// <summary>
        /// 向服务端ACK
        /// </summary>
        /// <param name="server"></param>
        /// <param name="timeout"></param>
        /// <param name="chunk"></param>
        /// <returns></returns>
        public ChunkAck Ack(PhysicalServer server, int timeout, ConsumerAckChunk chunk)
        {
            Guard.ArgumentNotNull(server, "server");
            Guard.ArgumentNotNull(chunk, "chunk");
            Guard.ArgumentNotNullOrEmpty(server.ServerDomainName, "server.ServerDomainName");
            Guard.ArgumentNotNullOrEmpty(server.ServerName, "server.ServerName");

            MetricUtil.Set(new AckRequestCountMetric { ServerHostName = server.ServerName });
            foreach (var ack in chunk.ConsumerAcks)
            {
                MetricUtil.Set(new AckMessageCountMetric { Consumer = ack.Uri });
            }

            ChunkAck chunkAck=null;
            var watch = new Stopwatch();
            DispatcherServiceWrapper.Client client=null;
            try
            {
                client = Client.CreateDispatcherServiceClient(server.ServerDomainName, timeout);
                watch.Start();
                chunkAck = client.Ack(chunk);

                return chunkAck;
            }
            catch {
                if (client != null) client.Dispose();
                throw;
            }
            finally
            {
                watch.Stop();
                var milliseconds = watch.ElapsedMilliseconds;
                MetricUtil.Set(new AckResponseCountMetric { ServerHostName = server.ServerName, Latency = milliseconds, StatusCode = chunkAck == null ? "0" : chunkAck.StatusCode.ToString() });
                MetricUtil.Set(new AckResponseLatencyMetric { ServerHostName = server.ServerName }, milliseconds);
            }
        }
예제 #6
0
        /// <summary>
        /// 向服务端拉数据
        /// </summary>
        /// <param name="server"></param>
        /// <param name="receiveTimeout"></param>
        /// <param name="request"></param>
        /// <returns></returns>
        public SubChunk Pulling(PhysicalServer server, int receiveTimeout, PullingRequest request)
        {
            Guard.ArgumentNotNull(server, "server");
            Guard.ArgumentNotNull(request, "request");
            Guard.ArgumentNotNullOrEmpty(request.Uri, "request.Uri");
            Guard.ArgumentNotNullOrEmpty(server.ServerDomainName, "server.ServerDomainName");
            Guard.ArgumentNotNullOrEmpty(server.ServerName, "server.ServerName");
            SubChunk chunk= null;
            bool isNotMessage = true;
            long milliseconds=0;
            string errortype = "";
            
            MetricUtil.Set(new PullingRequestCountMetric
            {
                ServerHostName = server.ServerName,
                Consumer = request.Uri
            });
            var statuscode = 0;
            var watch = new Stopwatch();
            DispatcherServiceWrapper.Client client = null;
#if DEBUG
            var index = countor.AtomicIncrementAndGet();
#endif
            try
            {
#if DEBUG
                debugLog.Write(string.Format("Start servicepulling->{0} server->{1}", index, server.ServerName), request.Uri);
#endif
                client = Client.CreateDispatcherServiceClient(server.ServerDomainName, receiveTimeout);
            
                watch.Start();
                chunk = client.Pulling(request);
                isNotMessage = (chunk == null || chunk.Messages == null || chunk.Messages.Count < 1);
                statuscode = chunk == null ? 0 : (int)chunk.StatusCode;
            }
            catch (WebException webException)
            {
                if (client != null) client.Dispose();
                statuscode = (int)Convert(webException.Status);
                errortype = "webexception";
                if (webException.Message.IndexOf("timed out", StringComparison.InvariantCultureIgnoreCase) > 0)
                {
                    errortype = "timeout";
                }
                throw webException;
            }
            catch (TTransportException te)
            {
                if (client != null) client.Dispose();
                errortype = "ttransportexception";
                
                if ((te.Message.IndexOf("timed out", StringComparison.InvariantCultureIgnoreCase) > 0)||
                    (te.InnerException != null 
                    && (te.InnerException.Message.IndexOf("timed out", StringComparison.InvariantCultureIgnoreCase) > 0)))
                {
                    errortype = "timeout";
                }

                if (te.InnerException != null && te.InnerException is WebException)
                {
                    var webex = (WebException)te.InnerException;
                    statuscode = (int)Convert(webex.Status);
                }
                else
                {
                    statuscode = (int)Convert(te.Type);
                }
                throw te;
            }
            catch (Exception ex)
            {
                if (client != null) client.Dispose();
                errortype = "systemexception";
                throw ex;
            }
            finally
            {
                watch.Stop();
                milliseconds = watch.ElapsedMilliseconds;
                if(!isNotMessage && chunk !=null && chunk.Messages !=null)
                {
                    var messageSize = 0;
                    var messageCount = chunk.Messages.Count;
                    chunk.Messages.Iterate(x => messageSize += x.Pub != null ? x.Pub.Size : 0);
                    MetricUtil.Set(new PullingMessageCountMetric { MessageSize = messageSize, Consumer = request.Uri }, messageCount);
                }
#if DEBUG
                var messageCount1 = (chunk == null || chunk.Messages == null) ? 0 : chunk.Messages.Count;
                debugLog.Write(string.Format("End servicepulling->{0} messagecount->{1} time->{2}", index, isNotMessage ? "0" : messageCount1.ToString(), milliseconds), request.Uri);
#endif
                MetricUtil.Set(new PullingResponseCountMetric
                {
                    Consumer = request.Uri,
                    ServerHostName = server.ServerName,
                    Latency = milliseconds,
                    StatusCode = statuscode.ToString(),
                    HasMessages = isNotMessage?"0":"1",
                    Error = errortype
                });
                MetricUtil.Set(new PullingResponseLatencyMetric { ServerHostName = server.ServerName, Consumer = request.Uri }, milliseconds);
            }
            return chunk;
        }
예제 #7
0
 /// <summary>
 /// 往SERVER请求加载数据
 /// </summary>
 /// <param name="uri"></param>
 /// <param name="batchSize"></param>
 /// <param name="receiveTimeout"></param>
 /// <param name="server"> </param>
 /// <param name="sync"> </param>
 private SubChunk Pulling(string uri, int batchSize, int receiveTimeout, PhysicalServer server,bool sync= true)
 {
     SubChunk chunk = null;
     try
     {
         _serverPool.AcquireTimeout = receiveTimeout;
         var service = _serverPool.Acquire(); //SERVER线程数控制
         if (MomoryManager.IsOutOfMaxMemorySize)
         {
             Logg.Write("当前内存超过最大使用内存", LogLevel.Warn, "Consumer.InputBuffer.LoadData",
                          new[]
                              {
                                  new KeyValue {Key = "Consumer", Value = uri},
                                  new KeyValue
                                      {Key = "MaxMemorySize", Value = MomoryManager.MaxMemorySize.ToString()},
                                  new KeyValue
                                      {
                                          Key = "CurrentMemorySize",
                                          Value = MomoryManager.CurrentMemorySize.ToString()
                                      }
                              });
             _serverPool.Release();
             return null; //内存控制
         }
         if (_cts != null && _cts.IsCancellationRequested)
         {
             _serverPool.Release();
             return null;
         }
         try
         {
             chunk = service.Pulling(server,
                                     receiveTimeout,
                                     new PullingRequest
                                         {
                                             AckTimeout = AckTimeout,
                                             Timestamp = Time.ToTimestamp(),
                                             Uri = uri,
                                             BatchSize = batchSize,
                                             ClientIP = Local.IPV4,
                                             ClientVersion = Version.AssemblyMajorVersion,
                                             Platform = "NET"
                                         });
             if (chunk == null) throw new Exception("pulling return null");
         }
         catch
         {
             MetricUtil.Set(new ExceptionCountMetric { Consumer = uri, HappenedWhere = ExceptionType.OnPulling.ToString() });
             throw;
         }
         finally
         {
             _serverPool.Release();
             _serverUriManager.EndPulling(uri, server,(chunk != null && chunk.Messages != null && chunk.Messages.Count > 0));
         }
     }
     catch (Exception ex)
     {
         Logg.Write(ex, LogLevel.Error,
                      Consts.Consumer_Title_LoadDataError,
                      new[]
                          {
                              new KeyValue {Key = "PullingRequestUri", Value = uri},
                              new KeyValue {Key = "ServerUri", Value = server == null ? "" : server.ServerDomainName},
                          });
     }
     return chunk;
 }