//上报处理结果 public void PushReport(CloudflareLogReport CloudflareLogReport) { if (cloudflareLogReports == null) { cloudflareLogReports = new ConcurrentBag <CloudflareLogReport>(); } cloudflareLogReports.Add(CloudflareLogReport); }
//取队列数据处理 public void Dequeue() { try { while (true) { //OnMessage(new MessageEventArgs("取队列数据处理开始")); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); KeyValuePair <DateTime, DateTime> keyValuePair = default(KeyValuePair <DateTime, DateTime>); if (keyValuePairs.TryDequeue(out keyValuePair)) { DateTime start = keyValuePair.Key; DateTime end = keyValuePair.Value; string time = string.Format("{0}-{1}", start.ToString("yyyyMMddHHmmss"), end.ToString("yyyyMMddHHmmss")); bool retry = false; List <CloudflareLog> cloudflareLogs = cloundFlareApiService.GetCloudflareLogs(start, end, out retry); while (retry == true) { cloudflareLogs = cloundFlareApiService.GetCloudflareLogs(start, end, out retry); } if (cloudflareLogs != null && cloudflareLogs.Count > 0) { concurrentBagCloudflareLogs.Add(cloudflareLogs); } stopwatch.Stop(); OnMessage(new MessageEventArgs("取出数据:" + time + "共" + cloudflareLogs.Count + "条,用时:" + stopwatch.ElapsedMilliseconds / 1000 + "秒")); stopwatch.Restart(); var result = cloudflareLogs.GroupBy(a => new { a.ClientRequestHost, a.ClientIP, a.ClientRequestURI }). Select(g => new CloudflareLogReportItem { ClientRequestHost = g.Key.ClientRequestHost, ClientIP = g.Key.ClientIP, ClientRequestURI = g.Key.ClientRequestURI, Count = g.Count() }); //List<CloudflareLogReportItem> CloudflareLogReportItemList = new List<CloudflareLogReportItem>(); //foreach (var group in result) //{ // CloudflareLogReportItemList.Add(new CloudflareLogReportItem // { // ClientRequestHost = group.ClientRequestHost, // ClientIP = group.ClientIP, // ClientRequestURI = group.ClientRequestURI, // Count = group.Count // }); //} //foreach (var group in result) //{ // roteLimitConfig.RateLimits.ForEach(c => { // CloudflareLogReportItemList.Add(new CloudflareLogReportItem // { // ClientRequestHost = group.Key.ClientRequestHost, // ClientIP = group.Key.ClientIP, // //ClientRequestURI = c.Url, // ClientRequestURI = group.Key.ClientRequestURI, // Count = group.Count(a => a.ClientRequestURI.ToLower().Contains(c.Url.ToLower())) // }); // }); //} CloudflareLogReport CloudflareLogReport = new CloudflareLogReport { Guid = Guid.NewGuid().ToString(), Time = time, Start = start, End = end, Size = cloudflareLogs.Count, CloudflareLogReportItems = result.ToArray() }; stopwatch.Stop(); OnMessage(new MessageEventArgs("分析" + time + "用时:" + stopwatch.ElapsedMilliseconds / 1000 + "秒")); PushReport(CloudflareLogReport); //存数据库 //logsController.InsertData(start, cloudflareLogs); } else { break; } //OnMessage(new MessageEventArgs("取队列数据处理结束")); } } catch (Exception e) { logger.Error(e.Message); } }
//获取实时报表 public CloudflareLogReport GetCloudflareLogReport() { CloudflareLogReport cloudflareLogReport = new CloudflareLogReport(); try { Stopwatch stopwatch = new Stopwatch(); var orderby = cloudflareLogReports.Where(a => a.Start >= startTime && a.End <= endTime).OrderBy(a => a.Time).ToList(); CloudflareLogReport minCloudflareLogReport = orderby?.First(); CloudflareLogReport maxCloudflareLogReport = orderby?.Last(); DateTime start = minCloudflareLogReport.Start; DateTime end = maxCloudflareLogReport.End; int size = orderby.Sum(a => a.Size); string time = string.Format("{0}-{1}", start.ToString("yyyyMMddHHmmss"), end.ToString("yyyyMMddHHmmss")); var cloudflareLogReportItemsMany = orderby.SelectMany(a => a.CloudflareLogReportItems).ToList(); //合并处理 var itemsManyGroup = cloudflareLogReportItemsMany.GroupBy(a => new { a.ClientIP, a.ClientRequestHost, a.ClientRequestURI }) .Select(g => new { g.Key.ClientRequestHost, g.Key.ClientIP, g.Key.ClientRequestURI, Ban = false, Count = g.Sum(c => c.Count) }).OrderByDescending(a => a.Count).ToList(); List <IpNeedToBan> ipNeedToBans = new List <IpNeedToBan>(); List <IpNeedToBan> ipNeedToBansHas = logsController.GetIpNeedToBans(); var banItems = new List <CloudflareLogReportItem>(); // 没有增加 N 校验 //var result = (from item in itemsManyGroup // from config in requestLimitConfigs // where item.ClientRequestURI.ToLower().Contains( config.Url.ToLower() ) // && ( ( item.Count /(float) ( (end - start).TotalSeconds * sample ) ) >= (config.LimitTimes /(float) config.Interval) ) // select item).OrderByDescending(a=>a.Count).ToList(); var result = (from item in itemsManyGroup from config in roteLimitConfig.RateLimits where item.ClientRequestURI.ToLower().Contains(config.Url.ToLower()) && ((item.Count / (float)((end - start).TotalSeconds * sample)) >= (config.LimitTimes * roteLimitConfig.TriggerRatio / (float)config.Interval)) select new { item, config.Id }).OrderByDescending(a => a.item.Count).ToList(); List <int> handleIdList = new List <int>(); foreach (var item in result) { var ban = false; //存储rotelimit的触发log var roteLimit = roteLimitConfig.RateLimits.FirstOrDefault(a => a.Id == item.Id); var ipNumber = 0; var action = "None";//None/Create/Delete var containRoteLimitList = result.Where(a => a.Id == item.Id).ToList(); ipNumber = containRoteLimitList.Count; //同时有N个IP达到触发某条规则时候开启本条规则或者创建 if (ipNumber >= roteLimitConfig.TriggerCreateNumber) { action = "Create"; ban = true; } if (!handleIdList.Contains(item.Id)) { handleIdList.Add(item.Id); if (action == "Create") { //首先找是否已经存在rotelimit var roteLimitList = cloundFlareApiService.GetRateLimitRuleList(); var rote = roteLimitList.FirstOrDefault(a => a.match.request.url == roteLimit.Url && a.period == roteLimit.LimitTimes && a.threshold == roteLimit.Interval); if (updateRateLimitForCloundflare) { if (rote != null && !string.IsNullOrEmpty(rote.id)) { //已经存在的rote 如果是 disable = false 则开启 否则不用处理 if (!rote.disabled) { rote.disabled = true; cloundFlareApiService.UpdateRateLimit(rote); } } else { // { // "id": "efc79e757a5d449aa8fb43820ce30347", // "disabled": false, // "description": "chatserver.comm100.com/chatwindowembedded.aspx", // "match": { // "request": { // "methods": [ // "_ALL_" // ], // "schemes": [ // "_ALL_" // ], // "url": "chatserver.comm100.com/chatwindowembedded.aspx" // }, // "response": { // "origin_traffic": true, // "headers": [ // { // "name": "Cf-Cache-Status", // "op": "ne", // "value": "HIT" // } // ] // } // }, // "login_protect": false, // "threshold": 2, // "period": 60, // "action": { // "mode": "challenge", // "timeout": 0 // } //} cloundFlareApiService.CreateRateLimit(new RateLimitRule { disabled = true, description = roteLimit.Url, match = new Match { request = new Request { methods = new string[] { "_ALL_" }, schemes = new string[] { "_ALL_" }, url = roteLimit.Url, }, response = new Response { origin_traffic = true, headers = new Header[] { new Header { name = "Cf-Cache-Status", op = "ne", value = "HIT", } } } }, login_protect = false, threshold = roteLimit.Interval, period = roteLimit.LimitTimes, action = new RateLimitAction { mode = "challenge", timeout = 0 } }); } } } TriggerlogsDto triggerlogsDto = triggerlogsAppService.Create(new TriggerlogsDto { RequestLimitConfigId = item.Id, RequestLimitConfigDetail = JsonConvert.SerializeObject(new { roteLimit, roteLimitConfig.TriggerRatio, roteLimitConfig.TriggerCreateNumber, roteLimitConfig.TriggerDeleteTime, }), IpNumber = ipNumber, TriggerTime = DateTime.Now, Action = action, Remark = "Add By Defense System", }); List <TriggerlogdetailsDto> triggerlogdetailsDtos = new List <TriggerlogdetailsDto>(); foreach (var itm in containRoteLimitList) { var dto = new TriggerlogdetailsDto { TriggerLogId = triggerlogsDto.Id, //StartTime = startTime, //EndTime = endTime, Size = size, Sample = sample, ClientIP = itm.item.ClientIP, ClientRequestHost = itm.item.ClientRequestHost, ClientRequestURI = itm.item.ClientRequestURI, Count = itm.item.Count, }; triggerlogdetailsDtos.Add(dto); //triggerlogdetailsAppService.Create(dto); } //triggerlogdetailsAppService.CreateBatch(triggerlogdetailsDtos); logsController.CreateTriggerlogdetailsDtoBatch(triggerlogdetailsDtos); } if (!banItems.Exists(a => a.ClientIP == item.item.ClientIP && a.ClientRequestHost == item.item.ClientRequestHost && a.ClientRequestURI == item.item.ClientRequestURI)) { banItems.Add(new CloudflareLogReportItem { ClientRequestHost = item.item.ClientRequestHost, ClientIP = item.item.ClientIP, ClientRequestURI = item.item.ClientRequestURI, Count = item.item.Count, Ban = ban }); } } if (resultSaveDb == true) { logsController.InsertResultDataBulk(ipNeedToBans); } cloudflareLogReport = new CloudflareLogReport { Guid = Guid.NewGuid().ToString(), Time = time, Start = start, End = end, Size = size, CloudflareLogReportItems = banItems.ToArray() }; stopwatch.Stop(); OnMessage(new MessageEventArgs("报表汇总用时:" + stopwatch.ElapsedMilliseconds / 1000 + "秒")); } catch (Exception e) { logger.Error(e.Message); } return(cloudflareLogReport); }
//查询区间报表 public CloudflareLogReport GetCloudflareLogReport(DateTime start, DateTime end) { CloudflareLogReport CloudflareLogReport = new CloudflareLogReport(); return(CloudflareLogReport); }
//获取实时报表 public CloudflareLogReport GetCloudflareLogReport() { CloudflareLogReport cloudflareLogReport = new CloudflareLogReport(); try { Stopwatch stopwatch = new Stopwatch(); var orderby = cloudflareLogReports.Where(a => a.Start >= startTime && a.End <= endTime).OrderBy(a => a.Time); CloudflareLogReport minCloudflareLogReport = orderby.First(); CloudflareLogReport maxCloudflareLogReport = orderby.Last(); DateTime start = minCloudflareLogReport.Start; DateTime end = maxCloudflareLogReport.End; int size = orderby.Sum(a => a.Size); string time = string.Format("{0}-{1}", start.ToString("yyyyMMddHHmmss"), end.ToString("yyyyMMddHHmmss")); var cloudflareLogReportItemsMany = orderby.SelectMany(a => a.CloudflareLogReportItems).ToList(); //合并处理 var itemsManyGroup = cloudflareLogReportItemsMany.GroupBy(a => new { a.ClientIP, a.ClientRequestHost, a.ClientRequestURI }) .Select(g => new { g.Key.ClientRequestHost, g.Key.ClientIP, g.Key.ClientRequestURI, Ban = false, Count = g.Sum(c => c.Count) }).OrderByDescending(a => a.Count).ToList(); List <IpNeedToBan> ipNeedToBans = new List <IpNeedToBan>(); List <IpNeedToBan> ipNeedToBansHas = logsController.GetIpNeedToBans(); var banItems = new List <CloudflareLogReportItem>(); var result = (from item in itemsManyGroup from config in requestLimitConfigs where item.ClientRequestURI == config.Url && ((item.Count / ((end - start).TotalSeconds * sample)) >= (config.LimitTimes / config.Interval)) select item).OrderByDescending(a => a.Count).ToList(); foreach (var item in result) { banItems.Add(new CloudflareLogReportItem { ClientRequestHost = item.ClientRequestHost, ClientIP = item.ClientIP, ClientRequestURI = item.ClientRequestURI, Count = item.Count, Ban = true }); if (!ipNeedToBansHas.Exists(a => a.IP == item.ClientIP)) { ipNeedToBans.Add(new IpNeedToBan { Host = item.ClientRequestHost, IP = item.ClientIP, RelatedURL = item.ClientRequestURI, HasBanned = false, RequestedTime = DateTime.Now, Remark = "" }); } } if (resultSaveDb == true) { logsController.InsertResultDataBulk(ipNeedToBans); } cloudflareLogReport = new CloudflareLogReport { Guid = Guid.NewGuid().ToString(), Time = time, Start = start, End = end, Size = size, CloudflareLogReportItems = banItems.ToArray() }; stopwatch.Stop(); OnMessage(new MessageEventArgs("报表汇总用时:" + stopwatch.ElapsedMilliseconds / 1000 + "秒")); } catch (Exception e) { logger.Error(e.Message); } return(cloudflareLogReport); }