コード例 #1
0
        private bool IfMatchConditionAccumulation(RateLimitEntity rateLimit, LogAnalyzeModel analyzeModel)
        {
            bool bResult = false;

            bResult = rateLimit.Url.EndsWith("*") ? ((analyzeModel.RequestUrl.ToLower().StartsWith(rateLimit.Url.Replace("*", "").ToLower())) && (analyzeModel.RequestCount >= rateLimit.EnlargementFactor * rateLimit.Threshold * accumulationSecond / (float)rateLimit.Period))
                : ((analyzeModel.RequestUrl.ToLower().Equals(rateLimit.Url.ToLower())) && (analyzeModel.RequestCount >= rateLimit.EnlargementFactor * rateLimit.Threshold * accumulationSecond / (float)rateLimit.Period));
            return(bResult);
        }
コード例 #2
0
        private AnalyzeResult AnalyzeRatelimit(List <CloudflareLog> cloudflareLogs, int accumulation = 1)
        {
            AnalyzeResult analyzeResult = null;

            if (cloudflareLogs != null && cloudflareLogs.Count > 0)
            {
                analyzeResult = new AnalyzeResult();
                var removedLogs = new List <LogAnalyzeModel>();
                //logger.Debug(JsonConvert.SerializeObject(new
                //{
                //    DataType = "0-CloudflareLogs",
                //    Value = cloudflareLogs,
                //}));

                var key = "AnalyzeRatelimit_GetZoneList_Key";
                List <ZoneEntity> zoneEntityList = Utils.GetMemoryCache(key, () =>
                {
                    string url     = awsGetZoneListApiUrl;
                    string content = HttpGet(url);
                    //logger.Debug(JsonConvert.SerializeObject(new
                    //{
                    //    DataType = "GetZoneList",
                    //    Value = content,
                    //}));
                    return(JsonConvert.DeserializeObject <List <ZoneEntity> >(content));
                }, 1440);

                CloudflareLog cloudflare = cloudflareLogs.FirstOrDefault();
                string        zoneId     = zoneEntityList.FirstOrDefault(a => (a.HostNames.Split(new string[] { Utils.Separator }, StringSplitOptions.RemoveEmptyEntries)).Contains(cloudflare.ClientRequestHost))?.ZoneId;//?

                //logger.Debug(JsonConvert.SerializeObject(new
                //{
                //    FirstOrDefault = cloudflare,
                //    ZoneId = zoneId,
                //}));

                if (!string.IsNullOrEmpty(zoneId))
                {
                    //每60分钟去获取一次RateLimit规则
                    key = "AnalyzeRatelimit_GetRatelimits_Key_" + zoneId;
                    List <RateLimitEntity> rateLimitEntities = Utils.GetMemoryCache(key, () =>
                    {
                        string url     = getRatelimitsApiUrl;
                        url            = url.Replace("{zoneId}", zoneId);
                        string content = HttpGet(url);
                        //logger.Debug(JsonConvert.SerializeObject(new
                        //{
                        //    DataType = "GetRatelimits",
                        //    ZoneId = zoneId,
                        //    Value = content,
                        //}));
                        return(JsonConvert.DeserializeObject <List <RateLimitEntity> >(content));
                    }, 5);

                    key = "AnalyzeRatelimit_GetWhiteList_Key_" + zoneId;
                    List <WhiteListModel> whiteListModels = Utils.GetMemoryCache(key, () =>
                    {
                        string url     = awsGetWhiteListApiUrl;
                        url            = url.Replace("{zoneId}", zoneId);
                        string content = HttpGet(url);
                        //logger.Debug(JsonConvert.SerializeObject(new
                        //{
                        //    DataType = "GetWhiteList",
                        //    ZoneId = zoneId,
                        //    Value = content,
                        //}));
                        return(JsonConvert.DeserializeObject <List <WhiteListModel> >(content));
                    }, 1440);

                    //获取1S规则
                    List <RateLimitEntity> rateLimitEntitiesSub = rateLimitEntities.Where(a => a.ZoneId == zoneId && a.Period == 1).ToList();
                    if (accumulation > 1)
                    {
                        rateLimitEntitiesSub = rateLimitEntities.Where(a => a.ZoneId == zoneId && a.Period > 1).ToList();
                    }

                    if (rateLimitEntitiesSub != null && rateLimitEntitiesSub.Count > 0)
                    {
                        List <string> ipWhiteList = new List <string>();
                        if (whiteListModels != null)
                        {
                            ipWhiteList = whiteListModels.Select(a => a.IP).ToList();
                        }

                        analyzeResult.result = new List <Result>();
                        var ruleResult = new Result();
                        //logger.Debug(JsonConvert.SerializeObject(new
                        //{
                        //    DataType = "1-CloudflareLogs",
                        //    Value = cloudflareLogs,
                        //}));

                        var logAnalyzeModelList = cloudflareLogs.Where(a => !ipWhiteList.Contains(a.ClientIP))
                                                  .Select(x =>
                        {
                            var model = new LogAnalyzeModel
                            {
                                IP             = x.ClientIP,
                                RequestHost    = x.ClientRequestHost,
                                RequestFullUrl = $"{x.ClientRequestHost}{x.ClientRequestURI}",
                                RequestUrl     =
                                    $"{x.ClientRequestHost}{(x.ClientRequestURI.IndexOf('?') > 0 ? x.ClientRequestURI.Substring(0, x.ClientRequestURI.IndexOf('?')) : x.ClientRequestURI)}"
                            };
                            return(model);
                        });

                        bool ifContainWildcard;
                        foreach (var rateLimit in rateLimitEntitiesSub)
                        {
                            //systemLogList.Add(new AuditLogEntity(zoneId, LogLevel.App,
                            //    $"Start analyzing rule [Url=[{rateLimit.Url}],Period=[{rateLimit.Period}],Threshold=[{rateLimit.Threshold}],EnlargementFactor=[{rateLimit.EnlargementFactor}]]"));
                            //抽取出所有ratelimit规则中的请求列表
                            ifContainWildcard = rateLimit.Url.EndsWith("*");
                            var logAnalyzeDetailList = ifContainWildcard
                                ? logAnalyzeModelList.Where(x => x.RequestUrl.ToLower().StartsWith(rateLimit.Url.ToLower().Replace("*", ""))).ToList()
                                : logAnalyzeModelList.Where(x => x.RequestUrl.ToLower().Equals(rateLimit.Url.ToLower())).ToList();

                            if (logAnalyzeDetailList.Count > 0)
                            {
                                //对IP的请求地址(包含querystring)进行分组
                                var ipRequestListIncludingQueryString = logAnalyzeDetailList.GroupBy(x => new { x.IP, x.RequestFullUrl }).Select(x => new LogAnalyzeModel()
                                {
                                    IP             = x.Key.IP,
                                    RequestFullUrl = x.Key.RequestFullUrl,
                                    RequestCount   = x.Count()
                                }).ToList();

                                //对IP的请求地址(不包含querystring)进行分组
                                var ipRequestList = logAnalyzeDetailList.GroupBy(x => new { x.IP }).Select(x => new LogAnalyzeModel()
                                {
                                    IP           = x.Key.IP,
                                    RequestUrl   = rateLimit.Url,
                                    RequestCount = x.Count()
                                }).ToList();

                                //抽取出所有违反规则的IP请求列表
                                var brokenRuleIpList = (from item in ipRequestList
                                                        where accumulation > 1 ? IfMatchConditionAccumulation(rateLimit, item) : IfMatchCondition(rateLimit, item)
                                                        select new LogAnalyzeModel()
                                {
                                    IP = item.IP,
                                    RequestUrl = item.RequestUrl,
                                    RequestCount = item.RequestCount,
                                    RateLimitId = rateLimit.ID,
                                    RateLimitTriggerIpCount = rateLimit.RateLimitTriggerIpCount
                                }).ToList();

                                var brokenIpCountList = brokenRuleIpList.GroupBy(x => new { x.RateLimitId, x.RequestUrl, x.RateLimitTriggerIpCount }).Select(x => new LogAnalyzeModel()
                                {
                                    RateLimitTriggerIpCount = x.Key.RateLimitTriggerIpCount,
                                    RateLimitId             = x.Key.RateLimitId,
                                    RequestUrl   = x.Key.RequestUrl,
                                    RequestCount = x.Count()
                                }).ToList();

                                if (brokenIpCountList.Count > 0)
                                {
                                    //抽取出超过IP数量的规则列表,需要新增或OPEN Cloudflare的Rate Limiting Rule
                                    var brokenRuleList = brokenIpCountList.Where(x => x.RateLimitTriggerIpCount <= x.RequestCount).ToList();

                                    if (brokenRuleList.Count > 0)
                                    {
                                        ruleResult = new Result()
                                        {
                                            RuleId                  = rateLimit.ID,
                                            Url                     = rateLimit.Url,
                                            Threshold               = rateLimit.Threshold,
                                            Period                  = rateLimit.Period,
                                            EnlargementFactor       = rateLimit.EnlargementFactor,
                                            RateLimitTriggerIpCount = rateLimit.RateLimitTriggerIpCount,
                                            BrokenIpList            = new List <BrokenIp>()
                                        };
                                        var brokenIpList = new List <BrokenIp>();
                                        foreach (var rule in brokenRuleIpList)
                                        {
                                            brokenIpList.Add(new BrokenIp()
                                            {
                                                IP             = rule.IP,
                                                RequestRecords = ipRequestListIncludingQueryString.Where(x => x.IP.Equals(rule.IP))
                                                                 .OrderByDescending(x => x.RequestCount).Select(x => new RequestRecord()
                                                {
                                                    FullUrl      = x.RequestFullUrl,
                                                    RequestCount = x.RequestCount,
                                                    HostName     = x.RequestHost
                                                }).ToList()
                                            });
                                            removedLogs = logAnalyzeModelList.ToList();
                                            removedLogs.RemoveAll(p => p.IP == rule.IP);
                                            logAnalyzeModelList = removedLogs;
                                        }
                                        ruleResult.BrokenIpList = brokenIpList;

                                        analyzeResult.result.Add(ruleResult);
                                    }
                                }
                            }

                            //删除已经触犯了当前ratelimit的所有url
                            removedLogs = logAnalyzeModelList.ToList();
                            removedLogs.RemoveAll(x => ifContainWildcard ? x.RequestUrl.ToLower().StartsWith(rateLimit.Url.ToLower().Replace("*", "")) : x.RequestUrl.ToLower().Equals(rateLimit.Url.ToLower()));
                            logAnalyzeModelList = removedLogs;
                        }


                        //logger.Debug(JsonConvert.SerializeObject(new
                        //{
                        //    DataType = "2-CloudflareLogs-!whitelist",
                        //    Value = logAnalyzeModelList,
                        //}));

                        //var itemsGroup = logAnalyzeModelList.GroupBy(a => new { a.IP, a.RequestHost, a.RequestUrl })
                        //                    .Select(g => new LogAnalyzeModel
                        //                    {
                        //                        RequestHost = g.Key.RequestHost,
                        //                        IP = g.Key.IP,
                        //                        RequestUrl = g.Key.RequestUrl,
                        //                        RequestCount = g.Count()
                        //                    }).ToList();

                        //var result =
                        //    (from analyzeModel in itemsGroup
                        //     from config in rateLimitEntitiesSub
                        //     where IfMatchCondition(config, analyzeModel)
                        //     select new LogAnalyzeModel
                        //     {
                        //         RequestHost = analyzeModel.RequestHost,
                        //         IP = analyzeModel.IP,
                        //         RequestUrl = analyzeModel.RequestUrl,
                        //         RequestCount = analyzeModel.RequestCount,
                        //         RateLimitId = config.ID,

                        //     });

                        //logger.Debug(JsonConvert.SerializeObject(new
                        //{
                        //    DataType = "3-CloudflareLogs-IfMatchCondition",
                        //    Value = result,
                        //}));

                        //List<Result> results = new List<Result>();
                        //if (result != null && result.Count() > 0)
                        //{
                        //    foreach (var item in result)
                        //    {
                        //        RateLimitEntity rateLimit = rateLimitEntitiesSub.FirstOrDefault(a => a.ID == item.RateLimitId);

                        //        int ruleId = rateLimit.ID;
                        //        int period = rateLimit.Period;
                        //        int threshold = rateLimit.Threshold;
                        //        string url = rateLimit.Url;
                        //        var enlargementFactor = rateLimit.EnlargementFactor;

                        //        //触发规则的 IP+Host 数据
                        //        List<LogAnalyzeModel> logAnalyzesList = logAnalyzeModelList.Where(a => a.RequestHost == item.RequestHost && a.IP == item.IP)
                        //            .GroupBy(a => a.RequestFullUrl)
                        //            .Select(g => new LogAnalyzeModel
                        //            {
                        //                RequestFullUrl = g.Key,
                        //                RequestCount = g.Count(),
                        //            }).ToList();

                        //        Result rst = results.FirstOrDefault(a => a.RuleId == ruleId);
                        //        if (rst == null)
                        //        {
                        //            List<BrokenIp> brokenIpList = new List<BrokenIp>();
                        //            rst = new Result
                        //            {
                        //                RuleId = ruleId,
                        //                Period = period,
                        //                Threshold = threshold,
                        //                EnlargementFactor = enlargementFactor,
                        //                Url = url,
                        //                BrokenIpList = brokenIpList,
                        //            };
                        //            results.Add(rst);
                        //        }
                        //        BrokenIp brokenIp = new BrokenIp
                        //        {
                        //            IP = item.IP,
                        //            RequestRecords = new List<RequestRecord>(),
                        //        };
                        //        rst.BrokenIpList.Add(brokenIp);
                        //        if (logAnalyzesList != null)
                        //        {
                        //            foreach (var logAnalyze in logAnalyzesList)
                        //            {
                        //                brokenIp.RequestRecords.Add(new RequestRecord
                        //                {
                        //                    FullUrl = logAnalyze.RequestFullUrl,
                        //                    RequestCount = logAnalyze.RequestCount,
                        //                });
                        //            }
                        //        }
                        //    }
                        //}
                        analyzeResult.ZoneId    = zoneId;
                        analyzeResult.timeStage = accumulation;
                        //analyzeResult.result = null;

                        //logger.Debug(JsonConvert.SerializeObject(new
                        //{
                        //    DataType = "4-CloudflareLogs-analyzeResult",
                        //    Value = analyzeResult,
                        //}));
                    }
                }
            }
            return(analyzeResult);
        }