private bool IfMatchConditionAccumulation(RateLimitEntity rateLimit, LogAnalyzeModel analyzeModel)
{
bool bResult = false;
bResult = rateLimit.Url.EndsWith("*") ? ((analyzeModel.RequestUrl.ToLower().StartsWith(rateLimit.Url.Replace("*", "").ToLower())) && (analyzeModel.RequestCount >= rateLimit.EnlargementFactor * rateLimit.Threshold * accumulationSecond / (float)rateLimit.Period))
: ((analyzeModel.RequestUrl.ToLower().Equals(rateLimit.Url.ToLower())) && (analyzeModel.RequestCount >= rateLimit.EnlargementFactor * rateLimit.Threshold * accumulationSecond / (float)rateLimit.Period));
return(bResult);
}
private AnalyzeResult AnalyzeRatelimit(List <CloudflareLog> cloudflareLogs, int accumulation = 1)
{
AnalyzeResult analyzeResult = null;
if (cloudflareLogs != null && cloudflareLogs.Count > 0)
{
analyzeResult = new AnalyzeResult();
var removedLogs = new List <LogAnalyzeModel>();
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "0-CloudflareLogs",
// Value = cloudflareLogs,
//}));
var key = "AnalyzeRatelimit_GetZoneList_Key";
List <ZoneEntity> zoneEntityList = Utils.GetMemoryCache(key, () =>
{
string url = awsGetZoneListApiUrl;
string content = HttpGet(url);
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "GetZoneList",
// Value = content,
//}));
return(JsonConvert.DeserializeObject <List <ZoneEntity> >(content));
}, 1440);
CloudflareLog cloudflare = cloudflareLogs.FirstOrDefault();
string zoneId = zoneEntityList.FirstOrDefault(a => (a.HostNames.Split(new string[] { Utils.Separator }, StringSplitOptions.RemoveEmptyEntries)).Contains(cloudflare.ClientRequestHost))?.ZoneId;//?
//logger.Debug(JsonConvert.SerializeObject(new
//{
// FirstOrDefault = cloudflare,
// ZoneId = zoneId,
//}));
if (!string.IsNullOrEmpty(zoneId))
{
//每60分钟去获取一次RateLimit规则
key = "AnalyzeRatelimit_GetRatelimits_Key_" + zoneId;
List <RateLimitEntity> rateLimitEntities = Utils.GetMemoryCache(key, () =>
{
string url = getRatelimitsApiUrl;
url = url.Replace("{zoneId}", zoneId);
string content = HttpGet(url);
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "GetRatelimits",
// ZoneId = zoneId,
// Value = content,
//}));
return(JsonConvert.DeserializeObject <List <RateLimitEntity> >(content));
}, 5);
key = "AnalyzeRatelimit_GetWhiteList_Key_" + zoneId;
List <WhiteListModel> whiteListModels = Utils.GetMemoryCache(key, () =>
{
string url = awsGetWhiteListApiUrl;
url = url.Replace("{zoneId}", zoneId);
string content = HttpGet(url);
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "GetWhiteList",
// ZoneId = zoneId,
// Value = content,
//}));
return(JsonConvert.DeserializeObject <List <WhiteListModel> >(content));
}, 1440);
//获取1S规则
List <RateLimitEntity> rateLimitEntitiesSub = rateLimitEntities.Where(a => a.ZoneId == zoneId && a.Period == 1).ToList();
if (accumulation > 1)
{
rateLimitEntitiesSub = rateLimitEntities.Where(a => a.ZoneId == zoneId && a.Period > 1).ToList();
}
if (rateLimitEntitiesSub != null && rateLimitEntitiesSub.Count > 0)
{
List <string> ipWhiteList = new List <string>();
if (whiteListModels != null)
{
ipWhiteList = whiteListModels.Select(a => a.IP).ToList();
}
analyzeResult.result = new List <Result>();
var ruleResult = new Result();
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "1-CloudflareLogs",
// Value = cloudflareLogs,
//}));
var logAnalyzeModelList = cloudflareLogs.Where(a => !ipWhiteList.Contains(a.ClientIP))
.Select(x =>
{
var model = new LogAnalyzeModel
{
IP = x.ClientIP,
RequestHost = x.ClientRequestHost,
RequestFullUrl = $"{x.ClientRequestHost}{x.ClientRequestURI}",
RequestUrl =
$"{x.ClientRequestHost}{(x.ClientRequestURI.IndexOf('?') > 0 ? x.ClientRequestURI.Substring(0, x.ClientRequestURI.IndexOf('?')) : x.ClientRequestURI)}"
};
return(model);
});
bool ifContainWildcard;
foreach (var rateLimit in rateLimitEntitiesSub)
{
//systemLogList.Add(new AuditLogEntity(zoneId, LogLevel.App,
// $"Start analyzing rule [Url=[{rateLimit.Url}],Period=[{rateLimit.Period}],Threshold=[{rateLimit.Threshold}],EnlargementFactor=[{rateLimit.EnlargementFactor}]]"));
//抽取出所有ratelimit规则中的请求列表
ifContainWildcard = rateLimit.Url.EndsWith("*");
var logAnalyzeDetailList = ifContainWildcard
? logAnalyzeModelList.Where(x => x.RequestUrl.ToLower().StartsWith(rateLimit.Url.ToLower().Replace("*", ""))).ToList()
: logAnalyzeModelList.Where(x => x.RequestUrl.ToLower().Equals(rateLimit.Url.ToLower())).ToList();
if (logAnalyzeDetailList.Count > 0)
{
//对IP的请求地址(包含querystring)进行分组
var ipRequestListIncludingQueryString = logAnalyzeDetailList.GroupBy(x => new { x.IP, x.RequestFullUrl }).Select(x => new LogAnalyzeModel()
{
IP = x.Key.IP,
RequestFullUrl = x.Key.RequestFullUrl,
RequestCount = x.Count()
}).ToList();
//对IP的请求地址(不包含querystring)进行分组
var ipRequestList = logAnalyzeDetailList.GroupBy(x => new { x.IP }).Select(x => new LogAnalyzeModel()
{
IP = x.Key.IP,
RequestUrl = rateLimit.Url,
RequestCount = x.Count()
}).ToList();
//抽取出所有违反规则的IP请求列表
var brokenRuleIpList = (from item in ipRequestList
where accumulation > 1 ? IfMatchConditionAccumulation(rateLimit, item) : IfMatchCondition(rateLimit, item)
select new LogAnalyzeModel()
{
IP = item.IP,
RequestUrl = item.RequestUrl,
RequestCount = item.RequestCount,
RateLimitId = rateLimit.ID,
RateLimitTriggerIpCount = rateLimit.RateLimitTriggerIpCount
}).ToList();
var brokenIpCountList = brokenRuleIpList.GroupBy(x => new { x.RateLimitId, x.RequestUrl, x.RateLimitTriggerIpCount }).Select(x => new LogAnalyzeModel()
{
RateLimitTriggerIpCount = x.Key.RateLimitTriggerIpCount,
RateLimitId = x.Key.RateLimitId,
RequestUrl = x.Key.RequestUrl,
RequestCount = x.Count()
}).ToList();
if (brokenIpCountList.Count > 0)
{
//抽取出超过IP数量的规则列表，需要新增或OPEN Cloudflare的Rate Limiting Rule
var brokenRuleList = brokenIpCountList.Where(x => x.RateLimitTriggerIpCount <= x.RequestCount).ToList();
if (brokenRuleList.Count > 0)
{
ruleResult = new Result()
{
RuleId = rateLimit.ID,
Url = rateLimit.Url,
Threshold = rateLimit.Threshold,
Period = rateLimit.Period,
EnlargementFactor = rateLimit.EnlargementFactor,
RateLimitTriggerIpCount = rateLimit.RateLimitTriggerIpCount,
BrokenIpList = new List <BrokenIp>()
};
var brokenIpList = new List <BrokenIp>();
foreach (var rule in brokenRuleIpList)
{
brokenIpList.Add(new BrokenIp()
{
IP = rule.IP,
RequestRecords = ipRequestListIncludingQueryString.Where(x => x.IP.Equals(rule.IP))
.OrderByDescending(x => x.RequestCount).Select(x => new RequestRecord()
{
FullUrl = x.RequestFullUrl,
RequestCount = x.RequestCount,
HostName = x.RequestHost
}).ToList()
});
removedLogs = logAnalyzeModelList.ToList();
removedLogs.RemoveAll(p => p.IP == rule.IP);
logAnalyzeModelList = removedLogs;
}
ruleResult.BrokenIpList = brokenIpList;
analyzeResult.result.Add(ruleResult);
}
}
}
//删除已经触犯了当前ratelimit的所有url
removedLogs = logAnalyzeModelList.ToList();
removedLogs.RemoveAll(x => ifContainWildcard ? x.RequestUrl.ToLower().StartsWith(rateLimit.Url.ToLower().Replace("*", "")) : x.RequestUrl.ToLower().Equals(rateLimit.Url.ToLower()));
logAnalyzeModelList = removedLogs;
}
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "2-CloudflareLogs-!whitelist",
// Value = logAnalyzeModelList,
//}));
//var itemsGroup = logAnalyzeModelList.GroupBy(a => new { a.IP, a.RequestHost, a.RequestUrl })
// .Select(g => new LogAnalyzeModel
// {
// RequestHost = g.Key.RequestHost,
// IP = g.Key.IP,
// RequestUrl = g.Key.RequestUrl,
// RequestCount = g.Count()
// }).ToList();
//var result =
// (from analyzeModel in itemsGroup
// from config in rateLimitEntitiesSub
// where IfMatchCondition(config, analyzeModel)
// select new LogAnalyzeModel
// {
// RequestHost = analyzeModel.RequestHost,
// IP = analyzeModel.IP,
// RequestUrl = analyzeModel.RequestUrl,
// RequestCount = analyzeModel.RequestCount,
// RateLimitId = config.ID,
// });
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "3-CloudflareLogs-IfMatchCondition",
// Value = result,
//}));
//List<Result> results = new List<Result>();
//if (result != null && result.Count() > 0)
//{
// foreach (var item in result)
// {
// RateLimitEntity rateLimit = rateLimitEntitiesSub.FirstOrDefault(a => a.ID == item.RateLimitId);
// int ruleId = rateLimit.ID;
// int period = rateLimit.Period;
// int threshold = rateLimit.Threshold;
// string url = rateLimit.Url;
// var enlargementFactor = rateLimit.EnlargementFactor;
// //触发规则的 IP+Host 数据
// List<LogAnalyzeModel> logAnalyzesList = logAnalyzeModelList.Where(a => a.RequestHost == item.RequestHost && a.IP == item.IP)
// .GroupBy(a => a.RequestFullUrl)
// .Select(g => new LogAnalyzeModel
// {
// RequestFullUrl = g.Key,
// RequestCount = g.Count(),
// }).ToList();
// Result rst = results.FirstOrDefault(a => a.RuleId == ruleId);
// if (rst == null)
// {
// List<BrokenIp> brokenIpList = new List<BrokenIp>();
// rst = new Result
// {
// RuleId = ruleId,
// Period = period,
// Threshold = threshold,
// EnlargementFactor = enlargementFactor,
// Url = url,
// BrokenIpList = brokenIpList,
// };
// results.Add(rst);
// }
// BrokenIp brokenIp = new BrokenIp
// {
// IP = item.IP,
// RequestRecords = new List<RequestRecord>(),
// };
// rst.BrokenIpList.Add(brokenIp);
// if (logAnalyzesList != null)
// {
// foreach (var logAnalyze in logAnalyzesList)
// {
// brokenIp.RequestRecords.Add(new RequestRecord
// {
// FullUrl = logAnalyze.RequestFullUrl,
// RequestCount = logAnalyze.RequestCount,
// });
// }
// }
// }
//}
analyzeResult.ZoneId = zoneId;
analyzeResult.timeStage = accumulation;
//analyzeResult.result = null;
//logger.Debug(JsonConvert.SerializeObject(new
//{
// DataType = "4-CloudflareLogs-analyzeResult",
// Value = analyzeResult,
//}));
}
}
}
return(analyzeResult);
}