private string BanIpByRateLimitRule(ZoneEntity zoneEntity, bool ifTestStage, CloudFlareApiService cloudflare, int timeStage, string requestHost, string ip, int period, int threshold, int requestCount)
{
var sbDetail = new StringBuilder();
if (ifTestStage)
{
sbDetail.AppendFormat("Ban IP [{0}] successfully.<br />", ip);
}
else
{
var cloudflareAccessRuleResponse = cloudflare.BanIp(ip, "Ban Ip By Attack Prevent Windows service!");
if (cloudflareAccessRuleResponse.Success)
{
sbDetail.AppendFormat("Ban IP [{0}] and add ban history successfully.<br />", ip);
BanIpHistoryBusiness.Add(new BanIpHistory()
{
IP = ip,
ZoneId = zoneEntity.ZoneId,
RuleId = 0,
Remark = string.Format("IP [{0}] visited [{2}] [{3}] times, time range：[{1}].<br /> Exceeded rate limit threshold(Period=[{4}],Threshold=[{5}]).",
ip, timeStage, requestHost, requestCount, period, threshold)
});
}
else
{
sbDetail.AppendFormat("Ban IP [{0}] failure, the reason is：[{1}].<br />", ip, cloudflareAccessRuleResponse.Errors.Length > 0 ? cloudflareAccessRuleResponse.Errors[0].message : "No error message from Cloudflare.");
}
}
return(sbDetail.ToString());
}
private List <AuditLogEntity> BanIp(ZoneEntity zone, CloudFlareApiService cloudflare, AnalyzeResult analyzeResult)
{
List <AuditLogEntity> auditLogEntities = new List <AuditLogEntity>();
foreach (var rst in analyzeResult.result)
{
foreach (var broken in rst.BrokenIpList)
{
StringBuilder sbDetail = new StringBuilder();
if (rst.Url.EndsWith("*"))
{
var count = broken.RequestRecords.Sum(a => a.RequestCount);
//logger.Debug(JsonConvert.SerializeObject(new {
// count= count,
// RequestRecords = broken.RequestRecords
//}));
sbDetail.Append($"IP [{broken.IP}] visited [{rst.Url}] [{count}] times, time range：[{analyzeResult.timeStage}].<br /> Exceeded rate limiting threshold(URL=[{rst.Url}],Period=[{rst.Period}],Threshold=[{rst.Threshold}],EnlargementFactor=[{rst.EnlargementFactor}])，details(only list the top 10 records)：<br />");
var top10 = broken.RequestRecords.OrderByDescending(a => a.RequestCount).Take(10);
foreach (var item in top10)
{
sbDetail.AppendFormat("[{0}] {1} times.<br />", item.FullUrl, item.RequestCount);
}
}
else
{
var count = broken.RequestRecords.Sum(a => a.RequestCount);
//logger.Debug(JsonConvert.SerializeObject(new
//{
// count = count,
// RequestRecords = broken.RequestRecords
//}));
sbDetail.Append($"IP [{broken.IP}] visited [{rst.Url}] [{count}] times, time range：[{analyzeResult.timeStage}].<br /> Exceeded rate limiting threshold(URL=[{rst.Url}],Period=[{rst.Period}],Threshold=[{rst.Threshold}],EnlargementFactor=[{rst.EnlargementFactor}]).<br />");
}
string banIpLog = BanIpByRateLimitRule(zone, zone.IfTestStage, cloudflare, analyzeResult.timeStage, broken.RequestRecords.FirstOrDefault().HostName, broken.IP, rst.Period, rst.Threshold, rst.BrokenIpList.Count);
if (!string.IsNullOrEmpty(banIpLog))
{
sbDetail.Append(banIpLog);
}
auditLogEntities.Add(new AuditLogEntity(zone.TableID, LogLevel.Audit, sbDetail.ToString()));
}
}
return(auditLogEntities);
}
private bool isStoreToday;//2020.7.10
public CloudflareLogHandleSercie(string zoneId, string authEmail, string authKey, double sample, DateTime start, DateTime end, bool isStoreToday = false)
{
this.zoneId = zoneId;
this.authEmail = authEmail;
this.authKey = authKey;
this.sample = sample;
this.startTime = start;
this.endTime = end;
this.isStoreToday = isStoreToday;
_cloudFlareApiService = new CloudFlareApiService();
logService = new Business.LogService();
if (dicCloudflareLogsToday == null)
{
dicCloudflareLogsToday = new ConcurrentDictionary <string, List <CloudflareLog> >();
}
}
private void Analyze(AnalyzeResult analyzeResult)
{
if (analyzeResult != null && analyzeResult.result != null)
{
List <AuditLogEntity> auditLogEntities = new List <AuditLogEntity>();
string key = "AnalyzeRatelimit_GetZoneList_Key";
List <ZoneEntity> zoneList = Utils.GetMemoryCache(key, () =>
{
return(ZoneBusiness.GetZoneList());
}, 1440);
string zoneID = analyzeResult.ZoneId;
var zone = zoneList.FirstOrDefault(a => a.ZoneId == zoneID);
if (zone != null)
{
string authEmail = zone.AuthEmail;
string authKey = zone.AuthKey;
var cloudflare = new CloudFlareApiService(zone.ZoneId, zone.AuthEmail, zone.AuthKey);
//发送警报
Warn(analyzeResult);
//开启RateLimit
var logs = OpenRageLimit(zone, cloudflare, analyzeResult);
auditLogEntities.AddRange(logs);
//Ban IP
//logs = BanIp(zone, cloudflare, analyzeResult);
//auditLogEntities.AddRange(logs);
//记录日志
InsertLogs(auditLogEntities);
}
}
}
private void GeneratedWhiteListReport(string title, ZoneEntity zone, List <List <CloudflareLog> > cloudflareLogs)
{
var cloundFlareApiService = new CloudFlareApiService();
var whiteList = cloundFlareApiService.GetAccessRuleList(zone.ZoneId, zone.AuthEmail, zone.AuthKey, EnumMode.whitelist);
var subWhiteList = whiteList.Where(a => a.notes.Contains("WHITELIST CLEINT'S IP ADDRESS SITEID"))
.Select(a => new WhiteListModel
{
IP = a.configurationValue
}).ToList();
//var subWhiteList = new List< WhiteListModel>(){
// new WhiteListModel {
// IP= "131.242.135.253",
// },
// new WhiteListModel {
// IP= "131.242.135.252",
// }
//};
var totalList = cloudflareLogs.SelectMany(a => a)
.Join(subWhiteList,
left => left.ClientIP,
right => right.IP,
(left, right) => new
{
left.ClientIP,
left.ClientRequestHost
})
.GroupBy(a => new { a.ClientIP, a.ClientRequestHost })
.Select(
g => new
{
IP = g.Key.ClientIP,
HostName = g.Key.ClientRequestHost,
Count = g.Count(),
})
.OrderByDescending(a => a.Count);
List <string> ipList = new List <string>();
foreach (var item in totalList)
{
//取不同的20个IP
if (!ipList.Contains(item.IP))
{
ipList.Add(item.IP);
}
if (ipList.Count > 20)
{
break;
}
int max = GetMax(cloudflareLogs, item.IP, item.HostName);
int min = GetMin(cloudflareLogs, item.IP, item.HostName);
int avg = GetAvg(cloudflareLogs, item.IP, item.HostName);
List <string> urls = GetTop5Urls(cloudflareLogs, item.IP, item.HostName);
string urlsJson = JsonConvert.SerializeObject(urls);
int?maxHistory = ActionReportBusiness.GetMaxForWhiteList(zone.ZoneId, item.IP, item.HostName);
int?minHistory = ActionReportBusiness.GetMinForWhiteList(zone.ZoneId, item.IP, item.HostName);
int?avgHistory = ActionReportBusiness.GetAvgForWhiteList(zone.ZoneId, item.IP, item.HostName);
string maxDisplay = string.Format("{0}({1})", max, maxHistory.HasValue ? maxHistory.Value.ToString() : nothing);
string minDisplay = string.Format("{0}({1})", min, minHistory.HasValue ? minHistory.Value.ToString() : nothing);
string avgDisplay = string.Format("{0}({1})", avg, avgHistory.HasValue ? avgHistory.Value.ToString() : nothing);
ActionReport report = new ActionReport
{
IP = item.IP,
HostName = item.HostName,
Max = max,
Min = min,
Avg = avg,
FullUrl = urlsJson,
Title = title,
ZoneId = zone.ZoneId,
Count = item.Count,
Mode = "WhiteList",
CreatedTime = DateTime.UtcNow,
MaxDisplay = maxDisplay,
MinDisplay = minDisplay,
AvgDisplay = avgDisplay,
Remark = "",
};
ActionReportBusiness.Add(report);
}
}
private void GeneratedActiveReport(string title, ZoneEntity zone, List <List <CloudflareLog> > cloudflareLogs)
{
var cloundFlareApiService = new CloudFlareApiService();
var whiteList = cloundFlareApiService.GetAccessRuleList(zone.ZoneId, zone.AuthEmail, zone.AuthKey, EnumMode.whitelist);
var whiteListIps = whiteList.Select(a => a.configurationValue);
var totalList = cloudflareLogs.SelectMany(a => a)
.GroupBy(a => new { a.ClientIP, a.ClientRequestHost })
.Select(
g => new
{
IP = g.Key.ClientIP,
HostName = g.Key.ClientRequestHost,
Count = g.Count(),
})
.Where(a => !whiteListIps.Contains(a.IP))
.OrderByDescending(a => a.Count);
List <string> ipList = new List <string>();
foreach (var item in totalList)
{
//取不同的20个IP
if (!ipList.Contains(item.IP))
{
ipList.Add(item.IP);
}
if (ipList.Count > 20)
{
break;
}
int max = GetMax(cloudflareLogs, item.IP, item.HostName);
int min = GetMin(cloudflareLogs, item.IP, item.HostName);
int avg = GetAvg(cloudflareLogs, item.IP, item.HostName);
List <string> urls = GetTop5Urls(cloudflareLogs, item.IP, item.HostName);
string urlsJson = JsonConvert.SerializeObject(urls);
int?maxHistory = ActionReportBusiness.GetMaxForAction(zone.ZoneId, item.IP, item.HostName);
int?minHistory = ActionReportBusiness.GetMinForAction(zone.ZoneId, item.IP, item.HostName);
int?avgHistory = ActionReportBusiness.GetAvgForAction(zone.ZoneId, item.IP, item.HostName);
string maxDisplay = string.Format("{0}({1})", max, maxHistory.HasValue ? maxHistory.Value.ToString() : nothing);
string minDisplay = string.Format("{0}({1})", min, minHistory.HasValue ? minHistory.Value.ToString() : nothing);
string avgDisplay = string.Format("{0}({1})", avg, avgHistory.HasValue ? avgHistory.Value.ToString() : nothing);
ActionReport report = new ActionReport
{
IP = item.IP,
HostName = item.HostName,
Max = max,
Min = min,
Avg = avg,
FullUrl = urlsJson,
Title = title,
ZoneId = zone.ZoneId,
Count = item.Count,
Mode = "Action",
CreatedTime = DateTime.UtcNow,
MaxDisplay = maxDisplay,
MinDisplay = minDisplay,
AvgDisplay = avgDisplay,
Remark = "",
};
ActionReportBusiness.Add(report);
}
}
private List <AuditLogEntity> OpenRageLimit(ZoneEntity zone, CloudFlareApiService cloudflare, AnalyzeResult analyzeResult)
{
List <AuditLogEntity> auditLogEntities = new List <AuditLogEntity>();
var sbDetail = new StringBuilder();
#region Open Rate Limiting Rule
foreach (var rst in analyzeResult.result)
{
// 更新 Rate Limit Trigger Time
RateLimitBusiness.TriggerRateLimit(new RateLimitEntity()
{
Url = rst.Url,
Period = rst.Period,
Threshold = rst.Threshold,
ZoneId = zone.ZoneId
});
sbDetail = new StringBuilder(
$"[{rst.BrokenIpList.Count}] IPs exceeded rate limiting threshold(Url=[{rst.Url}],Threshold=[{rst.Threshold}],Period=[{rst.Period}],EnlargementFactor=[{rst.EnlargementFactor}],RateLimitTriggerIpCount=[{rst.RateLimitTriggerIpCount}]), time range：[{analyzeResult.timeStage}], details：<br />");
foreach (var rule in rst.BrokenIpList)
{
sbDetail.AppendFormat("IP [{0}] visited [{1}] times.<br /> ", rule.IP, rule.RequestRecords.Sum(x => x.RequestCount));
}
//auditLogEntities.Add(new AuditLogEntity(zone.TableID, LogLevel.App, sbDetail.ToString()));
////sbDetail.AppendFormat("Start opening rate limiting rule in Cloudflare [URL=[{0}],Threshold=[{1}],Period=[{2}]].<br />", rateLimit.Url, rateLimit.Threshold, rateLimit.Period);
//sbDetail = new StringBuilder();
if (zone != null && zone.IfTestStage)
{
sbDetail.AppendFormat("Open rate limiting rule in Cloudflare [URL=[{0}],Threshold=[{1}],Period=[{2}]] successfully.<br />", rst.Url, rst.Threshold, rst.Period);
}
else
{
if (cloudflare.OpenRateLimit(rst.Url, rst.Threshold, rst.Period, out var errorLog))
{
sbDetail.AppendFormat("Open rate limiting rule in Cloudflare [URL=[{0}],Threshold=[{1}],Period=[{2}]] successfully.<br />", rst.Url, rst.Threshold, rst.Period);
}
else
{
sbDetail.AppendFormat(errorLog.Detail);
}
}
auditLogEntities.Add(new AuditLogEntity(zone.TableID, LogLevel.Audit, sbDetail.ToString()));
//Ban Ip
foreach (var broken in rst.BrokenIpList)
{
sbDetail = new StringBuilder();
if (rst.Url.EndsWith("*"))
{
var count = broken.RequestRecords.Sum(a => a.RequestCount);
sbDetail.Append($"IP [{broken.IP}] visited [{rst.Url}] [{count}] times, time range：[{analyzeResult.timeStage}].<br /> Exceeded rate limiting threshold(URL=[{rst.Url}],Period=[{rst.Period}],Threshold=[{rst.Threshold}],EnlargementFactor=[{rst.EnlargementFactor}])，details(only list the top 10 records)：<br />");
var top10 = broken.RequestRecords.OrderByDescending(a => a.RequestCount).Take(10);
foreach (var item in top10)
{
sbDetail.AppendFormat("[{0}] {1} times.<br />", item.FullUrl, item.RequestCount);
}
}
else
{
var count = broken.RequestRecords.Sum(a => a.RequestCount);
sbDetail.Append($"IP [{broken.IP}] visited [{rst.Url}] [{count}] times, time range：[{analyzeResult.timeStage}].<br /> Exceeded rate limiting threshold(URL=[{rst.Url}],Period=[{rst.Period}],Threshold=[{rst.Threshold}],EnlargementFactor=[{rst.EnlargementFactor}]).<br />");
}
string banIpLog = BanIpByRateLimitRule(zone, zone.IfTestStage, cloudflare, analyzeResult.timeStage, broken.RequestRecords.FirstOrDefault().HostName, broken.IP, rst.Period, rst.Threshold, rst.BrokenIpList.Count);
if (!string.IsNullOrEmpty(banIpLog))
{
sbDetail.Append(banIpLog);
}
auditLogEntities.Add(new AuditLogEntity(zone.TableID, LogLevel.Audit, sbDetail.ToString()));
}
}
#endregion
return(auditLogEntities);
}
