protected override void ApplicationStartup(TinyIoCContainer container, IPipelines pipelines)
{
Log.Debug("ApplicationStartup");
pipelines.EnableGzipCompression(new GzipCompressionSettings()
{
MinimumBytes = 1024
});
pipelines.AddHeadersAndLogging();
pipelines.AddRequestStashing();
// rate-limiting - uses remote address by default, but we can extend this to include user-agent and/or
// authenticated user if necessary
LeakyBucketRateLimiter.Enable(pipelines, new LeakyBucketRateLimiterConfiguration
{
// 100 requests per minute should be plenty
MaxNumberOfRequests = 100, RefreshRate = TimeSpan.FromMinutes(1)
});
// setup auth last so that rate-limited responses include authenticated user
// (uses AddItemToStartOfPipeline() to ensure it happens before rate-limiter)
var auth_method = Env.GetEnvironmentString("COMPUTE_AUTH_METHOD", "");
if (auth_method == "RHINO_ACCOUNT")
{
pipelines.AddAuthRhinoAccount();
}
else if (auth_method == "API_KEY")
{
pipelines.AddAuthApiKey();
}
base.ApplicationStartup(container, pipelines);
}
public ResponseDto.Root?Call(int num = 10)
{
var body = new RequestDto()
{
Num = num,
R18 = 2
};
var jsonSerializeSetting = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
DefaultValueHandling = DefaultValueHandling.Ignore,
ContractResolver = new CamelCasePropertyNamesContractResolver()
};
string responseStr = null;
using (var limiter = new LeakyBucketRateLimiter(nameof(SetuAPI), 1))
{
if (limiter.CanRequest())
{
responseStr = WebHelper.HttpPost("https://api.lolicon.app/setu/v2", JsonConvert.SerializeObject(body, jsonSerializeSetting),
WebHelper.WebContentType.Json);
}
}
return(responseStr == null ? null : JsonConvert.DeserializeObject <ResponseDto.Root>(responseStr, jsonSerializeSetting));
}
public void Should_throw_with_null_configuration()
{
var pipelines = A.Fake <IPipelines>();
var module = A.Fake <INancyModule>();
Assert.Throws <ArgumentNullException>(() => LeakyBucketRateLimiter.Enable(pipelines, null));
Assert.Throws <ArgumentNullException>(() => LeakyBucketRateLimiter.Enable(module, null));
}
public void Should_add_before_hook_in_module_when_enabled()
{
var module = A.Fake <INancyModule>();
LeakyBucketRateLimiter.Enable(module, new LeakyBucketRateLimiterConfiguration());
A.CallTo(() => module.Before.AddItemToStartOfPipeline(A <Func <NancyContext, Response> > .Ignored))
.MustHaveHappened(Repeated.Exactly.Once);
}
public void Should_add_before_request_hook_in_application_when_enabled()
{
var pipelines = A.Fake <IPipelines>();
LeakyBucketRateLimiter.Enable(pipelines, new LeakyBucketRateLimiterConfiguration());
A.CallTo(() => pipelines.BeforeRequest.AddItemToStartOfPipeline(A <Func <NancyContext, Response> > .Ignored))
.MustHaveHappened(Repeated.Exactly.Once);
}
private static INancyBootstrapper CreateBootstrapper(LeakyBucketRateLimiterConfiguration configuration)
{
return(new ConfigurableBootstrapper(config =>
{
config.ApplicationStartup((container, pipelines) =>
{
LeakyBucketRateLimiter.Enable(pipelines, configuration);
});
}));
}
/// <summary>
/// 调用罗马音API
/// </summary>
/// <param name="japanese"></param>
/// <returns></returns>
public string CallAPI(string japanese)
{
if (japanese.IsNullOrWhiteSpace())
{
return(null);
}
string result;
using (var limiter = new LeakyBucketRateLimiter(nameof(KouRomajiHelper), 1))
{
if (!limiter.CanRequest())
{
this.InheritError(limiter, "发生在" + nameof(KouRomajiHelper) + "中的" + nameof(CallAPI));
}
string data = "mode=japanese&q=" + HttpUtility.UrlEncode(japanese);
result = WebHelper.HttpPost("http://www.kawa.net/works/ajax/romanize/romanize.cgi ", data, WebHelper.WebContentType.General);
}
return(result);
}
public Root CallAPI(string str)
{
if (str.IsNullOrWhiteSpace())
{
return(null);
}
string result;
using (var limiter = new LeakyBucketRateLimiter(nameof(KouNbnhhsh), 2))
{
if (!limiter.CanRequest())
{
this.InheritError(limiter, "发生在" + nameof(KouNbnhhsh) + "中的" + nameof(CallAPI));
return(null);
}
result = WebHelper.HttpPost("https://lab.magiconch.com/api/nbnhhsh/guess/", "{\"text\":\"" + str + "\"}", WebHelper.WebContentType.Json);
}
result = "{\"result\":" + result + "}";
Root root = JsonConvert.DeserializeObject <Root>(result);
return(root);
}
public void Should_throw_with_null_module_with_configuration()
{
var config = new LeakyBucketRateLimiterConfiguration();
Assert.Throws <ArgumentNullException>(() => LeakyBucketRateLimiter.Enable((INancyModule)null, config));
}
public static void ConfigureRateLimiting(HttpFilterCollection filters)
{
#region Setting up the Redis rate limiter
var redisRateLimiterSettings = new RedisRateLimiterSettings();
ConfigureRateLimitingSettings(redisRateLimiterSettings);
var rateLimitCacheProvider = new LeakyBucketRateLimiter(
redisRateLimiterSettings.RateLimitRedisCacheConnectionString,
onException: (ex) =>
{
//_logger.LogError("Error connecting to Redis")
},
circuitBreaker: new DefaultCircuitBreaker(redisRateLimiterSettings.FaultThreshholdPerWindowDuration,
redisRateLimiterSettings.FaultWindowDurationInMilliseconds, redisRateLimiterSettings.CircuitOpenIntervalInSecs,
onCircuitException: (ex) =>
{
//_logger.LogError("Rate limiting circuit error")
},
onCircuitOpened: () =>
{
//_logger.LogWarning("Rate limiting circuit opened")
},
onCircuitClosed: () =>
{
//logger.LogWarning("Rate limiting circuit closed")
}),
countThrottledRequests: true
);
#endregion
var configSettings = new DomainElasticSearchLoggingOptions()
{
Enabled = true,
Url = "http://localhost:9200",
Application = "Public Adapter",
IndexFormat = "quotaaudits-{0:yyyy.MM.dd}"
};
var loggerConfig = new LoggerConfiguration()
.MinimumLevel.Information()
.Enrich.FromLogContext();
// if enabled in appsettings.json, logs will be shipped to Elasticsearch in a standard json format
if (configSettings.Enabled)
{
loggerConfig.WriteTo.DomainElasticsearch(configSettings);
}
var auditLogger = loggerConfig.CreateLogger();
var policyProvider = new ClientQuotaPolicyProvider();
filters.Add(new RateLimitingFilter(
new RateLimiter(rateLimitCacheProvider, new RateLimitingPolicyManager(policyProvider)),
filters,
onPostLimit: async(request, policy, result, actionContext) =>
{
var operationClass = CallClassification.RouteTemplateToClassMap[request.RouteTemplate];
var cost = CallClassification.CostPerClass[operationClass];
var clientId = result.CacheKey.RequestId;
// sns publish
if (result.State == ResultState.Success)
{
// sns publish
auditLogger.Information(
"Result {Result}: Request success for client {ClientId} and endpoint {Endpoint} with route {RouteTemplate} which is Class {Class} with Cost {Cost}",
"Success",
clientId,
request.Path,
request.RouteTemplate,
operationClass,
cost);
}
else if (result.State == ResultState.Throttled)
{
auditLogger.Information(
"Result {Result}: throttled for client {ClientId} and endpoint {Endpoint} with route {RouteTemplate} which is Class {Class} with Cost {Cost} by violating policy {ViolatedPolicy}",
"Throttled",
clientId,
request.Path,
request.RouteTemplate,
operationClass,
cost,
$"{policy.Name}:{result.CacheKey.AllowedConsumptionRate}");
}
else if (result.State == ResultState.ThrottledButCompensationFailed)
{
auditLogger.Information(
"Result {Result}: throttled but failed to compensate for client {ClientId} and endpoint {Endpoint} with route {RouteTemplate} which is Class {Class} with Cost {Cost} by violating policy {ViolatedPolicy}",
result.State,
clientId,
request.Path,
request.RouteTemplate,
operationClass,
cost,
$"{policy.Name}:{result.CacheKey.AllowedConsumptionRate}");
}
else if (result.State == ResultState.LimitApplicationFailed)
{
auditLogger.Information(
"Result {Result}: Free pass for client {ClientId} and endpoint {Endpoint} with route {RouteTemplate} which is Class {Class} with Cost {Cost}",
"FreePass",
clientId,
request.Path,
request.RouteTemplate,
operationClass,
cost);
}
else if (result.State == ResultState.NotApplicable)
{
auditLogger.Information(
"Result {Result}: Not Applicable for client {ClientId} and endpoint {Endpoint} with route {RouteTemplate} which is Class {Class} with Cost {Cost}",
ResultState.NotApplicable,
clientId,
request.Path,
request.RouteTemplate,
operationClass,
cost);
}
return(Decision.OK);
},
onPostLimitRevert: async(request, policy, result, actionContext) =>
{
var operationClass = CallClassification.RouteTemplateToClassMap[request.RouteTemplate];
if (result.State == ResultState.Success || result.State == ResultState.Throttled)
{
auditLogger.Information(
"Result {Result}: Limit Cost Reverted for client {ClientId} and endpoint {Endpoint} with route {RouteTemplate} which is Class {Class} with Cost {Cost}",
"SuccessCostReverted",
result.CacheKey.RequestId,
request.Path,
request.RouteTemplate,
operationClass,
-policy.CostPerCall);
}
else if (result.State == ResultState.LimitApplicationFailed)
{
auditLogger.Information(
"Result {Result}: Limit Cost Reverting failed for client {ClientId} and endpoint {Endpoint} with route {RouteTemplate} which is Class {Class} with Cost {Cost}",
"SuccessCostRevertingFailed",
result.CacheKey.RequestId,
request.Path,
request.RouteTemplate,
operationClass,
-policy.CostPerCall);
}
},
postOperationDecisionFuncAsync: async(request, policy, result, actionExecutedContext) =>
{
if (actionExecutedContext.Exception != null || (int)actionExecutedContext.Response.StatusCode >= 400)
{
return(Decision.REVERTSUCCESSCOST);
}
return(Decision.OK);
},
//getPolicyFuncAsync: policyProvider.GetPolicyAsync,
simulationMode: false));
}
