void handle_result(AwsHttpResult result)
{
bool retry = false;
bool failed = false;
string err_msg = "";
if (!result.successful())
{
failed = true;
err_msg = result.error();
retry = true;
}
else if (result.status_code() != 200)
{
failed = true;
err_msg = result.response_body();
retry = result.status_code() >= 500;
}
if (failed)
{
StdErrorOut.Instance.StdError(string.Format("Metrics upload failed: \n{0}\nRequest was: \n {1}", err_msg, result.context <string>()));
}
if (retry)
{
lock (mutex_)
retryable_requests.Add(result.context <string>());
}
}
void retry_not_expired(AwsHttpResult result)
{
retry_not_expired
(
result,
result.successful() ? result.status_code().ToString() : "Exception",
result.successful() ? result.response_body().Substring(0, 4096) : result.error()
);
}
void emit_metrics(AwsHttpResult result)
{
MetricsPutter metrics_putter = new MetricsPutter(metrics_manager_, result);
PutRecordsRequest prr = result.context <PutRecordsRequest>();
double num_urs = 0;
foreach (var kr in prr.Items())
{
metrics_putter.put(Names.UserRecordsPerKinesisRecord, kr.Items().Count, (ulong)kr.Items()[kr.Items().Count - 1].Predicted_shard());
num_urs += kr.Items().Count;
}
metrics_putter.put
(Names.RequestTime, result.duration_millis()).put
(Names.KinesisRecordsPerPutRecordsRequest, prr.Items().Count).put
(Names.UserRecordsPerPutRecordsRequest, num_urs);
string err_code = null;
if (result.successful())
{
var status_code = result.status_code();
if (status_code != 200)
{
// TODO parse the json (if any) to get the error code
err_code = "Http" + status_code;
}
}
else
{
err_code = result.error().Substring(0, 255);
}
if (err_code != null)
{
metrics_putter.put
(Names.ErrorsByCode, 1, 0, err_code).put
(Names.AllErrors, 1);
}
}
void handle_put_records_result(AwsHttpResult result)
{
emit_metrics(result);
try
{
if (result.successful())
{
var status_code = result.status_code();
if (status_code == 200)
{
on_200(result);
}
else if (status_code >= 500 && status_code < 600)
{
retry_not_expired(result);
}
else
{
// For PutRecords, errors that apply to individual kinesis records
// (like throttling, too big or bad format) come back in code 200s.
// This is different from plain old PutRecord, where those come back
// with code 400. As such, all the errors we want to retry on are
// handled in the 200 case. All 400 codes are therefore not retryable.
StdErrorOut.Instance.StdError(string.Format("PutRecords failed: {0}", result.response_body()));
fail(result);
}
}
else
{
retry_not_expired(result);
}
}
catch (Exception ex)
{
StdErrorOut.Instance.StdError(string.Format("Unexpected error encountered processing http result: {0}", ex.ToString()));
fail(result, "Unexpected Error", ex.ToString());
}
}
void fail(AwsHttpResult result)
{
fail(result,
result.successful() ? result.status_code().ToString() : "Exception",
result.successful() ? result.response_body().Substring(0, 4096) : result.error());
}
void update_callback(AwsHttpResult result)
{
if (!result.successful())
{
update_fail(result.error());
return;
}
if (result.status_code() != 200)
{
update_fail(result.response_body());
return;
}
try
{
dynamic json = System.Web.Helpers.Json.Decode(result.response_body());
var stream_description = json["StreamDescription"];
string stream_status = stream_description["StreamStatus"];
if (stream_status != "ACTIVE" && stream_status != "UPDATING")
{
string ss = "Stream status is " + stream_status;
throw new Exception(ss);
}
List <dynamic> shards = stream_description["Shards"];
for (int i = 0; i < shards.Count; i++)
{
// Check if the shard is closed, if so, do not use it.
if (shards[i]["SequenceNumberRange"]["EndingSequenceNumber"])
{
continue;
}
end_hash_key_to_shard_id.Add(new KeyValuePair <BigInteger, ulong>(BigInteger.Parse(shards[i]["HashKeyRange"]["EndingHashKey"]), shard_id_from_str(shards[i]["ShardId"])));
}
backoff = min_backoff;
if (stream_description["HasMoreShards"])
{
update(shards[shards.Count - 1]["ShardId"]);
return;
}
end_hash_key_to_shard_id.Sort();
lock (mutex_)
{
state = State.READY;
updated_at = DateTime.Now;
}
StdErrorOut.Instance.StdOut(LogLevel.info, string.Format("Successfully updated shard map for stream \"{0}\" found {1} shards", stream, end_hash_key_to_shard_id.Count));
}
catch (Exception ex)
{
update_fail(ex.ToString());
}
}
