public async Task <HttpResponseMessage> InvokeAsync(HttpRequestMessage req, TraceWriter log,
Func <CrawlResponse, object> requestBodyFunc,
Action <CrawlResponse, BlobContent> responseAction,
CancellationToken cancellationToken)
{
log.Info("Crawl." + this.containerName);
await this.InitializeAsync();
string reqBodyStr = null;
CrawlResponse reqBody = null;
BlobContent blobContent = null;
try
{
using (var operation = Services.TelemetryClient.StartOperation <DependencyTelemetry>("Crawl." + this.containerName))
{
reqBodyStr = await req.Content.ReadAsStringAsync();
reqBody = JsonConvert.DeserializeObject <CrawlResponse>(reqBodyStr);
operation.Telemetry.Target = this.endpoint;
operation.Telemetry.Properties.Add("AppId", reqBody.Site);
operation.Telemetry.Properties.Add("ActionId", reqBody.Id);
blobContent = await this.PostAsync(
log,
reqBody.Site,
reqBody.Id,
requestBodyFunc(reqBody),
reqBody.ForceRefresh,
cancellationToken);
if (blobContent != null)
{
operation.Telemetry.Properties.Add("Expires", blobContent.Expires.ToString(CultureInfo.InvariantCulture));
if (blobContent.Value != null)
{
responseAction(reqBody, blobContent);
operation.Telemetry.ResultCode = "OK";
}
}
return(req.CreateResponse(blobContent));
}
}
catch (Exception ex)
{
Services.TrackException(ex, req, log, reqBodyStr, reqBody, blobContent);
throw ex;
}
}
public static HttpResponseMessage CreateResponse(BlobContent blobContent)
{
blobContent.Output?.Add(new JProperty("_expires", blobContent.Expires));
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(
blobContent.Output?.ToString(Formatting.None) ?? string.Empty,
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
"application/json")
};
// Get replaced in deployed version
// response.Content.Headers.Expires = expires;
// response.Content.Headers.TryAddWithoutValidation("X-DecisionService-Expires", expires.ToString("ddd, dd MMM yyyy HH:mm:ss 'GMT'"));
return(response);
}
public static void TrackException(Exception ex, HttpRequestMessage req, TraceWriter log, string reqBodyStr, CrawlResponse reqBody, BlobContent blobContent = null)
{
var props = new Dictionary <string, string>
{
{ "Service", req.RequestUri.ToString() },
{ "Request", reqBodyStr }
};
if (reqBody != null)
{
props.Add("AppId", reqBody.Site);
props.Add("ActionId", reqBody.Id);
}
if (blobContent != null)
{
props.Add("Response", blobContent.Value);
}
TelemetryClient.TrackException(ex, props);
log.Error($"Request for AppId={reqBody?.Site} ActionId={reqBody?.Id} failed", ex);
}
public async Task <BlobContent> PostAsync(TraceWriter log, string site, string id, object request, bool forceRefresh, CancellationToken cancellationToken)
{
await this.InitializeAsync();
var stopwatch = Stopwatch.StartNew();
var cacheHit = true;
HttpResponseMessage responseMessage = null;
string body = null;
try
{
body = request as string;
string input;
string contentType;
if (body != null)
{
// if this is a raw string, we need to escape for storage
input = JsonConvert.SerializeObject(request);
contentType = "text/plain";
}
else
{
body = JsonConvert.SerializeObject(request);
input = body;
contentType = "application/json";
}
log.Trace(new TraceEvent(TraceLevel.Verbose,
$"Requesting {this.containerName} at {this.endpoint}: {body}"));
var blobCache = new BlobCache(this.storageConnectionString);
// lookup Azure Blob storage cache first
// have a 5min timeout for retries
BlobContent blobContent = null;
if (!forceRefresh)
{
blobContent = await blobCache.GetAsync(site, id, this.containerName, input, TimeSpan.FromMinutes(5), cancellationToken);
}
if (blobContent == null)
{
cacheHit = false;
var stopwatchReqeust = Stopwatch.StartNew();
// make the actual HTTP request
responseMessage = await this.client.PostAsync(
string.Empty,
new StringContent(
body,
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
contentType));
Services.TelemetryClient.TrackDependency(this.containerName, this.endpoint, this.containerName, null,
DateTime.UtcNow, stopwatchReqeust.Elapsed,
$"{responseMessage.StatusCode} {responseMessage.ReasonPhrase}", responseMessage.IsSuccessStatusCode);
log.Trace(new TraceEvent(TraceLevel.Verbose, $"Response: {responseMessage.StatusCode} {responseMessage.ReasonPhrase}"));
if (!responseMessage.IsSuccessStatusCode)
{
blobContent = new BlobContent
{
// TODO: random expiration
Expires = DateTime.UtcNow + TimeSpan.FromMinutes(1),
};
}
else
{
var responseStr = await responseMessage.Content.ReadAsStringAsync();
log.Trace(new TraceEvent(TraceLevel.Verbose, $"Result {this.containerName} at {this.endpoint}: {responseStr}"));
// once we got a response, cache for 3 days
// TODO: add configuration option
// TODO: add force refresh parameter
blobContent = await blobCache.PersistAsync(site, id, this.containerName, input, responseStr, TimeSpan.FromDays(3), cancellationToken);
}
}
return(blobContent);
}
finally
{
var props = new Dictionary <string, string>
{
{ "site", site },
{ "id", id },
{ "cacheHit", cacheHit.ToString() },
{ "StatusCode", responseMessage?.StatusCode.ToString() },
{ "Reason", responseMessage?.ReasonPhrase }
};
var sb = new StringBuilder(this.containerName);
if (responseMessage != null && responseMessage.StatusCode != System.Net.HttpStatusCode.OK)
{
props.Add("Request", body);
sb.Append(" failed");
}
Services.TelemetryClient.TrackEvent(
sb.ToString(),
props,
metrics: new Dictionary <string, double>
{
{ "requestTime", stopwatch.ElapsedMilliseconds }
});
}
}
public async Task <HttpResponseMessage> InvokeAsync(HttpRequestMessage req, TraceWriter log,
Func <CrawlResponse, object> requestBodyFunc,
Action <CrawlResponse, BlobContent> responseAction,
bool isPost,
CancellationToken cancellationToken)
{
log.Info("Crawl." + this.containerName);
await this.InitializeAsync();
string reqBodyStr = null;
CrawlResponse reqBody = null;
BlobContent blobContent = null;
try
{
using (var operation = Services.TelemetryClient.StartOperation <DependencyTelemetry>("Crawl." + this.containerName))
{
reqBodyStr = await req.Content.ReadAsStringAsync();
reqBody = JsonConvert.DeserializeObject <CrawlResponse>(reqBodyStr);
operation.Telemetry.Target = this.endpoint;
operation.Telemetry.Properties.Add("AppId", reqBody.Site);
operation.Telemetry.Properties.Add("ActionId", reqBody.Id);
var serviceRequestBody = requestBodyFunc(reqBody);
if (serviceRequestBody == null)
{
return(new HttpResponseMessage(System.Net.HttpStatusCode.OK)
{
Content = new StringContent(
string.Empty,
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
"application/json")
});
}
blobContent = await this.RequestAsync(
log,
reqBody.Site,
reqBody.Id,
serviceRequestBody,
reqBody.ForceRefresh,
isPost,
cancellationToken);
if (blobContent != null)
{
operation.Telemetry.Properties.Add("Expires", blobContent.Expires.ToString(CultureInfo.InvariantCulture));
if (blobContent.Value != null)
{
responseAction(reqBody, blobContent);
operation.Telemetry.ResultCode = "OK";
}
}
return(Services.CreateResponse(blobContent));
}
}
catch (Exception ex)
{
Services.TrackException(ex, req, log, reqBodyStr, reqBody, blobContent);
throw ex;
}
}