public async Task <HttpResponseMessage> InvokeAsync(HttpRequestMessage req, TraceWriter log, Func <CrawlResponse, object> requestBodyFunc, Action <CrawlResponse, BlobContent> responseAction, CancellationToken cancellationToken) { log.Info("Crawl." + this.containerName); await this.InitializeAsync(); string reqBodyStr = null; CrawlResponse reqBody = null; BlobContent blobContent = null; try { using (var operation = Services.TelemetryClient.StartOperation <DependencyTelemetry>("Crawl." + this.containerName)) { reqBodyStr = await req.Content.ReadAsStringAsync(); reqBody = JsonConvert.DeserializeObject <CrawlResponse>(reqBodyStr); operation.Telemetry.Target = this.endpoint; operation.Telemetry.Properties.Add("AppId", reqBody.Site); operation.Telemetry.Properties.Add("ActionId", reqBody.Id); blobContent = await this.PostAsync( log, reqBody.Site, reqBody.Id, requestBodyFunc(reqBody), reqBody.ForceRefresh, cancellationToken); if (blobContent != null) { operation.Telemetry.Properties.Add("Expires", blobContent.Expires.ToString(CultureInfo.InvariantCulture)); if (blobContent.Value != null) { responseAction(reqBody, blobContent); operation.Telemetry.ResultCode = "OK"; } } return(req.CreateResponse(blobContent)); } } catch (Exception ex) { Services.TrackException(ex, req, log, reqBodyStr, reqBody, blobContent); throw ex; } }
public static HttpResponseMessage CreateResponse(BlobContent blobContent) { blobContent.Output?.Add(new JProperty("_expires", blobContent.Expires)); var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent( blobContent.Output?.ToString(Formatting.None) ?? string.Empty, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), "application/json") }; // Get replaced in deployed version // response.Content.Headers.Expires = expires; // response.Content.Headers.TryAddWithoutValidation("X-DecisionService-Expires", expires.ToString("ddd, dd MMM yyyy HH:mm:ss 'GMT'")); return(response); }
public static void TrackException(Exception ex, HttpRequestMessage req, TraceWriter log, string reqBodyStr, CrawlResponse reqBody, BlobContent blobContent = null) { var props = new Dictionary <string, string> { { "Service", req.RequestUri.ToString() }, { "Request", reqBodyStr } }; if (reqBody != null) { props.Add("AppId", reqBody.Site); props.Add("ActionId", reqBody.Id); } if (blobContent != null) { props.Add("Response", blobContent.Value); } TelemetryClient.TrackException(ex, props); log.Error($"Request for AppId={reqBody?.Site} ActionId={reqBody?.Id} failed", ex); }
public async Task <BlobContent> PostAsync(TraceWriter log, string site, string id, object request, bool forceRefresh, CancellationToken cancellationToken) { await this.InitializeAsync(); var stopwatch = Stopwatch.StartNew(); var cacheHit = true; HttpResponseMessage responseMessage = null; string body = null; try { body = request as string; string input; string contentType; if (body != null) { // if this is a raw string, we need to escape for storage input = JsonConvert.SerializeObject(request); contentType = "text/plain"; } else { body = JsonConvert.SerializeObject(request); input = body; contentType = "application/json"; } log.Trace(new TraceEvent(TraceLevel.Verbose, $"Requesting {this.containerName} at {this.endpoint}: {body}")); var blobCache = new BlobCache(this.storageConnectionString); // lookup Azure Blob storage cache first // have a 5min timeout for retries BlobContent blobContent = null; if (!forceRefresh) { blobContent = await blobCache.GetAsync(site, id, this.containerName, input, TimeSpan.FromMinutes(5), cancellationToken); } if (blobContent == null) { cacheHit = false; var stopwatchReqeust = Stopwatch.StartNew(); // make the actual HTTP request responseMessage = await this.client.PostAsync( string.Empty, new StringContent( body, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), contentType)); Services.TelemetryClient.TrackDependency(this.containerName, this.endpoint, this.containerName, null, DateTime.UtcNow, stopwatchReqeust.Elapsed, $"{responseMessage.StatusCode} {responseMessage.ReasonPhrase}", responseMessage.IsSuccessStatusCode); log.Trace(new TraceEvent(TraceLevel.Verbose, $"Response: {responseMessage.StatusCode} {responseMessage.ReasonPhrase}")); if (!responseMessage.IsSuccessStatusCode) { blobContent = new BlobContent { // TODO: random expiration Expires = DateTime.UtcNow + TimeSpan.FromMinutes(1), }; } else { var responseStr = await responseMessage.Content.ReadAsStringAsync(); log.Trace(new TraceEvent(TraceLevel.Verbose, $"Result {this.containerName} at {this.endpoint}: {responseStr}")); // once we got a response, cache for 3 days // TODO: add configuration option // TODO: add force refresh parameter blobContent = await blobCache.PersistAsync(site, id, this.containerName, input, responseStr, TimeSpan.FromDays(3), cancellationToken); } } return(blobContent); } finally { var props = new Dictionary <string, string> { { "site", site }, { "id", id }, { "cacheHit", cacheHit.ToString() }, { "StatusCode", responseMessage?.StatusCode.ToString() }, { "Reason", responseMessage?.ReasonPhrase } }; var sb = new StringBuilder(this.containerName); if (responseMessage != null && responseMessage.StatusCode != System.Net.HttpStatusCode.OK) { props.Add("Request", body); sb.Append(" failed"); } Services.TelemetryClient.TrackEvent( sb.ToString(), props, metrics: new Dictionary <string, double> { { "requestTime", stopwatch.ElapsedMilliseconds } }); } }
public async Task <HttpResponseMessage> InvokeAsync(HttpRequestMessage req, TraceWriter log, Func <CrawlResponse, object> requestBodyFunc, Action <CrawlResponse, BlobContent> responseAction, bool isPost, CancellationToken cancellationToken) { log.Info("Crawl." + this.containerName); await this.InitializeAsync(); string reqBodyStr = null; CrawlResponse reqBody = null; BlobContent blobContent = null; try { using (var operation = Services.TelemetryClient.StartOperation <DependencyTelemetry>("Crawl." + this.containerName)) { reqBodyStr = await req.Content.ReadAsStringAsync(); reqBody = JsonConvert.DeserializeObject <CrawlResponse>(reqBodyStr); operation.Telemetry.Target = this.endpoint; operation.Telemetry.Properties.Add("AppId", reqBody.Site); operation.Telemetry.Properties.Add("ActionId", reqBody.Id); var serviceRequestBody = requestBodyFunc(reqBody); if (serviceRequestBody == null) { return(new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent( string.Empty, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), "application/json") }); } blobContent = await this.RequestAsync( log, reqBody.Site, reqBody.Id, serviceRequestBody, reqBody.ForceRefresh, isPost, cancellationToken); if (blobContent != null) { operation.Telemetry.Properties.Add("Expires", blobContent.Expires.ToString(CultureInfo.InvariantCulture)); if (blobContent.Value != null) { responseAction(reqBody, blobContent); operation.Telemetry.ResultCode = "OK"; } } return(Services.CreateResponse(blobContent)); } } catch (Exception ex) { Services.TrackException(ex, req, log, reqBodyStr, reqBody, blobContent); throw ex; } }