public async Task <HttpResponseMessage> InvokeAsync(HttpRequestMessage req, TraceWriter log,
                                                            Func <CrawlResponse, object> requestBodyFunc,
                                                            Action <CrawlResponse, BlobContent> responseAction,
                                                            CancellationToken cancellationToken)
        {
            log.Info("Crawl." + this.containerName);

            await this.InitializeAsync();

            string        reqBodyStr  = null;
            CrawlResponse reqBody     = null;
            BlobContent   blobContent = null;

            try
            {
                using (var operation = Services.TelemetryClient.StartOperation <DependencyTelemetry>("Crawl." + this.containerName))
                {
                    reqBodyStr = await req.Content.ReadAsStringAsync();

                    reqBody = JsonConvert.DeserializeObject <CrawlResponse>(reqBodyStr);

                    operation.Telemetry.Target = this.endpoint;
                    operation.Telemetry.Properties.Add("AppId", reqBody.Site);
                    operation.Telemetry.Properties.Add("ActionId", reqBody.Id);

                    blobContent = await this.PostAsync(
                        log,
                        reqBody.Site,
                        reqBody.Id,
                        requestBodyFunc(reqBody),
                        reqBody.ForceRefresh,
                        cancellationToken);

                    if (blobContent != null)
                    {
                        operation.Telemetry.Properties.Add("Expires", blobContent.Expires.ToString(CultureInfo.InvariantCulture));

                        if (blobContent.Value != null)
                        {
                            responseAction(reqBody, blobContent);

                            operation.Telemetry.ResultCode = "OK";
                        }
                    }

                    return(req.CreateResponse(blobContent));
                }
            }
            catch (Exception ex)
            {
                Services.TrackException(ex, req, log, reqBodyStr, reqBody, blobContent);
                throw ex;
            }
        }
Exemple #2
0
        public static HttpResponseMessage CreateResponse(BlobContent blobContent)
        {
            blobContent.Output?.Add(new JProperty("_expires", blobContent.Expires));

            var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
            {
                Content = new StringContent(
                    blobContent.Output?.ToString(Formatting.None) ?? string.Empty,
                    new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
                    "application/json")
            };

            // Get replaced in deployed version
            // response.Content.Headers.Expires = expires;

            // response.Content.Headers.TryAddWithoutValidation("X-DecisionService-Expires", expires.ToString("ddd, dd MMM yyyy HH:mm:ss 'GMT'"));

            return(response);
        }
Exemple #3
0
        public static void TrackException(Exception ex, HttpRequestMessage req, TraceWriter log, string reqBodyStr, CrawlResponse reqBody, BlobContent blobContent = null)
        {
            var props = new Dictionary <string, string>
            {
                { "Service", req.RequestUri.ToString() },
                { "Request", reqBodyStr }
            };

            if (reqBody != null)
            {
                props.Add("AppId", reqBody.Site);
                props.Add("ActionId", reqBody.Id);
            }

            if (blobContent != null)
            {
                props.Add("Response", blobContent.Value);
            }

            TelemetryClient.TrackException(ex, props);
            log.Error($"Request for AppId={reqBody?.Site} ActionId={reqBody?.Id} failed", ex);
        }
        public async Task <BlobContent> PostAsync(TraceWriter log, string site, string id, object request, bool forceRefresh, CancellationToken cancellationToken)
        {
            await this.InitializeAsync();

            var stopwatch = Stopwatch.StartNew();
            var cacheHit  = true;
            HttpResponseMessage responseMessage = null;
            string body = null;

            try
            {
                body = request as string;
                string input;
                string contentType;
                if (body != null)
                {
                    // if this is a raw string, we need to escape for storage
                    input       = JsonConvert.SerializeObject(request);
                    contentType = "text/plain";
                }
                else
                {
                    body        = JsonConvert.SerializeObject(request);
                    input       = body;
                    contentType = "application/json";
                }

                log.Trace(new TraceEvent(TraceLevel.Verbose,
                                         $"Requesting {this.containerName} at {this.endpoint}: {body}"));

                var blobCache = new BlobCache(this.storageConnectionString);

                // lookup Azure Blob storage cache first
                // have a 5min timeout for retries
                BlobContent blobContent = null;
                if (!forceRefresh)
                {
                    blobContent = await blobCache.GetAsync(site, id, this.containerName, input, TimeSpan.FromMinutes(5), cancellationToken);
                }

                if (blobContent == null)
                {
                    cacheHit = false;

                    var stopwatchReqeust = Stopwatch.StartNew();

                    // make the actual HTTP request
                    responseMessage = await this.client.PostAsync(
                        string.Empty,
                        new StringContent(
                            body,
                            new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
                            contentType));

                    Services.TelemetryClient.TrackDependency(this.containerName, this.endpoint, this.containerName, null,
                                                             DateTime.UtcNow, stopwatchReqeust.Elapsed,
                                                             $"{responseMessage.StatusCode} {responseMessage.ReasonPhrase}", responseMessage.IsSuccessStatusCode);

                    log.Trace(new TraceEvent(TraceLevel.Verbose, $"Response: {responseMessage.StatusCode} {responseMessage.ReasonPhrase}"));

                    if (!responseMessage.IsSuccessStatusCode)
                    {
                        blobContent = new BlobContent
                        {
                            // TODO: random expiration
                            Expires = DateTime.UtcNow + TimeSpan.FromMinutes(1),
                        };
                    }
                    else
                    {
                        var responseStr = await responseMessage.Content.ReadAsStringAsync();

                        log.Trace(new TraceEvent(TraceLevel.Verbose, $"Result {this.containerName} at {this.endpoint}: {responseStr}"));

                        // once we got a response, cache for 3 days
                        // TODO: add configuration option
                        // TODO: add force refresh parameter
                        blobContent = await blobCache.PersistAsync(site, id, this.containerName, input, responseStr, TimeSpan.FromDays(3), cancellationToken);
                    }
                }

                return(blobContent);
            }
            finally
            {
                var props = new Dictionary <string, string>
                {
                    { "site", site },
                    { "id", id },
                    { "cacheHit", cacheHit.ToString() },
                    { "StatusCode", responseMessage?.StatusCode.ToString() },
                    { "Reason", responseMessage?.ReasonPhrase }
                };

                var sb = new StringBuilder(this.containerName);
                if (responseMessage != null && responseMessage.StatusCode != System.Net.HttpStatusCode.OK)
                {
                    props.Add("Request", body);
                    sb.Append(" failed");
                }

                Services.TelemetryClient.TrackEvent(
                    sb.ToString(),
                    props,
                    metrics: new Dictionary <string, double>
                {
                    { "requestTime", stopwatch.ElapsedMilliseconds }
                });
            }
        }
Exemple #5
0
        public async Task <HttpResponseMessage> InvokeAsync(HttpRequestMessage req, TraceWriter log,
                                                            Func <CrawlResponse, object> requestBodyFunc,
                                                            Action <CrawlResponse, BlobContent> responseAction,
                                                            bool isPost,
                                                            CancellationToken cancellationToken)
        {
            log.Info("Crawl." + this.containerName);

            await this.InitializeAsync();

            string        reqBodyStr  = null;
            CrawlResponse reqBody     = null;
            BlobContent   blobContent = null;

            try
            {
                using (var operation = Services.TelemetryClient.StartOperation <DependencyTelemetry>("Crawl." + this.containerName))
                {
                    reqBodyStr = await req.Content.ReadAsStringAsync();

                    reqBody = JsonConvert.DeserializeObject <CrawlResponse>(reqBodyStr);

                    operation.Telemetry.Target = this.endpoint;
                    operation.Telemetry.Properties.Add("AppId", reqBody.Site);
                    operation.Telemetry.Properties.Add("ActionId", reqBody.Id);

                    var serviceRequestBody = requestBodyFunc(reqBody);

                    if (serviceRequestBody == null)
                    {
                        return(new HttpResponseMessage(System.Net.HttpStatusCode.OK)
                        {
                            Content = new StringContent(
                                string.Empty,
                                new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
                                "application/json")
                        });
                    }

                    blobContent = await this.RequestAsync(
                        log,
                        reqBody.Site,
                        reqBody.Id,
                        serviceRequestBody,
                        reqBody.ForceRefresh,
                        isPost,
                        cancellationToken);

                    if (blobContent != null)
                    {
                        operation.Telemetry.Properties.Add("Expires", blobContent.Expires.ToString(CultureInfo.InvariantCulture));

                        if (blobContent.Value != null)
                        {
                            responseAction(reqBody, blobContent);

                            operation.Telemetry.ResultCode = "OK";
                        }
                    }

                    return(Services.CreateResponse(blobContent));
                }
            }
            catch (Exception ex)
            {
                Services.TrackException(ex, req, log, reqBodyStr, reqBody, blobContent);
                throw ex;
            }
        }