private static CachedRpdeItem LastPageCachedRpdeItem(LastItem lastItem, string source) { return(new CachedRpdeItem { id = Utils.LAST_PAGE_ITEM_RESERVED_ID, modified = Utils.LAST_PAGE_ITEM_RESERVED_MODIFIED, deleted = false, data = JsonConvert.SerializeObject(lastItem, Newtonsoft.Json.Formatting.None, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }), kind = string.Empty, source = source, expiry = null }); }
public static async Task <HttpResponseMessage> Run( [HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "feeds/{source}")] HttpRequest req, string source, ILogger log) { long afterTimestamp = 0; string afterTimestampString = req.Query["afterTimestamp"]; if (afterTimestampString != null && !long.TryParse(afterTimestampString, out afterTimestamp)) { return(req.CreateErrorResponse(HttpStatusCode.BadRequest, "afterTimestamp must be an integer")); } string afterId = req.Query["afterId"]; try { var sw = new Stopwatch(); sw.Start(); StringBuilder str = new StringBuilder(); int itemCount = 0; LastItem lastItem = null; using (SqlConnection connection = new SqlConnection(SqlUtils.SqlDatabaseConnectionString)) { SqlCommand cmd = new SqlCommand("READ_ITEM_PAGE", connection); cmd.CommandType = CommandType.StoredProcedure; cmd.Parameters.Add( new SqlParameter() { ParameterName = "@source", SqlDbType = SqlDbType.VarChar, Value = source, }); cmd.Parameters.Add( new SqlParameter() { ParameterName = "@id", SqlDbType = SqlDbType.NVarChar, Value = afterId ?? "", }); cmd.Parameters.Add( new SqlParameter() { ParameterName = "@modified", SqlDbType = SqlDbType.BigInt, Value = afterTimestamp, }); connection.Open(); // This query will return one additional record (from the previous page) to check that the provided "source" value is valid // (So even the last page will return at least 1 record) SqlDataReader reader = await cmd.ExecuteReaderAsync(); // Call Read before accessing data. if (await reader.ReadAsync()) { var itemStrings = new List <string>(); //Skip the first row if it's the same as the query parameters (see comment above) if ((reader.GetString((int)ResultColumns.id) != afterId && reader.GetInt64((int)ResultColumns.modified) != afterTimestamp) || reader.Read()) { do { var timestamp = reader.GetInt64((int)ResultColumns.modified); if (timestamp != Utils.LAST_PAGE_ITEM_RESERVED_MODIFIED) { itemStrings.Add(reader.GetString((int)ResultColumns.data)); // Get the last item values for the next URL afterTimestamp = timestamp; afterId = reader.GetString((int)ResultColumns.id); } else { lastItem = JsonConvert.DeserializeObject <LastItem>(reader.GetString((int)ResultColumns.data)); } }while (await reader.ReadAsync()); } itemCount = itemStrings.Count; // Construct response using string concatenation instead of deserialisation for maximum efficiency str.Append("{\"next\":"); str.Append(JsonConvert.ToString($"{Utils.GetFeedUrl(source)}?afterTimestamp={afterTimestamp}&afterId={afterId}")); str.Append(",\"items\": ["); str.AppendJoin(',', itemStrings); str.Append("],\"license\":"); str.Append(JsonConvert.ToString(Utils.CC_BY_LICENSE)); str.Append("}"); // Call Close when done reading. reader.Close(); } else { // Call Close when done reading. reader.Close(); // Return 404 for invalid source, rather than just for last page return(req.CreateErrorResponse(HttpStatusCode.NotFound, $"'{source}' feed not found")); } } // Create response var resp = req.CreateJSONResponseFromString(HttpStatusCode.OK, str.ToString()); // Leverage long-term caching in CDN for data payload if (itemCount > 0) { // Partial pages for high-frequency data have a shorter expiry set, so that they can give way to full pages that will take their place after a time // This speeds up reading of feeds for new data users to get fully up-to-date if (lastItem?.RecommendedPollInterval != null && lastItem.RecommendedPollInterval < 60) { if (itemCount < 30) { // For the end of near-real-time feeds, use a short expiry time due to the high volume resp = resp.AsCachable(TimeSpan.FromMinutes(15)); } else { // For the beginning of near-real-time feeds, use a long-ish expiry to ensure data is not needlessly stockpiled resp = resp.AsCachable(TimeSpan.FromHours(4)); } } else { // Fuller pages (likely earlier in the feed) have a long expiry set resp = resp.AsCachable(TimeSpan.FromHours(12)); } } else { // The last page has a minimal expiry based on the underlying data's refresh frequency if (lastItem?.Expires != null) { // Add 2 seconds to expiry to account for proxy lag const int ESTIMATED_PROXY_LATENCY_SECONDS = 2; var expiresFromProxy = lastItem?.Expires?.AddSeconds(ESTIMATED_PROXY_LATENCY_SECONDS); if (expiresFromProxy < DateTimeOffset.UtcNow) { // If the expiry has passed, project it forward based on the poll interval if possible if (lastItem.RecommendedPollInterval != null && lastItem.RecommendedPollInterval != 0) { resp = resp.AsCachable(ProjectExpiryForward((DateTimeOffset)expiresFromProxy, (int)lastItem.RecommendedPollInterval)); resp.Headers.Add(Utils.RECOMMENDED_POLL_INTERVAL_HEADER, Convert.ToString(lastItem.RecommendedPollInterval)); } else { // Default cache expiry resp = resp.AsCachable(TimeSpan.FromSeconds(Utils.DEFAULT_POLL_INTERVAL)); } } else { resp = resp.AsCachable(expiresFromProxy); } } else if (lastItem?.MaxAge != null) { resp = resp.AsCachable((TimeSpan)lastItem.MaxAge); } else { // Default cache expiry resp = resp.AsCachable(TimeSpan.FromSeconds(Utils.DEFAULT_POLL_INTERVAL)); } } sw.Stop(); log.LogWarning($"GETPAGE TIMER {sw.ElapsedMilliseconds} ms."); return(resp); } catch (SqlException ex) { if (SqlUtils.SqlTransientErrorNumbers.Contains(ex.Number) || ex.Message.Contains("timeout", StringComparison.InvariantCultureIgnoreCase)) { log.LogWarning($"Throttle on GetPage, retry after {SqlUtils.SqlRetrySecondsRecommendation} seconds."); return(req.CreateTooManyRequestsResponse(TimeSpan.FromSeconds(SqlUtils.SqlRetrySecondsRecommendation))); } else { log.LogError("Error during GetPage: " + ex.ToString()); return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, ex.Message)); } } }