private static CachedRpdeItem LastPageCachedRpdeItem(LastItem lastItem, string source)
{
return(new CachedRpdeItem
{
id = Utils.LAST_PAGE_ITEM_RESERVED_ID,
modified = Utils.LAST_PAGE_ITEM_RESERVED_MODIFIED,
deleted = false,
data = JsonConvert.SerializeObject(lastItem, Newtonsoft.Json.Formatting.None,
new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
}),
kind = string.Empty,
source = source,
expiry = null
});
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "feeds/{source}")] HttpRequest req, string source,
ILogger log)
{
long afterTimestamp = 0;
string afterTimestampString = req.Query["afterTimestamp"];
if (afterTimestampString != null && !long.TryParse(afterTimestampString, out afterTimestamp))
{
return(req.CreateErrorResponse(HttpStatusCode.BadRequest, "afterTimestamp must be an integer"));
}
string afterId = req.Query["afterId"];
try
{
var sw = new Stopwatch();
sw.Start();
StringBuilder str = new StringBuilder();
int itemCount = 0;
LastItem lastItem = null;
using (SqlConnection connection = new SqlConnection(SqlUtils.SqlDatabaseConnectionString))
{
SqlCommand cmd = new SqlCommand("READ_ITEM_PAGE", connection);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add(
new SqlParameter()
{
ParameterName = "@source",
SqlDbType = SqlDbType.VarChar,
Value = source,
});
cmd.Parameters.Add(
new SqlParameter()
{
ParameterName = "@id",
SqlDbType = SqlDbType.NVarChar,
Value = afterId ?? "",
});
cmd.Parameters.Add(
new SqlParameter()
{
ParameterName = "@modified",
SqlDbType = SqlDbType.BigInt,
Value = afterTimestamp,
});
connection.Open();
// This query will return one additional record (from the previous page) to check that the provided "source" value is valid
// (So even the last page will return at least 1 record)
SqlDataReader reader = await cmd.ExecuteReaderAsync();
// Call Read before accessing data.
if (await reader.ReadAsync())
{
var itemStrings = new List <string>();
//Skip the first row if it's the same as the query parameters (see comment above)
if ((reader.GetString((int)ResultColumns.id) != afterId &&
reader.GetInt64((int)ResultColumns.modified) != afterTimestamp) || reader.Read())
{
do
{
var timestamp = reader.GetInt64((int)ResultColumns.modified);
if (timestamp != Utils.LAST_PAGE_ITEM_RESERVED_MODIFIED)
{
itemStrings.Add(reader.GetString((int)ResultColumns.data));
// Get the last item values for the next URL
afterTimestamp = timestamp;
afterId = reader.GetString((int)ResultColumns.id);
}
else
{
lastItem = JsonConvert.DeserializeObject <LastItem>(reader.GetString((int)ResultColumns.data));
}
}while (await reader.ReadAsync());
}
itemCount = itemStrings.Count;
// Construct response using string concatenation instead of deserialisation for maximum efficiency
str.Append("{\"next\":");
str.Append(JsonConvert.ToString($"{Utils.GetFeedUrl(source)}?afterTimestamp={afterTimestamp}&afterId={afterId}"));
str.Append(",\"items\": [");
str.AppendJoin(',', itemStrings);
str.Append("],\"license\":");
str.Append(JsonConvert.ToString(Utils.CC_BY_LICENSE));
str.Append("}");
// Call Close when done reading.
reader.Close();
}
else
{
// Call Close when done reading.
reader.Close();
// Return 404 for invalid source, rather than just for last page
return(req.CreateErrorResponse(HttpStatusCode.NotFound, $"'{source}' feed not found"));
}
}
// Create response
var resp = req.CreateJSONResponseFromString(HttpStatusCode.OK, str.ToString());
// Leverage long-term caching in CDN for data payload
if (itemCount > 0)
{
// Partial pages for high-frequency data have a shorter expiry set, so that they can give way to full pages that will take their place after a time
// This speeds up reading of feeds for new data users to get fully up-to-date
if (lastItem?.RecommendedPollInterval != null && lastItem.RecommendedPollInterval < 60)
{
if (itemCount < 30)
{
// For the end of near-real-time feeds, use a short expiry time due to the high volume
resp = resp.AsCachable(TimeSpan.FromMinutes(15));
}
else
{
// For the beginning of near-real-time feeds, use a long-ish expiry to ensure data is not needlessly stockpiled
resp = resp.AsCachable(TimeSpan.FromHours(4));
}
}
else
{
// Fuller pages (likely earlier in the feed) have a long expiry set
resp = resp.AsCachable(TimeSpan.FromHours(12));
}
}
else
{
// The last page has a minimal expiry based on the underlying data's refresh frequency
if (lastItem?.Expires != null)
{
// Add 2 seconds to expiry to account for proxy lag
const int ESTIMATED_PROXY_LATENCY_SECONDS = 2;
var expiresFromProxy = lastItem?.Expires?.AddSeconds(ESTIMATED_PROXY_LATENCY_SECONDS);
if (expiresFromProxy < DateTimeOffset.UtcNow)
{
// If the expiry has passed, project it forward based on the poll interval if possible
if (lastItem.RecommendedPollInterval != null && lastItem.RecommendedPollInterval != 0)
{
resp = resp.AsCachable(ProjectExpiryForward((DateTimeOffset)expiresFromProxy, (int)lastItem.RecommendedPollInterval));
resp.Headers.Add(Utils.RECOMMENDED_POLL_INTERVAL_HEADER, Convert.ToString(lastItem.RecommendedPollInterval));
}
else
{
// Default cache expiry
resp = resp.AsCachable(TimeSpan.FromSeconds(Utils.DEFAULT_POLL_INTERVAL));
}
}
else
{
resp = resp.AsCachable(expiresFromProxy);
}
}
else if (lastItem?.MaxAge != null)
{
resp = resp.AsCachable((TimeSpan)lastItem.MaxAge);
}
else
{
// Default cache expiry
resp = resp.AsCachable(TimeSpan.FromSeconds(Utils.DEFAULT_POLL_INTERVAL));
}
}
sw.Stop();
log.LogWarning($"GETPAGE TIMER {sw.ElapsedMilliseconds} ms.");
return(resp);
}
catch (SqlException ex)
{
if (SqlUtils.SqlTransientErrorNumbers.Contains(ex.Number) || ex.Message.Contains("timeout", StringComparison.InvariantCultureIgnoreCase))
{
log.LogWarning($"Throttle on GetPage, retry after {SqlUtils.SqlRetrySecondsRecommendation} seconds.");
return(req.CreateTooManyRequestsResponse(TimeSpan.FromSeconds(SqlUtils.SqlRetrySecondsRecommendation)));
}
else
{
log.LogError("Error during GetPage: " + ex.ToString());
return(req.CreateErrorResponse(HttpStatusCode.InternalServerError, ex.Message));
}
}
}
