public string RenderMessageWithContext(FeedState feedStateItem)
        {
            switch (ErrorCategory)
            {
            case ExpectedErrorCategory.Unauthorised401:
                return($"Feed attempting poll returned 401 and will be purged: '{feedStateItem.name}'.");

            case ExpectedErrorCategory.DuplicateMessageDetected:
                return($"Duplicate message detected for '{feedStateItem.name}': {feedStateItem.id}, and will be dropped.");

            case ExpectedErrorCategory.InvalidRPDEPage:
                return($"Invalid RPDE page received for '{feedStateItem.name}': {feedStateItem.nextUrl}");

            case ExpectedErrorCategory.PageFetchError:
                return($"Error retrieving page for '{feedStateItem.name}': {feedStateItem.nextUrl}");

            case ExpectedErrorCategory.SqlTransientError:
                return($"Throttle on PollFeed, retry after {SqlUtils.SqlRetrySecondsRecommendation} seconds.");

            case ExpectedErrorCategory.SqlUnexpectedError:
                return($"SQL Error {(this.InnerException as SqlException)?.Number} writing page to SQL Server.");

            case ExpectedErrorCategory.UnexpectedErrorDuringDatabaseWrite:
                return($"Unexpected error writing page to SQL Server.");

            case ExpectedErrorCategory.UnexpectedError:
                return($"Unexpected error.");

            case ExpectedErrorCategory.ForceClearProxyCache:
                return($"ClearProxyCache has been set in App Config. Deadlettering all feeds.");

            default:
                return($"Unexpected error for '{feedStateItem.name}': {feedStateItem.nextUrl}");
            }
        }
Пример #2
0
        public static async Task <IActionResult> Run(
            [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
            ILogger log)
        {
            log.LogInformation("C# HTTP trigger function processed a request.");

            string name = req.Query["name"];

            string ServiceBusConnectionString = Environment.GetEnvironmentVariable("ServiceBusConnection");
            var    queueClients = new List <IMessageReceiver> {
                new MessageReceiver(ServiceBusConnectionString, Utils.FEED_STATE_QUEUE_NAME),
                new MessageReceiver(ServiceBusConnectionString, Utils.FEED_STATE_QUEUE_NAME + "/$DeadLetterQueue"),
                new MessageReceiver(ServiceBusConnectionString, Utils.PURGE_QUEUE_NAME),
                new MessageReceiver(ServiceBusConnectionString, Utils.REGISTRATION_QUEUE_NAME)
            };
            List <MessageStatus> list = new List <MessageStatus>();

            foreach (var client in queueClients)
            {
                List <MessageStatus> lastRetrievedList = null;
                do
                {
                    lastRetrievedList = (await client.PeekAsync(100)).Select(m => new MessageStatus
                    {
                        queue            = client.Path,
                        feedState        = FeedState.DecodeFromMessage(m),
                        expiration       = m.ExpiresAtUtc,
                        delayedUntil     = m.ScheduledEnqueueTimeUtc,
                        locked           = m.SystemProperties.IsLockTokenSet,
                        lockedUntil      = m.SystemProperties.LockedUntilUtc,
                        retryAttempts    = m.SystemProperties.DeliveryCount,
                        deadLetterSource = m.SystemProperties.DeadLetterSource,
                        isReceived       = m.SystemProperties.IsReceived
                    }).ToList();
                    list.AddRange(lastRetrievedList);
                } while (lastRetrievedList.Count > 0);
            }

            return(new JsonResult(list.OrderBy(x => x.feedState.name).ToList(), new JsonSerializerSettings
            {
                Converters = new List <JsonConverter>()
                {
                    new StringEnumConverter()
                },
                Formatting = Formatting.Indented
            }));
        }
        public static async Task Run([ServiceBusTrigger(Utils.FEED_STATE_QUEUE_NAME + "/$DeadLetterQueue", Connection = "ServiceBusConnection")] Message message, MessageReceiver messageReceiver, string lockToken,
                                     [ServiceBus(Utils.PURGE_QUEUE_NAME, Connection = "ServiceBusConnection", EntityType = EntityType.Queue)] IAsyncCollector <Message> purgeQueueCollector,
                                     ILogger log)
        {
            var feedStateItem = FeedState.DecodeFromMessage(message);

            log.LogInformation($"DeadLetterQueue Trigger Started: {feedStateItem.name}");

            // Dead letter queue from PollFeed simply triggers a purge
            feedStateItem.ResetCounters();

            // Check lock exists, as close to a transaction as we can get
            if (await messageReceiver.RenewLockAsync(lockToken) != null)
            {
                await messageReceiver.CompleteAsync(lockToken);

                await purgeQueueCollector.AddAsync(feedStateItem.EncodeToMessage(0));
            }

            log.LogInformation($"DeadLetterQueue Trigger Complete: {feedStateItem.name}");
        }
Пример #4
0
        public static async Task <List <FeedState> > GetFeedStateFromQueues(string name = null, bool ignoreRegistrationQueue = false)
        {
            string ServiceBusConnectionString = Environment.GetEnvironmentVariable("ServiceBusConnection");
            var    queueClients = new List <IMessageReceiver> {
                new MessageReceiver(ServiceBusConnectionString, Utils.FEED_STATE_QUEUE_NAME),
                new MessageReceiver(ServiceBusConnectionString, Utils.FEED_STATE_QUEUE_NAME + "/$DeadLetterQueue"),
                new MessageReceiver(ServiceBusConnectionString, Utils.PURGE_QUEUE_NAME)
            };

            if (!ignoreRegistrationQueue)
            {
                queueClients.Add(
                    new MessageReceiver(ServiceBusConnectionString, Utils.REGISTRATION_QUEUE_NAME)
                    );
            }

            List <FeedState> list = new List <FeedState>();

            foreach (var client in queueClients)
            {
                List <FeedState> lastRetrievedList = null;
                do
                {
                    lastRetrievedList = (await client.PeekAsync(100)).Select(m => FeedState.DecodeFromMessage(m)).ToList();
                    list.AddRange(lastRetrievedList);
                } while (lastRetrievedList.Count > 0);
            }

            if (name == null)
            {
                return(list);
            }
            else
            {
                return(list.Where(fs => fs.name == name).ToList());
            }
        }
Пример #5
0
        public static async Task Run([ServiceBusTrigger(Utils.REGISTRATION_QUEUE_NAME, Connection = "ServiceBusConnection")] Message message, MessageReceiver messageReceiver, string lockToken,
                                     [ServiceBus(Utils.REGISTRATION_QUEUE_NAME, Connection = "ServiceBusConnection", EntityType = EntityType.Queue)] IAsyncCollector <Message> queueCollector,
                                     [ServiceBus(Utils.FEED_STATE_QUEUE_NAME, Connection = "ServiceBusConnection", EntityType = EntityType.Queue)] IAsyncCollector <Message> feedStateQueueCollector,
                                     ILogger log)
        {
            var feedStateItem = FeedState.DecodeFromMessage(message);

            // Dead letter queue from PollFeed simply triggers a purge

            log.LogInformation($"Registration Trigger Started: {feedStateItem.name}");

            // Double-check this feed doesn't already exist
            var matchingFeedStateList = await Utils.GetFeedStateFromQueues(feedStateItem.name, true);

            if (matchingFeedStateList.Count > 0)
            {
                if (matchingFeedStateList.First().url != feedStateItem.url)
                {
                    log.LogError($"Conflicting feed already registered with same name '{feedStateItem.name}' using different url '{matchingFeedStateList.First().url}', and will be dropped.");
                }
                else
                {
                    log.LogError($"Conflicting feed already registered with same name '{feedStateItem.name}' using same url '{matchingFeedStateList.First().url}', and will be dropped.");
                }
                await messageReceiver.CompleteAsync(lockToken);

                return;
            }

            // Attempt to get first page
            try
            {
                var result = await httpClient.GetAsync(feedStateItem.nextUrl);

                if (result.StatusCode == HttpStatusCode.Unauthorized)
                {
                    // Remove from queue on 401 (OWS key has changed)
                    log.LogWarning($"Feed attempting registration returned 401 and will be dropped: '{feedStateItem.name}'.");
                    DeleteFeedFromDatabase(feedStateItem.name);
                    await messageReceiver.CompleteAsync(lockToken);

                    return;
                }
                else
                {
                    var data = JsonConvert.DeserializeObject <RpdeFeed>(await result.Content.ReadAsStringAsync());

                    // check for "license" and valid data: "https://creativecommons.org/licenses/by/4.0/"
                    if (data?.license != Utils.CC_BY_LICENSE)
                    {
                        throw new ApplicationException($"Registration error while validating first page - dropping feed. Error retrieving license for '{feedStateItem.url}'.");
                    }
                }
            }
            catch (Exception ex)
            {
                feedStateItem.lastError = ex.ToString();

                // Retry registration three times over 30 minutes, then fail
                if (feedStateItem.registrationRetries > 3)
                {
                    log.LogError($"Registration error while validating first page. Dropping feed. Error retrieving '{feedStateItem.url}'. {ex.ToString()}");
                    DeleteFeedFromDatabase(feedStateItem.name);
                    await messageReceiver.CompleteAsync(lockToken);
                }
                else
                {
                    feedStateItem.registrationRetries++;
                    feedStateItem.totalErrors++;
                    log.LogWarning($"Registration error while validating first page. Retrying '{feedStateItem.name}' attempt {feedStateItem.registrationRetries}. Error retrieving '{feedStateItem.url}'. {ex.ToString()}");

                    // Check lock exists, as close to a transaction as we can get
                    if (await messageReceiver.RenewLockAsync(lockToken) != null)
                    {
                        await messageReceiver.CompleteAsync(lockToken);

                        await queueCollector.AddAsync(feedStateItem.EncodeToMessage(30));
                    }
                }
                return;
            }

            // Write the successful registration to the feeds table
            using (var db = new Database(SqlUtils.SqlDatabaseConnectionString, DatabaseType.SqlServer2012, SqlClientFactory.Instance))
            {
                db.Save <Feed>(new Feed
                {
                    source           = feedStateItem.name,
                    url              = feedStateItem.url,
                    datasetUrl       = feedStateItem.datasetUrl,
                    initialFeedState = feedStateItem
                });
            }

            // Restart the feed from the beginning
            feedStateItem.nextUrl = feedStateItem.url;
            feedStateItem.ResetCounters();

            // Check lock exists, as close to a transaction as we can get
            if (await messageReceiver.RenewLockAsync(lockToken) != null)
            {
                await messageReceiver.CompleteAsync(lockToken);

                await feedStateQueueCollector.AddAsync(feedStateItem.EncodeToMessage(0));
            }

            log.LogInformation($"Registration Trigger Promoting Feed: {feedStateItem.name}");
        }
 public string RenderMessageWithFullContext(FeedState feedStateItem, string retryContext)
 {
     return($"{RenderMessageWithContext(feedStateItem)}. {retryContext} {this.InnerException?.ToString()}");
 }
        public static async Task Run([ServiceBusTrigger(Utils.FEED_STATE_QUEUE_NAME, Connection = "ServiceBusConnection")] Message message, MessageReceiver messageReceiver, string lockToken,
                                     [ServiceBus(Utils.FEED_STATE_QUEUE_NAME, Connection = "ServiceBusConnection", EntityType = EntityType.Queue)] IAsyncCollector <Message> queueCollector,
                                     ILogger log)
        {
            var feedStateItem = FeedState.DecodeFromMessage(message);

            log.LogInformation($"PollQueueHandler queue trigger function processed message: {feedStateItem?.nextUrl}");

            // Increment poll requests before anything else
            feedStateItem.totalPollRequests++;
            feedStateItem.dateModified = DateTime.Now;

            SourcePage sourcePage;

            try
            {
                sourcePage = await ExecutePoll(feedStateItem.name, feedStateItem.nextUrl, feedStateItem.lastPageReads == 0, feedStateItem.deletedItemDaysToLive, log);
            }
            catch (Exception ex)
            {
                var expectedException = ExpectedPollException.ExpectTheUnexpected(ex);
                feedStateItem.retryStategy = new RetryStrategy(expectedException, feedStateItem?.retryStategy);
                feedStateItem.totalErrors++;

                if (feedStateItem.retryStategy.DeadLetter)
                {
                    log.LogError(expectedException.RenderMessageWithFullContext(feedStateItem, $"DEAD-LETTERING: '{feedStateItem.name}'."));
                    await messageReceiver.DeadLetterAsync(lockToken);
                }
                if (feedStateItem.retryStategy.DropImmediately)
                {
                    log.LogWarning(expectedException.RenderMessageWithFullContext(feedStateItem, $"Dropped message for '{feedStateItem.name}'."));
                    await messageReceiver.CompleteAsync(lockToken);
                }
                else
                {
                    feedStateItem.lastError = expectedException.RenderMessageWithFullContext(feedStateItem, $"Retrying '{feedStateItem.name}' attempt {feedStateItem.retryStategy.RetryCount} in {feedStateItem.retryStategy.DelaySeconds} seconds.");
                    log.LogWarning(feedStateItem.lastError);

                    var retryMsg = feedStateItem.EncodeToMessage(feedStateItem.retryStategy.DelaySeconds);

                    // Check lock exists, as close to a transaction as we can get
                    if (await messageReceiver.RenewLockAsync(lockToken) != null)
                    {
                        await messageReceiver.CompleteAsync(lockToken);

                        await queueCollector.AddAsync(retryMsg);
                    }
                }

                return;
            }

            // Update counters on success
            if (sourcePage.IsLastPage)
            {
                feedStateItem.lastPageReads++;
            }
            else
            {
                feedStateItem.lastPageReads = 0;
            }
            feedStateItem.retryStategy = null;
            feedStateItem.lastError    = null;
            feedStateItem.totalPagesRead++;
            feedStateItem.totalItemsRead += sourcePage.Content.items.Count;
            feedStateItem.nextUrl         = sourcePage.Content.next;

            Message newMessage;

            // If immediate poll is specified for last page, respect any Expires header provided for throttling
            if (sourcePage.IsLastPage)
            {
                if (sourcePage.LastPageDetails?.Expires != null)
                {
                    newMessage = feedStateItem.EncodeToMessage(sourcePage.LastPageDetails.Expires);
                }
                else if (sourcePage.LastPageDetails?.MaxAge != null)
                {
                    newMessage = feedStateItem.EncodeToMessage((int)sourcePage.LastPageDetails.MaxAge?.TotalSeconds);
                }
                else
                {
                    // Default last page polling interval
                    newMessage = feedStateItem.EncodeToMessage(Utils.DEFAULT_POLL_INTERVAL);
                }
            }
            else
            {
                // If not last page, get the next page immediately
                newMessage = feedStateItem.EncodeToMessage(0);
            }

            // These two operations should be in a transaction, but to save cost they ordered so that a failure will result in the polling stopping,
            // and the ResyncDroppedFeeds timer trigger will hence be required to ensure the system is still robust
            // using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
            // {
            //    await messageReceiver.CompleteAsync(lockToken);
            //    await queueCollector.AddAsync(newMessage);
            //    scope.Complete(); // declare the transaction done
            // }

            // Check lock exists, as close to a transaction as we can get
            if (await messageReceiver.RenewLockAsync(lockToken) != null)
            {
                await messageReceiver.CompleteAsync(lockToken);

                await queueCollector.AddAsync(newMessage);
            }
        }
        public static async Task Run([ServiceBusTrigger(Utils.PURGE_QUEUE_NAME, Connection = "ServiceBusConnection")] Message message, MessageReceiver messageReceiver, string lockToken,
                                     [ServiceBus(Utils.PURGE_QUEUE_NAME, Connection = "ServiceBusConnection", EntityType = EntityType.Queue)] IAsyncCollector <Message> queueCollector,
                                     [ServiceBus(Utils.REGISTRATION_QUEUE_NAME, Connection = "ServiceBusConnection", EntityType = EntityType.Queue)] IAsyncCollector <Message> registrationQueueCollector,
                                     ILogger log)
        {
            var feedStateItem = FeedState.DecodeFromMessage(message);

            log.LogInformation($"Purge Trigger Started: {feedStateItem.name}");

            try
            {
                int itemCount = 0;

                var sw = new Stopwatch();
                sw.Start();

                using (var db = new Database(SqlUtils.SqlDatabaseConnectionString, DatabaseType.SqlServer2012, SqlClientFactory.Instance))
                {
                    itemCount = await db.ExecuteAsync("DELETE_SOURCE @0", feedStateItem.name);
                }

                sw.Stop();

                log.LogWarning($"PURGE TIMER {feedStateItem.name}: Deleted {itemCount} items from source '{feedStateItem.name}' in {sw.ElapsedMilliseconds} ms");

                feedStateItem.purgedItems = itemCount;

                if (itemCount < 1000)
                {
                    log.LogInformation($"Purge complete for '{feedStateItem.name}'");

                    // Check lock exists, as close to a transaction as we can get
                    if (await messageReceiver.RenewLockAsync(lockToken) != null)
                    {
                        await messageReceiver.CompleteAsync(lockToken);

                        // Attempt re-registration unless the proxy cache is being cleared
                        if (Environment.GetEnvironmentVariable("ClearProxyCache")?.ToString() != "true")
                        {
                            feedStateItem.ResetCounters();
                            feedStateItem.totalPurgeCount++;
                            await registrationQueueCollector.AddAsync(feedStateItem.EncodeToMessage(1));
                        }
                        else
                        {
                            log.LogWarning($"Purge Successfully Cleaned: {feedStateItem.name}");
                        }
                    }
                }
                else
                {
                    feedStateItem.purgedItems += itemCount;

                    // Check lock exists, as close to a transaction as we can get
                    if (await messageReceiver.RenewLockAsync(lockToken) != null)
                    {
                        await messageReceiver.CompleteAsync(lockToken);

                        await queueCollector.AddAsync(feedStateItem.EncodeToMessage(1));
                    }
                }
            }
            catch (SqlException ex)
            {
                log.LogError($"Error during DELETE_SOURCE stored procedure {ex.Number}: " + ex.ToString());

                feedStateItem.lastError = ex.ToString();

                feedStateItem.purgeRetries++;

                TimeSpan timeSpan   = new TimeSpan(1, 0, 0);
                Random   randomTest = new Random();
                TimeSpan newSpan    = TimeSpan.FromMinutes(randomTest.Next(0, (int)timeSpan.TotalMinutes));

                int delaySeconds = (int)newSpan.TotalSeconds;

                log.LogWarning($"Unexpected error purging items: Retrying '{feedStateItem.name}' attempt {feedStateItem.purgeRetries} in {delaySeconds} seconds");

                // Check lock exists, as close to a transaction as we can get
                if (await messageReceiver.RenewLockAsync(lockToken) != null)
                {
                    await messageReceiver.CompleteAsync(lockToken);

                    await queueCollector.AddAsync(feedStateItem.EncodeToMessage(delaySeconds));
                }
            }

            log.LogInformation($"Purge Trigger Complete: {feedStateItem.name}");
        }