public async Task <bool> IsUrlVisited(string url) { var urlMD5 = NScrapyHelper.GetMD5FromString(url); var connection = RedisSchedulerContext.Current.Connection; var urlSetName = $"{RedisSchedulerContext.Current.ReceiverQueue}.VisitedURLMD5"; if (await connection.GetDatabase().SetContainsAsync(urlSetName, urlMD5)) { return(true); } await connection.GetDatabase().SetAddAsync(urlSetName, urlMD5); return(false); }
private void ListenToRedisTopic() { var connection = RedisSchedulerContext.Current.Connection; while (true) { var lockToken = Guid.NewGuid(); RedisResponseMessage responseMessage = null; RedisValue value = new RedisValue(); try { RedisSchedulerContext.Current.GetLock($"{RedisSchedulerContext.Current.ResponseQueue}.Lock", lockToken.ToString()); if (RedisSchedulerContext.Current.Connection.GetDatabase().ListLength(RedisSchedulerContext.Current.ResponseQueue) > 0) { value = RedisSchedulerContext.Current.Connection.GetDatabase().ListRightPop(RedisSchedulerContext.Current.ResponseQueue); } } catch (Exception ex) { NScrapyContext.CurrentContext.Log.Error($"Error getting value from Redis", ex); } finally { RedisSchedulerContext.Current.ReleaseLock($"{RedisSchedulerContext.Current.ResponseQueue}.Lock", lockToken.ToString()); } if (!value.HasValue) { continue; } responseMessage = JsonConvert.DeserializeObject <RedisResponseMessage>(value); if (registedCallback.ContainsKey(responseMessage.CallbacksFingerprint)) { var callback = registedCallback[responseMessage.CallbacksFingerprint]; var response = NScrapyHelper.DecompressResponse(responseMessage.Payload); var task = new Task(() => callback(response)); task.ContinueWith(t => NScrapyContext.CurrentContext.Log.Error($"Error Process Callback {responseMessage.CallbacksFingerprint}", task.Exception), TaskContinuationOptions.OnlyOnFaulted); task.Start(); } } }
private static void ProcessRequestAndSendBack(string responseQueueName, string requestMessage) { var requestObj = JsonConvert.DeserializeObject <RedisRequestMessage>(requestMessage); var request = new HttpRequest() { URL = requestObj.URL }; var result = Downloader.Downloader.SendRequestAsync(request); result.ContinueWith(async u => { var resultPayload = string.Empty; try { resultPayload = JsonConvert.SerializeObject(u.Result); } catch (Exception ex) { DownloaderContext.Context.Log.Error($"Serialize response for {request.URL} failed!", ex); } resultPayload = await NScrapyHelper.CompressString(resultPayload); var responseMessage = new RedisResponseMessage() { URL = requestObj.URL, CallbacksFingerprint = requestObj.CallbacksFingerprint, Payload = resultPayload }; //var publishResult = RedisManager.Connection.GetSubscriber().Publish(responseQueueName, $"{JsonConvert.SerializeObject(responseMessage)}"); var publishResult = RedisManager.Connection.GetDatabase().ListLeftPush(responseQueueName, $"{JsonConvert.SerializeObject(responseMessage)}"); DownloaderContext.Context.Log.Info($"Sending request to {request.URL} success!"); }, TaskContinuationOptions.OnlyOnRanToCompletion); result.ContinueWith(u => DownloaderContext.Context.Log.Info($"Sending request to {request.URL} failed", result.Exception.InnerException), TaskContinuationOptions.OnlyOnFaulted); }