예제 #1
0
        private void SaveResult(PollTarget target, PollResult result)
        {
            Storage._.Update(target);

            Storage._.Insert(result);

            Storage._.AppendToFeed(result);
        }
예제 #2
0
        private PollResult FetchUrl(PollTarget item)
        {
            var result     = new PollResult(item.Name);
            var statusCode = string.Empty;

            try
            {
                var target = item.Source;

                if (target.Host == Kimono.Host)
                {
                    var uri = new UriBuilder(target);
                    uri.Query = "apikey=" + Kimono.Key;
                    target    = uri.Uri;
                }

                C.Log("Issuing request to {0}...", target);
                var req = (HttpWebRequest)WebRequest.Create(target);
                req.Timeout = TIMEOUT;
                req.Method  = "GET";

                using (var resp = (HttpWebResponse)req.GetResponse())
                {
                    result.StatusCode = resp.StatusCode;

                    C.Log("Got response with status code {0} ({1})", (int)result.StatusCode, Enum.GetName(typeof(HttpStatusCode), result.StatusCode));
                    if (result.Success)
                    {
                        result.Content = Feed.ProcessJsonStream(result.RowKey, GetTransformFor(result.PartitionKey), resp.GetResponseStream());

                        if (result.Content == null)
                        {
                            result.StatusCode = HttpStatusCode.NoContent;
                            item.NextRun      = Time.Mins(60);

                            C.Log("Failed to process valid json for {0}, will try again at {1}.", target.AbsoluteUri, C.Localize(item.NextRun));
                        }
                        else
                        {
                            item.NextRun = (result.Content.NextPollAfter > C.CurrTime()) ?
                                           result.Content.NextPollAfter :
                                           Time.NextRunFor(item.Schedule);

                            C.Log("Processed response successfully!  Setting next run for {0}...", C.Localize(item.NextRun));
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                C.Log("Error getting content for url {0}: ", ex, item.Source);
            }

            return(result);
        }
예제 #3
0
        internal static int WriteInitialTargets()
        {
            var newMsgCt = 0;

            if (Lock._.AcquireLease())
            {
                C.Log("Lease acquired, role instance: {0}", C.Id);

                var targets    = Storage._.Query();
                var numTargets = targets.Count;

                C.Log("Checking queue for existing messages...");

                var msgs = Storage._.PeekMessages(targets.Count);

                if (msgs.Count < numTargets)
                {
                    C.Log("Found {0} messages for {1} targets.  Not all targets are presently queued - queueing all missing targets.", msgs.Count, numTargets);

                    IList <PollTarget> requeue;

                    if (msgs.Count == 0)
                    {
                        requeue = targets;
                    }
                    else
                    {
                        requeue = new List <PollTarget>();

                        foreach (var b in targets)
                        {
                            var found = false;

                            foreach (var m in msgs)
                            {
                                var a = PollTarget.ParseFromString(m.AsString);
                                if (a.PartitionKey == b.PartitionKey)
                                {
                                    found = true;
                                }
                            }

                            if (!found)
                            {
                                requeue.Add(b);
                            }
                        }
                    }

                    C.Log("Writing {0} messages...", requeue.Count);

                    Storage._.SaveMessages(requeue);
                    newMsgCt = requeue.Count;
                }
            }
            else
            {
                C.Log("Failed to acquire lease, role instance: {0}", C.Id);

                // wait a little while since another role has the lease and may be writing messages now.
                Thread.Sleep(15 * C.TO_MILLI);
            }

            return(newMsgCt);
        }