private void SaveResult(PollTarget target, PollResult result)
{
Storage._.Update(target);
Storage._.Insert(result);
Storage._.AppendToFeed(result);
}
private PollResult FetchUrl(PollTarget item)
{
var result = new PollResult(item.Name);
var statusCode = string.Empty;
try
{
var target = item.Source;
if (target.Host == Kimono.Host)
{
var uri = new UriBuilder(target);
uri.Query = "apikey=" + Kimono.Key;
target = uri.Uri;
}
C.Log("Issuing request to {0}...", target);
var req = (HttpWebRequest)WebRequest.Create(target);
req.Timeout = TIMEOUT;
req.Method = "GET";
using (var resp = (HttpWebResponse)req.GetResponse())
{
result.StatusCode = resp.StatusCode;
C.Log("Got response with status code {0} ({1})", (int)result.StatusCode, Enum.GetName(typeof(HttpStatusCode), result.StatusCode));
if (result.Success)
{
result.Content = Feed.ProcessJsonStream(result.RowKey, GetTransformFor(result.PartitionKey), resp.GetResponseStream());
if (result.Content == null)
{
result.StatusCode = HttpStatusCode.NoContent;
item.NextRun = Time.Mins(60);
C.Log("Failed to process valid json for {0}, will try again at {1}.", target.AbsoluteUri, C.Localize(item.NextRun));
}
else
{
item.NextRun = (result.Content.NextPollAfter > C.CurrTime()) ?
result.Content.NextPollAfter :
Time.NextRunFor(item.Schedule);
C.Log("Processed response successfully! Setting next run for {0}...", C.Localize(item.NextRun));
}
}
}
}
catch (Exception ex)
{
C.Log("Error getting content for url {0}: ", ex, item.Source);
}
return(result);
}
internal static int WriteInitialTargets()
{
var newMsgCt = 0;
if (Lock._.AcquireLease())
{
C.Log("Lease acquired, role instance: {0}", C.Id);
var targets = Storage._.Query();
var numTargets = targets.Count;
C.Log("Checking queue for existing messages...");
var msgs = Storage._.PeekMessages(targets.Count);
if (msgs.Count < numTargets)
{
C.Log("Found {0} messages for {1} targets. Not all targets are presently queued - queueing all missing targets.", msgs.Count, numTargets);
IList <PollTarget> requeue;
if (msgs.Count == 0)
{
requeue = targets;
}
else
{
requeue = new List <PollTarget>();
foreach (var b in targets)
{
var found = false;
foreach (var m in msgs)
{
var a = PollTarget.ParseFromString(m.AsString);
if (a.PartitionKey == b.PartitionKey)
{
found = true;
}
}
if (!found)
{
requeue.Add(b);
}
}
}
C.Log("Writing {0} messages...", requeue.Count);
Storage._.SaveMessages(requeue);
newMsgCt = requeue.Count;
}
}
else
{
C.Log("Failed to acquire lease, role instance: {0}", C.Id);
// wait a little while since another role has the lease and may be writing messages now.
Thread.Sleep(15 * C.TO_MILLI);
}
return(newMsgCt);
}
