private void Run(string apiKey, DateTime pFrom, DateTime pTo, int pNumberOfPosts)
{
var client = new LiveFeedServiceReference.LiveFeed2SoapClient();
var programStartTimestamp = DateTime.Now.ToUniversalTime();
Console.WriteLine("Program started: " + programStartTimestamp);
pFrom = pFrom.ToUniversalTime();
pTo = pTo.ToUniversalTime();
DateTime from = pFrom;
while (true)
{
// Check if we should quit
if (ShouldQuit)
return;
from = from.ToUniversalTime();
var to = pTo.ToUniversalTime();
var requestStopwatch = Stopwatch.StartNew();
try
{
Console.WriteLine("{0:u} - {1:u} - Trying to fetch data", from, to);
//fetch the data
XmlNode data = client.GetDataByPostCountAndTimespan(
apiKey,
from,
to,
pNumberOfPosts);
//parse the result and do something with it
DoSomethingWithResults(from, data);
var numberOfPosts = int.Parse(data.Attributes["noOfPosts"].InnerText);
_numberOfPostsSinceStartup += numberOfPosts;
Console.WriteLine("{0:u} - {1:u} - got {2} posts - total number of posts: {3}", from, to, numberOfPosts, _numberOfPostsSinceStartup);
if (numberOfPosts == 0)
{
// If we didn't get any posts in the specified timeinterval, we are finished
Console.WriteLine("Finished: total number of posts {0}", _numberOfPostsSinceStartup);
return;
}
else
{
// We got some posts, so lastPost and lastPostMs attributes should exist
// Use them to update the from parameter, and add 1 millisecond,
// so we don't refetech the ones we already have fetched.
var lastPostTs = DateTime.
Parse(data.Attributes["lastPost"].InnerText).
ToUniversalTime();
var lastPostMs = Double.
Parse(data.Attributes["lastPostMs"].InnerText);
from = lastPostTs.AddMilliseconds(lastPostMs + 1);
}
}
catch (Exception ex)
{
// Log the error
Console.WriteLine("Got some kind of exception.. " + ex.Message);
Console.WriteLine("Will retry in 30 seconds");
// Wait a bit and try again.
Thread.Sleep(30 * 1000);
}
}
}
public void LivefeedExample()
{
var client = new LiveFeedServiceReference.LiveFeed2SoapClient();
var programStartTimestamp = DateTime.Now.ToUniversalTime();
Console.WriteLine("Program started: " + programStartTimestamp);
while (true)
{
// Check if we should quit
if (ShouldQuit)
return;
// Initialize the from and to timestamps.
// The from-parameter is read from disk, if the file exists,
// otherwise it gets set to a default value of now minus 1 hour
var from = ReadFromTimestampFromDisk();
// The to-parameter is always set to now minus 5 minutes
// (5 minutes is chosen so the code works even if clocks
// are somewhat out-of-sync)
var to = DateTime.Now.ToUniversalTime().AddMinutes(-5);
var requestStopwatch = Stopwatch.StartNew();
try
{
//fetch the data
XmlNode data = client.GetDataByPostCountAndTimespan(
ApiKey,
from,
to,
MaxNoOfPosts);
//parse the result and do something with it
DoSomethingWithResults(data);
var numberOfPosts = int.Parse(data.Attributes["noOfPosts"].InnerText);
if (numberOfPosts == 0)
{
// If we didn't get any posts in the specified timeinterval, we set from to to and save it
from = to;
}
else
{
// We got some posts, so lastPost and lastPostMs attributes should exist
// Use them to update the from parameter, and add 1 millisecond,
// so we don't refetech the ones we already have fetched.
var lastPostTs = DateTime.
Parse(data.Attributes["lastPost"].InnerText).
ToUniversalTime();
var lastPostMs = Double.
Parse(data.Attributes["lastPostMs"].InnerText);
from = lastPostTs.AddMilliseconds(lastPostMs + 1);
}
WriteFromTimestampToDisk(from);
if (numberOfPosts >= MaxNoOfPosts)
{
// If we get max number of posts or more, we know there might be more to fetch, so no need to sleep.
Console.WriteLine("There is possibly more data available, trying again immediately");
}
else
{
var milliSecondsToSleep = 4 * 60 * 1000; //sleep 4 minutes
requestStopwatch.Stop();
// Adjust the time to sleep based on how long it took to fetch and process the last batch
milliSecondsToSleep = milliSecondsToSleep - (int)requestStopwatch.Elapsed.TotalMilliseconds;
if (milliSecondsToSleep > 0)
{
Console.WriteLine(
"Have fetched {0} posts between {1} and {2} since startup. Going to sleep for {3} ms.",
_numberOfPostsSinceStartup, programStartTimestamp, to, milliSecondsToSleep);
Thread.Sleep(milliSecondsToSleep);
}
}
}
catch (Exception ex)
{
// Log the error
Console.WriteLine("Got some kind of exception.. " + ex.Message);
// Wait one minute and try again.
Thread.Sleep(60*1000);
}
}
}