public IEnumerable <Tweet> StreamStatuses(TwitterConfig config)
{
DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));
var streamReader = ReadTweets(config);
while (keepRunning)
{
string line = null;
try { line = streamReader.ReadLine(); }
catch (Exception) { }
if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\""))
{
var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
result.RawJson = line;
yield return(result);
}
// Oops the Twitter has ended... or more likely some error have occurred.
// Reconnect to the twitter feed.
if (line == null)
{
streamReader = ReadTweets(config);
}
}
}
private void WriteToFile(string rawJsonLine, TwitterConfig config)
{
// need to deserialize and re-serialize to get rid of \u escaped text
dynamic jsonObject = JsonConvert.DeserializeObject(rawJsonLine);
var rawJson = JsonConvert.SerializeObject(jsonObject);
// Add a new line to the end of the raw JSON for easier separation in the file:
rawJson += Environment.NewLine;
if (config.CreateBigFile)
{
string path = Path.Combine(config.FolderName, config.BigFileName);
if (File.Exists(path))
{
var currentFileSize = new FileInfo(path).Length;
if (currentFileSize > config.FileSizeLimit)
{
// overwrite config and local variable
config.InitializeBigFileName();
path = Path.Combine(config.FolderName, config.BigFileName);
}
}
File.AppendAllText(path, rawJson, Encoding.UTF8);
}
else
{
string path = Path.Combine(config.FolderName, TwitterConfig.GetTweetFilename());
// Without Encoding it will be written as UTF-8 w/o BOM
File.WriteAllText(path, rawJson);
}
}
private static string CreateOAuthHeader(TwitterConfig config, string baseUrl)
{
var oauth_version = "1.0";
var oauth_signature_method = "HMAC-SHA1";
// unique request details
var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
var oauth_timestamp = Convert.ToInt64(
(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
.TotalSeconds).ToString();
// create oauth signature
var baseString = string.Format(
"oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
"oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
config.OAuthConsumerKey,
oauth_nonce,
oauth_signature_method,
oauth_timestamp,
config.OAuthToken,
oauth_version,
Uri.EscapeDataString(config.Keywords));
baseString = string.Concat("POST&", Uri.EscapeDataString(baseUrl), "&", Uri.EscapeDataString(baseString));
var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
"&", Uri.EscapeDataString(config.OAuthTokenSecret));
string oauth_signature;
using (var hasher = new HMACSHA1(Encoding.ASCII.GetBytes(compositeKey)))
{
oauth_signature = Convert.ToBase64String(
hasher.ComputeHash(Encoding.ASCII.GetBytes(baseString)));
}
// create the request header
return(string.Format(
"OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
"oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
"oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
"oauth_version=\"{6}\"",
Uri.EscapeDataString(oauth_nonce),
Uri.EscapeDataString(oauth_signature_method),
Uri.EscapeDataString(oauth_timestamp),
Uri.EscapeDataString(config.OAuthConsumerKey),
Uri.EscapeDataString(config.OAuthToken),
Uri.EscapeDataString(oauth_signature),
Uri.EscapeDataString(oauth_version)
));
}
static TextReader ReadTweets(TwitterConfig config)
{
const string TwitterBaseUrlStreamingTweets = "https://stream.twitter.com/1.1/statuses/filter.json";
string authHeader = CreateOAuthHeader(config, TwitterBaseUrlStreamingTweets);
// make the request
ServicePointManager.Expect100Continue = false;
var postBody = "track=" + HttpUtility.UrlEncode(config.Keywords);
string resource_url = TwitterBaseUrlStreamingTweets + "?" + postBody;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);
request.Headers.Add("Authorization", authHeader);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.PreAuthenticate = true;
request.AllowWriteStreamBuffering = true;
request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
// bail out and retry after 5 seconds
var responseTask = request.GetResponseAsync();
try
{
if (responseTask.Wait(5000))
{
return(new StreamReader(responseTask.Result.GetResponseStream(), Encoding.UTF8)); // TWE: added Encoding.UTF8
}
else
{
request.Abort();
return(StreamReader.Null);
}
}
catch (AggregateException ae)
{
WriteException("Exception while reading tweets: ", ae);
request.Abort();
return(StreamReader.Null);
}
}
static TextReader ReadTweets(TwitterConfig config)
{
var oauth_version = "1.0";
var oauth_signature_method = "HMAC-SHA1";
// unique request details
var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
var oauth_timestamp = Convert.ToInt64(
(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
.TotalSeconds).ToString();
var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json";
// create oauth signature
var baseString = string.Format(
"oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
"oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
config.OAuthConsumerKey,
oauth_nonce,
oauth_signature_method,
oauth_timestamp,
config.OAuthToken,
oauth_version,
Uri.EscapeDataString(config.Keywords));
baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString));
var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
"&", Uri.EscapeDataString(config.OAuthTokenSecret));
string oauth_signature;
using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey)))
{
oauth_signature = Convert.ToBase64String(
hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString)));
}
// create the request header
var authHeader = string.Format(
"OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
"oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
"oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
"oauth_version=\"{6}\"",
Uri.EscapeDataString(oauth_nonce),
Uri.EscapeDataString(oauth_signature_method),
Uri.EscapeDataString(oauth_timestamp),
Uri.EscapeDataString(config.OAuthConsumerKey),
Uri.EscapeDataString(config.OAuthToken),
Uri.EscapeDataString(oauth_signature),
Uri.EscapeDataString(oauth_version)
);
// make the request
ServicePointManager.Expect100Continue = false;
// updating TLS version "TLS < 1.2 is unsupported by Tweeter starting July 15, 2019"
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
var postBody = "track=" + HttpUtility.UrlEncode(config.Keywords);
resource_url += "?" + postBody;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);
request.Headers.Add("Authorization", authHeader);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.PreAuthenticate = true;
request.AllowWriteStreamBuffering = true;
request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
// bail out and retry after 5 seconds
var tresponse = request.GetResponseAsync();
if (tresponse.Wait(5000))
{
return(new StreamReader(tresponse.Result.GetResponseStream()));
}
else
{
request.Abort();
return(StreamReader.Null);
}
}
public IEnumerable <Tweet> StreamStatuses(TwitterConfig config)
{
DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));
var streamReader = ReadTweets(config);
while (keepRunning)
{
string line = null;
try { line = streamReader.ReadLine(); }
catch (Exception) { }
if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\"") && !line.StartsWith("{\"limit\""))
{
Debug.WriteLine(line);
// Sometimes the line is not correctly read which will end up in a serialization exception
dynamic result = null;
try
{
result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
numberTweets++;
if (result.CreatedAt == null)
{
Console.Write("potential limit");
}
else
{
if ((config.IncludeRetweets == false) && IsRetweet(result))
{
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine("{0} Retweet will not be processed ****", numberTweets);
Console.ForegroundColor = previousColor;
}
else
{
WriteToFile(line, config);
WriteToConsole(line);
}
}
}
catch (SerializationException ex1)
{
WriteException("Line could not be de-serialized: " + line, ex1);
}
catch (JsonSerializationException ex2)
{
WriteException("Line could not be de-serialized: " + line, ex2);
}
if (result != null)
{
yield return(result);
}
}
// Oops the Twitter has ended... or more likely some error have occurred.
// Reconnect to the twitter feed.
if (line == null)
{
Console.BackgroundColor = ConsoleColor.Red;
Console.WriteLine("Potential Limit reached... will reconnect to feed in an instant.");
Console.BackgroundColor = ConsoleColor.Black;
// As an exception we use Thread.Sleep here
Thread.Sleep(1000);
streamReader = ReadTweets(config);
}
}
}