static bool ParseUser(JsonObject juser, User user, HashSet <long> usersSeen)
{
try {
user.UpdateFromJson((JsonObject)juser);
if (!usersSeen.Contains(user.Id))
{
usersSeen.Add(user.Id);
Database.Main.Insert(user, "OR REPLACE");
}
} catch {
return(false);
}
return(true);
}
//
// Loads a single user from the stream
//
static public User LoadUser(Stream source)
{
JsonValue root;
try {
root = JsonValue.Load(source);
} catch (Exception e) {
Console.WriteLine(e);
return(null);
}
User user = new User();
user.UpdateFromJson((JsonObject)root);
lock (Database.Main)
Database.Main.Insert(user, "OR REPLACE");
return(user);
}
//
// Loads the users from the stream, as a convenience,
// returns the last user loaded (which during lookups is a single one)
//
// Requires datbase lock to be taken.
static public IEnumerable <User> UnlockedLoadUsers(Stream source)
{
JsonValue root;
try {
root = (JsonValue)JsonValue.Load(source);
} catch (Exception e) {
Console.WriteLine(e);
yield break;
}
foreach (JsonObject juser in root)
{
User user = new User();
user.UpdateFromJson(juser);
Database.Main.Insert(user, "OR REPLACE");
yield return(user);
}
}
static Tweet ParseTweet(Stream stream)
{
JsonObject jentry;
try {
jentry = (JsonObject)JsonValue.Load(stream);
} catch (Exception e) {
Console.WriteLine(e);
return(null);
}
try {
var user = new User();
user.UpdateFromJson((JsonObject)jentry ["user"]);
lock (Database.Main)
Database.Main.Insert(user, "OR REPLACE");
return(FromJsonEntry(jentry, user));
} catch (Exception e) {
Console.WriteLine(e);
return(null);
}
}
static Tweet FromJsonEntry(JsonObject jentry, User user)
{
var tweet = new Tweet()
{
Kind = TweetKind.Transient
};
if (!tweet.TryPopulate(jentry))
{
return(null);
}
PopulateUser(tweet, user);
if (tweet.Retweeter != null)
{
user = new User();
user.UpdateFromJson((JsonObject)(jentry ["retweeted_status"]["user"]));
lock (Database.Main)
Database.Main.Insert(user, "OR REPLACE");
}
return(tweet);
}
//
// Loads a single user from the stream
//
public static User LoadUser(Stream source)
{
JsonValue root;
try {
root = JsonValue.Load (source);
} catch (Exception e){
Console.WriteLine (e);
return null;
}
User user = new User ();
user.UpdateFromJson ((JsonObject) root);
lock (Database.Main)
Database.Main.Insert (user, "OR REPLACE");
return user;
}
//
// Loads the users from the stream, as a convenience,
// returns the last user loaded (which during lookups is a single one)
//
// Requires datbase lock to be taken.
public static IEnumerable<User> UnlockedLoadUsers(Stream source)
{
JsonValue root;
try {
root = (JsonValue) JsonValue.Load (source);
} catch (Exception e) {
Console.WriteLine (e);
yield break;
}
foreach (JsonObject juser in root){
User user = new User ();
user.UpdateFromJson (juser);
Database.Main.Insert (user, "OR REPLACE");
yield return user;
}
}
static bool ParseUser(JsonObject juser, User user, HashSet<long> usersSeen)
{
try {
user.UpdateFromJson ((JsonObject) juser);
if (!usersSeen.Contains (user.Id)){
usersSeen.Add (user.Id);
Database.Main.Insert (user, "OR REPLACE");
}
} catch {
return false;
}
return true;
}
static Tweet ParseTweet(Stream stream)
{
JsonObject jentry;
try {
jentry = (JsonObject) JsonValue.Load (stream);
} catch (Exception e) {
Console.WriteLine (e);
return null;
}
try {
var user = new User ();
user.UpdateFromJson ((JsonObject) jentry ["user"]);
lock (Database.Main)
Database.Main.Insert (user, "OR REPLACE");
return FromJsonEntry (jentry, user);
} catch (Exception e){
Console.WriteLine (e);
return null;
}
}
static Tweet FromJsonEntry(JsonObject jentry, User user)
{
var tweet = new Tweet () { Kind = TweetKind.Transient };
if (!tweet.TryPopulate (jentry))
return null;
PopulateUser (tweet, user);
if (tweet.Retweeter != null){
user = new User ();
user.UpdateFromJson ((JsonObject)(jentry ["retweeted_status"]["user"]));
lock (Database.Main)
Database.Main.Insert (user, "OR REPLACE");
}
return tweet;
}
//
// Alternative version that just parses the users and tweets and returns them as lists
// I thought it would be useful, but it is not. The JSon parsing is too fast, we
// only get bogged down with the actual sqlite insert
//
public static void ParseJson(Stream stream, int localAccount, TweetKind kind, out List<User> users, out List<Tweet> tweets)
{
JsonValue root;
string userKey;
try {
root = JsonValue.Load (stream);
if (kind == TweetKind.Direct)
userKey = "sender";
else
userKey = "user";
} catch (Exception e) {
Console.WriteLine (e);
tweets = null;
users = null;
return;
}
users = new List<User> (root.Count/4);
tweets = new List<Tweet> (root.Count);
var start = DateTime.UtcNow;
var usersSeen = new HashSet<long> ();
var user = new User ();
foreach (JsonObject jentry in root){
var juser = jentry [userKey];
bool result;
try {
user.UpdateFromJson ((JsonObject) juser);
if (!usersSeen.Contains (user.Id)){
usersSeen.Add (user.Id);
users.Add (user);
user = new User ();
}
} catch {
continue;
}
var tweet = new Tweet ();
if (kind == TweetKind.Direct)
result = tweet.TryPopulateDirect (jentry);
else
result = tweet.TryPopulate (jentry);
if (result){
PopulateUser (tweet, user);
tweets.Add (tweet);
}
// Repeat user loading for the retweet info
if (tweet.Retweeter != null){
user.UpdateFromJson ((JsonObject)(jentry ["retweeted_status"]["user"]));
if (!usersSeen.Contains (user.Id)){
usersSeen.Add (user.Id);
users.Add (user);
user = new User ();
}
}
}
var end = DateTime.UtcNow;
Console.WriteLine ("Parsing time for tweet stream: {0} for {1} tweets", end-start, tweets.Count);
}
//
// Alternative version that just parses the users and tweets and returns them as lists
// I thought it would be useful, but it is not. The JSon parsing is too fast, we
// only get bogged down with the actual sqlite insert
//
public static void ParseJson(Stream stream, int localAccount, TweetKind kind, out List <User> users, out List <Tweet> tweets)
{
JsonValue root;
string userKey;
try {
root = JsonValue.Load(stream);
if (kind == TweetKind.Direct)
{
userKey = "sender";
}
else
{
userKey = "user";
}
} catch (Exception e) {
Console.WriteLine(e);
tweets = null;
users = null;
return;
}
users = new List <User> (root.Count / 4);
tweets = new List <Tweet> (root.Count);
var start = DateTime.UtcNow;
var usersSeen = new HashSet <long> ();
var user = new User();
foreach (JsonObject jentry in root)
{
var juser = jentry [userKey];
bool result;
try {
user.UpdateFromJson((JsonObject)juser);
if (!usersSeen.Contains(user.Id))
{
usersSeen.Add(user.Id);
users.Add(user);
user = new User();
}
} catch {
continue;
}
var tweet = new Tweet();
if (kind == TweetKind.Direct)
{
result = tweet.TryPopulateDirect(jentry);
}
else
{
result = tweet.TryPopulate(jentry);
}
if (result)
{
PopulateUser(tweet, user);
tweets.Add(tweet);
}
// Repeat user loading for the retweet info
if (tweet.Retweeter != null)
{
user.UpdateFromJson((JsonObject)(jentry ["retweeted_status"]["user"]));
if (!usersSeen.Contains(user.Id))
{
usersSeen.Add(user.Id);
users.Add(user);
user = new User();
}
}
}
var end = DateTime.UtcNow;
Util.Log("Parsing time for tweet stream: {0} for {1} tweets", end - start, tweets.Count);
}