private void WriteUrl(Dictionary <string, object> obj, ArrayList um)
{
if (um != null)
{
for (int i = 0; i < um.Count; i++)
{
string url = (string)((Dictionary <string, object>)um[i])["url"];
if (url.StartsWith("http://t.co/"))
{
url = url.Substring(12);
}
else if (url.StartsWith("http://t.co/"))
{
url = url.Substring(13);
}
else
{
continue;
}
// [run_id] [smallint] NOT NULL
BulkWriter.WriteSmallInt(RunID);
// [tweet_id] [bigint] NOT NULL
BulkWriter.WriteBigInt(JsonUtil.GetInt64(obj, "id"));
// [user_id] [bigint] NOT NULL
BulkWriter.WriteBigInt(JsonUtil.GetInt64(obj, "user.id"));
// [url_id] [char](8) NOT NULL
BulkWriter.WriteChar(url, 8);
// [created_at] [datetime] NOT NULL
BulkWriter.WriteDateTime(JsonUtil.GetDateTime(obj, "created_at"));
// [expanded_url] [varchar](8000)
if (((Dictionary <string, object>)um[i]).ContainsKey("expanded_url"))
{
BulkWriter.WriteVarChar((string)((Dictionary <string, object>)um[i])["expanded_url"], 8000);
}
else
{
BulkWriter.WriteVarChar(null, 8000);
}
BulkWriter.EndLine();
}
}
}
private void MapOne(Dictionary <string, object> obj)
{
var user = (Dictionary <string, object>)obj["user"];
// [run_id] [smallint] NOT NULL
BulkWriter.WriteSmallInt(RunID);
// [user_id] [bigint] NOT NULL
BulkWriter.WriteBigInt(JsonUtil.GetInt64(user, "id"));
// [created_at] [datetime] NOT NULL
BulkWriter.WriteDateTime(JsonUtil.GetDateTime(user, "created_at"));
// [tweeted_at] [datetime] NOT NULL
BulkWriter.WriteDateTime(JsonUtil.GetDateTime(obj, "created_at"));
// [screen_name] [nvarchar](50) NOT NULL
BulkWriter.WriteVarChar(JsonUtil.GetString(user, "screen_name") ?? "", 50);
// [description] [nvarchar](160) NOT NULL
BulkWriter.WriteVarChar(Util.UnescapeText(JsonUtil.GetString(user, "description")) ?? "", 160);
// [favourites_count] [int] NOT NULL
BulkWriter.WriteInt(JsonUtil.GetNullableInt32(user, "favourites_count") ?? 0);
// [followers_count] [int] NOT NULL
BulkWriter.WriteInt(JsonUtil.GetNullableInt32(user, "followers_count") ?? 0);
// [friends_count] [int] NOT NULL
BulkWriter.WriteInt(JsonUtil.GetNullableInt32(user, "friends_count") ?? 0);
// [statuses_count] [int] NOT NULL
BulkWriter.WriteInt(JsonUtil.GetNullableInt32(user, "statuses_count") ?? 0);
// [geo_enabled] [bit] NOT NULL
BulkWriter.WriteBit(JsonUtil.GetNullableBoolean(user, "geo_enabled") ?? false);
// [lang] [char](5) NOT NULL,
BulkWriter.WriteChar(JsonUtil.GetString(user, "lang") ?? "", 5);
// [location] [nvarchar](100) NULL
BulkWriter.WriteVarChar(Util.UnescapeText(JsonUtil.GetString(user, "location")), 100);
// [name] [nvarchar](30) NOT NULL
BulkWriter.WriteVarChar(Util.UnescapeText(JsonUtil.GetString(user, "name")) ?? "", 30);
// [profile_background_color] [char](6) NOT NULL
BulkWriter.WriteChar(JsonUtil.GetString(user, "profile_background_color") ?? "000000", 6);
// [profile_text_color] [char](6) NOT NULL
BulkWriter.WriteChar(JsonUtil.GetString(user, "profile_text_color") ?? "000000", 6);
// [protected] [bit] NOT NULL
BulkWriter.WriteBit(JsonUtil.GetNullableBoolean(user, "protected") ?? false);
// [show_all_inline_media] [bit] NOT NULL
BulkWriter.WriteBit(JsonUtil.GetNullableBoolean(user, "show_all_inline_media") ?? false);
// [utc_offset] [int] NULL
BulkWriter.WriteNullableInt(JsonUtil.GetNullableInt32(user, "utc_offset"));
// [verified] [bit] NOT NULL
BulkWriter.WriteBit(JsonUtil.GetNullableBoolean(user, "verified") ?? false);
BulkWriter.EndLine();
}
private void MapOne(Dictionary <string, object> obj)
{
// [run_id] [smallint] NOT NULL
BulkWriter.WriteSmallInt(RunID);
// [tweet_id] [bigint] NOT NULL
BulkWriter.WriteBigInt(JsonUtil.GetInt64(obj, "id"));
// [created_at] [datetime] NOT NULL
BulkWriter.WriteDateTime(JsonUtil.GetDateTime(obj, "created_at"));
// [utc_offset] [int] NULL
BulkWriter.WriteNullableInt(JsonUtil.GetNullableInt32(obj, "user.utc_offset"));
// [user_id] [bigint] NOT NULL
BulkWriter.WriteBigInt(JsonUtil.GetInt64(obj, "user.id"));
// [place_id] [char](16) NULL
BulkWriter.WriteNullableChar(JsonUtil.GetString(obj, "place.id"), 16);
// [lon] [float] NULL
// [lat] [float] NULL
// [cx] [float] NOT NULL
// [cy] [float] NOT NULL
// [cz] [float] NOT NULL
// [htm_id] [bigint] NOT NULL
if (obj.ContainsKey("coordinates") && obj["coordinates"] != null)
{
string[] coords = (string[])((ArrayList)JsonUtil.GetValue(obj, "coordinates.coordinates")).ToArray(typeof(string));
var lon = double.Parse(coords[0], System.Globalization.CultureInfo.InvariantCulture);
var lat = double.Parse(coords[1], System.Globalization.CultureInfo.InvariantCulture);
var c = new Cartesian(lon, lat);
BulkWriter.WriteNullableFloat(lon);
BulkWriter.WriteNullableFloat(lat);
BulkWriter.WriteFloat(c.X);
BulkWriter.WriteFloat(c.Y);
BulkWriter.WriteFloat(c.Z);
BulkWriter.WriteBigInt(Trixel.CartesianToHid20(c));
}
else
{
BulkWriter.WriteNullableFloat(null);
BulkWriter.WriteNullableFloat(null);
BulkWriter.WriteFloat(0);
BulkWriter.WriteFloat(0);
BulkWriter.WriteFloat(0);
BulkWriter.WriteBigInt(0);
}
// [in_reply_to_tweet_id] [bigint] NULL
BulkWriter.WriteNullableBigInt(JsonUtil.GetNullableInt64(obj, "in_reply_to_status_id"));
// [in_reply_to_user_id] [bigint] NULL
BulkWriter.WriteNullableBigInt(JsonUtil.GetNullableInt64(obj, "in_reply_to_user_id"));
// [possibly_sensitive] [bit] NULL
// [possibly_sensitive_editable] [bit] NULL
if (obj.ContainsKey("possibly_sensitive") && obj.ContainsKey("possibly_sensitive_editable"))
{
BulkWriter.WriteNullableBit(JsonUtil.GetNullableBoolean(obj, "possibly_sensitive"));
BulkWriter.WriteNullableBit(JsonUtil.GetNullableBoolean(obj, "possibly_sensitive_editable"));
}
else
{
BulkWriter.WriteNullableBit(null);
BulkWriter.WriteNullableBit(null);
}
// [retweet_count] [int] NOT NULL
BulkWriter.WriteInt(JsonUtil.GetNullableInt32(obj, "retweet_count") ?? 0);
// [text] [nvarchar](150) NOT NULL
var text = Util.UnescapeText(System.Web.HttpUtility.HtmlDecode(JsonUtil.GetString(obj, "text")));
BulkWriter.WriteVarChar(text, 150);
// [truncated] [bit] NOT NULL
BulkWriter.WriteBit(JsonUtil.GetNullableBoolean(obj, "truncated") ?? false);
// [lang] [char](5) NOT NULL
BulkWriter.WriteChar(JsonUtil.GetString(obj, "user.lang") ?? "??", 5);
// [lang_word_count] [tinyint] NOT NULL
// [lang_guess1] [char](2) NOT NULL
// [lang_guess2] [char](2) NOT NULL
int words;
string lang1, lang2;
if (LanguageUtil.DetectLanguage(text, out words, out lang1, out lang2))
{
BulkWriter.WriteTinyInt((sbyte)words); // [lang_word_count]
BulkWriter.WriteChar(lang1, 2); // [lang_guess1]
BulkWriter.WriteChar(lang2, 2); // [lang_guess2]
}
else
{
BulkWriter.WriteTinyInt(0);
BulkWriter.WriteChar("??", 2);
BulkWriter.WriteChar("??", 2);
}
BulkWriter.EndLine();
}