/// <summary>
/// transforme les tweets en liste d'entrées de playlist
/// </summary>
/// <param name="youtube_client"></param>
/// <param name="soundcloud_client"></param>
/// <returns></returns>
public List<Music.PlayListEntry> getPlaylistEntriesFromTweets(Youtube.Youtube youtube_client, SoundCloud.SoundCloud soundcloud_client)
{
List<Music.PlayListEntry> res = new List<PlayListEntry>();
List<IMention> tweets;
try
{
tweets = (from t in this.getTweets().ToList() where t.CreatedAt >= this.last_update && t.Urls.Count() > 0 select t).ToList();
}
catch(NullReferenceException e)
{
Console.WriteLine("Une erreur est survenue lors de la récupération des tweets.");
return res;
}
Console.WriteLine("analizing tweets");
foreach(IMention t in tweets)
{
string user = t.CreatedBy.Name;
string url = t.Urls[0].ExpandedURL;
ITrack track = null;
if (Youtube.Youtube.isCompatible(url))
track = youtube_client.resolveTrack(url);
else if (SoundCloud.SoundCloud.isCompatible(url))
track = soundcloud_client.resolveTrack(url);
if(track != null)
res.Add(new PlayListEntry(track, user, false));
}
this.last_update = DateTime.Now;
return res;
}
public void LlamadasATwitterXCategoria()
{
if (listaCategorias != null && listaCategorias.Any())
{
listaCategorias = listaCategorias.Skip(1).ToList();
Task.Factory.StartNew(() => SearchStream(listaCategorias.Select(x => x.value).ToList()));
}
}
public Queue<TwitterFeed> CreateLiveFeed(List<string> peopleInput, List<string> tweetsInput)
{
var people = this.CreatePeople(peopleInput).OrderBy(x => x.Name).ToList();
var tweets = this.CreateTweets(people, tweetsInput).OrderBy(x => x.Order).ToList();
var liveFeed = this.CreateLiveFeed(people, tweets);
return liveFeed;
}
public IEnumerable<dynamic> GetLists(string twitterName)
{
var service = new TwitterService(CONSUMER_KEY, CONSUMER_SECRET);
var expandoList = new List<dynamic>();
var options = new ListListsForOptions();
options.ScreenName = "pparadis";
expandoList = service
.ListListsFor(options)
.Select(p => ToExpando(p))
.OrderBy(p => p.FullName)
.ToList();
return expandoList;
}
private static void LanzarPrograma(List<Category> listaCategorias, List<Province> listaProvincias)
{
Console.WriteLine("DEMO BIG DATA");
Console.WriteLine("Selecciona la opción que quieres ejecutar:");
Console.WriteLine("1 - Generar archivo Infojobs");
Console.WriteLine("2 - Generar archivos Twitter (Para salir pulse cualquier tecla)");
Console.WriteLine("3 - Procesar archivo Infojobs en HDInsight - Azure");
Console.WriteLine("4 - Procesar archivos Twitter en HDInsight - Azure");
Console.WriteLine("5 - Salir");
var linea = Console.ReadLine();
switch (linea)
{
case "1":
Console.WriteLine("Generando archivo de ofertas de Infojobs ...");
var infojobs = new InfoJobsRepository(listaCategorias, listaProvincias);
infojobs.ProcesarOfertasPorCategoria();
Console.WriteLine("Finalizado. Pulse cualquier tecla para salir");
break;
case "2":
Console.WriteLine("Generando archivo de tweets ...");
var twitter = new TwitterRepository(listaCategorias, listaProvincias);
twitter.LlamadasATwitterXCategoria();
break;
case "3":
Console.WriteLine("Procesando ofertas ...");
var hdinsight = new HDInsightRepository();
hdinsight.LanzarInfoJobsJob();
Console.WriteLine("Finalizado. Pulse cualquier tecla para salir");
break;
case "4":
Console.WriteLine("Procesando tweets ...");
var hdinsight2 = new HDInsightRepository();
hdinsight2.LanzarTwitterJob();
Console.WriteLine("Finalizado. Pulse cualquier tecla para salir");
break;
case "5":
return;
default:
Console.WriteLine("Opción no reconocida");
break;
}
Console.ReadLine();
Console.Clear();
LanzarPrograma(listaCategorias, listaProvincias);
}
private void SearchStream(List<string> tracks)
{
TwitterCredentials.SetCredentials(ACCESS_TOKEN, ACCESS_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET);
var filteredStream = Stream.CreateFilteredStream();
foreach (var track in tracks)
{
filteredStream.AddTrack(track);
}
filteredStream.MatchingTweetReceived += filteredStream_MatchingTweetReceived;
filteredStream.StartStreamMatchingAllConditions();
}
public FrmAddAccount()
{
InitializeComponent();
m_oaApi = new OAuthAPI();
m_alStatusPanels = new List<Panel>();
m_alStatusPanels.Add(pnlWelcome);
m_alStatusPanels.Add(pnlUrl);
m_alStatusPanels.Add(pnlPin);
m_alStatusPanels.Add(pnlDone);
m_oaApi.RequestTokenReceived += new OAuthAPI.OAuthCallbackHandler(m_oaApi_RequestTokenReceived);
m_oaApi.AccessTokenReceived += new OAuthAPI.OAuthCallbackHandler(m_oaApi_AccessTokenReceived);
}
private static byte[] GetMacAddressId()
{
List<string> lsAddresses = new List<string>();
StringBuilder sbAllAddrs = new StringBuilder();
foreach (NetworkInterface nic in NetworkInterface.GetAllNetworkInterfaces())
lsAddresses.Add(nic.GetPhysicalAddress().ToString());
lsAddresses.Sort();
foreach (string sAddr in lsAddresses)
sbAllAddrs.Append(sAddr);
return Encoding.ASCII.GetBytes(sbAllAddrs.ToString());
}
public List<Twit> ParseString(string json)
{
if (String.IsNullOrEmpty (json))
return new List<Twit> ();
var _tweets = new List<Twit>();
Newtonsoft.Json.Linq.JObject jObject = Newtonsoft.Json.Linq.JObject.Parse(json);
foreach (var current in jObject["statuses"])
{
var tweet = new Twit();
tweet.ImagePath = current["user"]["profile_image_url"].ToString();
tweet.Info = current["text"].ToString ();
tweet.PostDate = current["created_at"].ToString();
tweet.SmallInfo = current["user"]["description"].ToString ();
tweet.Name = current["user"]["name"].ToString ();
_tweets.Add(tweet);
}
return _tweets;
}
// Go through list of users and the people they are following
// Create new person and followers
private List<Person> CreatePeople(List<string> peopleInput)
{
var people = new List<Person>();
foreach (var follow in peopleInput)
{
var tempPerson = new Person();
var separation = follow.Split(new string[] { "follows" }, StringSplitOptions.None);
// check that the string has been split correctly
// if the string has been split correctly by the word follows the user and the people it follows will be created
if (separation.Length > 1)
{
var followSeparation = separation[1].Split(',');
// check is person already exists
// if it exists, add the people it follows
var existingPerson = people.FirstOrDefault(x => x.Name == separation[0]);
if (existingPerson != null)
{
existingPerson.Following.AddRange(followSeparation);
existingPerson.Following = existingPerson.Following.Distinct().ToList();
continue;
}
else
{
// create a new person
tempPerson.Name = separation[0].Trim();
tempPerson.Following = new List<string>();
tempPerson.Following.AddRange(followSeparation);
}
people.Add(tempPerson);
}
}
return people;
}
public TwitterRepository(List<Category> listaCategorias, List<Province> listaProvincias)
{
this.listaCategorias = listaCategorias;
this.listaProvincias = listaProvincias;
}
public void AddRange(List<Twit> twits)
{
_tableItems = twits;
}
public TableSource(List<Twit> items)
{
_tableItems = items;
}
// Goes through the list of tweets and assigns a person to a tweet
// adds a number to the tweet which is the order the tweets were "written"
private List<Tweet> CreateTweets(List<Person> people, List<string> tweetsInput)
{
var tweets = new List<Tweet>();
var count = 0; // count to keep track of the order of the tweets
foreach (var tweet in tweetsInput)
{
var separation = tweet.Split('>').ToArray();
// check that the string has been split correctly by >
// if it is not split correctly the tweets are not added
if (separation.Length > 1)
{
count++;
var actualTweet = new Tweet()
{
Person = people.FirstOrDefault(x => x.Name.Trim() == separation[0].Trim()),
Order = count
};
// check for 140 charachters
// if twwet in the character range, add the whole tweet to the list
if (separation[1].Length <= 140)
{
actualTweet.Description = separation[1];
}
// if the tweet is greater than 140 characters
else if (separation[1].Length > 140)
{
actualTweet.Description = separation[1].Substring(0, 140);
}
tweets.Add(actualTweet);
}
}
return tweets;
}
//Used for extracting data from API response
public static List<string> extractData(string outputText, string start, string end)
{
#region Find all the position of all mentions of start string
List<int> startPos = new List<int>();
List<int> stopPos = new List<int>();
int i = 0;
while ((i = outputText.IndexOf(start, i)) != -1)
{
startPos.Add(i);
i++;
}
#endregion
#region Do the same for end string
i = 0;
while ((i = outputText.IndexOf(end, i)) != -1)
{
stopPos.Add(i);
i++;
}
#endregion
//Data to return
List<string> returnMe = new List<string>();
//If we have a different number of start and end points, something has gone wrong
if (startPos.Count != stopPos.Count)
startPos.Remove(startPos[startPos.Count - 1]);//Try fixing
for (int j = startPos.Count - 1; j > -1; j--)
{
string output = "";
for (int c = startPos[j]; c < stopPos[j]; c++)
{
output += outputText[c];
}
if (start != ",\"entities\":")
{
#region Format output string
output = output.Replace(start, "");
//Remove emoji type things
output = output.Replace("\ud83c[\udf00-\udfff]", " ! ");
output = output.Replace("\\\"", "\"");
output = output.Replace("\\/", "/");
output = output.Replace("&", "&");
//Attempt at emoji removal
List<int> EmojisOrSimilar = new List<int>();
i = 0;
while ((i = output.IndexOf("\\u", i)) != -1)
{
EmojisOrSimilar.Add(i);
i++;
}
for (int u = EmojisOrSimilar.Count - 1; u > -1; u--)
{
//Emoji text is typically 6 characters long
output = output.Remove(EmojisOrSimilar[u], 6);
output.Insert(EmojisOrSimilar[u], "*!*");
}
#endregion
}
else if (output != "[]")
{
//Remove text from original input and return
//Remove each section of the string STARTING AT THE END AND WORKING BACK
outputText = outputText.Remove(startPos[j] + 1, output.Length);// + start.Length - 1);
output = null;
ammendOutputText = outputText;
}
returnMe.Add(output);
}
return returnMe;
}
// Goes through people list and tweet list and creates the twitter feed
private Queue<TwitterFeed> CreateLiveFeed(List<Person> people, List<Tweet> tweets)
{
var twitterFeedQueue = new Queue<TwitterFeed>();
// add all names
var users = people.Select(x => x.Name).ToList();
// add all following
users.AddRange(people.SelectMany(x => x.Following));
// sort users alphabetically
users = users.Select(x => x.Trim()).Distinct().OrderBy(x => x).ToList();
// Go through the list of people in alphabetical order
// add each person to the twitter feed
// enqueue the twitter feed in order
// for each person add either their own tweets or the people they are followings' tweets
foreach (var user in users)
{
var feed = new TwitterFeed();
var allTweets = new List<string>();
var person = people.FirstOrDefault(x => x.Name == user);
feed.Name = user;
if (person != null)
{
foreach (var tweet in tweets)
{
//check if it's person's tweet
if (tweet.Person != null)
{
if (tweet.Person.Name.Trim() == person.Name.Trim())
{
allTweets.Add("@" + tweet.Person.Name.Trim() + ": " + tweet.Description.Trim());
}
// check for the people followings' tweet
var followingTweet = person.Following.FirstOrDefault(x => x.Trim() == tweet.Person.Name.Trim());
if (followingTweet != null)
{
allTweets.Add("@" + tweet.Person.Name.Trim() + ": " + tweet.Description.Trim());
}
}
}
feed.AllTweets = allTweets;
}
twitterFeedQueue.Enqueue(feed);
}
return twitterFeedQueue;
}
public AsyncContent()
{
m_lacCallbacks = new List<AsyncContentCallback>();
}
public static tw_DateTime formatDateTime(string dateTime)
{
string temp = "";
List<string> date = new List<string>();
for (int k = 0; k < dateTime.Length; k++)
{
if (dateTime[k] != ' ')
temp += dateTime[k];
else
{
date.Add(temp);
temp = "";
}
if (k == dateTime.Length - 1)
date.Add(temp);
}
temp = "";
List<string> timeOfDay = new List<string>();
for (int k = 0; k < date[3].Length; k++)
{
if (date[3][k] != ':')
temp += date[3][k];
else
{
timeOfDay.Add(temp);
temp = "";
}
if (k == date[3].Length - 1)
timeOfDay.Add(temp);
}
Twitter.API.tw_DateTime time = new Twitter.API.tw_DateTime();
time.Weekday = date[0];
time.Month = date[1];
time.Day = int.Parse(date[2]);
time.Hour = int.Parse(timeOfDay[0]);
time.Minute = int.Parse(timeOfDay[1]);
time.Second = int.Parse(timeOfDay[2]);
time.Year = int.Parse(date[5]);
time.Offset = date[4];
return time;
}
public static void GetUserTimeline(string name, string AccessToken, int count, twitterButton caller)
{
Dictionary<string, string> headers = new Dictionary<string, string>();
headers["Authorization"] = "Bearer " + AccessToken;
WWW web = new WWW("https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name=" + name + "&count=" + count + "&trim_user=1" + "&include_rts=0&exclude_replies=true&contributor_details=false", null, headers);
while (!web.isDone)
{
Debug.Log("Grabbing timeline...");
}
//find user mentions
List<string> mentions = extractData(web.text, ",\"user_mentions\":", ",\"urls\":");
//remove if true
string extractMe;
if (ammendOutputText == null)
extractMe = web.text;
else
extractMe = ammendOutputText;
Debug.Log("Extracting profile data...");
List<string> dateTime = extractData(extractMe, "{\"created_at\":\"", "\",\"id\":");
List<string> text = extractData(extractMe, ",\"text\":\"", ",\"truncated\":");
List<string> favs = extractData(extractMe, "\"favorite_count\":", ",\"entities\":");
List<string> RTs = extractData(extractMe, "\"retweet_count\":", ",\"favorite_count\":");
List<string> userID = extractData(extractMe, "\"user\":{\"id\":", "\"},\"geo\":");
List<string> tweetID = extractData(extractMe, ",\"id\":", "\",\"text\":");
List<Tweet> tweets = new List<Tweet>();
//format datetime
//
//For each time a space is detected,
//do things with boop[2]
for (int i = 0; i < text.Count; i++)
{
Tweet thisTweet = new Tweet();
#region dateTime formating
string temp = "";
List<string> boop = new List<string>();
for (int k = 0; k < dateTime[i].Length; k++)
{
if (dateTime[i][k] != ' ')
temp += dateTime[i][k];
else
{
boop.Add(temp);
temp = "";
}
if (k == dateTime[i].Length - 1)
boop.Add(temp);
}
temp = "";
List<string> doop = new List<string>();
for (int k = 0; k < boop[3].Length; k++)
{
if (boop[3][k] != ':')
temp += boop[3][k];
else
{
doop.Add(temp);
temp = "";
}
if (k == boop[3].Length - 1)
doop.Add(temp);
}
tw_DateTime time = new tw_DateTime();
time.Weekday = boop[0];
time.Month = boop[1];
time.Day = boop[2];
time.Hour = doop[0];
time.Minute = doop[1];
time.Second = doop[2];
time.Year = boop[3];
time.Offset = boop[4];
#endregion
thisTweet.dateTime = time;
thisTweet.Text = text[i];
thisTweet.UserID = userID[i].Substring(0, userID[i].IndexOf(",\"id_str"));
thisTweet.RTs = int.Parse(RTs[i]);
thisTweet.Favs = int.Parse(favs[i]);
thisTweet.ID = tweetID[i].Substring(0, tweetID[i].IndexOf(",\"id_str"));
tweets.Add(thisTweet);
}
caller.tweets = tweets;
ammendOutputText = null;
}
public DC_FriendshipsLookupRequest()
{
screen_name = new List<string>();
user_id = new List<long>();
}
public void AddRange(List<Twit> twits)
{
_tableItems = twits;
_tableItems.Add (new Twit());
}
public static List<string> extractData(string outputText, string start, string end)
{
List<int> startPos = new List<int>();
List<int> stopPos = new List<int>();
int i = 0;
while ((i = outputText.IndexOf(start, i)) != -1)
{
startPos.Add(i);
i++;
}
i = 0;
while ((i = outputText.IndexOf(end, i)) != -1)
{
stopPos.Add(i);
i++;
}
List<string> returnMe = new List<string>();
for (int j = startPos.Count - 1; j > -1; j--)
{
string output = "";
for (int c = startPos[j]; c < stopPos[j]; c++)
{
output += outputText[c];
}
output = output.Replace(start, "");
output = output.Replace("\ud83c[\udf00-\udfff]", " ! ");
output = output.Replace("\\\"", "\"");
output = output.Replace("\\/", "/");
output = output.Replace("&", "&");
List<int> fuckyou = new List<int>();
i = 0;
while ((i = output.IndexOf("\\u", i)) != -1)
{
fuckyou.Add(i);
i++;
}
for (int u = fuckyou.Count - 1; u > -1; u--)
output = output.Remove(fuckyou[u], 6);
if (output != "[]" && start == ",\"user_mentions\":")
{
outputText = outputText.Remove(startPos[j] + 1 + start.Length, output.Length - 1);
output = null;
ammendOutputText = outputText;
}
returnMe.Add(output);
}
return returnMe;
}
public static List<string> extractData(string outputText, string start)
{
List<int> startPos = new List<int>();
int i = 0;
while ((i = outputText.IndexOf(start, i)) != -1)
{
startPos.Add(i);
i++;
}
List<string> returnMe = new List<string>();
for (int j = startPos.Count - 1; j > -1; j--)
{
string output = "";
int endPoint = outputText.Length;
if (j + 1 < startPos.Count)
{
endPoint = startPos[j + 1];
}
Debug.Log(endPoint);
for (int c = startPos[j]; c < endPoint; c++)
{
output += outputText[c];
}
output = output.Replace(start, "");
output = output.Replace("\\n", " ");
output = output.Replace("\\", "");
returnMe.Add(output);
}
return returnMe;
}
