/// <summary>
/// Gets the VoS tweets from the memory cache or twitter api
/// </summary>
/// <param name="ignoreCache">Whether or not to use the cache</param>
/// <returns>VoS tweets</returns>
private async Task <IEnumerable <ITweet> > GetVosTweets(bool ignoreCache = false)
{
if (!ignoreCache && MemoryCache.Default.Get("vosTweets") != null)
{
return((IEnumerable <ITweet>)MemoryCache.Default.Get("vosTweets"));
}
var tweets = await TimelineAsync.GetUserTimeline(3429940841, 10);
// Checking if request was successful
if (tweets == null)
{
throw new ControlledException("Tweets from @JagexClock could not be retrieved.");
}
// Getting only vos tweets
tweets = tweets.Where(t => t.Text.Matches(VosRegex)).Take(2);
// Getting invalidation time
DateTime invalTime = DateTime.Today.AddHours(DateTime.Now.TimeOfDay.TotalHours.Ceiling());
// Storing tweets
MemoryCache.Default.Set("vosTweets", tweets, new DateTimeOffset(invalTime));
return(tweets);
}
public async Task <List <TweetDTO> > GetTweets(string accountName)
{
var user = new UserIdentifier(accountName);
var timeLine = await TimelineAsync.GetUserTimeline(user);
return(TwitterMapper.Map(timeLine));
}
public static async Task <IEnumerable <Tweetinvi.Models.ITweet> > GetTwitters(string TwitterUserNmae)
{
TweetinviConfig.CurrentThreadSettings.ProxyConfig = new ProxyConfig("http://127.0.0.1:1080");
var appCreds = Auth.SetApplicationOnlyCredentials("dz3Kic4aQHxsx6SEcASG2A", "MHyyF5DMOZ5hkSnFG7ZqZNsrb5P2clv2K1tYSqKUYc", true);
Auth.InitializeApplicationOnlyCredentials(appCreds);
IEnumerable <Tweetinvi.Models.ITweet> tweets = await TimelineAsync.GetUserTimeline(TwitterUserNmae, 10);
return(tweets);
}
public async Task <int> GetRetweetsCountForUserAsync(string userId)
{
var userTimelineParam = new UserTimelineParameters
{
MaximumNumberOfTweetsToRetrieve = 200,
IncludeRTS = true,
TrimUser = true,
IncludeEntities = false,
ExcludeReplies = true,
};
var user = new UserIdentifier(long.Parse(userId));
var tweets = await Auth.ExecuteOperationWithCredentials(
this.credentials, () => TimelineAsync.GetUserTimeline(user, userTimelineParam));
return(tweets.Count(x => x.IsRetweet));
}
public virtual async Task <IEnumerable <Status> > GetUserTimelineAsync(StatusListParams statusListParams)
{
var userTimelineParam = new UserTimelineParameters
{
MaximumNumberOfTweetsToRetrieve = statusListParams.Count ?? 100,
IncludeRTS = true
};
if (!string.IsNullOrEmpty(statusListParams.MaxId))
{
userTimelineParam.MaxId = long.Parse(statusListParams.MaxId) - 1;
}
var userId = new UserIdentifier(long.Parse(statusListParams.CreatedByUserId));
var tweets = await Auth.ExecuteOperationWithCredentials(
this.credentials, () => TimelineAsync.GetUserTimeline(userId, userTimelineParam));
tweets = tweets ?? Enumerable.Empty <ITweet>();
var mapper = new StatusMapper();
return(tweets.Select(x => mapper.Map(x, new Status())));
}
public async Task <IEnumerable <ITweet> > LoadTimelineAsync(string username)
{
return(await TimelineAsync.GetUserTimeline(username));
}
protected async Task <TrainSet> ScheduleProcessTwitter(TrainSet data)
{
var collection = await _twitterCollectionsStore.Get(data.SourceId);
if (collection == null)
{
data.SetFailed("Can not find source data");
return(data);
}
var sources = await _twitterSourcesStore.GetBy(x => x.CollectionId == collection.Id);
if (sources == null || !sources.Any())
{
data.SetFailed("Can not find any twitter sources");
return(data);
}
var user = await _userStore.FindByIdAsync(collection.UserId);
if (user == null)
{
data.SetFailed("Can not find user data");
return(data);
}
var userTwitter = await _userSocialsStore.GetTwitter(user.Id);
if (userTwitter == null)
{
data.SetFailed("No twitter access token");
return(data);
}
try
{
OAuthTwitter(userTwitter);
}
catch
{
data.SetFailed("Error with twitter connections");
return(data);
}
// upload twitter data
int min = data.MinCount;
min = Math.Max(1, min);
int max = data.MaxCount;
max = Math.Max(100, max);
max = Math.Min(10000, max);
if (min > max)
{
var t = min;
max = min;
min = t;
}
int perSource = (int)Math.Ceiling((double)max / sources.Count);
var entity = new TrainSetModel {
};
var rawData = new StringBuilder();
int total = 0;
var regex = new Regex("http[s]?://[A-Za-z0-9._-]*");
foreach (var screen in sources)
{
long?lastId = null;
int count = 0;
var twetterUser = await UserAsync.GetUserFromId(screen.TwitterId);
while (perSource > count)
{
var @params = new UserTimelineParameters
{
MaximumNumberOfTweetsToRetrieve = 50,
};
if (lastId.HasValue)
{
@params.SinceId = lastId.Value;
}
var tweets = await TimelineAsync.GetUserTimeline(twetterUser, @params);
if (tweets == null || !tweets.Any())
{
break;
}
count += tweets.Count();
foreach (var t in tweets)
{
rawData.Append(regex.Replace(t.FullText, string.Empty));
}
}
total += count;
}
if (total < min)
{
data.SetFailed($"Not enough data avaliable. Avaliable : {total}. Minimum: {min}");
return(data);
}
WordBag wb = WordBag.CreateToWords(rawData.ToString(), data.InputWordsCount + 1);
_vReader.UploadBinary();
List <Tuple <string[], string[]> > stringList = new List <Tuple <string[], string[]> >();
List <Tuple <double[], double[]> > doubleList = new List <Tuple <double[], double[]> >();
foreach (var s in wb.Read())
{
var vectorList = new List <double[]>();
var wordList = new List <string>();
foreach (var ss in s)
{
var word = _vReader.Vocab.GetRepresentationOrNullFor(ss);
if (word == null)
{
break;
}
vectorList.Add(ss.Select(x => (double)x).ToArray());
wordList.Add(ss);
}
if (vectorList.Count < s.Length)
{
continue;
}
var tmpVector = new List <double>();
foreach (var i in vectorList.Take(data.InputWordsCount))
{
tmpVector.AddRange(i);
}
doubleList.Add(new Tuple <double[], double[]>(tmpVector.ToArray(), vectorList.Last().ToArray()));
stringList.Add(new Tuple <string[], string[]>(wordList.Take(wordList.Count - 1).ToArray(), new string[1] {
wordList.Last()
}));
}
entity.Data = doubleList.ToArray();
entity.StringSource = stringList.ToArray();
string dataString = JsonConvert.SerializeObject(entity);
await _storageBlobClient.SetContainer(CONTAINER_NAME, true);
var storageKey = await _storageBlobClient.WriteText(dataString);
if (string.IsNullOrWhiteSpace(storageKey))
{
data.SetFailed("Can not upload train set to storage");
return(data);
}
data.StorageKey = storageKey;
data.SetReady();
data.ExamplesCount = entity.Data.Count();
return(data);
}
public async Task <List <Models.Tweet> > GetTweets(List <FollowItem> items)
{
List <Models.Tweet> res = new List <Models.Tweet>();
//1. launch tasks
List <Task <IEnumerable <ITweet> > > toWait = new List <Task <IEnumerable <ITweet> > >();
Dictionary <Task, FollowItem> taskToItem = new Dictionary <Task, FollowItem>();
foreach (var item in items)
{
if (item.Type == FollowType.HASHTAG || item.Type == FollowType.SEARCH)
{
var tmpT = SearchAsync.SearchTweets(item.Id);
toWait.Add(tmpT);
taskToItem.Add(tmpT, item);
}
else if (item.Type == FollowType.USER)
{
var tmpT = TimelineAsync.GetUserTimeline(item.Id);
toWait.Add(tmpT);
taskToItem.Add(tmpT, item);
}
else
{
//TODO
}
}
//2. wait for completion
await Task.WhenAll(toWait);
//3. Produce results from TweetInvi models
foreach (var task in toWait)
{
if (task.IsCompleted && !task.IsFaulted && !task.IsCanceled && task.Result != null)
{
foreach (var tweet in task.Result)
{
if (String.IsNullOrEmpty(tweet.InReplyToScreenName))
{
res.Add(new Models.Tweet()
{
InnerURL = tweet.Urls.Select(x => x.ExpandedURL).ToList(),
Content = tweet.Text,
Date = tweet.CreatedAt,
URL = tweet.Url,
Id = tweet.IdStr,
AssociatedID = taskToItem[task].Id,
AssociatedFollowItem = taskToItem[task],
MediaURI = tweet.Media.Select(item => new MediaItem()
{
URI = item.MediaURL
}).ToList(),
User = new Models.User()
{
Name = tweet.CreatedBy.Name,
ImageURI = tweet.CreatedBy.ProfileImageUrl400x400
}
});
}
}
}
}
//4. order results
res = res.OrderBy(x => new DateTimeOffset(x.Date)).Reverse().ToList();
return(res);
}
