private static List <Entry> GetEntries(TimelineTweets response)
{
if (response.Timeline.Instructions[0].ClearCache != null && response.Timeline.Instructions.Count > 1)
{
return(response.Timeline.Instructions[1].AddEntries.Entries);
}
return(response.Timeline.Instructions[0].AddEntries.Entries);
}
private bool CheckPostAge(TimelineTweets response)
{
var entries = GetEntries(response);
if (entries == null || entries.Count == 0)
{
return(false);
}
var id = entries[0]?.SortIndex;
if (id == null)
{
return(false);
}
ulong highestPostId;
_ = ulong.TryParse(id, out highestPostId);
return(highestPostId >= GetLastPostId());
}
private async Task AddUrlsToDownloadListAsync(TimelineTweets document)
{
Users = document.GlobalObjects.Users;
var lastPostId = GetLastPostId();
foreach (Entry entry in GetEntries(document))
{
var cursorType = entry.Content.Operation?.Cursor.CursorType;
if (cursorType != null)
{
continue;
}
if (!entry.EntryId.ToLower().StartsWith("tweet-", StringComparison.InvariantCultureIgnoreCase) &&
!entry.EntryId.ToLower().StartsWith("sq-i-t-", StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
if (!document.GlobalObjects.Tweets.ContainsKey(entry.Content.Item.Content.Tweet.Id))
{
Logger.Warning("tweet-id {0} of blog {1} not found", entry.Content.Item.Content.Tweet.Id, twUser.Data.User.Id);
continue;
}
Tweet post = document.GlobalObjects.Tweets[entry.Content.Item.Content.Tweet.Id];
try
{
if (CheckIfShouldStop())
{
break;
}
CheckIfShouldPause();
if (lastPostId > 0 && ulong.TryParse(post.IdStr, out var postId) && postId < lastPostId)
{
continue;
}
if (!PostWithinTimeSpan(post))
{
continue;
}
if (!CheckIfContainsTaggedPost(post))
{
continue;
}
if (!CheckIfDownloadRebloggedPosts(post))
{
continue;
}
try
{
AddPhotoUrlToDownloadList(post);
AddVideoUrlToDownloadList(post);
AddGifUrlToDownloadList(post);
AddTextUrlToDownloadList(post);
}
catch (NullReferenceException e)
{
Logger.Verbose("TwitterCrawler.AddUrlsToDownloadListAsync: {0}", e);
}
}
catch (Exception e)
{
Logger.Error("TwitterCrawler.AddUrlsToDownloadListAsync: {0}", e);
ShellService.ShowError(e, "{0}: Error parsing tweet!", Blog.Name);
}
}
await Task.CompletedTask;
}
