private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
{
var tweets = new ExtractedTweet[0];
// Don't retrieve TL if protected
var userView = _twitterUserService.GetUser(user.Acct);
if (userView == null || userView.Protected)
{
return(tweets);
}
try
{
if (user.LastTweetPostedId == -1)
{
tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
}
else
{
tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
}
}
catch (Exception e)
{
_logger.LogError(e, "Error retrieving TL of {Username} from {LastTweetPostedId}, purging user from cache", user.Acct, user.LastTweetPostedId);
_twitterUserService.PurgeUser(user.Acct);
}
return(tweets);
}
public async Task ProcessAsync_Test()
{
#region Stubs
var user = new SyncTwitterUser
{
Id = 1
};
var tweet1 = new ExtractedTweet
{
Id = 36
};
var tweet2 = new ExtractedTweet
{
Id = 37
};
var follower1 = new Follower
{
FollowingsSyncStatus = new Dictionary <int, long>
{
{ 1, 37 }
}
};
var usersWithTweets = new UserWithTweetsToSync
{
Tweets = new []
{
tweet1,
tweet2
},
Followers = new []
{
follower1
},
User = user
};
var loggerMock = new Mock <ILogger <SaveProgressionProcessor> >();
#endregion
#region Mocks
var twitterUserDalMock = new Mock <ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.UpdateTwitterUserAsync(
It.Is <int>(y => y == user.Id),
It.Is <long>(y => y == tweet2.Id),
It.Is <long>(y => y == tweet2.Id),
It.IsAny <DateTime>()
))
.Returns(Task.CompletedTask);
#endregion
var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object);
await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
#region Validations
twitterUserDalMock.VerifyAll();
loggerMock.VerifyAll();
#endregion
}
public async Task ProcessAsync_RemoveFollower()
{
#region Stubs
var twitter = new SyncTwitterUser
{
Id = 24,
Acct = "my-acct"
};
var followers = new List <Follower>
{
new Follower
{
Id = 48,
Followings = new List <int> {
24
},
FollowingsSyncStatus = new Dictionary <int, long> {
{ 24, 1024 }
}
}
};
#endregion
#region Mocks
var followersDalMock = new Mock <IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowersAsync(
It.Is <int>(y => y == 24)))
.ReturnsAsync(followers.ToArray());
followersDalMock
.Setup(x => x.DeleteFollowerAsync(
It.Is <int>(y => y == 48)))
.Returns(Task.CompletedTask);
var twitterUserDalMock = new Mock <ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.DeleteTwitterUserAsync(
It.Is <int>(y => y == 24)))
.Returns(Task.CompletedTask);
var rejectFollowingActionMock = new Mock <IRejectFollowingAction>(MockBehavior.Strict);
rejectFollowingActionMock
.Setup(x => x.ProcessAsync(
It.Is <Follower>(y => y.Id == 48),
It.Is <SyncTwitterUser>(y => y.Acct == twitter.Acct)))
.Returns(Task.CompletedTask);
#endregion
var action = new RemoveTwitterAccountAction(followersDalMock.Object, twitterUserDalMock.Object, rejectFollowingActionMock.Object);
await action.ProcessAsync(twitter);
#region Validations
followersDalMock.VerifyAll();
twitterUserDalMock.VerifyAll();
rejectFollowingActionMock.VerifyAll();
#endregion
}
public async Task ProcessAsync_UserSync_Test()
{
#region Stubs
var user1 = new SyncTwitterUser
{
Id = 1,
Acct = "acct",
LastTweetPostedId = 46,
LastTweetSynchronizedForAllFollowersId = 46
};
var users = new[]
{
user1
};
var tweets = new[]
{
new ExtractedTweet
{
Id = 47
},
new ExtractedTweet
{
Id = 48
},
new ExtractedTweet
{
Id = 49
}
};
#endregion
#region Mocks
var twitterServiceMock = new Mock <ITwitterTweetsService>(MockBehavior.Strict);
twitterServiceMock
.Setup(x => x.GetTimeline(
It.Is <string>(y => y == user1.Acct),
It.Is <int>(y => y == 200),
It.Is <long>(y => y == user1.LastTweetSynchronizedForAllFollowersId)
))
.Returns(tweets);
var twitterUserDalMock = new Mock <ITwitterUserDal>(MockBehavior.Strict);
#endregion
var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
var usersResult = await processor.ProcessAsync(users, CancellationToken.None);
#region Validations
twitterServiceMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(users.Length, usersResult.Length);
Assert.AreEqual(users[0].Acct, usersResult[0].User.Acct);
Assert.AreEqual(tweets.Length, usersResult[0].Tweets.Length);
#endregion
}
public async Task ExecuteAsync_UserExists_TwitterExists_Test()
{
#region Stubs
var username = "******";
var domain = "m.s";
var twitterName = "handle";
var followerInbox = "/user/testest";
var inbox = "/inbox";
var actorId = "actorUrl";
var follower = new Follower
{
Id = 1,
Acct = username,
Host = domain,
SharedInboxRoute = followerInbox,
InboxRoute = inbox,
Followings = new List <int>(),
FollowingsSyncStatus = new Dictionary <int, long>()
};
var twitterUser = new SyncTwitterUser
{
Id = 2,
Acct = twitterName,
LastTweetPostedId = -1,
LastTweetSynchronizedForAllFollowersId = -1
};
#endregion
#region Mocks
var followersDalMock = new Mock <IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowerAsync(username, domain))
.ReturnsAsync(follower);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is <Follower>(y => y.Followings.Contains(twitterUser.Id) &&
y.FollowingsSyncStatus[twitterUser.Id] == -1)
))
.Returns(Task.CompletedTask);
var twitterUserDalMock = new Mock <ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUserAsync(twitterName))
.ReturnsAsync(twitterUser);
#endregion
var action = new ProcessFollowUser(followersDalMock.Object, twitterUserDalMock.Object);
await action.ExecuteAsync(username, domain, twitterName, followerInbox, inbox, actorId);
#region Validations
followersDalMock.VerifyAll();
twitterUserDalMock.VerifyAll();
#endregion
}
public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, string host, Follower[] followersPerInstance)
{
var userId = user.Id;
var inbox = followersPerInstance.First().SharedInboxRoute;
var fromStatusId = followersPerInstance
.Max(x => x.FollowingsSyncStatus[userId]);
var tweetsToSend = tweets
.Where(x => x.Id > fromStatusId)
.OrderBy(x => x.Id)
.ToList();
var syncStatus = fromStatusId;
try
{
foreach (var tweet in tweetsToSend)
{
try
{
if (!tweet.IsReply ||
tweet.IsReply && tweet.IsThread ||
_settings.PublishReplies)
{
var note = _statusService.GetStatus(user.Acct, tweet);
await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox);
}
}
catch (ArgumentException e)
{
if (e.Message.Contains("Invalid pattern") && e.Message.Contains("at offset")) //Regex exception
{
_logger.LogError(e, "Can't parse {MessageContent} from Tweet {Id}", tweet.MessageContent, tweet.Id);
}
else
{
throw;
}
}
syncStatus = tweet.Id;
}
}
finally
{
if (syncStatus != fromStatusId)
{
foreach (var f in followersPerInstance)
{
f.FollowingsSyncStatus[userId] = syncStatus;
await _followersDal.UpdateFollowerAsync(f);
}
}
}
}
public async Task ProcessAsync_UserNotSync_Test()
{
#region Stubs
var user1 = new SyncTwitterUser
{
Id = 1,
Acct = "acct",
LastTweetPostedId = -1
};
var users = new[]
{
user1
};
var tweets = new[]
{
new ExtractedTweet
{
Id = 47
}
};
#endregion
#region Mocks
var twitterServiceMock = new Mock <ITwitterTweetsService>(MockBehavior.Strict);
twitterServiceMock
.Setup(x => x.GetTimeline(
It.Is <string>(y => y == user1.Acct),
It.Is <int>(y => y == 1),
It.Is <long>(y => y == -1)
))
.Returns(tweets);
var twitterUserDalMock = new Mock <ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.UpdateTwitterUserAsync(
It.Is <int>(y => y == user1.Id),
It.Is <long>(y => y == tweets.Last().Id),
It.Is <long>(y => y == tweets.Last().Id),
It.IsAny <DateTime>()
))
.Returns(Task.CompletedTask);
#endregion
var processor = new RetrieveTweetsProcessor(twitterServiceMock.Object, twitterUserDalMock.Object);
var usersResult = await processor.ProcessAsync(users, CancellationToken.None);
#region Validations
twitterServiceMock.VerifyAll();
twitterUserDalMock.VerifyAll();
Assert.AreEqual(0, usersResult.Length);
#endregion
}
private ExtractedTweet[] RetrieveNewTweets(SyncTwitterUser user)
{
ExtractedTweet[] tweets;
if (user.LastTweetPostedId == -1)
{
tweets = _twitterTweetsService.GetTimeline(user.Acct, 1);
}
else
{
tweets = _twitterTweetsService.GetTimeline(user.Acct, 200, user.LastTweetSynchronizedForAllFollowersId);
}
return(tweets);
}
public async Task ProcessAsync(Follower follower, SyncTwitterUser twitterUser)
{
try
{
var activityFollowing = new ActivityFollow
{
type = "Follow",
actor = follower.ActorId,
apObject = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct)
};
await _userService.SendRejectFollowAsync(activityFollowing, follower.Host);
}
catch (Exception) { }
}
public async Task ProcessAsync_Exception()
{
#region Stubs
var follower = new Follower
{
Followings = new List <int>
{
24
},
Host = "host"
};
var settings = new InstanceSettings
{
Domain = "domain"
};
var twitterUser = new SyncTwitterUser
{
Id = 24,
Acct = "acct"
};
#endregion
#region Mocks
var userServiceMock = new Mock <IUserService>(MockBehavior.Strict);
userServiceMock
.Setup(x => x.SendRejectFollowAsync(
It.Is <ActivityFollow>(y => y.type == "Follow"),
It.IsNotNull <string>()
))
.Throws(new Exception());
#endregion
var action = new RejectFollowingAction(userServiceMock.Object, settings);
await action.ProcessAsync(follower, twitterUser);
#region Validations
userServiceMock.VerifyAll();
#endregion
}
public async Task ProcessAsync(SyncTwitterUser twitterUser)
{
// Check Followers
var twitterUserId = twitterUser.Id;
var followers = await _followersDal.GetFollowersAsync(twitterUserId);
// Remove all Followers
foreach (var follower in followers)
{
// Perform undo following to user instance
await _rejectFollowingAction.ProcessAsync(follower, twitterUser);
// Remove following from DB
if (follower.Followings.Contains(twitterUserId))
{
follower.Followings.Remove(twitterUserId);
}
if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
{
follower.FollowingsSyncStatus.Remove(twitterUserId);
}
if (follower.Followings.Any())
{
await _followersDal.UpdateFollowerAsync(follower);
}
else
{
await _followersDal.DeleteFollowerAsync(follower.Id);
}
}
// Remove twitter user
await _twitterUserDal.DeleteTwitterUserAsync(twitterUserId);
}
private async Task ProcessFollowersWithInbox(ExtractedTweet[] tweets, List <Follower> followerWtInbox, SyncTwitterUser user)
{
foreach (var follower in followerWtInbox)
{
try
{
await _sendTweetsToInboxTask.ExecuteAsync(tweets, follower, user);
}
catch (Exception e)
{
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.InboxRoute);
}
}
}
public async Task ExecuteAsync_MultiFollows_Test()
{
#region Stubs
var username = "******";
var domain = "m.s";
var twitterName = "handle";
var follower = new Follower
{
Id = 1,
Acct = username,
Host = domain,
Followings = new List <int> {
2, 3
},
FollowingsSyncStatus = new Dictionary <int, long> {
{ 2, 460 }, { 3, 563 }
}
};
var twitterUser = new SyncTwitterUser
{
Id = 2,
Acct = twitterName,
LastTweetPostedId = 460,
LastTweetSynchronizedForAllFollowersId = 460
};
var followerList = new List <Follower>
{
new Follower(),
new Follower()
};
#endregion
#region Mocks
var followersDalMock = new Mock <IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.GetFollowerAsync(username, domain))
.ReturnsAsync(follower);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is <Follower>(y => !y.Followings.Contains(twitterUser.Id) &&
!y.FollowingsSyncStatus.ContainsKey(twitterUser.Id))
))
.Returns(Task.CompletedTask);
followersDalMock
.Setup(x => x.GetFollowersAsync(twitterUser.Id))
.ReturnsAsync(followerList.ToArray());
var twitterUserDalMock = new Mock <ITwitterUserDal>(MockBehavior.Strict);
twitterUserDalMock
.Setup(x => x.GetTwitterUserAsync(twitterName))
.ReturnsAsync(twitterUser);
#endregion
var action = new ProcessUndoFollowUser(followersDalMock.Object, twitterUserDalMock.Object);
await action.ExecuteAsync(username, domain, twitterName);
#region Validations
followersDalMock.VerifyAll();
twitterUserDalMock.VerifyAll();
#endregion
}
public async Task ExecuteAsync_SingleTweet_ArgumentException_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List <ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
}
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary <int, long> {
{ twitterUserId, 9 }
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock <IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock <IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is <string>(y => y == twitterHandle),
It.Is <ExtractedTweet>(y => y.Id == tweetId)))
.Throws(new ArgumentException());
var followersDalMock = new Mock <IFollowersDal>(MockBehavior.Strict);
var loggerMock = new Mock <ILogger <SendTweetsToInboxTask> >();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
try
{
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
}
finally
{
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
}
public async Task ExecuteAsync_MultipleTweets_Error_Test()
{
#region Stubs
var tweetId1 = 10;
var tweetId2 = 11;
var tweetId3 = 12;
var tweets = new List <ExtractedTweet>();
foreach (var tweetId in new[] { tweetId1, tweetId2, tweetId3 })
{
tweets.Add(new ExtractedTweet
{
Id = tweetId
});
}
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary <int, long> {
{ twitterUserId, 10 }
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock <IActivityPubService>(MockBehavior.Strict);
activityPubService
.Setup(x => x.PostNewNoteActivity(
It.Is <Note>(y => y.id == tweetId2.ToString()),
It.Is <string>(y => y == twitterHandle),
It.Is <string>(y => y == tweetId2.ToString()),
It.Is <string>(y => y == host),
It.Is <string>(y => y == inbox)))
.Returns(Task.CompletedTask);
activityPubService
.Setup(x => x.PostNewNoteActivity(
It.Is <Note>(y => y.id == tweetId3.ToString()),
It.Is <string>(y => y == twitterHandle),
It.Is <string>(y => y == tweetId3.ToString()),
It.Is <string>(y => y == host),
It.Is <string>(y => y == inbox)))
.Throws(new HttpRequestException());
var statusServiceMock = new Mock <IStatusService>(MockBehavior.Strict);
foreach (var tweetId in new[] { tweetId2, tweetId3 })
{
statusServiceMock
.Setup(x => x.GetStatus(
It.Is <string>(y => y == twitterHandle),
It.Is <ExtractedTweet>(y => y.Id == tweetId)))
.Returns(new Note {
id = tweetId.ToString()
});
}
var followersDalMock = new Mock <IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is <Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId2)))
.Returns(Task.CompletedTask);
var loggerMock = new Mock <ILogger <SendTweetsToInboxTask> >();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
try
{
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
}
finally
{
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
}
public async Task ExecuteAsync_SingleTweet_PublishReply_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List <ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
IsReply = true,
IsThread = false
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/user/inbox";
var follower = new Follower
{
Id = 1,
Host = host,
InboxRoute = inbox,
FollowingsSyncStatus = new Dictionary <int, long> {
{ twitterUserId, 9 }
}
};
var settings = new InstanceSettings
{
PublishReplies = true
};
#endregion
#region Mocks
var activityPubService = new Mock <IActivityPubService>(MockBehavior.Strict);
activityPubService
.Setup(x => x.PostNewNoteActivity(
It.Is <Note>(y => y.id == noteId),
It.Is <string>(y => y == twitterHandle),
It.Is <string>(y => y == tweetId.ToString()),
It.Is <string>(y => y == host),
It.Is <string>(y => y == inbox)))
.Returns(Task.CompletedTask);
var statusServiceMock = new Mock <IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is <string>(y => y == twitterHandle),
It.Is <ExtractedTweet>(y => y.Id == tweetId)))
.Returns(note);
var followersDalMock = new Mock <IFollowersDal>(MockBehavior.Strict);
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is <Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
var loggerMock = new Mock <ILogger <SendTweetsToInboxTask> >();
#endregion
var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser);
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}
private async Task ProcessFollowersWithSharedInbox(ExtractedTweet[] tweets, List <Follower> followers, SyncTwitterUser user)
{
var followersPerInstances = followers.GroupBy(x => x.Host);
foreach (var followersPerInstance in followersPerInstances)
{
try
{
await _sendTweetsToSharedInbox.ExecuteAsync(tweets, user, followersPerInstance.Key, followersPerInstance.ToArray());
}
catch (Exception e)
{
var follower = followersPerInstance.First();
_logger.LogError(e, "Posting to {Host}{Route} failed", follower.Host, follower.SharedInboxRoute);
}
}
}
public async Task ExecuteAsync_SingleTweet_ParsingError_Test()
{
#region Stubs
var tweetId = 10;
var tweets = new List <ExtractedTweet>
{
new ExtractedTweet
{
Id = tweetId,
}
};
var noteId = "noteId";
var note = new Note()
{
id = noteId
};
var twitterHandle = "Test";
var twitterUserId = 7;
var twitterUser = new SyncTwitterUser
{
Id = twitterUserId,
Acct = twitterHandle
};
var host = "domain.ext";
var inbox = "/inbox";
var followers = new List <Follower>
{
new Follower
{
Id = 1,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary <int, long> {
{ twitterUserId, 9 }
}
},
new Follower
{
Id = 2,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary <int, long> {
{ twitterUserId, 8 }
}
},
new Follower
{
Id = 3,
Host = host,
SharedInboxRoute = inbox,
FollowingsSyncStatus = new Dictionary <int, long> {
{ twitterUserId, 7 }
}
}
};
var settings = new InstanceSettings
{
PublishReplies = false
};
#endregion
#region Mocks
var activityPubService = new Mock <IActivityPubService>(MockBehavior.Strict);
var statusServiceMock = new Mock <IStatusService>(MockBehavior.Strict);
statusServiceMock
.Setup(x => x.GetStatus(
It.Is <string>(y => y == twitterHandle),
It.Is <ExtractedTweet>(y => y.Id == tweetId)))
.Throws(new ArgumentException("Invalid pattern blabla at offset 9"));
var followersDalMock = new Mock <IFollowersDal>(MockBehavior.Strict);
foreach (var follower in followers)
{
followersDalMock
.Setup(x => x.UpdateFollowerAsync(
It.Is <Follower>(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId)))
.Returns(Task.CompletedTask);
}
var loggerMock = new Mock <ILogger <SendTweetsToSharedInboxTask> >();
#endregion
var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object);
await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray());
#region Validations
activityPubService.VerifyAll();
statusServiceMock.VerifyAll();
followersDalMock.VerifyAll();
#endregion
}