public async Task ProcessUser(Rest.Yammer.User user, Guid sourceId, Credential creds)
{
var received = Events.User.From(user, sourceId);
var existing = _store.GetUser(sourceId, received.Id);
if (existing == null)
{
_store.Add(sourceId, received);
}
await _events.Sync(creds, sourceId, received, existing, Math.Min(received.KnownSince.ToUnixTimeMilliseconds(), (existing?.KnownSince ?? DateTimeOffset.MaxValue).ToUnixTimeMilliseconds()));
}
private async Task Process(Credential creds, Guid sourceId, IUserChatsCollectionPage userChatsCollectionPage)
{
foreach (var chat in userChatsCollectionPage)
{
var received = Events.Topic.From(chat);
var existing = _store.GetTopic(sourceId, received.Id);
if (existing == null)
{
_store.Add(sourceId, received);
}
await _events.Sync(creds, sourceId, received, existing, received.LastUpdated.ToUnixTimeMilliseconds());
}
}
public async Task <Unit> Handle(LinkedInProcessCommand request, CancellationToken cancellationToken)
{
if (!_store.IsHydrated)
{
throw new Exception("Store must be hydrated first");
}
var creds = _credentials.Get(request.SourceId, request.Username);
var count = await _events.GetCount(creds.Username, request.SourceId, "JsonPayload");
var currentCount = 0;
await _events.ReadForward(creds, request.SourceId, count, async (events, totalEvents) => {
var bodies = events
.Where(p => p.EventName == "JsonPayload")
.Select(p => p.Body)
.ToList();
foreach (var body in bodies)
{
currentCount++;
await _progress.Set(currentCount, totalEvents);
var payload = JsonConvert.DeserializeObject <JsonPayload>(body);
var uri = new Uri($"https://linkedin.com{payload.Url}");
if (uri.LocalPath.EndsWith("connections"))
{
var values = JsonConvert.DeserializeObject <Connections>(payload.Json);
foreach (var user in values.Included.Where(u => u.Type == "com.linkedin.voyager.identity.shared.MiniProfile"))
{
var scraped = new User {
Id = user.PublicIdentifier,
KnownSince = DateTimeOffset.FromUnixTimeMilliseconds(values.Included.Single(u => u.MiniProfile == user.EntityUrn).CreatedAt ?? 0),
Name = $"{user.FirstName} {user.LastName}",
Network = Network.LinkedIn,
Description = user.Occupation
};
var existing = _store.GetUser(request.SourceId, scraped.Id);
await _events.Sync(creds, request.SourceId, scraped, existing, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
}
}
}
return(events.Last().Id);
});
_logger.LogInformation("Completed processing");
return(Unit.Value);
}