public async Task <Unit> Handle(YammerProcessCommand request, CancellationToken cancellationToken)
{
if (!_store.IsHydrated)
{
throw new Exception("Store must be hydrated first");
}
var creds = _credentials.Get(request.SourceId, request.Username);
var count = await _events.GetCount(creds.Username, request.SourceId, "RestApiRequest");
var currentCount = 0;
await _events.ReadForward(creds, request.SourceId, count, async (events, totalCount) => {
var bodies = events
.Where(p => p.EventName == "RestApiRequest")
.Select(p => p.Body)
.ToList();
foreach (var body in bodies)
{
currentCount++;
await _progress.Set(currentCount, totalCount);
dynamic json = JsonConvert.DeserializeObject(body);
if (json.response != null)
{
foreach (var message in json.response.messages)
{
await ProcessMessage(Rest.Yammer.Message.FromJson(message.ToString()), request.SourceId, creds);
}
foreach (var reference in json.response.references)
{
switch (reference.type.ToString())
{
case "user": await ProcessUser(Rest.Yammer.User.FromJson(reference.ToString()), request.SourceId, creds);
break;
case "message": await ProcessMessage(Rest.Yammer.Message.FromJson(reference.ToString()), request.SourceId, creds);
break;
default:
_logger.LogWarning($"Unknown reference type: {reference.type}");
break;
}
}
}
}
return(events.Last().Id);
});
_logger.LogInformation("Completed processing");
return(Unit.Value);
}
public async Task <Unit> Handle(LinkedInProcessCommand request, CancellationToken cancellationToken)
{
if (!_store.IsHydrated)
{
throw new Exception("Store must be hydrated first");
}
var creds = _credentials.Get(request.SourceId, request.Username);
var count = await _events.GetCount(creds.Username, request.SourceId, "JsonPayload");
var currentCount = 0;
await _events.ReadForward(creds, request.SourceId, count, async (events, totalEvents) => {
var bodies = events
.Where(p => p.EventName == "JsonPayload")
.Select(p => p.Body)
.ToList();
foreach (var body in bodies)
{
currentCount++;
await _progress.Set(currentCount, totalEvents);
var payload = JsonConvert.DeserializeObject <JsonPayload>(body);
var uri = new Uri($"https://linkedin.com{payload.Url}");
if (uri.LocalPath.EndsWith("connections"))
{
var values = JsonConvert.DeserializeObject <Connections>(payload.Json);
foreach (var user in values.Included.Where(u => u.Type == "com.linkedin.voyager.identity.shared.MiniProfile"))
{
var scraped = new User {
Id = user.PublicIdentifier,
KnownSince = DateTimeOffset.FromUnixTimeMilliseconds(values.Included.Single(u => u.MiniProfile == user.EntityUrn).CreatedAt ?? 0),
Name = $"{user.FirstName} {user.LastName}",
Network = Network.LinkedIn,
Description = user.Occupation
};
var existing = _store.GetUser(request.SourceId, scraped.Id);
await _events.Sync(creds, request.SourceId, scraped, existing, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
}
}
}
return(events.Last().Id);
});
_logger.LogInformation("Completed processing");
return(Unit.Value);
}
public async Task <Result> Handle(TeamsProcessCommand request, CancellationToken cancellationToken)
{
if (!_store.IsHydrated)
{
_logger.LogInformation("Hydrating store...");
await _mediator.Send(new HydrateCommand { LabyrinthUsername = request.LabyrinthUsername, LabyrinthPassword = request.LabyrinthPassword });
}
var creds = new Credential(request.LabyrinthUsername, request.LabyrinthPassword);
var count = await _events.GetCount(creds.Username, request.SourceId, "RestApiRequest");
var currentCount = 0;
await _events.ReadForward(creds, request.SourceId, count, async (events, totalEvents) => {
var requests = events
.Where(p => p.EventName == "RestApiRequest")
.ToList();
foreach (var apiRequest in requests)
{
currentCount++;
await _progress.Set(currentCount, totalEvents);
var payload = JsonConvert.DeserializeObject <RestApiRequest>(apiRequest.Body);
if (payload.Category == TeamsRequestTypes.Chats)
{
await Process(creds, request.SourceId, JsonConvert.DeserializeObject <IUserChatsCollectionPage>(payload.Response));
}
else if (payload.Category == TeamsRequestTypes.ChatMessages)
{
dynamic data = JObject.Parse(payload.Data);
await ProcessMessages(creds, request.SourceId, (string)data.id, JsonConvert.DeserializeObject <IChatMessagesCollectionPage>(payload.Response), creds.ExternalSecret);
}
else
{
throw new NotImplementedException(payload.Category);
}
}
return(events.Last().Id);
});
_progress.Completed("Completed processing");
return(Result.Ok());
}
private async Task Hydrate <T>(Credential credential, string entityType, Guid sourceId, string eventType, Dictionary <string, T> dictionary)
where T : IExternalEntity
{
var count = await _events.GetCount(credential.Username, sourceId, eventType);
if (count == 0)
{
return;
}
_logger.LogInformation($"Hydrating {entityType}...");
var sw = Stopwatch.StartNew();
await _progress.New();
Func <Event[], int, Task <int> > eventProcessor = FillFromEvents <T>(dictionary);
await _events.ReadForward(credential, sourceId, count, eventProcessor);
Log(sw.Elapsed, sourceId, dictionary.Count);
}