public async Task <Unit> Handle(YammerProcessCommand request, CancellationToken cancellationToken) { if (!_store.IsHydrated) { throw new Exception("Store must be hydrated first"); } var creds = _credentials.Get(request.SourceId, request.Username); var count = await _events.GetCount(creds.Username, request.SourceId, "RestApiRequest"); var currentCount = 0; await _events.ReadForward(creds, request.SourceId, count, async (events, totalCount) => { var bodies = events .Where(p => p.EventName == "RestApiRequest") .Select(p => p.Body) .ToList(); foreach (var body in bodies) { currentCount++; await _progress.Set(currentCount, totalCount); dynamic json = JsonConvert.DeserializeObject(body); if (json.response != null) { foreach (var message in json.response.messages) { await ProcessMessage(Rest.Yammer.Message.FromJson(message.ToString()), request.SourceId, creds); } foreach (var reference in json.response.references) { switch (reference.type.ToString()) { case "user": await ProcessUser(Rest.Yammer.User.FromJson(reference.ToString()), request.SourceId, creds); break; case "message": await ProcessMessage(Rest.Yammer.Message.FromJson(reference.ToString()), request.SourceId, creds); break; default: _logger.LogWarning($"Unknown reference type: {reference.type}"); break; } } } } return(events.Last().Id); }); _logger.LogInformation("Completed processing"); return(Unit.Value); }
public async Task <Unit> Handle(LinkedInProcessCommand request, CancellationToken cancellationToken) { if (!_store.IsHydrated) { throw new Exception("Store must be hydrated first"); } var creds = _credentials.Get(request.SourceId, request.Username); var count = await _events.GetCount(creds.Username, request.SourceId, "JsonPayload"); var currentCount = 0; await _events.ReadForward(creds, request.SourceId, count, async (events, totalEvents) => { var bodies = events .Where(p => p.EventName == "JsonPayload") .Select(p => p.Body) .ToList(); foreach (var body in bodies) { currentCount++; await _progress.Set(currentCount, totalEvents); var payload = JsonConvert.DeserializeObject <JsonPayload>(body); var uri = new Uri($"https://linkedin.com{payload.Url}"); if (uri.LocalPath.EndsWith("connections")) { var values = JsonConvert.DeserializeObject <Connections>(payload.Json); foreach (var user in values.Included.Where(u => u.Type == "com.linkedin.voyager.identity.shared.MiniProfile")) { var scraped = new User { Id = user.PublicIdentifier, KnownSince = DateTimeOffset.FromUnixTimeMilliseconds(values.Included.Single(u => u.MiniProfile == user.EntityUrn).CreatedAt ?? 0), Name = $"{user.FirstName} {user.LastName}", Network = Network.LinkedIn, Description = user.Occupation }; var existing = _store.GetUser(request.SourceId, scraped.Id); await _events.Sync(creds, request.SourceId, scraped, existing, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); } } } return(events.Last().Id); }); _logger.LogInformation("Completed processing"); return(Unit.Value); }
public async Task <Result> Handle(TeamsProcessCommand request, CancellationToken cancellationToken) { if (!_store.IsHydrated) { _logger.LogInformation("Hydrating store..."); await _mediator.Send(new HydrateCommand { LabyrinthUsername = request.LabyrinthUsername, LabyrinthPassword = request.LabyrinthPassword }); } var creds = new Credential(request.LabyrinthUsername, request.LabyrinthPassword); var count = await _events.GetCount(creds.Username, request.SourceId, "RestApiRequest"); var currentCount = 0; await _events.ReadForward(creds, request.SourceId, count, async (events, totalEvents) => { var requests = events .Where(p => p.EventName == "RestApiRequest") .ToList(); foreach (var apiRequest in requests) { currentCount++; await _progress.Set(currentCount, totalEvents); var payload = JsonConvert.DeserializeObject <RestApiRequest>(apiRequest.Body); if (payload.Category == TeamsRequestTypes.Chats) { await Process(creds, request.SourceId, JsonConvert.DeserializeObject <IUserChatsCollectionPage>(payload.Response)); } else if (payload.Category == TeamsRequestTypes.ChatMessages) { dynamic data = JObject.Parse(payload.Data); await ProcessMessages(creds, request.SourceId, (string)data.id, JsonConvert.DeserializeObject <IChatMessagesCollectionPage>(payload.Response), creds.ExternalSecret); } else { throw new NotImplementedException(payload.Category); } } return(events.Last().Id); }); _progress.Completed("Completed processing"); return(Result.Ok()); }
private async Task Hydrate <T>(Credential credential, string entityType, Guid sourceId, string eventType, Dictionary <string, T> dictionary) where T : IExternalEntity { var count = await _events.GetCount(credential.Username, sourceId, eventType); if (count == 0) { return; } _logger.LogInformation($"Hydrating {entityType}..."); var sw = Stopwatch.StartNew(); await _progress.New(); Func <Event[], int, Task <int> > eventProcessor = FillFromEvents <T>(dictionary); await _events.ReadForward(credential, sourceId, count, eventProcessor); Log(sw.Elapsed, sourceId, dictionary.Count); }
public async Task <Result> Handle(PurgeEventsCommand request, CancellationToken cancellationToken) { if (await _keys.TestPassword(request.Username, request.Password)) { var credential = new Credential(request.Username, request.Password); var count = await _events.GetCount(request.Username, request.SourceId, request.Events); var events = request.Events.Length > 0 ? string.Join(", ", request.Events) : "(all)"; _logger.LogInformation($"Purging {count} {events} events from {request.SourceId}"); await _events.Delete(request.Username, credential.Password, request.SourceId, request.Events); return(Result.Ok()); } else { return(Result.Error("Bad password")); } }
public async Task <Result <NextPagedResult <Event> > > Handle(GetEventsQuery request, CancellationToken cancellationToken) { var auth = await _mediator.Send(new AuthorizeQuery { Username = request.Username, Password = request.Password }); if (auth.IsError) { return(new Result <NextPagedResult <Event> > { IsError = true, Message = auth.Message }); } var events = await _events.Paginate(new Credential(request.Username, request.Password), request.SourceId, request.LastId, request.PageSize); var totalRows = await _events.GetCount(request.Username, request.SourceId, request.EventTypes); return(new Result <NextPagedResult <Event> >(new NextPagedResult <Event> { LastId = events.Rows.Any() ? events.Rows.Last().Id : 0, PageSize = request.PageSize, Rows = events.Rows.ToArray(), TotalRows = totalRows, })); }