private static void ScrapeUserData() { using (var dbContext = new SlackTrackerContext()) { string cursor = null; MembersListResponse memberResponse; do { _logger.Log(LogLevel.Info, "Cursor: " + cursor); memberResponse = _client.GetUsers(cursor); var memberResponses = memberResponse.members; if (memberResponses != null) { foreach (var member in memberResponses) { _logger.Info($"Looking for Member {member.real_name}."); if (dbContext.Users.Count(c => c.Id.Equals(member.id) && c.EmailAddress == null) != 0) { _logger.Info($"Member { member.real_name}, found missing e-mail address. Updating"); var user = dbContext.Users.Single(c => c.Id.Equals(member.id)); user.EmailAddress = member.profile.email; user.Timezone = member.tz; user.TimezoneLabel = member.tz_label; user.ImagePath = member.profile.image_original; dbContext.Update(user); dbContext.SaveChanges(); } else if (dbContext.Users.Count(c => c.Id.Equals(member.id)).Equals(0)) { _logger.Log(LogLevel.Info, $"Adding User {member.name}, {member.real_name}."); dbContext.Users.Add(new Users() { Id = member.id, Username = member.name, Name = member.real_name, EmailAddress = member.profile.email, Timezone = member.tz, TimezoneLabel = member.tz_label, ImagePath = member.profile.image_original }); dbContext.SaveChanges(); } else { _logger.Log(LogLevel.Info, $"User {member.name}, {member.real_name} found. No updates required"); } } } if (memberResponse.response_metadata != null && !string.IsNullOrWhiteSpace(memberResponse.response_metadata.next_cursor)) { cursor = memberResponse.response_metadata.next_cursor; } } while (memberResponse.response_metadata != null && !string.IsNullOrWhiteSpace(memberResponse.response_metadata.next_cursor) ); } }
private static void ScrapeChannelData(string slackServer, string authoriztionToken) { _logger.Log(LogLevel.Info, "Scrapping Channels"); using (var dbContext = new SlackTrackerContext()) { foreach (var channel in _client.GetChannels().channels) { _logger.Log(LogLevel.Info, $"Looking for Channel {channel.name}."); if (dbContext.Channels.Count(c => c.Id.Equals(channel.id)).Equals(0)) { _logger.Log(LogLevel.Info, $"Adding Channel {channel.name}."); dbContext.Channels.Add(new Channels() { Id = channel.id, Name = channel.name }); dbContext.SaveChanges(); } } string cursor; CultureInfo cultureInfo = new CultureInfo("en-US"); Calendar calendar = cultureInfo.Calendar; _logger.Log(LogLevel.Info, "Populating Historical Message Data"); foreach (var channel in dbContext.Channels) { _logger.Log(LogLevel.Info, $"Checking Channel {channel.Name}"); var oldest = DateTime.Parse("1/1/2019").ToUniversalTime(); var latest = DateTime.UtcNow; _logger.Log(LogLevel.Info, $"Checking Channel {channel.Name}, Between {oldest} and {latest}."); cursor = null; ConversationHistoryResponse messages; do { messages = _client.GetMessageHistory(channel.Id, oldest, latest, cursor); if (messages.messages != null) { _logger.Log(LogLevel.Info, $"Received {messages.messages.Count()} messages."); if (messages.messages != null) { foreach (var message in messages.messages.Where(m => !string.IsNullOrWhiteSpace(m.client_msg_id))) { if (dbContext.MessageHistory.Count(m => m.Id.Equals(message.client_msg_id)) .Equals(0)) { _logger.Log(LogLevel.Info, $"Inserting message {message.client_msg_id} into Channel {channel.Name}."); dbContext.MessageHistory.Add(new MessageHistory() { Id = message.client_msg_id, User = message.user, Timestamp = message.ts.FromSlackTimestamp(), Channel = channel.Id, Text = message.text }); dbContext.SaveChanges(); } } } } else { _logger.Log(LogLevel.Info, "No Messages found."); } if (messages.has_more) { cursor = messages.response_metadata.next_cursor; } } while (messages.has_more); } } }