private DataFeed GetFeeds(IVkDataProvider dataProvider, LikeShareType type, string entityVkId, VkGroup vkGroup, ref int offsetCounter) { var sharesFeed = dataProvider.GetShares(vkGroup.Id.ToString(), entityVkId, type, offsetCounter); if (sharesFeed == null) { return(null); } this.log.DebugFormat("Shares feed is received: {0}", sharesFeed.Feed); sharesFeed.ParentObjectId = entityVkId; sharesFeed.LikeShareType = (int)type; string newFeed = this.responseMapper.MapResponseObject(sharesFeed); var dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = newFeed, VkGroupId = vkGroup.Id, RelatedObjectId = entityVkId, Type = DataFeedType.MemberShares }; if (sharesFeed.users[0].uid == null || sharesFeed.users[0].uid.Length == 0) { return(null); } offsetCounter += sharesFeed.users.Length; return(dataFeed); }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { int offsetCounter = 0; DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType); while (true) { var posts = dataProvider.GetWallPosts(vkGroup.Id.ToString(), offsetCounter); this.log.DebugFormat("Posts feed is received: {0}", posts.Feed); if (posts.post == null || posts.post.Length == 0) { break; } DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = posts.Feed, VkGroupId = vkGroup.Id, Type = DataFeedType.WallPosts }; yield return(dataFeed); offsetCounter += posts.post.Length; if (dateLimit.HasValue && posts.post[posts.post.Length - 1].date.FromUnixTimestamp() < dateLimit) { break; } } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { int offsetCounter = 0; DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType); while (true) { var topicFeed = dataProvider.GetTopics(vkGroup.Id.ToString(), offsetCounter); this.log.DebugFormat("Topics feed is received: {0}", topicFeed.Feed); if (topicFeed.topics == null || topicFeed.topics.Length == 0 || topicFeed.topics[0].topic == null || topicFeed.topics[0].topic.Length == 0) { break; } DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = topicFeed.Feed, VkGroupId = vkGroup.Id, Type = DataFeedType.Topic }; yield return(dataFeed); offsetCounter += topicFeed.topics[0].topic.Length; if (dateLimit.HasValue && topicFeed.topics[topicFeed.topics.Length - 1].topic[0].created.FromUnixTimestamp() < dateLimit) { break; } } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { IList <string> albumIds = this.photoRepository.GetGroupAlbumIds(vkGroup.Id); foreach (var albumId in albumIds) { var photos = dataProvider.GetPhotoAlbumDetails(vkGroup.Id.ToString(), albumId, 0); this.log.DebugFormat("Photos feed is received: {0}", photos.Feed); if (photos.photo == null || photos.photo.Length == 0) { continue; } DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = photos.Feed, VkGroupId = vkGroup.Id, Type = DataFeedType.PhotoAlbumDetails }; yield return(dataFeed); } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { DateTime? dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType); IList <Video> videos = this.videoRepository.GetVideosByVkGroupId(vkGroup.Id).Where(v => !dateLimit.HasValue || v.PostedDate > dateLimit).ToList(); foreach (var video in videos) { var videoLikes = dataProvider.GetLikes(vkGroup.Id.ToString(), video.VkId, LikeShareType.Video, 0); if (videoLikes == null) { continue; } this.log.DebugFormat("Video comments feed is received: {0}", videoLikes.Feed); videoLikes.ParentObjectId = video.VkId; string newFeed = this.responseMapper.MapResponseObject(videoLikes); DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = newFeed, VkGroupId = vkGroup.Id, RelatedObjectId = video.VkId, Type = DataFeedType.VideoLikes }; yield return(dataFeed); } }
public void SaveCountriesAndCities() { this.log.Debug("Places parsing started"); IVkDataProvider vkDataProvider = this.vkConnectionBuilder.GetVkDataProvider(); CountriesReponse countries = vkDataProvider.GetCountries(); this.SaveCountries(countries); this.log.Debug("Countries are saved"); int offset = 0; while (true) { CitiesReponse cities = vkDataProvider.GetCities(offset); this.SaveCities(cities); if (cities != null && cities.city != null && cities.city.Length > 0 && this.NonEmptyTitleExists(cities)) { offset += cities.city.Length; } else { break; } } this.log.Debug("Places parsing finished"); }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { int offsetCounter = 0; DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType); while (true) { var videos = dataProvider.GetVideos(vkGroup.Id.ToString(), offsetCounter); this.log.DebugFormat("Video feed is received: {0}", videos.Feed); if (videos.video == null || videos.video.Length == 0) { break; } DataFeed dataFeed = new DataFeed() { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = videos.Feed, VkGroupId = vkGroup.Id, Type = DataFeedType.Video }; yield return(dataFeed); offsetCounter += videos.video.Length; if (dateLimit.HasValue && videos.video[videos.video.Length - 1].date.FromUnixTimestamp() < dateLimit.Value) { break; } } }
private void ProcessQueueItem(FeedQueueItem queueItem, IVkDataProvider vkDataProvider, FeedProcessingConfig processingConfig) { try { if (!this.feedProviders.ContainsKey(queueItem.QueueItemType)) { throw new ArgumentException(string.Format("Unsupported feed type provided: \"{0}\"", queueItem.QueueItemType)); } var feedProvider = this.feedProviders[queueItem.QueueItemType]; var vkGroup = this.groupRepository.GetGroupById(queueItem.VkGroupId); if (vkGroup == null) { this.log.InfoFormat("Group with Id = \"{0}\" not found. Processing stopped.", queueItem.VkGroupId); return; } this.log.InfoFormat("Fetching of feed '{0}' for VkGroupId {1} is started", queueItem.QueueItemType, queueItem.VkGroupId); using (ICommandSender commandSender = Factory.GetInstance <ICommandSender>().Open(processingConfig.OutputQueueId)) { foreach (var dataFeed in feedProvider.GetFeeds(vkDataProvider, vkGroup)) { dataFeed.TtlInMinutes = processingConfig.TtlInMinutes; commandSender.SendCommand(dataFeed); } var terminator = new DataFeed { IsSequenceTerminator = true, ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), SendingDate = this.dateTimeHelper.GetDateTimeNow(), VkGroupId = vkGroup.Id, Type = feedProvider.ProvidedDataType, FetchingServer = this.webUtilities.GetServerName(), FetchingProcess = this.webUtilities.GetApplicationPoolName() }; commandSender.SendCommand(terminator); } this.log.InfoFormat("Fetching of feed '{0}' for VkGroupId {1} is finished", queueItem.QueueItemType, queueItem.VkGroupId); } catch (Exception exc) { this.log.Error(string.Format("Fetching of feed '{0}' for VkGroupId {1} is FAILED. Reason: {2}", queueItem.QueueItemType, queueItem.VkGroupId, exc)); } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { var memberFeed = dataProvider.GetGroupMemberIds(vkGroup.Id.ToString(), 0); this.log.DebugFormat("Users feed is received: {0}", memberFeed.Feed); DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = memberFeed.Feed, VkGroupId = vkGroup.Id, Type = DataFeedType.MembersCount }; yield return(dataFeed); }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { IList <long> memberVkIds = this.listRepository.GetMemberVkIds(vkGroup.Id); foreach (var memberId in memberVkIds) { int offsetCounter = 0; IList <response> feedsPerOneMember = new List <response>(); while (true) { var subscriptionFeed = dataProvider.GetMemberSubscriptions(memberId.ToString(), offsetCounter); this.log.DebugFormat("Member subscription feed is received: {0}", subscriptionFeed.Feed); if (subscriptionFeed.group != null && subscriptionFeed.group.Length > 0) { feedsPerOneMember.Add(subscriptionFeed); } if (subscriptionFeed.group == null || subscriptionFeed.group.Length == 0) { var dataFeed = this.Join(vkGroup.Id, memberId, feedsPerOneMember); if (dataFeed != null) { yield return(dataFeed); } break; } if (subscriptionFeed.group.Length < this.dataLimits.MemberSubscriptionLimits) { var dataFeed = this.Join(vkGroup.Id, memberId, feedsPerOneMember); if (dataFeed != null) { yield return(dataFeed); } break; } offsetCounter += subscriptionFeed.group.Length; } } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { int offsetCounter = 0; while (true) { var memberFeed = dataProvider.GetGroupMemberIds(vkGroup.Id.ToString(), offsetCounter); this.log.DebugFormat("Users feed is received: {0}", memberFeed.Feed); if (memberFeed.users == null || memberFeed.users.Length == 0) { break; } if (memberFeed.users[0].uid == null || memberFeed.users[0].uid.Length == 0) { break; } var userIds = memberFeed.users[0].uid.Select(u => u.Value).ToList(); var maxFetches = (int)Math.Floor((double)userIds.Count / CONST_MembersToFetchAtOnce); for (int i = 0; i < maxFetches; i++) { List <string> ids = userIds.Skip(i * CONST_MembersToFetchAtOnce).Take(CONST_MembersToFetchAtOnce).ToList(); if (ids.Count == 0) { break; } var userProfiles = dataProvider.GetUserProfiles(ids); DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = userProfiles.Feed, VkGroupId = vkGroup.Id, Type = DataFeedType.MembersInfo }; yield return(dataFeed); } offsetCounter += userIds.Count; } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { DateTime? dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType); IList <Topic> topics = this.topicRepository.GetTopicsByVkGroupId(vkGroup.Id).Where(p => !dateLimit.HasValue || p.PostedDate > dateLimit).ToList(); foreach (var topic in topics) { int offsetCounter = 0; while (true) { var topicComments = dataProvider.GetTopicComments(vkGroup.Id.ToString(), topic.VkId, offsetCounter); this.log.DebugFormat("Topic comments feed is received: {0}", topicComments.Feed); if (topicComments.Items == null || topicComments.Items.Length == 0 || topicComments.Items[0].comment == null || topicComments.Items[0].comment.Length == 0) { break; } // if last comment in this pack is was created before dateLimit, we ignore the whole pack, but still trying to find the latest one responseComments lastCommentGroup = topicComments.Items[topicComments.Items.Length - 1]; if (dateLimit.HasValue && lastCommentGroup.comment[lastCommentGroup.comment.Length - 1].date.FromUnixTimestamp() < dateLimit.Value) { continue; } DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = topicComments.Feed, VkGroupId = vkGroup.Id, RelatedObjectId = topic.VkId, Type = DataFeedType.TopicComment }; yield return(dataFeed); offsetCounter += topicComments.Items[0].comment.Length; } } }
public void ProcessNextQueueItem() { this.log.Info("GetFeedsFromVKAction process started"); var processingConfig = this.configProvider.GetConfigurationSection <FeedProcessingConfig>(); var selector = !string.IsNullOrWhiteSpace(processingConfig.FeedFilter) ? processingConfig.FeedFilter : null; using (ICommandReceiver commandReceiver = Factory.GetInstance <ICommandReceiver>().Open(processingConfig.InputQueueId, selector)) { IVkDataProvider vkDataProvider = this.vkConnectionBuilder.GetVkDataProvider(); for (int i = 0; i < CONST_BatchProcessingSize; i++) { FeedQueueItem queueItem = null; try { queueItem = commandReceiver.GetCommand <FeedQueueItem>(); if (queueItem == null) { this.log.Info("No items in queue found. Processing stopped."); return; } this.ProcessQueueItem(queueItem, vkDataProvider, processingConfig); } finally { if (queueItem != null) { queueItem.MarkAsCompleted(); using (ICommandSender commandSender = Factory.GetInstance <ICommandSender>().Open(processingConfig.InputQueueId)) { commandSender.SendCommand(queueItem.Copy()); } } } } } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { DateTime? dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType); IList <Post> posts = this.postRepository.GetPostsByVkGroupId(vkGroup.Id).Where(p => !dateLimit.HasValue || p.PostedDate > dateLimit).ToList(); foreach (var post in posts) { int offsetCounter = 0; while (true) { var wallPostComments = dataProvider.GetWallPostComments(post.VkId, vkGroup.Id.ToString(), offsetCounter); this.log.DebugFormat("Post comments feed is received: {0}", wallPostComments.Feed); if (wallPostComments.comment == null || wallPostComments.comment.Length == 0) { break; } // if last comment in this pack is was created before dateLimit, we ignore the whole pack, but still trying to find the latest one if (dateLimit.HasValue && wallPostComments.comment[wallPostComments.comment.Length - 1].date.FromUnixTimestamp() < dateLimit.Value) { continue; } DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = wallPostComments.Feed, VkGroupId = vkGroup.Id, RelatedObjectId = post.VkId, Type = DataFeedType.WallPostComments }; yield return(dataFeed); offsetCounter += wallPostComments.comment.Length; } } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType); IList <LikeShareType> types = Enum.GetValues(typeof(LikeShareType)).Cast <LikeShareType>().Where(t => t != LikeShareType.Undefined).ToList(); foreach (var type in types) { int offsetCounter = 0; IEnumerable <string> entities = this.GetEntityVkIds(type, vkGroup, dateLimit); foreach (var entity in entities) { var dataFeed = this.GetFeeds(dataProvider, type, entity, vkGroup, ref offsetCounter); if (dataFeed == null) { continue; } yield return(dataFeed); } } }
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup) { var admins = dataProvider.GetGroupAministrators((-vkGroup.Id).ToString(), (Ix.Palantir.Vkontakte.API.VkGroupType)(int) vkGroup.Type); this.log.DebugFormat("Administrators feed received"); if (string.IsNullOrWhiteSpace(admins.Page)) { return(new List <DataFeed>()); } DataFeed dataFeed = new DataFeed { ReceivedAt = this.dateTimeHelper.GetDateTimeNow(), Feed = admins.Page, VkGroupId = vkGroup.Id, Type = DataFeedType.Administrators }; return(new List <DataFeed> { dataFeed }); }