private DataFeed GetFeeds(IVkDataProvider dataProvider, LikeShareType type, string entityVkId, VkGroup vkGroup, ref int offsetCounter)
{
var sharesFeed = dataProvider.GetShares(vkGroup.Id.ToString(), entityVkId, type, offsetCounter);
if (sharesFeed == null)
{
return(null);
}
this.log.DebugFormat("Shares feed is received: {0}", sharesFeed.Feed);
sharesFeed.ParentObjectId = entityVkId;
sharesFeed.LikeShareType = (int)type;
string newFeed = this.responseMapper.MapResponseObject(sharesFeed);
var dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = newFeed,
VkGroupId = vkGroup.Id,
RelatedObjectId = entityVkId,
Type = DataFeedType.MemberShares
};
if (sharesFeed.users[0].uid == null || sharesFeed.users[0].uid.Length == 0)
{
return(null);
}
offsetCounter += sharesFeed.users.Length;
return(dataFeed);
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
int offsetCounter = 0;
DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
while (true)
{
var posts = dataProvider.GetWallPosts(vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Posts feed is received: {0}", posts.Feed);
if (posts.post == null || posts.post.Length == 0)
{
break;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = posts.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.WallPosts
};
yield return(dataFeed);
offsetCounter += posts.post.Length;
if (dateLimit.HasValue && posts.post[posts.post.Length - 1].date.FromUnixTimestamp() < dateLimit)
{
break;
}
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
int offsetCounter = 0;
DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
while (true)
{
var topicFeed = dataProvider.GetTopics(vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Topics feed is received: {0}", topicFeed.Feed);
if (topicFeed.topics == null || topicFeed.topics.Length == 0 || topicFeed.topics[0].topic == null || topicFeed.topics[0].topic.Length == 0)
{
break;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = topicFeed.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.Topic
};
yield return(dataFeed);
offsetCounter += topicFeed.topics[0].topic.Length;
if (dateLimit.HasValue && topicFeed.topics[topicFeed.topics.Length - 1].topic[0].created.FromUnixTimestamp() < dateLimit)
{
break;
}
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
IList <string> albumIds = this.photoRepository.GetGroupAlbumIds(vkGroup.Id);
foreach (var albumId in albumIds)
{
var photos = dataProvider.GetPhotoAlbumDetails(vkGroup.Id.ToString(), albumId, 0);
this.log.DebugFormat("Photos feed is received: {0}", photos.Feed);
if (photos.photo == null || photos.photo.Length == 0)
{
continue;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = photos.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.PhotoAlbumDetails
};
yield return(dataFeed);
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
DateTime? dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
IList <Video> videos = this.videoRepository.GetVideosByVkGroupId(vkGroup.Id).Where(v => !dateLimit.HasValue || v.PostedDate > dateLimit).ToList();
foreach (var video in videos)
{
var videoLikes = dataProvider.GetLikes(vkGroup.Id.ToString(), video.VkId, LikeShareType.Video, 0);
if (videoLikes == null)
{
continue;
}
this.log.DebugFormat("Video comments feed is received: {0}", videoLikes.Feed);
videoLikes.ParentObjectId = video.VkId;
string newFeed = this.responseMapper.MapResponseObject(videoLikes);
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = newFeed,
VkGroupId = vkGroup.Id,
RelatedObjectId = video.VkId,
Type = DataFeedType.VideoLikes
};
yield return(dataFeed);
}
}
public void SaveCountriesAndCities()
{
this.log.Debug("Places parsing started");
IVkDataProvider vkDataProvider = this.vkConnectionBuilder.GetVkDataProvider();
CountriesReponse countries = vkDataProvider.GetCountries();
this.SaveCountries(countries);
this.log.Debug("Countries are saved");
int offset = 0;
while (true)
{
CitiesReponse cities = vkDataProvider.GetCities(offset);
this.SaveCities(cities);
if (cities != null && cities.city != null && cities.city.Length > 0 && this.NonEmptyTitleExists(cities))
{
offset += cities.city.Length;
}
else
{
break;
}
}
this.log.Debug("Places parsing finished");
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
int offsetCounter = 0;
DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
while (true)
{
var videos = dataProvider.GetVideos(vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Video feed is received: {0}", videos.Feed);
if (videos.video == null || videos.video.Length == 0)
{
break;
}
DataFeed dataFeed = new DataFeed()
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = videos.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.Video
};
yield return(dataFeed);
offsetCounter += videos.video.Length;
if (dateLimit.HasValue && videos.video[videos.video.Length - 1].date.FromUnixTimestamp() < dateLimit.Value)
{
break;
}
}
}
private void ProcessQueueItem(FeedQueueItem queueItem, IVkDataProvider vkDataProvider, FeedProcessingConfig processingConfig)
{
try
{
if (!this.feedProviders.ContainsKey(queueItem.QueueItemType))
{
throw new ArgumentException(string.Format("Unsupported feed type provided: \"{0}\"", queueItem.QueueItemType));
}
var feedProvider = this.feedProviders[queueItem.QueueItemType];
var vkGroup = this.groupRepository.GetGroupById(queueItem.VkGroupId);
if (vkGroup == null)
{
this.log.InfoFormat("Group with Id = \"{0}\" not found. Processing stopped.", queueItem.VkGroupId);
return;
}
this.log.InfoFormat("Fetching of feed '{0}' for VkGroupId {1} is started", queueItem.QueueItemType, queueItem.VkGroupId);
using (ICommandSender commandSender = Factory.GetInstance <ICommandSender>().Open(processingConfig.OutputQueueId))
{
foreach (var dataFeed in feedProvider.GetFeeds(vkDataProvider, vkGroup))
{
dataFeed.TtlInMinutes = processingConfig.TtlInMinutes;
commandSender.SendCommand(dataFeed);
}
var terminator = new DataFeed
{
IsSequenceTerminator = true,
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
SendingDate = this.dateTimeHelper.GetDateTimeNow(),
VkGroupId = vkGroup.Id,
Type = feedProvider.ProvidedDataType,
FetchingServer = this.webUtilities.GetServerName(),
FetchingProcess = this.webUtilities.GetApplicationPoolName()
};
commandSender.SendCommand(terminator);
}
this.log.InfoFormat("Fetching of feed '{0}' for VkGroupId {1} is finished", queueItem.QueueItemType, queueItem.VkGroupId);
}
catch (Exception exc)
{
this.log.Error(string.Format("Fetching of feed '{0}' for VkGroupId {1} is FAILED. Reason: {2}", queueItem.QueueItemType, queueItem.VkGroupId, exc));
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
var memberFeed = dataProvider.GetGroupMemberIds(vkGroup.Id.ToString(), 0);
this.log.DebugFormat("Users feed is received: {0}", memberFeed.Feed);
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = memberFeed.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.MembersCount
};
yield return(dataFeed);
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
IList <long> memberVkIds = this.listRepository.GetMemberVkIds(vkGroup.Id);
foreach (var memberId in memberVkIds)
{
int offsetCounter = 0;
IList <response> feedsPerOneMember = new List <response>();
while (true)
{
var subscriptionFeed = dataProvider.GetMemberSubscriptions(memberId.ToString(), offsetCounter);
this.log.DebugFormat("Member subscription feed is received: {0}", subscriptionFeed.Feed);
if (subscriptionFeed.group != null && subscriptionFeed.group.Length > 0)
{
feedsPerOneMember.Add(subscriptionFeed);
}
if (subscriptionFeed.group == null || subscriptionFeed.group.Length == 0)
{
var dataFeed = this.Join(vkGroup.Id, memberId, feedsPerOneMember);
if (dataFeed != null)
{
yield return(dataFeed);
}
break;
}
if (subscriptionFeed.group.Length < this.dataLimits.MemberSubscriptionLimits)
{
var dataFeed = this.Join(vkGroup.Id, memberId, feedsPerOneMember);
if (dataFeed != null)
{
yield return(dataFeed);
}
break;
}
offsetCounter += subscriptionFeed.group.Length;
}
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
int offsetCounter = 0;
while (true)
{
var memberFeed = dataProvider.GetGroupMemberIds(vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Users feed is received: {0}", memberFeed.Feed);
if (memberFeed.users == null || memberFeed.users.Length == 0)
{
break;
}
if (memberFeed.users[0].uid == null || memberFeed.users[0].uid.Length == 0)
{
break;
}
var userIds = memberFeed.users[0].uid.Select(u => u.Value).ToList();
var maxFetches = (int)Math.Floor((double)userIds.Count / CONST_MembersToFetchAtOnce);
for (int i = 0; i < maxFetches; i++)
{
List <string> ids = userIds.Skip(i * CONST_MembersToFetchAtOnce).Take(CONST_MembersToFetchAtOnce).ToList();
if (ids.Count == 0)
{
break;
}
var userProfiles = dataProvider.GetUserProfiles(ids);
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = userProfiles.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.MembersInfo
};
yield return(dataFeed);
}
offsetCounter += userIds.Count;
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
DateTime? dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
IList <Topic> topics = this.topicRepository.GetTopicsByVkGroupId(vkGroup.Id).Where(p => !dateLimit.HasValue || p.PostedDate > dateLimit).ToList();
foreach (var topic in topics)
{
int offsetCounter = 0;
while (true)
{
var topicComments = dataProvider.GetTopicComments(vkGroup.Id.ToString(), topic.VkId, offsetCounter);
this.log.DebugFormat("Topic comments feed is received: {0}", topicComments.Feed);
if (topicComments.Items == null || topicComments.Items.Length == 0 || topicComments.Items[0].comment == null || topicComments.Items[0].comment.Length == 0)
{
break;
}
// if last comment in this pack is was created before dateLimit, we ignore the whole pack, but still trying to find the latest one
responseComments lastCommentGroup = topicComments.Items[topicComments.Items.Length - 1];
if (dateLimit.HasValue && lastCommentGroup.comment[lastCommentGroup.comment.Length - 1].date.FromUnixTimestamp() < dateLimit.Value)
{
continue;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = topicComments.Feed,
VkGroupId = vkGroup.Id,
RelatedObjectId = topic.VkId,
Type = DataFeedType.TopicComment
};
yield return(dataFeed);
offsetCounter += topicComments.Items[0].comment.Length;
}
}
}
public void ProcessNextQueueItem()
{
this.log.Info("GetFeedsFromVKAction process started");
var processingConfig = this.configProvider.GetConfigurationSection <FeedProcessingConfig>();
var selector = !string.IsNullOrWhiteSpace(processingConfig.FeedFilter) ? processingConfig.FeedFilter : null;
using (ICommandReceiver commandReceiver = Factory.GetInstance <ICommandReceiver>().Open(processingConfig.InputQueueId, selector))
{
IVkDataProvider vkDataProvider = this.vkConnectionBuilder.GetVkDataProvider();
for (int i = 0; i < CONST_BatchProcessingSize; i++)
{
FeedQueueItem queueItem = null;
try
{
queueItem = commandReceiver.GetCommand <FeedQueueItem>();
if (queueItem == null)
{
this.log.Info("No items in queue found. Processing stopped.");
return;
}
this.ProcessQueueItem(queueItem, vkDataProvider, processingConfig);
}
finally
{
if (queueItem != null)
{
queueItem.MarkAsCompleted();
using (ICommandSender commandSender = Factory.GetInstance <ICommandSender>().Open(processingConfig.InputQueueId))
{
commandSender.SendCommand(queueItem.Copy());
}
}
}
}
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
DateTime? dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
IList <Post> posts = this.postRepository.GetPostsByVkGroupId(vkGroup.Id).Where(p => !dateLimit.HasValue || p.PostedDate > dateLimit).ToList();
foreach (var post in posts)
{
int offsetCounter = 0;
while (true)
{
var wallPostComments = dataProvider.GetWallPostComments(post.VkId, vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Post comments feed is received: {0}", wallPostComments.Feed);
if (wallPostComments.comment == null || wallPostComments.comment.Length == 0)
{
break;
}
// if last comment in this pack is was created before dateLimit, we ignore the whole pack, but still trying to find the latest one
if (dateLimit.HasValue && wallPostComments.comment[wallPostComments.comment.Length - 1].date.FromUnixTimestamp() < dateLimit.Value)
{
continue;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = wallPostComments.Feed,
VkGroupId = vkGroup.Id,
RelatedObjectId = post.VkId,
Type = DataFeedType.WallPostComments
};
yield return(dataFeed);
offsetCounter += wallPostComments.comment.Length;
}
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
IList <LikeShareType> types = Enum.GetValues(typeof(LikeShareType)).Cast <LikeShareType>().Where(t => t != LikeShareType.Undefined).ToList();
foreach (var type in types)
{
int offsetCounter = 0;
IEnumerable <string> entities = this.GetEntityVkIds(type, vkGroup, dateLimit);
foreach (var entity in entities)
{
var dataFeed = this.GetFeeds(dataProvider, type, entity, vkGroup, ref offsetCounter);
if (dataFeed == null)
{
continue;
}
yield return(dataFeed);
}
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
var admins = dataProvider.GetGroupAministrators((-vkGroup.Id).ToString(), (Ix.Palantir.Vkontakte.API.VkGroupType)(int) vkGroup.Type);
this.log.DebugFormat("Administrators feed received");
if (string.IsNullOrWhiteSpace(admins.Page))
{
return(new List <DataFeed>());
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = admins.Page,
VkGroupId = vkGroup.Id,
Type = DataFeedType.Administrators
};
return(new List <DataFeed> {
dataFeed
});
}
