private void SaveNewMember(responseUser memberData, VkGroup @group)
{
Member member = this.memberMapper.CreateMember(memberData, @group);
this.memberMapper.UpdateMemberFields(member, memberData);
this.memberRepository.SaveMember(member);
}
public IList <IList <GroupedObjectWithPercents <string> > > GetInteractionFrequency(int projectId, int count)
{
VkGroup vkGroup = this.projectRepository.GetVkGroup(projectId);
var userCountInGroup = this.rawDataRepository.GetMembersCount(vkGroup.Id);
var userList = this.statisticsProvider.GetActiveUsers(projectId, null, 0).ToList();
var userListActiveCount = userList.Count;
var strictF = new List <GroupedObjectWithPercents <string> >();
var f = new List <GroupedObjectWithPercents <string> >();
if (count != 0)
{
f.Add(new GroupedObjectWithPercents <string>("1", userList.Count, userCountInGroup, userListActiveCount));
var usersMadeOneActionCount = userList.Count(x => (x.CommentCount + x.LikeCount + x.PostCount + x.ShareCount) == 1);
strictF.Add(new GroupedObjectWithPercents <string>("1", usersMadeOneActionCount, userCountInGroup, userListActiveCount));
for (var i = 2; i <= count; i++)
{
f.Add(new GroupedObjectWithPercents <string>(i.ToString(), f[i - 2].Value - strictF[i - 2].Value, userCountInGroup, userListActiveCount));
var usersMadeActions = userList.Count(x => (x.CommentCount + x.LikeCount + x.PostCount + x.ShareCount) == i);
strictF.Add(new GroupedObjectWithPercents <string>(i.ToString(), usersMadeActions, userCountInGroup, userListActiveCount));
}
}
var result = new List <IList <GroupedObjectWithPercents <string> > >();
result.Add(f.ToList());
result.Add(strictF.ToList());
return(result);
}
private void ProcessPost(string vkTopicId, responseCommentsComment comment, VkGroup group)
{
var savedComment = this.topicRepository.GetTopicComment(group.Id, comment.id);
if (savedComment != null)
{
this.log.DebugFormat("Topic comment with VkId={0} is already in database", comment.id);
return;
}
if (this.processingStrategy.IsLimitedProcessingEnabled(group.Id, DataFeedType.TopicComment) &&
comment.date.FromUnixTimestamp().AddMonths(this.processingStrategy.GetMonthLimit()) < DateTime.UtcNow)
{
this.log.DebugFormat("Fetched topic comment with VkId={0} is created more than {1} months ago. Skipping.", comment.id, this.processingStrategy.GetMonthLimit());
return;
}
savedComment = new TopicComment
{
VkId = comment.id,
VkTopicId = vkTopicId,
VkGroupId = group.Id,
PostedDate = comment.date.FromUnixTimestamp(),
CreatorId = long.Parse(comment.from_id)
};
this.topicRepository.SaveComment(savedComment);
this.log.DebugFormat("Topic comment with VkId={0} is not found in database. Saved with Id={1}", savedComment.VkId, savedComment.Id);
}
private void ProcessFeed(DataFeed dataFeed)
{
VkGroup group = this.groupRepository.GetGroupById(dataFeed.VkGroupId);
IFeedProcessor processor = this.processorFactory.Create(dataFeed.Type);
using (var transaction = this.transactionProvider.CreateTransaction().Begin())
{
this.log.InfoFormat("Processing data feed with VkGroupId={0} ({1}) is started", dataFeed.VkGroupId, dataFeed.Type);
try
{
processor.Process(dataFeed, group);
transaction.Commit();
this.log.InfoFormat("Processing data feed with VkGroupId={0} ({1}) is finished", dataFeed.VkGroupId, dataFeed.Type);
}
catch (Exception exc)
{
transaction.Rollback();
this.log.ErrorFormat("Data feed with VkGroupId={0} ({1}) is failed to be processed. Reason: {2}", dataFeed.VkGroupId, dataFeed.Type, exc.ToString());
throw;
}
finally
{
dataFeed.MarkAsCompleted();
this.log.InfoFormat("Data feed with Id={0} ({1}) is deleted", dataFeed.VkGroupId, dataFeed.Type);
}
}
}
private void ProcessVideo(responseVideo video, VkGroup group)
{
Video savedVideo = this.videoRepository.GetVideo(group.Id, video.vid);
if (savedVideo != null)
{
this.log.DebugFormat("Video with VkId={0} is already in database", video.vid);
return;
}
if (this.processingStrategy.IsLimitedProcessingEnabled(group.Id, DataFeedType.Video) &&
video.date.FromUnixTimestamp().AddMonths(this.processingStrategy.GetMonthLimit()) < DateTime.UtcNow)
{
this.log.DebugFormat("Fetched video with VkId={0} is created more than {1} months ago. Skipping.", video.vid, this.processingStrategy.GetMonthLimit());
return;
}
savedVideo = new Video
{
VkGroupId = group.Id,
PostedDate = video.date.FromUnixTimestamp(),
VkId = video.vid,
Title = video.title,
Description = video.description,
Duration = int.Parse(video.duration)
};
this.videoRepository.Save(savedVideo);
this.log.DebugFormat("Video with VkId={0} is not found in database. Saved with Id={1}", savedVideo.VkId, savedVideo.Id);
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
int offsetCounter = 0;
DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
while (true)
{
var videos = dataProvider.GetVideos(vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Video feed is received: {0}", videos.Feed);
if (videos.video == null || videos.video.Length == 0)
{
break;
}
DataFeed dataFeed = new DataFeed()
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = videos.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.Video
};
yield return(dataFeed);
offsetCounter += videos.video.Length;
if (dateLimit.HasValue && videos.video[videos.video.Length - 1].date.FromUnixTimestamp() < dateLimit.Value)
{
break;
}
}
}
private void ProcessPhoto(responsePhoto photo, VkGroup group)
{
var savedPhoto = this.photoRepository.GetPhotoByIdInAlbum(group.Id, photo.aid, photo.pid);
if (savedPhoto != null)
{
this.log.DebugFormat("Photo with VkId={0} is already in database", photo.pid);
return;
}
if (this.processingStrategy.IsLimitedProcessingEnabled(group.Id, DataFeedType.Photo) &&
photo.created.FromUnixTimestamp().AddMonths(this.processingStrategy.GetMonthLimit()) < DateTime.UtcNow)
{
this.log.DebugFormat("Fetched photo with VkId={0} is created more than {1} months ago. Skipping.", photo.pid, this.processingStrategy.GetMonthLimit());
return;
}
savedPhoto = new Photo
{
VkGroupId = group.Id,
PostedDate = photo.created.FromUnixTimestamp(),
VkId = photo.pid,
AlbumId = photo.aid,
Text = photo.text
};
this.photoRepository.Save(savedPhoto);
this.log.DebugFormat("Photo with VkId={0} is not found in database. Saved with Id={1}", savedPhoto.VkId, savedPhoto.Id);
}
private void ProcessPost(string vkPostId, responseComment comment, VkGroup group)
{
var savedComment = this.postRepository.GetPostComment(group.Id, comment.cid);
if (savedComment != null)
{
this.log.DebugFormat("Post comment with VkId={0} is already in database", comment.cid);
return;
}
if (this.processingStrategy.IsLimitedProcessingEnabled(group.Id, DataFeedType.WallPostComments) &&
comment.date.FromUnixTimestamp().AddMonths(this.processingStrategy.GetMonthLimit()) < DateTime.UtcNow)
{
this.log.DebugFormat("Fetched post comment with VkId={0} is created more than {1} months ago. Skipping.", comment.cid, this.processingStrategy.GetMonthLimit());
return;
}
savedComment = new PostComment
{
VkId = comment.cid,
VkPostId = vkPostId,
VkGroupId = group.Id,
PostedDate = comment.date.FromUnixTimestamp(),
CreatorId = long.Parse(comment.uid),
ReplyToUserId = string.IsNullOrWhiteSpace(comment.reply_to_uid) ? null : (long?)long.Parse(comment.reply_to_uid),
ReplyToVkId = string.IsNullOrWhiteSpace(comment.reply_to_cid) ? null : (int?)int.Parse(comment.reply_to_cid),
};
this.postRepository.SaveComment(savedComment);
this.log.DebugFormat("Post comment with VkId={0} is not found in database. Saved with Id={1}", savedComment.VkId, savedComment.Id);
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
int offsetCounter = 0;
DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
while (true)
{
var posts = dataProvider.GetWallPosts(vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Posts feed is received: {0}", posts.Feed);
if (posts.post == null || posts.post.Length == 0)
{
break;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = posts.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.WallPosts
};
yield return(dataFeed);
offsetCounter += posts.post.Length;
if (dateLimit.HasValue && posts.post[posts.post.Length - 1].date.FromUnixTimestamp() < dateLimit)
{
break;
}
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
IList <string> albumIds = this.photoRepository.GetGroupAlbumIds(vkGroup.Id);
foreach (var albumId in albumIds)
{
var photos = dataProvider.GetPhotoAlbumDetails(vkGroup.Id.ToString(), albumId, 0);
this.log.DebugFormat("Photos feed is received: {0}", photos.Feed);
if (photos.photo == null || photos.photo.Length == 0)
{
continue;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = photos.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.PhotoAlbumDetails
};
yield return(dataFeed);
}
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
DateTime? dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
IList <Video> videos = this.videoRepository.GetVideosByVkGroupId(vkGroup.Id).Where(v => !dateLimit.HasValue || v.PostedDate > dateLimit).ToList();
foreach (var video in videos)
{
var videoLikes = dataProvider.GetLikes(vkGroup.Id.ToString(), video.VkId, LikeShareType.Video, 0);
if (videoLikes == null)
{
continue;
}
this.log.DebugFormat("Video comments feed is received: {0}", videoLikes.Feed);
videoLikes.ParentObjectId = video.VkId;
string newFeed = this.responseMapper.MapResponseObject(videoLikes);
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = newFeed,
VkGroupId = vkGroup.Id,
RelatedObjectId = video.VkId,
Type = DataFeedType.VideoLikes
};
yield return(dataFeed);
}
}
private void ProcessShare(response feed, VkGroup vkGroup)
{
if (feed.users[0].uid == null || feed.users[0].uid.Length == 0)
{
return;
}
foreach (var memberId in feed.users[0].uid)
{
var share = this.memberLikeSharesRepository.GetShare(vkGroup.Id, memberId.Value, int.Parse(feed.ParentObjectId), (LikeShareType)feed.LikeShareType);
if (share != null)
{
this.log.DebugFormat("MemberShare for MemberId={0}, ItemId={1}, ItemType={2} is already in database", memberId.Value, feed.ParentObjectId, feed.LikeShareType);
continue;
}
share = new MemberShare
{
ItemId = int.Parse(feed.ParentObjectId),
VkGroupId = vkGroup.Id,
ItemType = (LikeShareType)feed.LikeShareType,
VkId = memberId.Value
};
this.memberLikeSharesRepository.SaveShare(share);
}
}
public async Task <VkNewsResponse> Get(string sourceIds = null, string filters = null, int count = 0, string startFrom = null)
{
var parameters = new Dictionary <string, string>();
if (!string.IsNullOrEmpty(sourceIds))
{
parameters.Add("source_ids", sourceIds);
}
if (!string.IsNullOrEmpty(filters))
{
parameters.Add("filters", filters);
}
if (count > 0)
{
parameters.Add("count", count.ToString());
}
if (!string.IsNullOrEmpty(startFrom))
{
parameters.Add("start_from", startFrom);
}
_vkontakte.SignMethod(parameters);
var response = await VkRequest.GetAsync(VkConst.MethodBase + "newsfeed.get", parameters);
if (response.SelectToken("response.items") != null)
{
var result = new VkNewsResponse((from n in response["response"]["items"] select VkNewsEntry.FromJson(n)).ToList());
if (response["response"]["profiles"] != null)
{
var users = (from n in response["response"]["profiles"] select VkProfile.FromJson(n)).ToList();
foreach (var entry in result.Items)
{
entry.Author = users.FirstOrDefault(u => u.Id == entry.SourceId);
}
}
if (response["response"]["groups"] != null)
{
var groups = (from n in response["response"]["groups"] select VkGroup.FromJson(n)).ToList();
foreach (var entry in result.Items.Where(e => e.Author == null))
{
entry.Author = groups.FirstOrDefault(g => g.Id == Math.Abs(entry.SourceId));
}
}
if (response["response"]["next_from"] != null)
{
result.NextFrom = response["response"]["next_from"].Value <string>();
}
return(result);
}
return(null);
}
private DomainModel.Project CreateProject(CreateProjectCommand project)
{
int vkGroupId = this.groupInfoProvider.GetVkGroupId(project.Url);
VkGroup group = this.vkGroupRepository.GetGroupById(vkGroupId);
bool isGroupExists = group != null;
DomainModel.Project projectToCreate = new DomainModel.Project
{
Title = project.Title,
CreationDate = this.dateTimeHelper.GetDateTimeNow(),
AccountId = project.AccountId,
VkGroup = group ?? new VkGroup
{
Id = vkGroupId,
Name = this.groupInfoProvider.GetVkGroupName(project.Url),
Url = project.Url,
Type = VkGroupType.Group
}
};
using (ITransactionScope transaction = this.unitOfWorkProvider.CreateTransaction().Begin())
{
this.projectRepository.Save(projectToCreate);
if (!isGroupExists)
{
this.AddGroupRelatedData(projectToCreate);
}
transaction.Commit();
}
return(projectToCreate);
}
public IEnumerable <DataFeed> GetFeeds(IVkDataProvider dataProvider, VkGroup vkGroup)
{
int offsetCounter = 0;
DateTime?dateLimit = this.strategy.GetDateLimit(vkGroup.Id, this.ProvidedDataType);
while (true)
{
var topicFeed = dataProvider.GetTopics(vkGroup.Id.ToString(), offsetCounter);
this.log.DebugFormat("Topics feed is received: {0}", topicFeed.Feed);
if (topicFeed.topics == null || topicFeed.topics.Length == 0 || topicFeed.topics[0].topic == null || topicFeed.topics[0].topic.Length == 0)
{
break;
}
DataFeed dataFeed = new DataFeed
{
ReceivedAt = this.dateTimeHelper.GetDateTimeNow(),
Feed = topicFeed.Feed,
VkGroupId = vkGroup.Id,
Type = DataFeedType.Topic
};
yield return(dataFeed);
offsetCounter += topicFeed.topics[0].topic.Length;
if (dateLimit.HasValue && topicFeed.topics[topicFeed.topics.Length - 1].topic[0].created.FromUnixTimestamp() < dateLimit)
{
break;
}
}
}
public async Task <VkItemsResponse <VkWallEntry> > Get(long ownerId, string filter, int count = 0, int offset = 0)
{
var parameters = new Dictionary <string, string>();
if (ownerId != 0)
{
parameters.Add("owner_id", ownerId.ToString());
}
if (!string.IsNullOrEmpty(filter))
{
parameters.Add("filter", filter);
}
if (count > 0)
{
parameters.Add("count", count.ToString());
}
if (offset > 0)
{
parameters.Add("offset", offset.ToString());
}
parameters.Add("extended", "1");
_vkontakte.SignMethod(parameters);
var response = await VkRequest.GetAsync(VkConst.MethodBase + "wall.get", parameters);
if (response.SelectToken("response.items") != null)
{
var result = new VkItemsResponse <VkWallEntry>();
result.Items = (from p in response["response"]["items"] where p.HasValues select VkWallEntry.FromJson(p)).ToList();
result.TotalCount = response["response"]["count"].Value <int>();
if (response["response"]["profiles"] != null)
{
var users = (from n in response["response"]["profiles"] select VkProfile.FromJson(n)).ToList();
foreach (var entry in result.Items)
{
entry.Author = users.FirstOrDefault(u => u.Id == entry.SourceId);
}
}
if (response["response"]["groups"] != null)
{
var groups = (from n in response["response"]["groups"] select VkGroup.FromJson(n)).ToList();
foreach (var entry in result.Items.Where(e => e.Author == null))
{
entry.Author = groups.FirstOrDefault(g => g.Id == Math.Abs(entry.SourceId));
}
}
return(result);
}
return(null);
}
private string GenerateFileName(ExportReportCommand exportCommand)
{
VkGroup vkGroup = this.vkGroupRepository.GetGroupById(exportCommand.VkGroupId);
return(!exportCommand.DateRange.IsSpecified
? string.Format("export_of_{0}.xslx", vkGroup.Name)
: string.Format("export_of_{0}_from_{1}_to_{2}.xlsx", vkGroup.Name, exportCommand.DateRange.From.ToString(CONST_DateTimeFormat), exportCommand.DateRange.To.ToString(CONST_DateTimeFormat)));
}
public void Process(DataFeed dataFeed, VkGroup group)
{
var feed = this.responseMapper.MapResponse <response>(dataFeed.Feed);
foreach (var post in feed.post)
{
this.ProcessPost(post, group);
}
}
public void Process(DataFeed dataFeed, VkGroup group)
{
var feed = this.responseMapper.MapResponse <response>(dataFeed.Feed);
foreach (var member in feed.user)
{
this.ProcessUser(member, group);
}
}
public void Process(DataFeed dataFeed, VkGroup group)
{
var feed = this.responseMapper.MapResponse <response>(dataFeed.Feed);
foreach (var comment in feed.comment)
{
this.ProcessPost(dataFeed.RelatedObjectId, comment, group);
}
}
public async Task <VkItemsResponse <VkWallEntry> > GetPosts(int count = 0, int offset = 0, bool extended = true)
{
if (_vkontakte.AccessToken == null || string.IsNullOrEmpty(_vkontakte.AccessToken.Token) || _vkontakte.AccessToken.HasExpired)
{
throw new Exception("Access token is not valid.");
}
var parameters = new Dictionary <string, string>();
if (count > 0)
{
parameters.Add("count", count.ToString());
}
if (offset > 0)
{
parameters.Add("offset", offset.ToString());
}
if (extended)
{
parameters.Add("extended", "1");
}
_vkontakte.SignMethod(parameters);
var response = await new VkRequest(new Uri(VkConst.MethodBase + "fave.getPosts"), parameters).Execute();
VkErrorProcessor.ProcessError(response);
if (response.SelectToken("response.items") != null)
{
var result = new VkItemsResponse <VkWallEntry>((from n in response["response"]["items"] select VkWallEntry.FromJson(n)).ToList());
if (response["response"]["profiles"] != null)
{
var users = (from n in response["response"]["profiles"] select VkProfile.FromJson(n)).ToList();
foreach (var entry in result.Items)
{
entry.Author = users.FirstOrDefault(u => u.Id == entry.SourceId);
}
}
if (response["response"]["groups"] != null)
{
var groups = (from n in response["response"]["groups"] select VkGroup.FromJson(n)).ToList();
foreach (var entry in result.Items.Where(e => e.Author == null))
{
entry.Author = groups.FirstOrDefault(g => g.Id == Math.Abs(entry.SourceId));
}
}
return(result);
}
return(null);
}
public void Process(DataFeed dataFeed, VkGroup group)
{
var feed = this.responseMapper.MapResponse <response>(dataFeed.Feed);
foreach (var video in feed.video)
{
this.ProcessVideo(video, group);
}
}
public IList <PostEntityInfo> GetMostPopularPosts(int projectId, DateRange dateRange)
{
using (this.unitOfWorkProvider.CreateUnitOfWork())
{
VkGroup vkGroup = this.projectRepository.GetVkGroup(projectId);
var posts = this.statisticsProvider.GetMostPopularPosts(projectId, PopularBy.LikesAndComments, dateRange);
return(posts.Select(x => this.CreatePostEntityInfo(vkGroup.Id, x)).ToList());
}
}
public IEnumerable <IEnumerable <PointInTime> > GetInteractionRate(int projectId, DateRange range, Periodicity periodicity)
{
VkGroup vkGroup = this.projectRepository.GetVkGroup(projectId);
using (this.unitOfWorkProvider.CreateUnitOfWork())
{
return(this.irChartDataProvider.GetInteractionRate(vkGroup.Id, range, periodicity));
}
}
public void Process(DataFeed dataFeed, VkGroup group)
{
var feed = this.responseMapper.MapResponse <response>(dataFeed.Feed);
foreach (var photo in feed.photo)
{
this.ProcessPhotoDetails(photo, group);
}
}
private void ProcessVideo(response feed, VkGroup group)
{
var savedVideo = this.videoRepository.GetVideo(group.Id, feed.ParentObjectId);
if (savedVideo != null)
{
this.UpdateExistingVideo(feed, savedVideo);
}
}
public VkGroup GetVkGroup(int projectId)
{
if (!this.groupsMap.ContainsKey(projectId))
{
VkGroup vkGroup = this.internalRepository.GetVkGroup(projectId);
this.groupsMap.Add(projectId, vkGroup);
}
return(this.groupsMap[projectId]);
}
internal static VkCatalogBlock FromJson(JToken json)
{
if (json == null)
{
throw new ArgumentException("Json can not be null.");
}
var result = new VkCatalogBlock();
result.Id = json["id"].Value <int>();
result.Title = json["title"].Value <string>();
result.Subtitle = json["subtitle"].Value <string>();
result.Type = json["type"].Value <string>();
result.Count = json["count"].Value <int>();
result.Source = json["source"].Value <string>();
if (json["audios"] != null)
{
result.Audios = json["audios"].Select(VkAudio.FromJson).ToList();
}
if (json["thumbs"] != null)
{
result.Thumbs = json["thumbs"].ToObject <List <VkThumb> >();
}
if (json["owners"] != null)
{
result.Owners = json["owners"].Select(x =>
{
if (x["type"].Value <string>() != "profile")
{
return((VkProfileBase)VkGroup.FromJson(x));
}
else
{
return((VkProfileBase)VkProfile.FromJson(x));
}
}).ToList();
}
if (json["playlists"] != null)
{
result.Playlists = json["playlists"].Select(VkPlaylist.FromJson).ToList();
}
if (json["extended_playlists"] != null)
{
result.ExtendedPlaylists = json["extended_playlists"].Select(VkExtendedPlaylist.FromJson).ToList();
}
return(result);
}
public Member CreateMember(responseUser memberData, VkGroup group)
{
Member member = new Member()
{
IsDeleted = false,
VkGroupId = group.Id,
VkMemberId = long.Parse(memberData.uid),
};
return(member);
}
private FeedQueueItem GetQueueItem(VkGroup @group, QueueItemType dataFeedType)
{
FeedQueueItem item = new FeedQueueItem()
{
VkGroupId = @group.Id,
QueueItemType = dataFeedType,
CreationDate = this.dateTimeHelper.GetDateTimeNow(),
};
return(item);
}
