// Async workers private void GroupsWork(Object sender, DoWorkEventArgs args) { // Do not access the form's BackgroundWorker reference directly. // Instead, use the reference provided by the sender parameter. var bw = sender as BackgroundWorker; // Extract the argument. var param = args.Argument as GroupPostsParam; var groupId = param.gid; if (param.from <= param.to) { postsFromDate = param.from; postsToDate = param.to; } else { postsFromDate = param.to; postsToDate = param.from; } var context = new VkRestApi.VkRestContext(userId, authToken); var sb = new StringBuilder(); isRunning = true; // gather group statistics if (param.justStats) { bw.ReportProgress(1, "Getting group stats"); sb.Length = 0; sb.Append("group_id=").Append(Math.Abs(groupId).ToString()).Append("&"); sb.Append("date_from=").Append(this.postsFromDate.ToString("yyyy-MM-dd")).Append("&"); sb.Append("date_to=").Append(this.postsToDate.ToString("yyyy-MM-dd")); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); // call VK REST API vkRestApi.CallVkFunction(VkFunction.StatsGet, context); return; } // create stream writers // 1) group posts // group posts IEntity e = new Post(); String fileName = Utils.GenerateFileName(WorkingFolderTextBox.Text, groupId, e); groupPostsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupPostsWriter, e); postsWithComments.Clear(); // reset comments reference list likes.Clear(); // reset likes posters.Clear(); // reset posters posterIds.Clear(); // clear poster ids visitorIds.Clear(); // clear visitor ids totalCount = 0; currentOffset = 0; step = 1; long timeLastCall = 0; // get group posts 100 at a time and store them in the file // request group posts while (isRunning) { if (bw.CancellationPending) break; if(currentOffset > totalCount) { // done break; } bw.ReportProgress(step, "Getting " + currentOffset + " posts out of " + totalCount); sb.Length = 0; sb.Append("owner_id=").Append(groupId.ToString()).Append("&"); sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); context.Cookie = currentOffset.ToString(); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.WallGet, context); // wait for the user data ReadyEvent.WaitOne(); currentOffset += POSTS_PER_REQUEST; } groupPostsWriter.Close(); if (postsWithComments.Count > 0) { // group comments e = new Comment(); fileName = Utils.GenerateFileName(WorkingFolderTextBox.Text, groupId, e); groupCommentsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupCommentsWriter, e); // request group comments bw.ReportProgress(-1, "Getting comments"); step = 10000 / postsWithComments.Count; timeLastCall = 0; for (var i = 0; i < postsWithComments.Count; i++) { isRunning = true; totalCount = 0; currentOffset = 0; bw.ReportProgress(step, "Getting " + (i + 1) + " post comments out of " + postsWithComments.Count); while (isRunning) { if (bw.CancellationPending) break; if (currentOffset > totalCount) { // done break; } sb.Length = 0; sb.Append("owner_id=").Append(groupId.ToString()).Append("&"); // group id sb.Append("post_id=").Append(postsWithComments[i]).Append("&"); // post id sb.Append("need_likes=").Append(1).Append("&"); // request likes info sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); context.Cookie = postsWithComments[i].ToString(); // pass post id as a cookie Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.WallGetComments, context); // wait for the user data ReadyEvent.WaitOne(); currentOffset += POSTS_PER_REQUEST; } } groupCommentsWriter.Close(); } // process the likers - they will be added to the posters if (likes.Count > 0) { // request likers ids bw.ReportProgress(-1, "Getting likers"); step = 10000 / likes.Count; timeLastCall = 0; for (int i = 0; i < this.likes.Count; i++) { isRunning = true; //this.totalCount = 0; //this.currentOffset = 0; bw.ReportProgress(step, "Getting " + (i + 1) + " likes out of " + this.likes.Count); if (bw.CancellationPending) break; sb.Length = 0; sb.Append("type=").Append(likes[i].type).Append("&"); // group id sb.Append("owner_id=").Append(likes[i].owner_id).Append("&"); // group id sb.Append("item_id=").Append(likes[i].item_id).Append("&"); // post id sb.Append("count=").Append(LIKES_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.LikesGetList, context); // wait for the user data ReadyEvent.WaitOne(); } } // now collect visitors (not members, who left a post or a comment or a like) foreach (var p in posterIds) { if (!memberIds.Contains(p)) { // this is a visitor poster visitorIds.Add(p); } } if (visitorIds.Count > 0) { // group visitors profiles e = new Profile(); fileName = Utils.GenerateFileName(this.WorkingFolderTextBox.Text, groupId, e, "visitor"); groupVisitorsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupVisitorsWriter, e); // request visitors info bw.ReportProgress(-1, "Getting visitors"); step = 10000 / visitorIds.Count; timeLastCall = 0; for (var i = 0; i < visitorIds.Count; i += 100) { isRunning = true; //this.totalCount = 0; //this.currentOffset = 0; bw.ReportProgress(step, "Getting " + (i + 1) + " visitors out of " + visitorIds.Count); if (bw.CancellationPending) break; sb.Length = 0; sb.Append("user_ids="); for (var j = i; j < visitorIds.Count && j < i + 100; ++j) { sb.Append(visitorIds[j]).Append(","); // users } sb.Append("&").Append("fields=").Append(PROFILE_FIELDS); context.Parameters = sb.ToString(); Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.UsersGet, context); // wait for the user data ReadyEvent.WaitOne(); } groupVisitorsWriter.Close(); } //args.Result = TimeConsumingOperation(bw, arg); // If the operation was canceled by the user, // set the DoWorkEventArgs.Cancel property to true. if (bw.CancellationPending) { args.Cancel = true; } }
// process group posts private void OnWallGet(JObject data) { if (data[VkRestApi.ResponseBody] == null) { isRunning = false; return; } if(totalCount == 0) { totalCount = data[VkRestApi.ResponseBody]["count"].ToObject<long>(); if (totalCount == 0) { isRunning = false; return; } step = (int)(10000 * POSTS_PER_REQUEST / totalCount); } // now calculate items in response var count = data[VkRestApi.ResponseBody]["items"].Count(); //this.backgroundGroupsWorker.ReportProgress(0, "Processing next " + count + " posts out of " + totalCount); var gId = (long)(isGroup ? decimal.Negate(groupId) : groupId); var posts = new List<Post>(); // process response body for (var i = 0; i < count; ++i) { var postObj = data[VkRestApi.ResponseBody]["items"][i].ToObject<JObject>(); // see if post is in the range var dt = Utils.getDateField("date", postObj); if(dt < postsFromDate || dt > postsToDate) { continue; } var post = new Post(); post.id = Utils.getLongField("id", postObj); post.owner_id = Utils.getLongField("owner_id", postObj); post.from_id = Utils.getLongField("from_id", postObj); post.signer_id = Utils.getLongField("signer_id", postObj); // post date post.date = Utils.getStringDateField("date", postObj); // post_type post.post_type = Utils.getStringField("post_type", postObj); // comments post.comments = Utils.getLongField("comments", "count", postObj); if (post.comments > 0) { postsWithComments.Add(post.id); // add post's id to the ref list for comments processing } // likes post.likes = Utils.getLongField("likes", "count", postObj); if (post.likes > 0) { var like = new Like(); like.type = "post"; like.owner_id = gId; like.item_id = post.id; likes.Add(like); } // reposts post.reposts = Utils.getLongField("reposts", "count", postObj); // attachments count if(postObj["attachments"] != null) { post.attachments = postObj["attachments"].ToArray().Length; } // post text post.text = Utils.getTextField("text", postObj); // update posters if different from the group if (post.from_id != gId) { if (!posters.ContainsKey(post.from_id)) { posters[post.from_id] = new Poster(); } posters[post.from_id].posts += 1; // increment number of posts posters[post.from_id].rec_likes += post.likes; // add to the poster ids posterIds.Add(post.from_id); } // if post has a signer - update posters if (post.signer_id > 0 && post.signer_id != post.from_id) { if (!posters.ContainsKey(post.signer_id)) { posters[post.signer_id] = new Poster(); } posters[post.signer_id].posts += 1; // increment number of posts posters[post.signer_id].rec_likes += post.likes; // add to the poster ids posterIds.Add(post.signer_id); } posts.Add(post); } if (posts.Count > 0) { // save the posts list Utils.PrintFileContent(groupPostsWriter, posts); } }
// Async workers private void GroupsWork(Object sender, DoWorkEventArgs args) { // Do not access the form's BackgroundWorker reference directly. // Instead, use the reference provided by the sender parameter. var bw = sender as BackgroundWorker; // Extract the argument. var param = args.Argument as GroupPostsParam; if (bw == null || param == null) { throw new ArgumentException("Illegal arguments for Group Work"); } if (param.From <= param.To) { postsFromDate = param.From; postsToDate = param.To; } else { postsFromDate = param.To; postsToDate = param.From; } // working directory var workingDir = WorkingFolderTextBox.Text; // clear all postsWithComments.Clear(); // reset reference list likes.Clear(); // reset likes posters.Clear(); // reset posters posterIds.Clear(); // clear poster ids visitorIds.Clear(); // clear visitor ids postInfo.Clear(); // clear post infos boardPostInfo.Clear(); // reset boards post infos topics.Clear(); // clear topics infos u2uMatrix.Clear(); // clear u2u matrix var context = new VkRestApi.VkRestContext(this.userId, this.authToken); var sb = new StringBuilder(); long timeLastCall = 0; networkEntities = param.GroupWall ? "wall" : ""; networkEntities += param.GroupTopics ? "-topics" : ""; // gather group statistics if (param.JustStats) { ResetCountersAndGetReady(); bw.ReportProgress(1, "Getting group stats"); sb.Length = 0; // TODO check if it takes negative number sb.Append("group_id=").Append(groupId).Append("&"); sb.Append("date_from=").Append(this.postsFromDate.ToString("yyyy-MM-dd")).Append("&"); sb.Append("date_to=").Append(this.postsToDate.ToString("yyyy-MM-dd")); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); // call VK REST API vkRestApi.CallVkFunction(VkFunction.StatsGet, context); return; } IEntity e; string fileName; if (param.GroupWall) { // group posts e = new Post(); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupPostsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupPostsWriter, e); e = new PostCopyHistory(0, null); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupPostsCopyHistoryWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupPostsCopyHistoryWriter, e); ResetCountersAndGetReady(); // request group posts bw.ReportProgress(-1, "Getting posts"); while (ShellContinue(bw)) { sb.Length = 0; sb.Append("owner_id=").Append(groupId.ToString()).Append("&"); sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); context.Cookie = currentOffset.ToString(); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.WallGet, context); // wait for the user data ReadyEvent.WaitOne(); bw.ReportProgress(step, "Getting " + currentOffset + " posts out of " + totalCount); } groupPostsWriter.Close(); groupPostsCopyHistoryWriter.Close(); if (postsWithComments.Count > 0 && !bw.CancellationPending) { // group comments e = new Comment(); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupCommentsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupCommentsWriter, e); // request group comments bw.ReportProgress(-1, "Getting comments"); timeLastCall = 0; for (int i = 0; i < postsWithComments.Count && !bw.CancellationPending; i++) { ResetCountersAndGetReady(); step = CalcStep(postsWithComments.Count); bw.ReportProgress(step, "Getting " + (i + 1) + " comments out of " + postsWithComments.Count); while (ShellContinue(bw)) { sb.Length = 0; sb.Append("owner_id=").Append(groupId).Append("&"); // group id sb.Append("post_id=").Append(postsWithComments[i]).Append("&"); // post id sb.Append("need_likes=").Append(1).Append("&"); // request likes info sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); context.Cookie = postsWithComments[i].ToString(); // pass post id as a cookie Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.WallGetComments, context); // wait for the user data ReadyEvent.WaitOne(); } } groupCommentsWriter.Close(); } } if (param.GroupTopics) { // process group board topics and comments e = new BoardTopic(); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupBoardTopicsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupBoardTopicsWriter, e); e = new Comment(); fileName = Utils.GenerateFileName(workingDir, groupId, e, "board"); groupBoardCommentsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupBoardCommentsWriter, e); ResetCountersAndGetReady(); bw.ReportProgress(-1, "Getting board topics"); // get group board topics while (ShellContinue(bw)) { sb.Length = 0; sb.Append("group_id=").Append(Math.Abs(groupId)).Append("&"); sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); context.Cookie = currentOffset.ToString(); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.BoardGetTopics, context); // wait for the user data ReadyEvent.WaitOne(); bw.ReportProgress(step, "Getting board topics " + currentOffset + " out of " + totalCount); } bw.ReportProgress(-1, "Getting board comments"); // collect comments from all board topics for (var i = 0; i < topics.Count; i++) { if (bw.CancellationPending) break; // canceled if (topics[i].is_closed || topics[i].comments == 0) continue; // empty or closed topic - ignore ResetCountersAndGetReady(); step = CalcStep(topics.Count); bw.ReportProgress(step, "Getting comments for board " + i + " / " + topics.Count); while (ShellContinue(bw)) { sb.Length = 0; sb.Append("group_id=").Append(Math.Abs(groupId)).Append("&"); // group id sb.Append("topic_id=").Append(topics[i].id).Append("&"); // post id sb.Append("need_likes=").Append(1).Append("&"); // need likes count sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); context.Cookie = topics[i].id.ToString(); // pass topic id as a cookie Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.BoardGetComments, context); // wait for the user data ReadyEvent.WaitOne(); bw.ReportProgress(0, "Getting comments " + currentOffset + " / " + totalCount + " for board " + i + " / " + topics.Count); } } groupBoardTopicsWriter.Close(); groupBoardCommentsWriter.Close(); } // process the likes from wall and board topics if (likes.Count > 0 && !bw.CancellationPending) { // request likers ids ResetCountersAndGetReady(); bw.ReportProgress(-1, "Getting likers"); timeLastCall = 0; step = CalcStep(likes.Count); for (var i = 0; i < likes.Count; i++) { isRunning = true; bw.ReportProgress(step, "Getting likes " + (i + 1) + " / " + likes.Count); if (bw.CancellationPending) break; sb.Length = 0; sb.Append("type=").Append(likes[i].type).Append("&"); // type sb.Append("owner_id=").Append(likes[i].owner_id).Append("&"); // group id sb.Append("item_id=").Append(likes[i].item_id).Append("&"); // post id sb.Append("count=").Append(LIKES_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); context.Cookie = likes[i].item_id.ToString(); // pass post/comment id as a cookie Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.LikesGetList, context); // wait for the user data ReadyEvent.WaitOne(); } } // now collect info about posters (users or visitors who left post, comment or like) visitorIds.AddRange(posterIds); if (visitorIds.Count > 0 && !bw.CancellationPending) { // group visitors profiles e = new Profile(); fileName = Utils.GenerateFileName(workingDir, groupId, e, networkEntities); groupVisitorsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupVisitorsWriter, e); // request visitors info bw.ReportProgress(-1, "Getting visitors"); step = CalcStep(visitorIds.Count, 100); timeLastCall = 0; for (var i = 0; i < visitorIds.Count; i += 100) { isRunning = true; bw.ReportProgress(step, "Getting " + (i + 1) + " visitors out of " + visitorIds.Count); if (bw.CancellationPending) break; sb.Length = 0; sb.Append("user_ids="); for (var j = i; j < visitorIds.Count && j < i + 100; ++j) { sb.Append(visitorIds[j]).Append(","); // users } sb.Append("&").Append("fields=").Append(PROFILE_FIELDS); context.Parameters = sb.ToString(); Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.UsersGet, context); // wait for the user data ReadyEvent.WaitOne(); } groupVisitorsWriter.Close(); } // update group posts/likes count (posted from group id) Poster groupPoster; if (posters.TryGetValue((long)this.groupId, out groupPoster)) { var attr = dictionaryFromPoster(groupPoster); // update poster vertex attributes contentNetworkAnalyzer.UpdateVertexAttributes((long)groupId, attr); } // If the operation was canceled by the user, // set the DoWorkEventArgs.Cancel property to true. args.Cancel = bw.CancellationPending; // complete the job //args.Result = }
private void ProcessCopyHistory(long id, IEnumerable<JToken> arr) { var postCopyHistory = new List<PostCopyHistory>(); foreach (var token in arr) { var postObj = token.ToObject<JObject>(); var post = new Post(); post.id = Utils.getLongField("id", postObj); post.owner_id = Utils.getLongField("owner_id", postObj); post.from_id = Utils.getLongField("from_id", postObj); post.signer_id = Utils.getLongField("signer_id", postObj); post.date = Utils.getStringDateField("date", postObj); post.post_type = Utils.getStringField("post_type", postObj); post.comments = Utils.getLongField("comments", "count", postObj); post.likes = Utils.getLongField("likes", "count", postObj); post.reposts = Utils.getLongField("reposts", "count", postObj); // attachments count if (postObj["attachments"] != null) { post.attachments = postObj["attachments"].ToArray().Length; } // post text post.text = Utils.getTextField("text", postObj); postCopyHistory.Add(new PostCopyHistory(id, post)); } Utils.PrintFileContent(groupPostsCopyHistoryWriter, postCopyHistory); }
// process group posts private void OnWallGet(JObject data) { if (data[VkRestApi.ResponseBody] == null) { isRunning = false; return; } if (totalCount == 0) { totalCount = data[VkRestApi.ResponseBody]["count"].ToObject<int>(); if (totalCount == 0) { isRunning = false; return; } step = CalcStep(totalCount, POSTS_PER_REQUEST ); } // calculate items in response int count = data[VkRestApi.ResponseBody]["items"].Count(); if (!CheckAndIncrement(count)) return; var posts = new List<Post>(); // process response body for (int i = 0; i < count; ++i) { var postObj = data[VkRestApi.ResponseBody]["items"][i].ToObject<JObject>(); // see if post is in the range var dt = Utils.getDateField("date", postObj); if (dt < this.postsFromDate || dt > this.postsToDate) { continue; } var post = new Post(); post.id = Utils.getLongField("id", postObj); post.owner_id = Utils.getLongField("owner_id", postObj); post.from_id = Utils.getLongField("from_id", postObj); post.signer_id = Utils.getLongField("signer_id", postObj); // post date post.date = Utils.getStringDateField("date", postObj); // post_type post.post_type = Utils.getStringField("post_type", postObj); // comments post.comments = Utils.getLongField("comments", "count", postObj); if (post.comments > 0) { this.postsWithComments.Add(post.id); // add post's id to the ref list for comments processing } // likes post.likes = Utils.getLongField("likes", "count", postObj); if (post.likes > 0) { var like = new Like(); like.type = "post"; like.owner_id = (long)this.groupId; like.item_id = post.id; this.likes.Add(like); } // reposts post.reposts = Utils.getLongField("reposts", "count", postObj); // attachments count if (postObj["attachments"] != null) { post.attachments = postObj["attachments"].ToArray().Length; } // post text post.text = Utils.getTextField("text", postObj); // post may have a copy_history field, which is repost from other community - save it in a file var copyHistory = Utils.GetArray("copy_history", postObj); if (copyHistory != null) { ProcessCopyHistory(post.id, copyHistory); } // if post has a signer - update posters with a signer if (post.signer_id > 0) { if (!posters.ContainsKey(post.signer_id)) { posters[post.signer_id] = new Poster(); } posters[post.signer_id].Posts += 1; // increment number of posts posters[post.signer_id].RecLikes += post.likes; // add to the poster ids posterIds.Add(post.signer_id); } else { if (!posters.ContainsKey(post.from_id)) { posters[post.from_id] = new Poster(); } posters[post.from_id].Posts += 1; // increment number of posts posters[post.from_id].RecLikes += post.likes; // add to the poster ids if different from the group if(post.from_id != this.groupId) posterIds.Add(post.from_id); } posts.Add(post); // add to the post infos if (!postInfo.ContainsKey(post.id)) { var uid = post.signer_id > 0 ? post.signer_id : post.from_id; postInfo[post.id] = new PostInfo(post.id, uid); } } // save the posts list Utils.PrintFileContent(groupPostsWriter, posts); }
public PostCopyHistory(long postId, Post post) { Postid = postId; Post = post; }