// Async workers private void GroupsWork(Object sender, DoWorkEventArgs args) { // Do not access the form's BackgroundWorker reference directly. // Instead, use the reference provided by the sender parameter. var bw = sender as BackgroundWorker; // Extract the argument. var param = args.Argument as GroupPostsParam; if (bw == null || param == null) { throw new ArgumentException("Illegal arguments for Group Work"); } if (param.From <= param.To) { postsFromDate = param.From; postsToDate = param.To; } else { postsFromDate = param.To; postsToDate = param.From; } // working directory var workingDir = WorkingFolderTextBox.Text; // clear all postsWithComments.Clear(); // reset reference list likes.Clear(); // reset likes posters.Clear(); // reset posters posterIds.Clear(); // clear poster ids visitorIds.Clear(); // clear visitor ids postInfo.Clear(); // clear post infos boardPostInfo.Clear(); // reset boards post infos topics.Clear(); // clear topics infos u2uMatrix.Clear(); // clear u2u matrix var context = new VkRestApi.VkRestContext(this.userId, this.authToken); var sb = new StringBuilder(); long timeLastCall = 0; networkEntities = param.GroupWall ? "wall" : ""; networkEntities += param.GroupTopics ? "-topics" : ""; // gather group statistics if (param.JustStats) { ResetCountersAndGetReady(); bw.ReportProgress(1, "Getting group stats"); sb.Length = 0; // TODO check if it takes negative number sb.Append("group_id=").Append(groupId).Append("&"); sb.Append("date_from=").Append(this.postsFromDate.ToString("yyyy-MM-dd")).Append("&"); sb.Append("date_to=").Append(this.postsToDate.ToString("yyyy-MM-dd")); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); // call VK REST API vkRestApi.CallVkFunction(VkFunction.StatsGet, context); return; } IEntity e; string fileName; if (param.GroupWall) { // group posts e = new Post(); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupPostsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupPostsWriter, e); e = new PostCopyHistory(0, null); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupPostsCopyHistoryWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupPostsCopyHistoryWriter, e); ResetCountersAndGetReady(); // request group posts bw.ReportProgress(-1, "Getting posts"); while (ShellContinue(bw)) { sb.Length = 0; sb.Append("owner_id=").Append(groupId.ToString()).Append("&"); sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); context.Cookie = currentOffset.ToString(); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.WallGet, context); // wait for the user data ReadyEvent.WaitOne(); bw.ReportProgress(step, "Getting " + currentOffset + " posts out of " + totalCount); } groupPostsWriter.Close(); groupPostsCopyHistoryWriter.Close(); if (postsWithComments.Count > 0 && !bw.CancellationPending) { // group comments e = new Comment(); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupCommentsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupCommentsWriter, e); // request group comments bw.ReportProgress(-1, "Getting comments"); timeLastCall = 0; for (int i = 0; i < postsWithComments.Count && !bw.CancellationPending; i++) { ResetCountersAndGetReady(); step = CalcStep(postsWithComments.Count); bw.ReportProgress(step, "Getting " + (i + 1) + " comments out of " + postsWithComments.Count); while (ShellContinue(bw)) { sb.Length = 0; sb.Append("owner_id=").Append(groupId).Append("&"); // group id sb.Append("post_id=").Append(postsWithComments[i]).Append("&"); // post id sb.Append("need_likes=").Append(1).Append("&"); // request likes info sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); context.Cookie = postsWithComments[i].ToString(); // pass post id as a cookie Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.WallGetComments, context); // wait for the user data ReadyEvent.WaitOne(); } } groupCommentsWriter.Close(); } } if (param.GroupTopics) { // process group board topics and comments e = new BoardTopic(); fileName = Utils.GenerateFileName(workingDir, groupId, e); groupBoardTopicsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupBoardTopicsWriter, e); e = new Comment(); fileName = Utils.GenerateFileName(workingDir, groupId, e, "board"); groupBoardCommentsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupBoardCommentsWriter, e); ResetCountersAndGetReady(); bw.ReportProgress(-1, "Getting board topics"); // get group board topics while (ShellContinue(bw)) { sb.Length = 0; sb.Append("group_id=").Append(Math.Abs(groupId)).Append("&"); sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); Debug.WriteLine("Download parameters: " + context.Parameters); context.Cookie = currentOffset.ToString(); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.BoardGetTopics, context); // wait for the user data ReadyEvent.WaitOne(); bw.ReportProgress(step, "Getting board topics " + currentOffset + " out of " + totalCount); } bw.ReportProgress(-1, "Getting board comments"); // collect comments from all board topics for (var i = 0; i < topics.Count; i++) { if (bw.CancellationPending) break; // canceled if (topics[i].is_closed || topics[i].comments == 0) continue; // empty or closed topic - ignore ResetCountersAndGetReady(); step = CalcStep(topics.Count); bw.ReportProgress(step, "Getting comments for board " + i + " / " + topics.Count); while (ShellContinue(bw)) { sb.Length = 0; sb.Append("group_id=").Append(Math.Abs(groupId)).Append("&"); // group id sb.Append("topic_id=").Append(topics[i].id).Append("&"); // post id sb.Append("need_likes=").Append(1).Append("&"); // need likes count sb.Append("offset=").Append(currentOffset).Append("&"); sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); context.Cookie = topics[i].id.ToString(); // pass topic id as a cookie Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.BoardGetComments, context); // wait for the user data ReadyEvent.WaitOne(); bw.ReportProgress(0, "Getting comments " + currentOffset + " / " + totalCount + " for board " + i + " / " + topics.Count); } } groupBoardTopicsWriter.Close(); groupBoardCommentsWriter.Close(); } // process the likes from wall and board topics if (likes.Count > 0 && !bw.CancellationPending) { // request likers ids ResetCountersAndGetReady(); bw.ReportProgress(-1, "Getting likers"); timeLastCall = 0; step = CalcStep(likes.Count); for (var i = 0; i < likes.Count; i++) { isRunning = true; bw.ReportProgress(step, "Getting likes " + (i + 1) + " / " + likes.Count); if (bw.CancellationPending) break; sb.Length = 0; sb.Append("type=").Append(likes[i].type).Append("&"); // type sb.Append("owner_id=").Append(likes[i].owner_id).Append("&"); // group id sb.Append("item_id=").Append(likes[i].item_id).Append("&"); // post id sb.Append("count=").Append(LIKES_PER_REQUEST).Append("&"); context.Parameters = sb.ToString(); context.Cookie = likes[i].item_id.ToString(); // pass post/comment id as a cookie Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.LikesGetList, context); // wait for the user data ReadyEvent.WaitOne(); } } // now collect info about posters (users or visitors who left post, comment or like) visitorIds.AddRange(posterIds); if (visitorIds.Count > 0 && !bw.CancellationPending) { // group visitors profiles e = new Profile(); fileName = Utils.GenerateFileName(workingDir, groupId, e, networkEntities); groupVisitorsWriter = File.CreateText(fileName); Utils.PrintFileHeader(groupVisitorsWriter, e); // request visitors info bw.ReportProgress(-1, "Getting visitors"); step = CalcStep(visitorIds.Count, 100); timeLastCall = 0; for (var i = 0; i < visitorIds.Count; i += 100) { isRunning = true; bw.ReportProgress(step, "Getting " + (i + 1) + " visitors out of " + visitorIds.Count); if (bw.CancellationPending) break; sb.Length = 0; sb.Append("user_ids="); for (var j = i; j < visitorIds.Count && j < i + 100; ++j) { sb.Append(visitorIds[j]).Append(","); // users } sb.Append("&").Append("fields=").Append(PROFILE_FIELDS); context.Parameters = sb.ToString(); Debug.WriteLine("Request parameters: " + context.Parameters); // play nice, sleep for 1/3 sec to stay within 3 requests/second limits timeLastCall = Utils.sleepTime(timeLastCall); // call VK REST API vkRestApi.CallVkFunction(VkFunction.UsersGet, context); // wait for the user data ReadyEvent.WaitOne(); } groupVisitorsWriter.Close(); } // update group posts/likes count (posted from group id) Poster groupPoster; if (posters.TryGetValue((long)this.groupId, out groupPoster)) { var attr = dictionaryFromPoster(groupPoster); // update poster vertex attributes contentNetworkAnalyzer.UpdateVertexAttributes((long)groupId, attr); } // If the operation was canceled by the user, // set the DoWorkEventArgs.Cancel property to true. args.Cancel = bw.CancellationPending; // complete the job //args.Result = }
// process group board topics private void OnBoardGetTopics(JObject data, string cookie) { if (data[VkRestApi.ResponseBody] == null) { isRunning = false; return; } if (totalCount == 0) { totalCount = data[VkRestApi.ResponseBody]["count"].ToObject<int>(); if (totalCount == 0) { isRunning = false; return; } step = CalcStep(totalCount, POSTS_PER_REQUEST); } // calculate items in response var count = data[VkRestApi.ResponseBody]["items"].Count(); if (!CheckAndIncrement(count)) return; // process a cookie // process response body for (int i = 0; i < count; ++i) { var obj = data[VkRestApi.ResponseBody]["items"][i].ToObject<JObject>(); var topic = new BoardTopic(); topic.id = Utils.getLongField("id", obj); topic.title = Utils.getStringField("title", obj); topic.created = Utils.getStringDateField("created", obj); topic.created_by = Utils.getLongField("created_by", obj); topic.updated = Utils.getStringDateField("update", obj); topic.updated_by = Utils.getLongField("updated_by", obj); topic.is_closed = Utils.getLongField("is_closed", obj) == 1; topic.is_fixed = Utils.getLongField("is_fixed", obj) == 1; topic.comments = Utils.getLongField("comments", obj); // according to the spec - we don't consider board creators/editors as posters topics.Add(topic); } // save the board topics Utils.PrintFileContent(groupBoardTopicsWriter, topics); }