// Async workers
private void GroupsWork(Object sender, DoWorkEventArgs args)
{
// Do not access the form's BackgroundWorker reference directly.
// Instead, use the reference provided by the sender parameter.
var bw = sender as BackgroundWorker;
// Extract the argument.
var param = args.Argument as GroupPostsParam;
var groupId = param.gid;
if (param.from <= param.to)
{
postsFromDate = param.from;
postsToDate = param.to;
}
else
{
postsFromDate = param.to;
postsToDate = param.from;
}
var context = new VkRestApi.VkRestContext(userId, authToken);
var sb = new StringBuilder();
isRunning = true;
// gather group statistics
if (param.justStats)
{
bw.ReportProgress(1, "Getting group stats");
sb.Length = 0;
sb.Append("group_id=").Append(Math.Abs(groupId).ToString()).Append("&");
sb.Append("date_from=").Append(this.postsFromDate.ToString("yyyy-MM-dd")).Append("&");
sb.Append("date_to=").Append(this.postsToDate.ToString("yyyy-MM-dd"));
context.Parameters = sb.ToString();
Debug.WriteLine("Download parameters: " + context.Parameters);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.StatsGet, context);
return;
}
// create stream writers
// 1) group posts
// group posts
IEntity e = new Post();
String fileName = Utils.GenerateFileName(WorkingFolderTextBox.Text, groupId, e);
groupPostsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupPostsWriter, e);
postsWithComments.Clear(); // reset comments reference list
likes.Clear(); // reset likes
posters.Clear(); // reset posters
posterIds.Clear(); // clear poster ids
visitorIds.Clear(); // clear visitor ids
totalCount = 0;
currentOffset = 0;
step = 1;
long timeLastCall = 0;
// get group posts 100 at a time and store them in the file
// request group posts
while (isRunning)
{
if (bw.CancellationPending)
break;
if(currentOffset > totalCount)
{
// done
break;
}
bw.ReportProgress(step, "Getting " + currentOffset + " posts out of " + totalCount);
sb.Length = 0;
sb.Append("owner_id=").Append(groupId.ToString()).Append("&");
sb.Append("offset=").Append(currentOffset).Append("&");
sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
Debug.WriteLine("Download parameters: " + context.Parameters);
context.Cookie = currentOffset.ToString();
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.WallGet, context);
// wait for the user data
ReadyEvent.WaitOne();
currentOffset += POSTS_PER_REQUEST;
}
groupPostsWriter.Close();
if (postsWithComments.Count > 0)
{
// group comments
e = new Comment();
fileName = Utils.GenerateFileName(WorkingFolderTextBox.Text, groupId, e);
groupCommentsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupCommentsWriter, e);
// request group comments
bw.ReportProgress(-1, "Getting comments");
step = 10000 / postsWithComments.Count;
timeLastCall = 0;
for (var i = 0; i < postsWithComments.Count; i++)
{
isRunning = true;
totalCount = 0;
currentOffset = 0;
bw.ReportProgress(step, "Getting " + (i + 1) + " post comments out of " + postsWithComments.Count);
while (isRunning)
{
if (bw.CancellationPending)
break;
if (currentOffset > totalCount)
{
// done
break;
}
sb.Length = 0;
sb.Append("owner_id=").Append(groupId.ToString()).Append("&"); // group id
sb.Append("post_id=").Append(postsWithComments[i]).Append("&"); // post id
sb.Append("need_likes=").Append(1).Append("&"); // request likes info
sb.Append("offset=").Append(currentOffset).Append("&");
sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
context.Cookie = postsWithComments[i].ToString(); // pass post id as a cookie
Debug.WriteLine("Request parameters: " + context.Parameters);
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.WallGetComments, context);
// wait for the user data
ReadyEvent.WaitOne();
currentOffset += POSTS_PER_REQUEST;
}
}
groupCommentsWriter.Close();
}
// process the likers - they will be added to the posters
if (likes.Count > 0)
{
// request likers ids
bw.ReportProgress(-1, "Getting likers");
step = 10000 / likes.Count;
timeLastCall = 0;
for (int i = 0; i < this.likes.Count; i++)
{
isRunning = true;
//this.totalCount = 0;
//this.currentOffset = 0;
bw.ReportProgress(step, "Getting " + (i + 1) + " likes out of " + this.likes.Count);
if (bw.CancellationPending)
break;
sb.Length = 0;
sb.Append("type=").Append(likes[i].type).Append("&"); // group id
sb.Append("owner_id=").Append(likes[i].owner_id).Append("&"); // group id
sb.Append("item_id=").Append(likes[i].item_id).Append("&"); // post id
sb.Append("count=").Append(LIKES_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
Debug.WriteLine("Request parameters: " + context.Parameters);
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.LikesGetList, context);
// wait for the user data
ReadyEvent.WaitOne();
}
}
// now collect visitors (not members, who left a post or a comment or a like)
foreach (var p in posterIds)
{
if (!memberIds.Contains(p))
{
// this is a visitor poster
visitorIds.Add(p);
}
}
if (visitorIds.Count > 0)
{
// group visitors profiles
e = new Profile();
fileName = Utils.GenerateFileName(this.WorkingFolderTextBox.Text, groupId, e, "visitor");
groupVisitorsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupVisitorsWriter, e);
// request visitors info
bw.ReportProgress(-1, "Getting visitors");
step = 10000 / visitorIds.Count;
timeLastCall = 0;
for (var i = 0; i < visitorIds.Count; i += 100)
{
isRunning = true;
//this.totalCount = 0;
//this.currentOffset = 0;
bw.ReportProgress(step, "Getting " + (i + 1) + " visitors out of " + visitorIds.Count);
if (bw.CancellationPending)
break;
sb.Length = 0;
sb.Append("user_ids=");
for (var j = i; j < visitorIds.Count && j < i + 100; ++j)
{
sb.Append(visitorIds[j]).Append(","); // users
}
sb.Append("&").Append("fields=").Append(PROFILE_FIELDS);
context.Parameters = sb.ToString();
Debug.WriteLine("Request parameters: " + context.Parameters);
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.UsersGet, context);
// wait for the user data
ReadyEvent.WaitOne();
}
groupVisitorsWriter.Close();
}
//args.Result = TimeConsumingOperation(bw, arg);
// If the operation was canceled by the user,
// set the DoWorkEventArgs.Cancel property to true.
if (bw.CancellationPending)
{
args.Cancel = true;
}
}
// Async workers
private void GroupsWork(Object sender, DoWorkEventArgs args)
{
// Do not access the form's BackgroundWorker reference directly.
// Instead, use the reference provided by the sender parameter.
var bw = sender as BackgroundWorker;
// Extract the argument.
var param = args.Argument as GroupPostsParam;
if (bw == null ||
param == null)
{
throw new ArgumentException("Illegal arguments for Group Work");
}
if (param.From <= param.To)
{
postsFromDate = param.From;
postsToDate = param.To;
}
else
{
postsFromDate = param.To;
postsToDate = param.From;
}
// working directory
var workingDir = WorkingFolderTextBox.Text;
// clear all
postsWithComments.Clear(); // reset reference list
likes.Clear(); // reset likes
posters.Clear(); // reset posters
posterIds.Clear(); // clear poster ids
visitorIds.Clear(); // clear visitor ids
postInfo.Clear(); // clear post infos
boardPostInfo.Clear(); // reset boards post infos
topics.Clear(); // clear topics infos
u2uMatrix.Clear(); // clear u2u matrix
var context = new VkRestApi.VkRestContext(this.userId, this.authToken);
var sb = new StringBuilder();
long timeLastCall = 0;
networkEntities = param.GroupWall ? "wall" : "";
networkEntities += param.GroupTopics ? "-topics" : "";
// gather group statistics
if (param.JustStats)
{
ResetCountersAndGetReady();
bw.ReportProgress(1, "Getting group stats");
sb.Length = 0;
// TODO check if it takes negative number
sb.Append("group_id=").Append(groupId).Append("&");
sb.Append("date_from=").Append(this.postsFromDate.ToString("yyyy-MM-dd")).Append("&");
sb.Append("date_to=").Append(this.postsToDate.ToString("yyyy-MM-dd"));
context.Parameters = sb.ToString();
Debug.WriteLine("Download parameters: " + context.Parameters);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.StatsGet, context);
return;
}
IEntity e;
string fileName;
if (param.GroupWall)
{
// group posts
e = new Post();
fileName = Utils.GenerateFileName(workingDir, groupId, e);
groupPostsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupPostsWriter, e);
e = new PostCopyHistory(0, null);
fileName = Utils.GenerateFileName(workingDir, groupId, e);
groupPostsCopyHistoryWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupPostsCopyHistoryWriter, e);
ResetCountersAndGetReady();
// request group posts
bw.ReportProgress(-1, "Getting posts");
while (ShellContinue(bw))
{
sb.Length = 0;
sb.Append("owner_id=").Append(groupId.ToString()).Append("&");
sb.Append("offset=").Append(currentOffset).Append("&");
sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
Debug.WriteLine("Download parameters: " + context.Parameters);
context.Cookie = currentOffset.ToString();
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.WallGet, context);
// wait for the user data
ReadyEvent.WaitOne();
bw.ReportProgress(step, "Getting " + currentOffset + " posts out of " + totalCount);
}
groupPostsWriter.Close();
groupPostsCopyHistoryWriter.Close();
if (postsWithComments.Count > 0 &&
!bw.CancellationPending)
{
// group comments
e = new Comment();
fileName = Utils.GenerateFileName(workingDir, groupId, e);
groupCommentsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupCommentsWriter, e);
// request group comments
bw.ReportProgress(-1, "Getting comments");
timeLastCall = 0;
for (int i = 0; i < postsWithComments.Count && !bw.CancellationPending; i++)
{
ResetCountersAndGetReady();
step = CalcStep(postsWithComments.Count);
bw.ReportProgress(step,
"Getting " + (i + 1) + " comments out of " + postsWithComments.Count);
while (ShellContinue(bw))
{
sb.Length = 0;
sb.Append("owner_id=").Append(groupId).Append("&"); // group id
sb.Append("post_id=").Append(postsWithComments[i]).Append("&"); // post id
sb.Append("need_likes=").Append(1).Append("&"); // request likes info
sb.Append("offset=").Append(currentOffset).Append("&");
sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
context.Cookie = postsWithComments[i].ToString(); // pass post id as a cookie
Debug.WriteLine("Request parameters: " + context.Parameters);
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.WallGetComments, context);
// wait for the user data
ReadyEvent.WaitOne();
}
}
groupCommentsWriter.Close();
}
}
if (param.GroupTopics)
{
// process group board topics and comments
e = new BoardTopic();
fileName = Utils.GenerateFileName(workingDir, groupId, e);
groupBoardTopicsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupBoardTopicsWriter, e);
e = new Comment();
fileName = Utils.GenerateFileName(workingDir, groupId, e, "board");
groupBoardCommentsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupBoardCommentsWriter, e);
ResetCountersAndGetReady();
bw.ReportProgress(-1, "Getting board topics");
// get group board topics
while (ShellContinue(bw))
{
sb.Length = 0;
sb.Append("group_id=").Append(Math.Abs(groupId)).Append("&");
sb.Append("offset=").Append(currentOffset).Append("&");
sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
Debug.WriteLine("Download parameters: " + context.Parameters);
context.Cookie = currentOffset.ToString();
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.BoardGetTopics, context);
// wait for the user data
ReadyEvent.WaitOne();
bw.ReportProgress(step, "Getting board topics " + currentOffset + " out of " + totalCount);
}
bw.ReportProgress(-1, "Getting board comments");
// collect comments from all board topics
for (var i = 0; i < topics.Count; i++)
{
if (bw.CancellationPending)
break; // canceled
if (topics[i].is_closed || topics[i].comments == 0)
continue; // empty or closed topic - ignore
ResetCountersAndGetReady();
step = CalcStep(topics.Count);
bw.ReportProgress(step, "Getting comments for board " + i + " / " + topics.Count);
while (ShellContinue(bw))
{
sb.Length = 0;
sb.Append("group_id=").Append(Math.Abs(groupId)).Append("&"); // group id
sb.Append("topic_id=").Append(topics[i].id).Append("&"); // post id
sb.Append("need_likes=").Append(1).Append("&"); // need likes count
sb.Append("offset=").Append(currentOffset).Append("&");
sb.Append("count=").Append(POSTS_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
context.Cookie = topics[i].id.ToString(); // pass topic id as a cookie
Debug.WriteLine("Request parameters: " + context.Parameters);
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.BoardGetComments, context);
// wait for the user data
ReadyEvent.WaitOne();
bw.ReportProgress(0,
"Getting comments " + currentOffset + " / " + totalCount + " for board " + i + " / " +
topics.Count);
}
}
groupBoardTopicsWriter.Close();
groupBoardCommentsWriter.Close();
}
// process the likes from wall and board topics
if (likes.Count > 0 &&
!bw.CancellationPending)
{
// request likers ids
ResetCountersAndGetReady();
bw.ReportProgress(-1, "Getting likers");
timeLastCall = 0;
step = CalcStep(likes.Count);
for (var i = 0; i < likes.Count; i++)
{
isRunning = true;
bw.ReportProgress(step, "Getting likes " + (i + 1) + " / " + likes.Count);
if (bw.CancellationPending)
break;
sb.Length = 0;
sb.Append("type=").Append(likes[i].type).Append("&"); // type
sb.Append("owner_id=").Append(likes[i].owner_id).Append("&"); // group id
sb.Append("item_id=").Append(likes[i].item_id).Append("&"); // post id
sb.Append("count=").Append(LIKES_PER_REQUEST).Append("&");
context.Parameters = sb.ToString();
context.Cookie = likes[i].item_id.ToString(); // pass post/comment id as a cookie
Debug.WriteLine("Request parameters: " + context.Parameters);
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.LikesGetList, context);
// wait for the user data
ReadyEvent.WaitOne();
}
}
// now collect info about posters (users or visitors who left post, comment or like)
visitorIds.AddRange(posterIds);
if (visitorIds.Count > 0 &&
!bw.CancellationPending)
{
// group visitors profiles
e = new Profile();
fileName = Utils.GenerateFileName(workingDir, groupId, e, networkEntities);
groupVisitorsWriter = File.CreateText(fileName);
Utils.PrintFileHeader(groupVisitorsWriter, e);
// request visitors info
bw.ReportProgress(-1, "Getting visitors");
step = CalcStep(visitorIds.Count, 100);
timeLastCall = 0;
for (var i = 0; i < visitorIds.Count; i += 100)
{
isRunning = true;
bw.ReportProgress(step, "Getting " + (i + 1) + " visitors out of " + visitorIds.Count);
if (bw.CancellationPending)
break;
sb.Length = 0;
sb.Append("user_ids=");
for (var j = i; j < visitorIds.Count && j < i + 100; ++j)
{
sb.Append(visitorIds[j]).Append(","); // users
}
sb.Append("&").Append("fields=").Append(PROFILE_FIELDS);
context.Parameters = sb.ToString();
Debug.WriteLine("Request parameters: " + context.Parameters);
// play nice, sleep for 1/3 sec to stay within 3 requests/second limits
timeLastCall = Utils.sleepTime(timeLastCall);
// call VK REST API
vkRestApi.CallVkFunction(VkFunction.UsersGet, context);
// wait for the user data
ReadyEvent.WaitOne();
}
groupVisitorsWriter.Close();
}
// update group posts/likes count (posted from group id)
Poster groupPoster;
if (posters.TryGetValue((long)this.groupId, out groupPoster))
{
var attr = dictionaryFromPoster(groupPoster);
// update poster vertex attributes
contentNetworkAnalyzer.UpdateVertexAttributes((long)groupId, attr);
}
// If the operation was canceled by the user,
// set the DoWorkEventArgs.Cancel property to true.
args.Cancel = bw.CancellationPending;
// complete the job
//args.Result =
}
// process group comments
private void OnWallGetComments(JObject data, String cookie)
{
if (data[VkRestApi.ResponseBody] == null)
{
isRunning = false;
return;
}
if (totalCount == 0)
{
totalCount = data[VkRestApi.ResponseBody]["count"].ToObject<long>();
}
// now calculate items in response
var count = data[VkRestApi.ResponseBody]["items"].Count();
var gId = (long)(isGroup ? decimal.Negate(groupId) : groupId);
var comments = new List<Comment>();
// process response body
for (var i = 0; i < count; ++i)
{
var postObj = data[VkRestApi.ResponseBody]["items"][i].ToObject<JObject>();
var comment = new Comment();
comment.id = Utils.getLongField("id", postObj);
comment.post_id = Convert.ToInt64(cookie); // passed as a cookie
comment.from_id = Utils.getLongField("from_id", postObj);
// post date
comment.date = Utils.getStringDateField("date", postObj);
comment.reply_to_uid = Utils.getLongField("reply_to_uid", postObj);
comment.reply_to_cid = Utils.getLongField("reply_to_cid", postObj);
// likes/dislikes
comment.likes = Utils.getLongField("likes", "count", postObj);
if (comment.likes > 0)
{
var like = new Like();
like.type = "comment";
like.owner_id = gId;
like.item_id = comment.id;
likes.Add(like);
}
// attachments count
if (postObj["attachments"] != null)
{
comment.attachments = postObj["attachments"].ToArray().Length;
}
// post text
comment.text = Utils.getTextField("text", postObj);
// update posters
if (comment.from_id != gId)
{
if (!posters.ContainsKey(comment.from_id))
{
posters[comment.from_id] = new Poster();
}
posters[comment.from_id].comments += 1; // increment number of comments
posters[comment.from_id].rec_likes += comment.likes;
// add to the poster ids
posterIds.Add(comment.from_id);
}
comments.Add(comment);
}
if (comments.Count > 0)
{
// save the posts list
Utils.PrintFileContent(groupCommentsWriter, comments);
}
}
// process group board topic comments
private void OnBoardGetComments(JObject data, string cookie)
{
if (data[VkRestApi.ResponseBody] == null)
{
isRunning = false;
return;
}
if (totalCount == 0)
{
totalCount = data[VkRestApi.ResponseBody]["count"].ToObject<int>();
if (totalCount == 0)
{
isRunning = false;
return;
}
step = CalcStep(totalCount, POSTS_PER_REQUEST);
}
// calculate items in response
var count = data[VkRestApi.ResponseBody]["items"].Count();
if (!CheckAndIncrement(count))
return;
var comments = new List<Comment>();
var board_id = Convert.ToInt64(cookie); // passed as a cookie
// process response body
for (int i = 0; i < count; ++i)
{
var postObj = data[VkRestApi.ResponseBody]["items"][i].ToObject<JObject>();
var comment = new Comment();
comment.id = Utils.getLongField("id", postObj);
comment.post_id = board_id;
comment.from_id = Utils.getLongField("from_id", postObj);
// post date
comment.date = Utils.getStringDateField("date", postObj);
comment.reply_to_uid = Utils.getLongField("reply_to_uid", postObj);
comment.reply_to_cid = Utils.getLongField("reply_to_cid", postObj);
comment.likes = Utils.getLongField("likes", "count", postObj);
// likes
if (comment.likes > 0)
{
var like = new Like();
like.type = "topic_comment";
like.owner_id = (long)this.groupId;
like.item_id = comment.id;
likes.Add(like);
}
// attachments count
if (postObj["attachments"] != null)
{
comment.attachments = postObj["attachments"].ToArray().Length;
}
// post text
comment.text = Utils.getTextField("text", postObj);
// update posters
if (!posters.ContainsKey(comment.from_id))
{
posters[comment.from_id] = new Poster();
}
posters[comment.from_id].BoardComments += 1; // increment number of board comments
posters[comment.from_id].RecLikes += comment.likes;
// add to the poster ids
if (comment.from_id != this.groupId)
posterIds.Add(comment.from_id);
comments.Add(comment);
// add to board post info
if (!boardPostInfo.ContainsKey(comment.id))
{
boardPostInfo[comment.id] = new PostInfo(comment.id, comment.from_id);
}
// update u2u matrix for replies
var to = ParseCommentForReplyTo(comment.text);
if (to > 0)
{
UpdateU2UMatrix(comment.from_id, to, true); // reply is comment
posters[comment.from_id].Comments += 1;
// there must by a poster down there - update posters
if (!posters.ContainsKey(to))
{
posters[to] = new Poster();
}
posters[to].RecComments += 1; // increment number of received comments
}
}
// save the board comments
Utils.PrintFileContent(groupBoardCommentsWriter, comments);
}