private void Populate(int index, int totalCount, ref IPeople person, bool isFollower, string personId, int count)
{
var reportText = isFollower ? "followers" : "followings";
var urlpage = isFollower ? "followers" : "following";
var urlBase = "https://www.quirky.com/api/v1/user_profile/{0}/{1}?paginated_options%5Bfollows%5D%5Buse_cursor%5D=true&paginated_options%5Bfollows%5D%5Bper_page%5D=20&paginated_options%5Bfollows%5D%5Border_column%5D=created_at&paginated_options%5Bfollows%5D%5Border%5D=desc";
var baseUrl = string.Format(urlBase, personId, urlpage);
var urlCursorAddition = "&paginated_options%5Bfollows%5D%5Bcursor%5D={0}";
var hasMore = true;
var firstIteration = true;
string cursor = null;
var scrapedCount = 0;
while (hasMore)
{
var url = baseUrl;
if (firstIteration) firstIteration = false; // First iteration has no pagination cursor
else url += string.Format(urlCursorAddition, cursor);
var json = Helper.GetXHRJson(url);
var jsonObj = JsonConvert.DeserializeObject(json) as JObject;
hasMore = jsonObj["paginated_meta"]["follows"].Value<bool>("has_next_page");
var arr = jsonObj["data"].Value<JArray>("follows");
scrapedCount += arr.Count;
if (hasMore)
{
cursor = arr.Last.Value<string>("created_at");
cursor = Helper.EncodeQuirkyDate(cursor);
if (cursor == null) hasMore = false;
}
// Get followers
var users = jsonObj["data"].Value<JArray>("users");
foreach (var user in users)
{
var personName = user.Value<string>("name");
var personUrl = string.Format(PeopleScraper.USER_URL_FORMAT, user.Value<string>("id"));
var fellow = new People
{
Name = personName,
URL = personUrl
};
if (isFollower)
person.AddFollower(fellow);
else
person.AddFollowing(fellow);
}
ReportProgress(index, totalCount,
string.Format("Scraping {0}'s {1}... Scraped: {2}/{3} {1}. Progress: {4}/{5}", person.Name, reportText, scrapedCount, count, index, totalCount));
}
ReportProgress(index + 1, totalCount,
string.Format("Completed scraping {0}'s {1}. Scraped: {2}/{3} {1}. Progress: {4}/{5}", person.Name, reportText, scrapedCount, count, index + 1, totalCount));
}