//----------------------------------------------------------------------
// ToDo.LoadFollowerTableFromBlob
//
// Input : blobContainer - blob container which has data file we uploaded
// followerTable - empty follower table
// Return : Number of inserted entries (records) into follower table
// Purpose : Load followerTable with follower entries which are composed from blob object
// Note :
//
// * To read a blob file line by line:
//
// StreamReader sr = new StreamReader(blobContainer.GetBlobReference(BlobAddressUri).OpenRead());
// String line;
// while ((line = sr.ReadLine()) != null)
// { ...
//
// * Format of the line of input data:
//
// user_id \t follower_id
//
// * To insert an entry to follower table:
//
// followerTable.AddFollowerEntry(new FollowerEntry(userID, followerID, partitionKey));
// followerTable.SaveChanges();
//----------------------------------------------------------------------
public static int LoadFollowerTableFromBlob(CloudBlobContainer blobContainer, FollowerEntryDataSource followerTable)
{
int numFollowerEntries = 0;
StreamReader sr = new StreamReader(blobContainer.GetBlockBlobReference(ToDo.blobAddressUri).OpenRead());
String line;
while ((line = sr.ReadLine()) != null)
{
numFollowerEntries++;
String [] split = line.Split(new Char [] {'\t'});
int User = Convert.ToInt32(split[0]);
int Follower = Convert.ToInt32(split[1]);
// Partition Key = (User + Follower) % #(Worker Role Instances)
// Users who are mutually friends to each other are contained in the same partition
string PartitionKey = ((User + Follower) % ToDo.NumWorkerRoleInstances).ToString();
followerTable.AddFollowerEntry(new FollowerEntry(User, Follower, PartitionKey));
followerTable.SaveChages();
}
return numFollowerEntries;
}
protected void LoadFollowerTableFromBlobButton_Click(object sender, EventArgs e)
{
FollowerEntryDataSource followerTable = new FollowerEntryDataSource();
followerTable.DeleteTable();
followerTable.CreateTable();
loadStopwatch.Restart();
int numFollowerEntries = ToDo.LoadFollowerTableFromBlob(blobContainer, followerTable);
loadStopwatch.Stop();
System.Diagnostics.Trace.TraceInformation("Loading completed. (" + loadStopwatch.Elapsed.TotalSeconds + " sec)");
LoadFollowerTableFromBlobLabel.Text = "Loading completed. (" + loadStopwatch.Elapsed.TotalSeconds + " sec)";
LoadFollowerTableFromBlobTTD.InnerText = "" + loadStopwatch.Elapsed.TotalSeconds;
LoadFollowerTableFromBlobRTD.InnerText = "" + numFollowerEntries;
}
//----------------------------------------------------------------------
// ToDo.FindMutualFriendsInEachWorkerRole
//
// Input : instanceIdx - worker role instance index number in string type,
// which begins with 0, i.g., "0", "1", "2", ...
// responseQueue - queue data structure to send a complete message to web role
// Return : mutual friend pairs found within a worker role
// Purpose : Retrieve some of entries from follwer table
// and find mutual friends among them,
// In other words, do a self-join on a part of follower table.
// Note : This method may be called several times in concurrent by web role(ToDo.FindMutualFriends)
// When we put together all of results of this method calls,
// we should obtain exact and complete mutual friend pairs.
//
// * To bring some of entries in follower table:
//
// FollowerEntryDataSource followerTable = new FollowerEntryDataSource();
// List<FollowerEntry> partition = followerTable.getEntryList(PartitionKey);
//----------------------------------------------------------------------
public static List<MutualFriend> FindMutualFriendsInEachWorkerRole(string instanceIdx, CloudQueue responseQueue)
{
List<MutualFriend> mutualFriends = new List<MutualFriend>();
FollowerEntryDataSource followerTable = new FollowerEntryDataSource();
if (isNestedLoopsJoin)
{
// Nested Loops Join
List<FollowerEntry> S = followerTable.getEntryList(instanceIdx);
List<FollowerEntry> R = S;
foreach (FollowerEntry r in R)
{
foreach (FollowerEntry s in S)
{
if (r.User == s.Follower && r.Follower == s.User)
{
mutualFriends.Add(new MutualFriend(s.User, s.Follower));
}
}
}
}
else
{
// Sort Merge Join
FollowerEntry[] S = followerTable.getEntryList(instanceIdx).ToArray();
FollowerEntry[] R = S;
Array.Sort(S,
delegate(FollowerEntry fe1, FollowerEntry fe2)
{
return (fe1.User + fe1.Follower).CompareTo(fe2.User + fe2.Follower);
});
Array.Sort(R,
delegate(FollowerEntry fe1, FollowerEntry fe2)
{
return (fe1.User + fe1.Follower).CompareTo(fe2.User + fe2.Follower);
});
int Tr = 0; // ranges over R
int Ts = 0; // ranges over S
int Gs = 0; // start of current S-partition
while (Tr < R.Length && Gs < S.Length)
{
// continue scan of R
while ((R[Tr].User + R[Tr].Follower) < (S[Gs].User + S[Gs].Follower))
Tr++;
// continue scan of S
while ((R[Tr].User + R[Tr].Follower) > (S[Gs].User + S[Gs].Follower))
Gs++;
if (Tr >= R.Length || Gs >= S.Length)
break;
Ts = Gs; // Needed in case Tri != Gsj
// process current R partition
while ((R[Tr].User + R[Tr].Follower) == (S[Gs].User + S[Gs].Follower))
{
// reset S partition scan
Ts = Gs;
// process current R tuple
while ((S[Ts].User + S[Ts].Follower) == (R[Tr].User + R[Tr].Follower))
{
// output joined tuples
if (S[Ts].Follower == R[Tr].User && S[Ts].User == R[Tr].Follower)
mutualFriends.Add(new MutualFriend(S[Ts].User, S[Ts].Follower));
Ts++; // advance S partition scan
// check if Ts == eof
if (Ts >= S.Length)
break; // then escape this loop (no more tuples to read in S!)
}
Tr++; // advance scan of R
// check if Tr == eof
if (Tr >= R.Length)
break; // then escape this loop (no more tuples to read in R!)
} // done with current R partition
// initialize search for next R partition
Gs = Ts;
}
}
return mutualFriends;
}
