public void RecalculateCentroidsTest2()
{
KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 0, null, 1, 10, DateTime.Now);
KMeansJob_Accessor target = new KMeansJob_Accessor(jobData, "server");
target.InitializeStorage();
byte[] cBytes = new byte[Centroid.Size];
using (BlobStream cStream = target.Centroids.OpenRead())
{
cStream.Read(cBytes, 0, cBytes.Length);
}
Centroid cOriginal = Centroid.FromByteArray(cBytes);
target.totalPointsProcessedDataByCentroid[cOriginal.ID] = new PointsProcessedData();
target.RecalculateCentroids();
byte[] cBytesNew = new byte[Centroid.Size];
using (BlobStream cStreamNew = target.Centroids.OpenRead())
{
cStreamNew.Read(cBytesNew, 0, cBytesNew.Length);
}
Centroid cNew = Centroid.FromByteArray(cBytesNew);
Assert.AreEqual(cOriginal.ID, cNew.ID);
Assert.AreEqual(cNew.X, 0);
Assert.AreEqual(cNew.Y, 0);
}
public void EnqueueMessagePollForMessageTest()
{
string queueName = Guid.NewGuid().ToString();
KMeansJobData message = new KMeansJobData(Guid.NewGuid(), 1, null, 2, 10, DateTime.Now);
bool async = false;
AzureHelper.EnqueueMessage(queueName, message, async);
KMeansJobData foundMessage = null;
AzureHelper.ExponentialBackoff(() =>
AzureHelper.PollForMessage <KMeansJobData>(queueName, msg =>
{
foundMessage = msg;
return(true);
}),
firstDelayMilliseconds: 100,
backoffFactor: 2,
maxDelay: 1000,
retryLimit: 5
);
Assert.AreNotEqual(null, foundMessage);
Assert.AreEqual(message.JobID, foundMessage.JobID);
Assert.AreEqual(message.K, foundMessage.K);
}
/// <summary>
/// Handles a request for a new k-means job. Sets up a new job and starts it off.
/// </summary>
/// <param name="message">The job request. Must be of type KMeansJobData.</param>
private bool ProcessNewJob(KMeansJobData jobData)
{
System.Diagnostics.Trace.TraceInformation("[ServerRole] ProcessNewJob(jobID={0})", jobData.JobID);
jobs[jobData.JobID] = new KMeansJob(jobData, "server");
jobs[jobData.JobID].InitializeStorage();
jobs[jobData.JobID].EnqueueTasks(workers.Values);
return(true);
}
public void ProcessWorkerResponseTest()
{
KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 4, null, 2, 10, DateTime.Now);
KMeansJob_Accessor target = new KMeansJob_Accessor(jobData, "server");
target.InitializeStorage();
// Upload a block with an arbitrary ClusterPoint, so we can verify it gets copied
ClusterPoint arbitraryPoint = new ClusterPoint(1, 2, Guid.NewGuid());
List <string> blockList;
using (ObjectCachedBlockWriter <ClusterPoint> pointPartitionWriteStream = new ObjectCachedBlockWriter <ClusterPoint>(target.Points, point => point.ToByteArray(), ClusterPoint.Size,
Environment.GetEnvironmentVariable("TEMP") + @"\" + Guid.NewGuid().ToString()))
{
pointPartitionWriteStream.Write(arbitraryPoint);
pointPartitionWriteStream.FlushBlock();
blockList = pointPartitionWriteStream.BlockList;
}
KMeansTaskData taskData = new KMeansTaskData(jobData, Guid.NewGuid(), 0, 1, target.Centroids.Uri, DateTime.Now, 0, null);
target.tasks.Clear();
target.tasks.Add(new KMeansTask(taskData));
KMeansTaskResult taskResult = new KMeansTaskResult(taskData);
CloudBlob pointsBlockListBlob = AzureHelper.CreateBlob(jobData.JobID.ToString(), Guid.NewGuid().ToString());
using (Stream stream = pointsBlockListBlob.OpenWrite())
{
BinaryFormatter bf = new BinaryFormatter();
bf.Serialize(stream, blockList);
}
taskResult.PointsBlockListBlob = pointsBlockListBlob.Uri;
taskResult.NumPointsChanged = 2;
Guid centroidID = Guid.NewGuid();
taskResult.PointsProcessedDataByCentroid = new Dictionary <Guid, PointsProcessedData> {
{ centroidID, new PointsProcessedData()
{
NumPointsProcessed = 2,
PartialPointSum = new Point(1, 2)
} }
};
target.ProcessWorkerResponse(taskResult, new List <Worker>());
// Verify that the first ClusterPoint in Points is indeed equal to arbitraryPoint
using (ObjectStreamReader <ClusterPoint> pointsStream = new ObjectStreamReader <ClusterPoint>(target.Points, ClusterPoint.FromByteArray, ClusterPoint.Size))
{
ClusterPoint point = pointsStream.First();
Assert.AreEqual(arbitraryPoint.X, point.X);
Assert.AreEqual(arbitraryPoint.Y, point.Y);
Assert.AreEqual(arbitraryPoint.CentroidID, point.CentroidID);
}
}
public void ToFromBinaryTest()
{
KMeansJobData target = new KMeansJobData(Guid.NewGuid(), 1, null, 2, 10, DateTime.Now);
byte[] bytes = target.ToBinary();
KMeansJobData targetNew = KMeansJobData.FromMessage(new CloudQueueMessage(bytes)) as KMeansJobData;
Assert.AreEqual(target.JobID, targetNew.JobID);
Assert.AreEqual(target.K, targetNew.K);
Assert.AreEqual(target.N, targetNew.N);
}
protected void Run_Click(object sender, EventArgs e)
{
FreezeUI();
ClearIndicators();
Guid jobID = Guid.NewGuid();
Status.Text = string.Format("Running job {0}.", jobID);
DownloadLog.NavigateUrl = string.Format("Log.aspx?JobID={0}", jobID);
DownloadLog.Enabled = true;
Session["jobID"] = jobID;
Session["lastLogRefreshTime"] = DateTime.MinValue;
Session["allLogs"] = new List <PerformanceLog>();
Uri pointsBlobUri = null;
if (PointsFile.HasFile)
{
CloudBlob pointsBlob = AzureHelper.CreateBlob(jobID.ToString(), AzureHelper.PointsBlob);
using (BlobStream stream = pointsBlob.OpenWrite())
{
PointsFile.FileContent.CopyTo(stream);
}
pointsBlobUri = pointsBlob.Uri;
}
else if (!string.IsNullOrEmpty(PointsBlob.Text))
{
CloudBlob pointsBlob = AzureHelper.CreateBlob(jobID.ToString(), AzureHelper.PointsBlob);
pointsBlob.CopyFromBlob(AzureHelper.GetBlob(new Uri(PointsBlob.Text)));
pointsBlobUri = pointsBlob.Uri;
}
int nInt = 0, kInt = 0, maxIterationCountInt = 0;
int.TryParse(N.Text, out nInt);
int.TryParse(K.Text, out kInt);
int.TryParse(MaxIterationCount.Text, out maxIterationCountInt);
KMeansJobData jobData = new KMeansJobData(jobID, nInt, pointsBlobUri, kInt, maxIterationCountInt, DateTime.Now)
{
ProgressEmail = ProgressEmail.Text
};
AzureHelper.EnqueueMessage(AzureHelper.ServerRequestQueue, jobData);
WaitForResults();
}
public void AzureMessageEnqueueDequeueTest()
{
CloudQueue queue = AzureHelper.StorageAccount.CreateCloudQueueClient().GetQueueReference(AzureHelper.ServerRequestQueue);
queue.CreateIfNotExist();
queue.Clear();
AzureMessage message = new KMeansJobData(Guid.NewGuid(), 1, null, 2, 10, DateTime.Now);
queue.AddMessage(new CloudQueueMessage(message.ToBinary()));
Thread.Sleep(2000);
AzureMessage received = KMeansJobData.FromMessage(queue.GetMessage());
KMeansJobData messageCast = message as KMeansJobData,
receivedCast = received as KMeansJobData;
Assert.AreEqual(messageCast.JobID, receivedCast.JobID);
Assert.AreEqual(messageCast.K, receivedCast.K);
Assert.AreEqual(messageCast.N, receivedCast.N);
}
public void MultiIterationJobTest() // TODO: make this unit test check things in more detail
{
KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 4, null, 2, 2, DateTime.Now);
List <Worker> workers = new List <Worker> {
new Worker("a", "", 1),
new Worker("b", "", 1)
};
KMeansJob_Accessor job = new KMeansJob_Accessor(jobData, "server");
// First iteration
job.InitializeStorage();
job.EnqueueTasks(workers);
for (int i = 0; i < jobData.MaxIterationCount; i++)
{
CheckWorkerRequests(job,
(from task in job.tasks
where task.Running
select task.TaskData),
workers.Count, job.Points);
// Create the worker results and send them to the job
List <KMeansTaskResult> results = new List <KMeansTaskResult>();
foreach (var task in job.tasks)
{
var taskResult = new KMeansTaskResult(task.TaskData);
taskResult.NumPointsChanged = 1;
results.Add(taskResult);
}
foreach (var result in results)
{
job.ProcessWorkerResponse(result, workers);
}
}
}
public void InitializeStorageTest()
{
KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 2, null, 4, 10, DateTime.Now);
KMeansJob target = new KMeansJob(jobData, "server");
target.InitializeStorage();
// Verify that the created containers and blobs actually exist
CloudBlobClient client = AzureHelper.StorageAccount.CreateCloudBlobClient();
CloudBlobContainer c = null;
try
{
c = client.GetContainerReference(jobData.JobID.ToString());
c.FetchAttributes();
}
catch (StorageClientException e)
{
if (e.ErrorCode == StorageErrorCode.ResourceNotFound)
{
Assert.Fail();
}
else
{
throw;
}
}
CloudBlob points = null, centroids = null;
try
{
points = c.GetBlobReference("points");
points.FetchAttributes();
centroids = c.GetBlobReference("centroids");
centroids.FetchAttributes();
}
catch (StorageClientException e)
{
if (e.ErrorCode == StorageErrorCode.ResourceNotFound)
{
Assert.Fail();
}
else
{
throw;
}
}
// Verify that unpacking a ClusterPoint actually yields a point with coordinates [-50, 50) and a null centroidID
byte[] pointBytes;
using (BlobStream pointsStream = points.OpenRead())
{
pointBytes = new byte[ClusterPoint.Size];
pointsStream.Read(pointBytes, 0, ClusterPoint.Size);
}
ClusterPoint p = ClusterPoint.FromByteArray(pointBytes);
Assert.IsTrue(p.X >= -50 && p.X < 50);
Assert.IsTrue(p.Y >= -50 && p.Y < 50);
Assert.AreEqual(p.CentroidID, Guid.Empty);
// Verify that the blobs are the correct length
Assert.AreEqual(points.Properties.Length, ClusterPoint.Size * jobData.N);
Assert.AreEqual(centroids.Properties.Length, Centroid.Size * jobData.K);
}
