protected void Run_Click(object sender, EventArgs e)
{
FreezeUI();
ClearIndicators();
Guid jobID = Guid.NewGuid();
Status.Text = string.Format("Running job {0}.", jobID);
DownloadLog.NavigateUrl = string.Format("Log.aspx?JobID={0}", jobID);
DownloadLog.Enabled = true;
Session["jobID"] = jobID;
Session["lastLogRefreshTime"] = DateTime.MinValue;
Session["allLogs"] = new List <PerformanceLog>();
Uri pointsBlobUri = null;
if (PointsFile.HasFile)
{
CloudBlob pointsBlob = AzureHelper.CreateBlob(jobID.ToString(), AzureHelper.PointsBlob);
using (BlobStream stream = pointsBlob.OpenWrite())
{
PointsFile.FileContent.CopyTo(stream);
}
pointsBlobUri = pointsBlob.Uri;
}
else if (!string.IsNullOrEmpty(PointsBlob.Text))
{
CloudBlob pointsBlob = AzureHelper.CreateBlob(jobID.ToString(), AzureHelper.PointsBlob);
pointsBlob.CopyFromBlob(AzureHelper.GetBlob(new Uri(PointsBlob.Text)));
pointsBlobUri = pointsBlob.Uri;
}
int nInt = 0, kInt = 0, maxIterationCountInt = 0;
int.TryParse(N.Text, out nInt);
int.TryParse(K.Text, out kInt);
int.TryParse(MaxIterationCount.Text, out maxIterationCountInt);
KMeansJobData jobData = new KMeansJobData(jobID, nInt, pointsBlobUri, kInt, maxIterationCountInt, DateTime.Now)
{
ProgressEmail = ProgressEmail.Text
};
AzureHelper.EnqueueMessage(AzureHelper.ServerRequestQueue, jobData);
WaitForResults();
}
public void GetBlobTest()
{
string containerName = "foo";
string blobName = "bar";
CloudBlobContainer container = AzureHelper.StorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
container.CreateIfNotExist();
CloudBlob blob = container.GetBlobReference(blobName);
using (BlobStream stream = blob.OpenWrite())
{
byte[] bytes = new System.Text.UTF8Encoding().GetBytes("hello world");
stream.Write(bytes, 0, bytes.Length);
}
Assert.AreEqual(blob.Properties.Length, AzureHelper.GetBlob(blob.Uri).Properties.Length);
}
private void UpdateUIFromLogs(bool final, IEnumerable <PerformanceLog> logs)
{
// Show all logs
Stats.Text = string.Empty;
foreach (PerformanceLog log in logs)
{
Stats.Text += string.Format("<tr><td>{0}</td><td>{1}</td><td>{2}</td></tr>",
log.IterationCount,
log.MethodName,
(log.EndTime - log.StartTime).TotalSeconds);
}
// Show the group stats
var logsByMethod = logs.GroupBy(log => log.MethodName);
StatsSummary.Text = string.Empty;
foreach (IGrouping <string, PerformanceLog> logGroup in logsByMethod)
{
StatsSummary.Text += string.Format("<tr><td>{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}</td></tr>",
logGroup.Key,
logGroup.Min(log => (log.EndTime - log.StartTime).TotalSeconds),
logGroup.Average(log => (log.EndTime - log.StartTime).TotalSeconds),
logGroup.Max(log => (log.EndTime - log.StartTime).TotalSeconds),
logGroup.Count());
}
// Update the points and centroids displays
try
{
UpdatePointsCentroids(
AzureHelper.GetBlob(logs.First().PartitionKey, AzureHelper.PointsBlob),
AzureHelper.GetBlob(logs.First().PartitionKey, AzureHelper.CentroidsBlob),
final);
}
catch (StorageClientException e)
{
Trace.Write("Information", "Updating points and centroids failed. Will try again later.", e);
}
catch (IOException e)
{
Trace.Write("Information", "Updating points and centroids failed. Will try again later.", e);
}
}
public void CreateBlobGetBlobTest()
{
string containerName = Guid.NewGuid().ToString();
string blobName = Guid.NewGuid().ToString();
CloudBlockBlob blob = AzureHelper.CreateBlob(containerName, blobName);
byte[] bytes = new byte[] { 1, 2, 3 };
using (Stream stream = blob.OpenWrite())
{
stream.Write(bytes, 0, bytes.Length);
}
CloudBlockBlob foundBlob = AzureHelper.GetBlob(blob.Uri);
using (Stream stream = foundBlob.OpenRead())
{
foreach (int b in bytes)
{
Assert.AreEqual(b, stream.ReadByte());
}
}
}
public void ProcessPointsTest()
{
CloudBlobContainer container = AzureHelper.StorageAccount.CreateCloudBlobClient().GetContainerReference("test");
container.CreateIfNotExist();
CloudBlob points = container.GetBlobReference(Guid.NewGuid().ToString());
CloudBlob centroids = container.GetBlobReference(Guid.NewGuid().ToString());
const int NumPoints = 100, NumCentroids = 10;
using (ObjectStreamWriter <ClusterPoint> pointStream = new ObjectStreamWriter <ClusterPoint>(points, point => point.ToByteArray(), ClusterPoint.Size))
{
for (int i = 0; i < NumPoints; i++)
{
pointStream.Write(new ClusterPoint(1, 2, Guid.Empty));
}
}
Guid centroidID = Guid.NewGuid();
using (ObjectStreamWriter <Centroid> stream = new ObjectStreamWriter <Centroid>(centroids, point => point.ToByteArray(), Centroid.Size))
{
stream.Write(new Centroid(centroidID, 3, 4));
for (int i = 0; i < NumCentroids - 1; i++)
{
stream.Write(new Centroid(Guid.NewGuid(), 1000, 1000));
}
}
KMeansTaskProcessor_Accessor target = new KMeansTaskProcessor_Accessor(new KMeansTaskData(Guid.NewGuid(), Guid.NewGuid(), NumPoints, points.Uri, NumCentroids, 1, 0, 0, centroids.Uri, DateTime.UtcNow, DateTime.UtcNow, 0, null));
System.Diagnostics.Trace.WriteLine("Entering InitializeCentroids");
target.InitializeCentroids();
System.Diagnostics.Trace.WriteLine("Entering ProcessPoints");
System.Diagnostics.Trace.WriteLine("ProcessPoints took " + AzureHelper.Time(() =>
{
target.ProcessPoints();
}).TotalSeconds + " seconds");
// Commit the blocks
CloudBlockBlob newPointsBlob = AzureHelper.GetBlob(target.TaskResult.Points);
using (Stream stream = AzureHelper.GetBlob(target.TaskResult.PointsBlockListBlob).OpenRead())
{
BinaryFormatter bf = new BinaryFormatter();
List <string> pointsBlockList = bf.Deserialize(stream) as List <string>;
newPointsBlob.PutBlockList(pointsBlockList);
}
using (ObjectStreamReader <ClusterPoint> stream = new ObjectStreamReader <ClusterPoint>(newPointsBlob, ClusterPoint.FromByteArray, ClusterPoint.Size))
{
foreach (ClusterPoint p in stream)
{
Assert.AreEqual(centroidID, p.CentroidID);
}
}
Assert.AreEqual(NumPoints, target.TaskResult.NumPointsChanged);
Assert.IsTrue(target.TaskResult.PointsProcessedDataByCentroid.ContainsKey(centroidID));
Assert.AreEqual(NumPoints, target.TaskResult.PointsProcessedDataByCentroid[centroidID].NumPointsProcessed);
const double Epsilon = 0.0001;
Assert.IsTrue(Math.Abs((1 * NumPoints) - target.TaskResult.PointsProcessedDataByCentroid[centroidID].PartialPointSum.X) < Epsilon);
Assert.IsTrue(Math.Abs((2 * NumPoints) - target.TaskResult.PointsProcessedDataByCentroid[centroidID].PartialPointSum.Y) < Epsilon);
}
