예제 #1
0
        public KMeansJob(KMeansJobData jobData, string machineID)
        {
            this.jobData   = jobData;
            this.MachineID = machineID;

            this.IterationCount = 0;
        }
예제 #2
0
        public KMeansJob(KMeansJobData jobData, string machineID)
        {
            this.jobData = jobData;
            this.MachineID = machineID;

            this.IterationCount = 0;
        }
예제 #3
0
 public KMeansJobData(KMeansJobData jobData)
 {
     JobID = jobData.JobID;
     N = jobData.N;
     Points = jobData.Points;
     K = jobData.K;
     MaxIterationCount = jobData.MaxIterationCount;
     JobStartTime = jobData.JobStartTime;
 }
예제 #4
0
 public KMeansJobData(KMeansJobData jobData)
 {
     JobID             = jobData.JobID;
     N                 = jobData.N;
     Points            = jobData.Points;
     K                 = jobData.K;
     MaxIterationCount = jobData.MaxIterationCount;
     JobStartTime      = jobData.JobStartTime;
 }
예제 #5
0
        /// <summary>
        /// Handles a request for a new k-means job. Sets up a new job and starts it off.
        /// </summary>
        /// <param name="message">The job request. Must be of type KMeansJobData.</param>
        private bool ProcessNewJob(KMeansJobData jobData)
        {
            System.Diagnostics.Trace.TraceInformation("[ServerRole] ProcessNewJob(jobID={0})", jobData.JobID);

            jobs[jobData.JobID] = new KMeansJob(jobData, "server");
            jobs[jobData.JobID].InitializeStorage();
            jobs[jobData.JobID].EnqueueTasks(workers.Values);

            return true;
        }
예제 #6
0
        public void ToFromBinaryTest()
        {
            KMeansJobData target = new KMeansJobData(Guid.NewGuid(), 1, null, 2, 10, DateTime.Now);
            byte[] bytes = target.ToBinary();
            KMeansJobData targetNew = KMeansJobData.FromMessage(new CloudQueueMessage(bytes)) as KMeansJobData;

            Assert.AreEqual(target.JobID, targetNew.JobID);
            Assert.AreEqual(target.K, targetNew.K);
            Assert.AreEqual(target.N, targetNew.N);
        }
예제 #7
0
 public KMeansTaskData(KMeansJobData job, Guid taskID, int partitionNumber, int m, Uri centroids, DateTime taskStartTime, int iteration, string buddyGroup)
     : base(job)
 {
     this.TaskID          = taskID;
     this.PartitionNumber = partitionNumber;
     this.M             = m;
     this.Centroids     = centroids;
     this.TaskStartTime = TaskStartTime;
     this.Iteration     = iteration;
     this.BuddyGroup    = buddyGroup;
 }
예제 #8
0
 public KMeansTaskData(KMeansJobData job, Guid taskID, int partitionNumber, int m, Uri centroids, DateTime taskStartTime, int iteration, string buddyGroup)
     : base(job)
 {
     this.TaskID = taskID;
     this.PartitionNumber = partitionNumber;
     this.M = m;
     this.Centroids = centroids;
     this.TaskStartTime = TaskStartTime;
     this.Iteration = iteration;
     this.BuddyGroup = buddyGroup;
 }
예제 #9
0
        protected void Run_Click(object sender, EventArgs e)
        {
            FreezeUI();
            ClearIndicators();
            Guid jobID = Guid.NewGuid();
            Status.Text = string.Format("Running job {0}.", jobID);
            DownloadLog.NavigateUrl = string.Format("Log.aspx?JobID={0}", jobID);
            DownloadLog.Enabled = true;

            Session["jobID"] = jobID;
            Session["lastLogRefreshTime"] = DateTime.MinValue;
            Session["allLogs"] = new List<PerformanceLog>();

            Uri pointsBlobUri = null;
            if (PointsFile.HasFile)
            {
                CloudBlob pointsBlob = AzureHelper.CreateBlob(jobID.ToString(), AzureHelper.PointsBlob);
                using (BlobStream stream = pointsBlob.OpenWrite())
                {
                    PointsFile.FileContent.CopyTo(stream);
                }
                pointsBlobUri = pointsBlob.Uri;
            }
            else if (!string.IsNullOrEmpty(PointsBlob.Text))
            {
                CloudBlob pointsBlob = AzureHelper.CreateBlob(jobID.ToString(), AzureHelper.PointsBlob);
                pointsBlob.CopyFromBlob(AzureHelper.GetBlob(new Uri(PointsBlob.Text)));
                pointsBlobUri = pointsBlob.Uri;
            }

            int nInt = 0, kInt = 0, maxIterationCountInt = 0;
            int.TryParse(N.Text, out nInt);
            int.TryParse(K.Text, out kInt);
            int.TryParse(MaxIterationCount.Text, out maxIterationCountInt);

            KMeansJobData jobData = new KMeansJobData(jobID, nInt, pointsBlobUri, kInt, maxIterationCountInt, DateTime.Now)
            {
                ProgressEmail = ProgressEmail.Text
            };
            AzureHelper.EnqueueMessage(AzureHelper.ServerRequestQueue, jobData);

            WaitForResults();
        }
예제 #10
0
 public KMeansJobResult(KMeansJobData jobData, Uri centroids)
     : base(jobData)
 {
     this.Centroids = centroids;
 }
예제 #11
0
 public KMeansJobResult(KMeansJobData jobData, Uri centroids)
     : base(jobData)
 {
     this.Centroids = centroids;
 }
예제 #12
0
        public void InitializeStorageTest()
        {
            KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 2, null, 4, 10, DateTime.Now);
            KMeansJob target = new KMeansJob(jobData, "server");
            target.InitializeStorage();

            // Verify that the created containers and blobs actually exist
            CloudBlobClient client = AzureHelper.StorageAccount.CreateCloudBlobClient();

            CloudBlobContainer c = null;
            try
            {
                c = client.GetContainerReference(jobData.JobID.ToString());
                c.FetchAttributes();
            }
            catch (StorageClientException e)
            {
                if (e.ErrorCode == StorageErrorCode.ResourceNotFound)
                {
                    Assert.Fail();
                }
                else
                {
                    throw;
                }
            }

            CloudBlob points = null, centroids = null;
            try
            {
                points = c.GetBlobReference("points");
                points.FetchAttributes();
                centroids = c.GetBlobReference("centroids");
                centroids.FetchAttributes();
            }
            catch (StorageClientException e)
            {
                if (e.ErrorCode == StorageErrorCode.ResourceNotFound)
                {
                    Assert.Fail();
                }
                else
                {
                    throw;
                }
            }

            // Verify that unpacking a ClusterPoint actually yields a point with coordinates [-50, 50) and a null centroidID
            byte[] pointBytes;
            using (BlobStream pointsStream = points.OpenRead())
            {
                pointBytes = new byte[ClusterPoint.Size];
                pointsStream.Read(pointBytes, 0, ClusterPoint.Size);
            }
            ClusterPoint p = ClusterPoint.FromByteArray(pointBytes);

            Assert.IsTrue(p.X >= -50 && p.X < 50);
            Assert.IsTrue(p.Y >= -50 && p.Y < 50);
            Assert.AreEqual(p.CentroidID, Guid.Empty);

            // Verify that the blobs are the correct length
            Assert.AreEqual(points.Properties.Length, ClusterPoint.Size * jobData.N);
            Assert.AreEqual(centroids.Properties.Length, Centroid.Size * jobData.K);
        }
예제 #13
0
        public void RecalculateCentroidsTest2()
        {
            KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 0, null, 1, 10, DateTime.Now);
            KMeansJob_Accessor target = new KMeansJob_Accessor(jobData, "server");
            target.InitializeStorage();

            byte[] cBytes = new byte[Centroid.Size];
            using (BlobStream cStream = target.Centroids.OpenRead())
            {
                cStream.Read(cBytes, 0, cBytes.Length);
            }
            Centroid cOriginal = Centroid.FromByteArray(cBytes);

            target.totalPointsProcessedDataByCentroid[cOriginal.ID] = new PointsProcessedData();

            target.RecalculateCentroids();

            byte[] cBytesNew = new byte[Centroid.Size];
            using (BlobStream cStreamNew = target.Centroids.OpenRead())
            {
                cStreamNew.Read(cBytesNew, 0, cBytesNew.Length);
            }
            Centroid cNew = Centroid.FromByteArray(cBytesNew);

            Assert.AreEqual(cOriginal.ID, cNew.ID);
            Assert.AreEqual(cNew.X, 0);
            Assert.AreEqual(cNew.Y, 0);
        }
예제 #14
0
        public void ProcessWorkerResponseTest()
        {
            KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 4, null, 2, 10, DateTime.Now);
            KMeansJob_Accessor target = new KMeansJob_Accessor(jobData, "server");
            target.InitializeStorage();

            // Upload a block with an arbitrary ClusterPoint, so we can verify it gets copied
            ClusterPoint arbitraryPoint = new ClusterPoint(1, 2, Guid.NewGuid());
            List<string> blockList;
            using (ObjectCachedBlockWriter<ClusterPoint> pointPartitionWriteStream = new ObjectCachedBlockWriter<ClusterPoint>(target.Points, point => point.ToByteArray(), ClusterPoint.Size,
                Environment.GetEnvironmentVariable("TEMP") + @"\" + Guid.NewGuid().ToString()))
            {
                pointPartitionWriteStream.Write(arbitraryPoint);
                pointPartitionWriteStream.FlushBlock();
                blockList = pointPartitionWriteStream.BlockList;
            }

            KMeansTaskData taskData = new KMeansTaskData(jobData, Guid.NewGuid(), 0, 1, target.Centroids.Uri, DateTime.Now, 0, null);

            target.tasks.Clear();
            target.tasks.Add(new KMeansTask(taskData));

            KMeansTaskResult taskResult = new KMeansTaskResult(taskData);
            CloudBlob pointsBlockListBlob = AzureHelper.CreateBlob(jobData.JobID.ToString(), Guid.NewGuid().ToString());
            using (Stream stream = pointsBlockListBlob.OpenWrite())
            {
                BinaryFormatter bf = new BinaryFormatter();
                bf.Serialize(stream, blockList);
            }
            taskResult.PointsBlockListBlob = pointsBlockListBlob.Uri;
            taskResult.NumPointsChanged = 2;
            Guid centroidID = Guid.NewGuid();
            taskResult.PointsProcessedDataByCentroid = new Dictionary<Guid, PointsProcessedData> {
                { centroidID, new PointsProcessedData() {
                        NumPointsProcessed = 2,
                        PartialPointSum = new Point(1, 2)
                    }
                }
            };
            target.ProcessWorkerResponse(taskResult, new List<Worker>());

            // Verify that the first ClusterPoint in Points is indeed equal to arbitraryPoint
            using (ObjectStreamReader<ClusterPoint> pointsStream = new ObjectStreamReader<ClusterPoint>(target.Points, ClusterPoint.FromByteArray, ClusterPoint.Size))
            {
                ClusterPoint point = pointsStream.First();
                Assert.AreEqual(arbitraryPoint.X, point.X);
                Assert.AreEqual(arbitraryPoint.Y, point.Y);
                Assert.AreEqual(arbitraryPoint.CentroidID, point.CentroidID);
            }
        }
예제 #15
0
        public void MultiIterationJobTest()
        {
            KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 4, null, 2, 2, DateTime.Now);
            List<Worker> workers = new List<Worker> {
                new Worker("a", "", 1),
                new Worker("b", "", 1)
            };
            KMeansJob_Accessor job = new KMeansJob_Accessor(jobData, "server");

            // First iteration
            job.InitializeStorage();
            job.EnqueueTasks(workers);

            for (int i = 0; i < jobData.MaxIterationCount; i++)
            {
                CheckWorkerRequests(job,
                    (from task in job.tasks
                    where task.Running
                    select task.TaskData),
                    workers.Count, job.Points);

                // Create the worker results and send them to the job
                List<KMeansTaskResult> results = new List<KMeansTaskResult>();
                foreach (var task in job.tasks)
                {
                    var taskResult = new KMeansTaskResult(task.TaskData);
                    taskResult.NumPointsChanged = 1;
                    results.Add(taskResult);
                }

                foreach (var result in results)
                {
                    job.ProcessWorkerResponse(result, workers);
                }
            }
        }