public void ProcessWorkerResponseTest()
        {
            KMeansJobData      jobData = new KMeansJobData(Guid.NewGuid(), 4, null, 2, 10, DateTime.Now);
            KMeansJob_Accessor target  = new KMeansJob_Accessor(jobData, "server");

            target.InitializeStorage();

            // Upload a block with an arbitrary ClusterPoint, so we can verify it gets copied
            ClusterPoint  arbitraryPoint = new ClusterPoint(1, 2, Guid.NewGuid());
            List <string> blockList;

            using (ObjectCachedBlockWriter <ClusterPoint> pointPartitionWriteStream = new ObjectCachedBlockWriter <ClusterPoint>(target.Points, point => point.ToByteArray(), ClusterPoint.Size,
                                                                                                                                 Environment.GetEnvironmentVariable("TEMP") + @"\" + Guid.NewGuid().ToString()))
            {
                pointPartitionWriteStream.Write(arbitraryPoint);
                pointPartitionWriteStream.FlushBlock();
                blockList = pointPartitionWriteStream.BlockList;
            }

            KMeansTaskData taskData = new KMeansTaskData(jobData, Guid.NewGuid(), 0, 1, target.Centroids.Uri, DateTime.Now, 0, null);

            target.tasks.Clear();
            target.tasks.Add(new KMeansTask(taskData));

            KMeansTaskResult taskResult          = new KMeansTaskResult(taskData);
            CloudBlob        pointsBlockListBlob = AzureHelper.CreateBlob(jobData.JobID.ToString(), Guid.NewGuid().ToString());

            using (Stream stream = pointsBlockListBlob.OpenWrite())
            {
                BinaryFormatter bf = new BinaryFormatter();
                bf.Serialize(stream, blockList);
            }
            taskResult.PointsBlockListBlob = pointsBlockListBlob.Uri;
            taskResult.NumPointsChanged    = 2;
            Guid centroidID = Guid.NewGuid();

            taskResult.PointsProcessedDataByCentroid = new Dictionary <Guid, PointsProcessedData> {
                { centroidID, new PointsProcessedData()
                  {
                      NumPointsProcessed = 2,
                      PartialPointSum    = new Point(1, 2)
                  } }
            };
            target.ProcessWorkerResponse(taskResult, new List <Worker>());

            // Verify that the first ClusterPoint in Points is indeed equal to arbitraryPoint
            using (ObjectStreamReader <ClusterPoint> pointsStream = new ObjectStreamReader <ClusterPoint>(target.Points, ClusterPoint.FromByteArray, ClusterPoint.Size))
            {
                ClusterPoint point = pointsStream.First();
                Assert.AreEqual(arbitraryPoint.X, point.X);
                Assert.AreEqual(arbitraryPoint.Y, point.Y);
                Assert.AreEqual(arbitraryPoint.CentroidID, point.CentroidID);
            }
        }
示例#2
0
        /// <summary>
        /// Handles a worker response as part of a running k-means job. Looks up the appropriate job and passes the worker's response to it.
        /// </summary>
        /// <param name="message">The worker response. Must be of type KMeansTaskResult.</param>
        private bool ProcessWorkerResponse(KMeansTaskResult taskResult)
        {
            taskResult.RestorePointsProcessedDataByCentroid();

            // Make sure the job belongs to this server
            if (!jobs.ContainsKey(taskResult.JobID))
            {
                return(false);
            }

            System.Diagnostics.Trace.TraceInformation("[ServerRole] ProcessWorkerResponse(jobID={0}, taskID={1}, iterationCount={2})", taskResult.JobID, taskResult.TaskID, jobs[taskResult.JobID].IterationCount);

            return(jobs[taskResult.JobID].ProcessWorkerResponse(taskResult, workers.Values));
        }
示例#3
0
        public void SavePointsProcessedDataByCentroidTest()
        {
            KMeansTaskData   task   = new KMeansTaskData(Guid.NewGuid(), Guid.NewGuid(), 1, null, 2, 3, 10, 0, null, DateTime.Now, DateTime.Now, 0, null);
            KMeansTaskResult target = new KMeansTaskResult(task);

            target.PointsProcessedDataByCentroid[Guid.NewGuid()] = new PointsProcessedData()
            {
                NumPointsProcessed = 100,
                PartialPointSum    = new Point(10, -10)
            };
            target.PointsProcessedDataByCentroid[Guid.NewGuid()] = new PointsProcessedData()
            {
                NumPointsProcessed = 100,
                PartialPointSum    = new Point(10, -10)
            };
            target.SavePointsProcessedDataByCentroid();

            foreach (KeyValuePair <Guid, PointsProcessedData> pair in target.PointsProcessedDataByCentroid)
            {
                Assert.IsTrue(target.PointsProcessedDataByCentroidList.Contains(pair));
            }
        }
        public void MultiIterationJobTest() // TODO: make this unit test check things in more detail
        {
            KMeansJobData jobData = new KMeansJobData(Guid.NewGuid(), 4, null, 2, 2, DateTime.Now);
            List <Worker> workers = new List <Worker> {
                new Worker("a", "", 1),
                new Worker("b", "", 1)
            };
            KMeansJob_Accessor job = new KMeansJob_Accessor(jobData, "server");

            // First iteration
            job.InitializeStorage();
            job.EnqueueTasks(workers);

            for (int i = 0; i < jobData.MaxIterationCount; i++)
            {
                CheckWorkerRequests(job,
                                    (from task in job.tasks
                                     where task.Running
                                     select task.TaskData),
                                    workers.Count, job.Points);

                // Create the worker results and send them to the job
                List <KMeansTaskResult> results = new List <KMeansTaskResult>();
                foreach (var task in job.tasks)
                {
                    var taskResult = new KMeansTaskResult(task.TaskData);
                    taskResult.NumPointsChanged = 1;
                    results.Add(taskResult);
                }

                foreach (var result in results)
                {
                    job.ProcessWorkerResponse(result, workers);
                }
            }
        }