private void CreateDirectories(SlicingOptions slicingOptions) { outputPath = Path.Combine(RoleEnvironment.GetLocalResource("output").RootPath, Guid.NewGuid().ToString()); inputPath = Path.Combine(RoleEnvironment.GetLocalResource("input").RootPath, slicingOptions.CloudResultContainer); Directory.CreateDirectory(outputPath); Directory.CreateDirectory(inputPath); }
private void CheckForComplete(SlicingOptions options, CubeManager manager) { int expectedResults = options.TextureSliceX * options.TextureSliceY; if (StorageUtilities.GetWorkCompletedCount(TableClient, options.CloudResultPath, options.CloudResultContainer) != expectedResults) { return; } var workResults = StorageUtilities.GetWorkCompletedMetadata(TableClient, options.CloudResultPath, options.CloudResultContainer); // Write metadata CubeMetadata metadata = new CubeMetadata(options.CubeGrid) { WorldBounds = manager.ObjInstance.Size, VirtualWorldBounds = options.ForceCubicalCubes ? manager.ObjInstance.CubicalSize : manager.ObjInstance.Size, VertexCount = manager.ObjInstance.VertexList.Count }; // Configure texture slicing metadata if (!string.IsNullOrEmpty(options.Texture) && (options.TextureSliceX + options.TextureSliceY) > 2) { metadata.TextureSetSize = new Vector2(options.TextureSliceX, options.TextureSliceY); } else { metadata.TextureSetSize = new Vector2(1, 1); } var resultsList = workResults.Select(w => SerializationUtilities.DecodeMetadataFromBase64( Texture.GetCubeListFromTextureTile(options.TextureSliceY, options.TextureSliceX, w.TextureTileX, w.TextureTileY, manager.ObjInstance), w.MetadataBase64)); foreach (var result in resultsList) { foreach (var cube in result.Keys) { metadata.CubeExists[cube.X, cube.Y, cube.Z] = result[cube]; } } // Write out some json metadata string metadataPath = Path.Combine(outputPath, "metadata.json"); if (File.Exists(metadataPath)) { File.Delete(metadataPath); } string metadataString = JsonConvert.SerializeObject(metadata); File.WriteAllText(metadataPath, metadataString); StorageUtilities.UpdateSetCompleted(TableClient, options.SetKey); }
private static WorkEntity TrackWorkStarted(DateTime startTime, SlicingOptions slicingOptions) { var workTrackingEntity = new WorkEntity(slicingOptions.CloudResultPath, slicingOptions.CloudResultContainer, slicingOptions.TextureTile.X, slicingOptions.TextureTile.Y) { StartTime = startTime, }; StorageUtilities.InsertWorkStartedMetadata(TableClient, workTrackingEntity); return(workTrackingEntity); }
private async Task VerifySourceDataAsync(SlicingOptions slicingOptions, CancellationToken cancellationToken) { if (!File.Exists(slicingOptions.Obj)) { await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Obj, slicingOptions.CloudObjPath, cancellationToken).ConfigureAwait(false); } if (!File.Exists(slicingOptions.Texture)) { await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Texture, slicingOptions.CloudTexturePath, cancellationToken).ConfigureAwait(false); } }
private void UploadResultData(SlicingOptions slicingOptions) { var files = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories); foreach (var file in files) { StorageUtilities.UploadBlob( BlobClient, file, Path.Combine(slicingOptions.CloudResultPath, file.Replace(outputPath, string.Empty).TrimStart(new char[] { '\\', '/' })).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), slicingOptions.CloudResultContainer); File.Delete(file); } }
private Texture GetTestTexture() { var options = new SlicingOptions { GenerateObj = true, Texture = "texture.jpg", TextureScale = 1, TextureSliceX = 2, TextureSliceY = 2, ForceCubicalCubes = false, Obj = "model.obj", CubeGrid = new Vector3(2, 2, 2) }; CubeManager manager = new CubeManager(options); return(new Texture(manager.ObjInstance)); }
private CubeManager GetLoadedManager() { var options = new SlicingOptions { GenerateObj = true, Texture = "texture.jpg", TextureScale = 1, TextureSliceX = 2, TextureSliceY = 2, ForceCubicalCubes = false, Obj = "model.obj", CubeGrid = new Vector3(2, 2, 2) }; CubeManager manager = new CubeManager(options); return(manager); }
// Note: This will probably only work well with OBJ files generated by Pix4D // as I have only supported the subset of data types it outputs. static void Main(string[] args) { Options opt; // Setup a timer for all operations Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); try { opt = CliParser.Parse <Options>(args); foreach (string path in opt.Input) { // Check if we are processing an image or a mesh if (Path.GetExtension(path).ToUpper().EndsWith("JPG")) { ImageTile tiler = new ImageTile(path, opt.XSize, opt.YSize); tiler.GenerateTiles(opt.OutputPath); } else if (Path.GetExtension(path).ToUpper().EndsWith("OBJ")) { // Generate subfolders named after input file // if multiple input files are provided string outputPath; if (opt.Input.Count == 1) { outputPath = opt.OutputPath; } else { outputPath = Path.Combine(opt.OutputPath, Path.GetFileNameWithoutExtension(path)); } if (opt.ForceCubical) { int longestGridSide = Math.Max(Math.Max(opt.XSize, opt.YSize), opt.ZSize); opt.XSize = opt.YSize = opt.ZSize = longestGridSide; Console.WriteLine("Due to -ForceCubical grid size is now {0},{0},{0}", longestGridSide); } var options = new SlicingOptions { OverrideMtl = opt.MtlOverride, GenerateEbo = opt.Ebo, GenerateOpenCtm = opt.OpenCtm, Debug = opt.Debug, GenerateObj = opt.Obj, Texture = opt.Texture, Obj = path, WriteMtl = opt.WriteMtl, TextureScale = opt.ScaleTexture, TextureSliceX = opt.TextureXSize, TextureSliceY = opt.TextureYSize, ForceCubicalCubes = opt.ForceCubical, CubeGrid = new Vector3 { X = opt.XSize, Y = opt.YSize, Z = opt.ZSize } }; CubeManager manager = new CubeManager(options); if (opt.MarkupUV) { Texture tex = new Texture(manager.ObjInstance); tex.MarkupTextureFaces(opt.Texture); } else { manager.GenerateCubes(outputPath, options); } } else { Console.WriteLine("PyriteCli only accepts .jpg and .obj files for input."); } } } catch (ParserExit) { return; } catch (ParseException ex) { Console.WriteLine("usage: PyriteCli --help\n" + ex.ToString()); } stopwatch.Stop(); Trace.TraceInformation(stopwatch.Elapsed.ToString()); }
// Entry point public static int Main(string[] args, ISimpleLineWriter output) { var start = DateTime.Now; var slicingOptions = new SlicingOptions(args); // we use it to parse the clousot options, and to split the work var clousot = new NewCCI2Driver(slicingOptions.remainingArgs.ToArray(), output); if (!clousot.CheckOptions()) { return(-1); } var errorCode = 0; WorkerPool workerPool = null; IQueue queue = null; IWorkerFactory localWorkerFactory = null; List <IWorkerFactory> remoteWorkerFactories = null; Func <SliceDefinition, ISliceWorkResult> workOnSlice = null; Dictionary <ISliceId, int> failedRegressions = null; var localWorkers = slicingOptions.LocalWorkers; var remoteWorkers = slicingOptions.RemoteWorkers; var workers = localWorkers + remoteWorkers; // If we have workers, we create a Worker factory if (workers > 0) { IDB db; if (remoteWorkers > 0 || clousot.options.useCache) { // Use Clousot Database db = new StdDB(clousot.options); } else { // In-memory database db = new MemorySingletonDB(); } workerPool = new WorkerPool("Worker", canCancel: false); queue = new FIFOQueue(workerPool, db); // When a job is done, it choses which jobs to put in the queue (e.g., analyze the dependencies) var scheduler = clousot.options.InferObjectInvariantsOnlyForReadonlyFields? new LazySchedulerForObjectInvariants(queue, db) : // use LazySchedulerForObjectInvariants for global fixpoint computation including object invariants new LazyScheduler(queue, db); // use LazyScheduler for the global fixpoint computation //var scheduler = new NoloopScheduler(queue); // Usual cache var clousotDB = slicingOptions.useDB ? db : null; var argsForWorker = clousot.argsForWorker.ToFList(); if (localWorkers > 0) { if (slicingOptions.cci1) { localWorkerFactory = new Clousot1WorkerFactory(scheduler, argsForWorker, output, clousotDB); } else { localWorkerFactory = new Clousot2WorkerFactory(scheduler, argsForWorker, output, clousotDB); } // TODO: use a lighter version of ClousotMain since options are already checked here } if (remoteWorkers > 0) // so far 1 factory per remote worker but we can do better { // TODO: specifiy, for each address the number of workers // We have a list, because we can have several addresses remoteWorkerFactories = slicingOptions.serviceAddress.Select(addr => new Clousot2SWorkerFactory(scheduler, argsForWorker, output, addr)).ToList <IWorkerFactory>(); } if (clousot.options.IsRegression) { failedRegressions = new Dictionary <ISliceId, int>(); } // fail if any work fails scheduler.OnWorkDone += (sliceId, returnCode) => { if (errorCode >= 0) { if (returnCode < 0) // special error code, keep only one { errorCode = returnCode; } else { int prevValue; if (clousot.options.IsRegression) { lock (failedRegressions) { Contract.Assume(failedRegressions != null); if (failedRegressions.TryGetValue(sliceId, out prevValue)) { output.WriteLine("[Regression] We already analyzed {0} with outcome {1}. Now we update the outcome to {2}", sliceId.Dll, prevValue, returnCode); } failedRegressions[sliceId] = returnCode; } } errorCode += returnCode; // regression error count, additive } } }; // What we do for each slice. Two things: // 1. Register the slice in the db // 2. Add to the queue (via the scheduler, who decides how to do it) workOnSlice = sliceDef => { var sliceId = db.RegisterSlice(sliceDef); scheduler.FeedQueue(new ISliceId[] { sliceId }); return(null); }; } ISlicerResult slicerResult = null; if (slicingOptions.sliceFirst) { slicerResult = clousot.SplitWork(workOnSlice); output.WriteLine("Slicing time: {0}", DateTime.Now - start); } if (workerPool != null) { if (localWorkerFactory != null) { for (var i = 0; i < localWorkers; i++) { workerPool.CreateWorker(localWorkerFactory); } } if (remoteWorkerFactories != null) { foreach (var factory in remoteWorkerFactories) { workerPool.CreateWorker(factory); } } } if (!slicingOptions.sliceFirst) { slicerResult = clousot.SplitWork(workOnSlice); output.WriteLine("Slicing time and thread creation time : {0}", DateTime.Now - start); } if (workerPool != null) { // workerPool != null ==> queue != null Contract.Assume(queue != null); workerPool.WaitAllAnd(queue.EmptyQueueWaitHandle); // Something else can arrive at the queue, so we want to stop all of them workerPool.StopAll(); } if (slicerResult != null) { var errors = slicerResult.GetErrors(); if (errors.Any()) { foreach (var errMessage in errors) { output.WriteLine(errMessage); } errorCode = errors.Count(); } } output.WriteLine("Total analysis time: {0}", DateTime.Now - start); var returnValue = errorCode; if (failedRegressions != null && clousot.options.IsRegression && errorCode >= 0) { returnValue = failedRegressions.Where(pair => pair.Value != 0).Select(pair => Math.Abs(pair.Value)).Sum(); } #if DEBUG if (clousot.options.IsRegression) { Console.WriteLine("[Regression] Returned value {0}", returnValue); } #endif return(returnValue); }
// TODO: make sure that for the tilted prints no brim or skirt are printed public string Slice(string stlFilePath, string configFile, SlicingOptions options = null) { throw new System.NotImplementedException(); }
// Entry point public static int Main(string[] args, ISimpleLineWriter output) { var start = DateTime.Now; var slicingOptions = new SlicingOptions(args); // we use it to parse the clousot options, and to split the work var clousot = new NewCCI2Driver(slicingOptions.remainingArgs.ToArray(), output); if (!clousot.CheckOptions()) { return -1; } var errorCode = 0; WorkerPool workerPool = null; IQueue queue = null; IWorkerFactory localWorkerFactory = null; List<IWorkerFactory> remoteWorkerFactories = null; Func<SliceDefinition, ISliceWorkResult> workOnSlice = null; Dictionary<ISliceId, int> failedRegressions = null; var localWorkers = slicingOptions.LocalWorkers; var remoteWorkers = slicingOptions.RemoteWorkers; var workers = localWorkers + remoteWorkers; // If we have workers, we create a Worker factory if (workers > 0) { IDB db; if (remoteWorkers > 0 || clousot.options.useCache) { // Use Clousot Database db = new StdDB(clousot.options); } else { // In-memory database db = new MemorySingletonDB(); } workerPool = new WorkerPool("Worker", canCancel: false); queue = new FIFOQueue(workerPool, db); // When a job is done, it choses which jobs to put in the queue (e.g., analyze the dependencies) var scheduler = clousot.options.InferObjectInvariantsOnlyForReadonlyFields? new LazySchedulerForObjectInvariants(queue, db) : // use LazySchedulerForObjectInvariants for global fixpoint computation including object invariants new LazyScheduler(queue, db); // use LazyScheduler for the global fixpoint computation //var scheduler = new NoloopScheduler(queue); // Usual cache var clousotDB = slicingOptions.useDB ? db : null; var argsForWorker = clousot.argsForWorker.ToFList(); if (localWorkers > 0) { if (slicingOptions.cci1) localWorkerFactory = new Clousot1WorkerFactory(scheduler, argsForWorker, output, clousotDB); else localWorkerFactory = new Clousot2WorkerFactory(scheduler, argsForWorker, output, clousotDB); // TODO: use a lighter version of ClousotMain since options are already checked here } if (remoteWorkers > 0) // so far 1 factory per remote worker but we can do better { // TODO: specifiy, for each address the number of workers // We have a list, because we can have several addresses remoteWorkerFactories = slicingOptions.serviceAddress.Select(addr => new Clousot2SWorkerFactory(scheduler, argsForWorker, output, addr)).ToList<IWorkerFactory>(); } if (clousot.options.IsRegression) { failedRegressions = new Dictionary<ISliceId, int>(); } // fail if any work fails scheduler.OnWorkDone += (sliceId, returnCode) => { if (errorCode >= 0) { if (returnCode < 0) // special error code, keep only one { errorCode = returnCode; } else { int prevValue; if (clousot.options.IsRegression) { lock (failedRegressions) { Contract.Assume(failedRegressions != null); if (failedRegressions.TryGetValue(sliceId, out prevValue)) { output.WriteLine("[Regression] We already analyzed {0} with outcome {1}. Now we update the outcome to {2}", sliceId.Dll, prevValue, returnCode); } failedRegressions[sliceId] = returnCode; } } errorCode += returnCode; // regression error count, additive } } }; // What we do for each slice. Two things: // 1. Register the slice in the db // 2. Add to the queue (via the scheduler, who decides how to do it) workOnSlice = sliceDef => { var sliceId = db.RegisterSlice(sliceDef); scheduler.FeedQueue(new ISliceId[] { sliceId }); return null; }; } ISlicerResult slicerResult = null; if (slicingOptions.sliceFirst) { slicerResult = clousot.SplitWork(workOnSlice); output.WriteLine("Slicing time: {0}", DateTime.Now - start); } if (workerPool != null) { if (localWorkerFactory != null) for (var i = 0; i < localWorkers; i++) workerPool.CreateWorker(localWorkerFactory); if (remoteWorkerFactories != null) foreach (var factory in remoteWorkerFactories) workerPool.CreateWorker(factory); } if (!slicingOptions.sliceFirst) { slicerResult = clousot.SplitWork(workOnSlice); output.WriteLine("Slicing time and thread creation time : {0}", DateTime.Now - start); } if (workerPool != null) { // workerPool != null ==> queue != null Contract.Assume(queue != null); workerPool.WaitAllAnd(queue.EmptyQueueWaitHandle); // Something else can arrive at the queue, so we want to stop all of them workerPool.StopAll(); } if (slicerResult != null) { var errors = slicerResult.GetErrors(); if (errors.Any()) { foreach (var errMessage in errors) output.WriteLine(errMessage); errorCode = errors.Count(); } } output.WriteLine("Total analysis time: {0}", DateTime.Now - start); var returnValue = errorCode; if (failedRegressions != null && clousot.options.IsRegression && errorCode >= 0) { returnValue = failedRegressions.Where(pair => pair.Value != 0).Select(pair => Math.Abs(pair.Value)).Sum(); } #if DEBUG if (clousot.options.IsRegression) { Console.WriteLine("[Regression] Returned value {0}", returnValue); } #endif return returnValue; }
public async Task DoWorkAsync(CancellationToken cancellationToken) { CloudQueueMessage retrievedMessage; DateTime startTime; try { // Get the next message retrievedMessage = await WorkQueue.GetMessageAsync(TimeSpan.FromHours(5), null, null, cancellationToken).ConfigureAwait(false); if (retrievedMessage == null) { return; } startTime = DateTime.UtcNow; } catch { return; } var messageContents = retrievedMessage.AsString; SlicingOptions slicingOptions = null; try { // Get the message slicingOptions = JsonConvert.DeserializeObject <SlicingOptions>(messageContents); // Make some fresh directories CreateDirectories(slicingOptions); // Populate input file locations slicingOptions.Obj = Path.Combine(inputPath, slicingOptions.Obj); slicingOptions.Texture = Path.Combine(inputPath, slicingOptions.Texture); // Track work started var workTrackingEntity = TrackWorkStarted(startTime, slicingOptions); // ** Prep Trace.TraceInformation("Syncing data"); await VerifySourceDataAsync(slicingOptions, cancellationToken); // ** Run Trace.TraceInformation("Starting Processing"); CubeManager manager = new CubeManager(slicingOptions); if (!string.IsNullOrEmpty(slicingOptions.Texture)) { slicingOptions.TextureInstance = new Texture(manager.ObjInstance, slicingOptions.Texture); } var vertexCounts = await manager.GenerateCubesForTextureTileAsync(outputPath, slicingOptions.TextureTile, slicingOptions, cancellationToken).ConfigureAwait(false); // Track work completed TrackWorkCompleted(workTrackingEntity, vertexCounts); // ** Check if set is complete CheckForComplete(slicingOptions, manager); // ** Cleanup slicingOptions.TextureInstance?.Dispose(); Trace.TraceInformation("Writing Results"); UploadResultData(slicingOptions); WorkQueue.DeleteMessage(retrievedMessage); } catch (Exception ex) { Trace.TraceError(ex.ToString()); // Release texure if we have one slicingOptions?.TextureInstance?.Dispose(); // Either delete this message or make it visible again for retry if (retrievedMessage.DequeueCount > 3) { if (slicingOptions != null) { Trace.TraceError($"Maximum Dequeue count hit for message \"{retrievedMessage.AsString}\". Failing set {slicingOptions.SetKey}"); StorageUtilities.UpdateSetFailed(TableClient, slicingOptions.SetKey, ex.ToString()); } WorkQueue.DeleteMessage(retrievedMessage); } else { WorkQueue.UpdateMessage(retrievedMessage, TimeSpan.FromSeconds(10), MessageUpdateFields.Visibility); } } finally { if (Directory.Exists(outputPath)) { Directory.Delete(outputPath, true); } if (Directory.Exists(inputPath)) { Directory.Delete(inputPath, true); } } }
static void Main(string[] args) { // Get storage account CloudStorageAccount storageAccount = CloudStorageAccount.Parse(Settings.Default.StorageConnectionString); // Create the clients CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient(); BlobClient = storageAccount.CreateCloudBlobClient(); TableClient = storageAccount.CreateCloudTableClient(); // Retrieve a reference to a queue WorkQueue = queueClient.GetQueueReference(Settings.Default.Queue); // Create the queue if it doesn't already exist WorkQueue.CreateIfNotExists(); CloudOptions opt; try { opt = CliParser.Parse <CloudOptions>(args); if (opt.ForceCubical) { int longestGridSide = Math.Max(Math.Max(opt.XSize, opt.YSize), opt.ZSize); opt.XSize = opt.YSize = opt.ZSize = longestGridSide; Console.WriteLine("Due to -ForceCubical grid size is now {0},{0},{0}", longestGridSide); } var options = new SlicingOptions { OverrideMtl = opt.MtlOverride, GenerateEbo = opt.Ebo, GenerateOpenCtm = opt.OpenCtm, Debug = opt.Debug, GenerateObj = opt.Obj, Texture = Path.GetFileName(opt.Texture), Obj = Path.GetFileName(opt.Input.First()), WriteMtl = opt.WriteMtl, TextureScale = opt.ScaleTexture, TextureSliceX = opt.TextureXSize, TextureSliceY = opt.TextureYSize, ForceCubicalCubes = opt.ForceCubical, CubeGrid = new Vector3 { X = opt.XSize, Y = opt.YSize, Z = opt.ZSize } }; string objPath; if (opt.Input.First().StartsWith("http")) { objPath = opt.Input.First(); } else { objPath = StorageUtilities.UploadBlob(BlobClient, opt.Input.First(), Guid.NewGuid().ToString(), "processingdata"); } string texPath; if (opt.Texture.StartsWith("http")) { texPath = opt.Texture; } else { texPath = StorageUtilities.UploadBlob(BlobClient, opt.Texture, Guid.NewGuid().ToString(), "processingdata"); } options.CloudObjPath = objPath; options.CloudTexturePath = texPath; options.CloudResultContainer = opt.OutputContainer; options.CloudResultPath = opt.OutputPath; // Get texture set size Vector2 setSize; if (!string.IsNullOrEmpty(options.Texture) && (options.TextureSliceX + options.TextureSliceY) > 2) { setSize = new Vector2(options.TextureSliceX, options.TextureSliceY); } else { setSize = new Vector2(1, 1); } // Queue work var setEntity = new SetEntity("Set", DateTime.UtcNow) { ResultPath = options.CloudResultPath, ResultContainer = options.CloudResultContainer, TextureTilesX = setSize.X, TextureTilesY = setSize.Y }; options.SetKey = setEntity.RowKey; StorageUtilities.InsertSetMetadata(TableClient, setEntity); SpatialUtilities.EnumerateSpace(setSize, (x, y) => { options.TextureTile = new Vector2(x, y); string message = JsonConvert.SerializeObject(options); WorkQueue.AddMessage(new CloudQueueMessage(message)); }); } catch (ParserExit) { return; } catch (ParseException) { Console.WriteLine("usage: PyriteCli --help"); } }