private void CreateDirectories(SlicingOptions slicingOptions)
{
outputPath = Path.Combine(RoleEnvironment.GetLocalResource("output").RootPath, Guid.NewGuid().ToString());
inputPath = Path.Combine(RoleEnvironment.GetLocalResource("input").RootPath, slicingOptions.CloudResultContainer);
Directory.CreateDirectory(outputPath);
Directory.CreateDirectory(inputPath);
}
private void CheckForComplete(SlicingOptions options, CubeManager manager)
{
int expectedResults = options.TextureSliceX * options.TextureSliceY;
if (StorageUtilities.GetWorkCompletedCount(TableClient, options.CloudResultPath, options.CloudResultContainer) != expectedResults)
{
return;
}
var workResults = StorageUtilities.GetWorkCompletedMetadata(TableClient, options.CloudResultPath, options.CloudResultContainer);
// Write metadata
CubeMetadata metadata = new CubeMetadata(options.CubeGrid)
{
WorldBounds = manager.ObjInstance.Size,
VirtualWorldBounds = options.ForceCubicalCubes ? manager.ObjInstance.CubicalSize : manager.ObjInstance.Size,
VertexCount = manager.ObjInstance.VertexList.Count
};
// Configure texture slicing metadata
if (!string.IsNullOrEmpty(options.Texture) && (options.TextureSliceX + options.TextureSliceY) > 2)
{
metadata.TextureSetSize = new Vector2(options.TextureSliceX, options.TextureSliceY);
}
else
{
metadata.TextureSetSize = new Vector2(1, 1);
}
var resultsList = workResults.Select(w =>
SerializationUtilities.DecodeMetadataFromBase64(
Texture.GetCubeListFromTextureTile(options.TextureSliceY, options.TextureSliceX, w.TextureTileX, w.TextureTileY, manager.ObjInstance),
w.MetadataBase64));
foreach (var result in resultsList)
{
foreach (var cube in result.Keys)
{
metadata.CubeExists[cube.X, cube.Y, cube.Z] = result[cube];
}
}
// Write out some json metadata
string metadataPath = Path.Combine(outputPath, "metadata.json");
if (File.Exists(metadataPath))
{
File.Delete(metadataPath);
}
string metadataString = JsonConvert.SerializeObject(metadata);
File.WriteAllText(metadataPath, metadataString);
StorageUtilities.UpdateSetCompleted(TableClient, options.SetKey);
}
private static WorkEntity TrackWorkStarted(DateTime startTime, SlicingOptions slicingOptions)
{
var workTrackingEntity = new WorkEntity(slicingOptions.CloudResultPath, slicingOptions.CloudResultContainer, slicingOptions.TextureTile.X, slicingOptions.TextureTile.Y)
{
StartTime = startTime,
};
StorageUtilities.InsertWorkStartedMetadata(TableClient, workTrackingEntity);
return(workTrackingEntity);
}
private async Task VerifySourceDataAsync(SlicingOptions slicingOptions, CancellationToken cancellationToken)
{
if (!File.Exists(slicingOptions.Obj))
{
await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Obj, slicingOptions.CloudObjPath, cancellationToken).ConfigureAwait(false);
}
if (!File.Exists(slicingOptions.Texture))
{
await StorageUtilities.DownloadBlobAsync(BlobClient, slicingOptions.Texture, slicingOptions.CloudTexturePath, cancellationToken).ConfigureAwait(false);
}
}
private void UploadResultData(SlicingOptions slicingOptions)
{
var files = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories);
foreach (var file in files)
{
StorageUtilities.UploadBlob(
BlobClient,
file,
Path.Combine(slicingOptions.CloudResultPath, file.Replace(outputPath, string.Empty).TrimStart(new char[] { '\\', '/' })).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar),
slicingOptions.CloudResultContainer);
File.Delete(file);
}
}
private Texture GetTestTexture()
{
var options = new SlicingOptions
{
GenerateObj = true,
Texture = "texture.jpg",
TextureScale = 1,
TextureSliceX = 2,
TextureSliceY = 2,
ForceCubicalCubes = false,
Obj = "model.obj",
CubeGrid = new Vector3(2, 2, 2)
};
CubeManager manager = new CubeManager(options);
return(new Texture(manager.ObjInstance));
}
private CubeManager GetLoadedManager()
{
var options = new SlicingOptions
{
GenerateObj = true,
Texture = "texture.jpg",
TextureScale = 1,
TextureSliceX = 2,
TextureSliceY = 2,
ForceCubicalCubes = false,
Obj = "model.obj",
CubeGrid = new Vector3(2, 2, 2)
};
CubeManager manager = new CubeManager(options);
return(manager);
}
// Note: This will probably only work well with OBJ files generated by Pix4D
// as I have only supported the subset of data types it outputs.
static void Main(string[] args)
{
Options opt;
// Setup a timer for all operations
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
try
{
opt = CliParser.Parse <Options>(args);
foreach (string path in opt.Input)
{
// Check if we are processing an image or a mesh
if (Path.GetExtension(path).ToUpper().EndsWith("JPG"))
{
ImageTile tiler = new ImageTile(path, opt.XSize, opt.YSize);
tiler.GenerateTiles(opt.OutputPath);
}
else if (Path.GetExtension(path).ToUpper().EndsWith("OBJ"))
{
// Generate subfolders named after input file
// if multiple input files are provided
string outputPath;
if (opt.Input.Count == 1)
{
outputPath = opt.OutputPath;
}
else
{
outputPath = Path.Combine(opt.OutputPath, Path.GetFileNameWithoutExtension(path));
}
if (opt.ForceCubical)
{
int longestGridSide = Math.Max(Math.Max(opt.XSize, opt.YSize), opt.ZSize);
opt.XSize = opt.YSize = opt.ZSize = longestGridSide;
Console.WriteLine("Due to -ForceCubical grid size is now {0},{0},{0}", longestGridSide);
}
var options = new SlicingOptions
{
OverrideMtl = opt.MtlOverride,
GenerateEbo = opt.Ebo,
GenerateOpenCtm = opt.OpenCtm,
Debug = opt.Debug,
GenerateObj = opt.Obj,
Texture = opt.Texture,
Obj = path,
WriteMtl = opt.WriteMtl,
TextureScale = opt.ScaleTexture,
TextureSliceX = opt.TextureXSize,
TextureSliceY = opt.TextureYSize,
ForceCubicalCubes = opt.ForceCubical,
CubeGrid = new Vector3 {
X = opt.XSize, Y = opt.YSize, Z = opt.ZSize
}
};
CubeManager manager = new CubeManager(options);
if (opt.MarkupUV)
{
Texture tex = new Texture(manager.ObjInstance);
tex.MarkupTextureFaces(opt.Texture);
}
else
{
manager.GenerateCubes(outputPath, options);
}
}
else
{
Console.WriteLine("PyriteCli only accepts .jpg and .obj files for input.");
}
}
}
catch (ParserExit)
{
return;
}
catch (ParseException ex)
{
Console.WriteLine("usage: PyriteCli --help\n" + ex.ToString());
}
stopwatch.Stop();
Trace.TraceInformation(stopwatch.Elapsed.ToString());
}
// Entry point
public static int Main(string[] args, ISimpleLineWriter output)
{
var start = DateTime.Now;
var slicingOptions = new SlicingOptions(args);
// we use it to parse the clousot options, and to split the work
var clousot = new NewCCI2Driver(slicingOptions.remainingArgs.ToArray(), output);
if (!clousot.CheckOptions())
{
return(-1);
}
var errorCode = 0;
WorkerPool workerPool = null;
IQueue queue = null;
IWorkerFactory localWorkerFactory = null;
List <IWorkerFactory> remoteWorkerFactories = null;
Func <SliceDefinition, ISliceWorkResult> workOnSlice = null;
Dictionary <ISliceId, int> failedRegressions = null;
var localWorkers = slicingOptions.LocalWorkers;
var remoteWorkers = slicingOptions.RemoteWorkers;
var workers = localWorkers + remoteWorkers;
// If we have workers, we create a Worker factory
if (workers > 0)
{
IDB db;
if (remoteWorkers > 0 || clousot.options.useCache)
{
// Use Clousot Database
db = new StdDB(clousot.options);
}
else
{
// In-memory database
db = new MemorySingletonDB();
}
workerPool = new WorkerPool("Worker", canCancel: false);
queue = new FIFOQueue(workerPool, db);
// When a job is done, it choses which jobs to put in the queue (e.g., analyze the dependencies)
var scheduler =
clousot.options.InferObjectInvariantsOnlyForReadonlyFields?
new LazySchedulerForObjectInvariants(queue, db) : // use LazySchedulerForObjectInvariants for global fixpoint computation including object invariants
new LazyScheduler(queue, db); // use LazyScheduler for the global fixpoint computation
//var scheduler = new NoloopScheduler(queue);
// Usual cache
var clousotDB = slicingOptions.useDB ? db : null;
var argsForWorker = clousot.argsForWorker.ToFList();
if (localWorkers > 0)
{
if (slicingOptions.cci1)
{
localWorkerFactory = new Clousot1WorkerFactory(scheduler, argsForWorker, output, clousotDB);
}
else
{
localWorkerFactory = new Clousot2WorkerFactory(scheduler, argsForWorker, output, clousotDB);
}
// TODO: use a lighter version of ClousotMain since options are already checked here
}
if (remoteWorkers > 0) // so far 1 factory per remote worker but we can do better
{
// TODO: specifiy, for each address the number of workers
// We have a list, because we can have several addresses
remoteWorkerFactories = slicingOptions.serviceAddress.Select(addr => new Clousot2SWorkerFactory(scheduler, argsForWorker, output, addr)).ToList <IWorkerFactory>();
}
if (clousot.options.IsRegression)
{
failedRegressions = new Dictionary <ISliceId, int>();
}
// fail if any work fails
scheduler.OnWorkDone += (sliceId, returnCode) =>
{
if (errorCode >= 0)
{
if (returnCode < 0) // special error code, keep only one
{
errorCode = returnCode;
}
else
{
int prevValue;
if (clousot.options.IsRegression)
{
lock (failedRegressions)
{
Contract.Assume(failedRegressions != null);
if (failedRegressions.TryGetValue(sliceId, out prevValue))
{
output.WriteLine("[Regression] We already analyzed {0} with outcome {1}. Now we update the outcome to {2}", sliceId.Dll, prevValue, returnCode);
}
failedRegressions[sliceId] = returnCode;
}
}
errorCode += returnCode; // regression error count, additive
}
}
};
// What we do for each slice. Two things:
// 1. Register the slice in the db
// 2. Add to the queue (via the scheduler, who decides how to do it)
workOnSlice = sliceDef =>
{
var sliceId = db.RegisterSlice(sliceDef);
scheduler.FeedQueue(new ISliceId[] { sliceId });
return(null);
};
}
ISlicerResult slicerResult = null;
if (slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time: {0}", DateTime.Now - start);
}
if (workerPool != null)
{
if (localWorkerFactory != null)
{
for (var i = 0; i < localWorkers; i++)
{
workerPool.CreateWorker(localWorkerFactory);
}
}
if (remoteWorkerFactories != null)
{
foreach (var factory in remoteWorkerFactories)
{
workerPool.CreateWorker(factory);
}
}
}
if (!slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time and thread creation time : {0}", DateTime.Now - start);
}
if (workerPool != null)
{
// workerPool != null ==> queue != null
Contract.Assume(queue != null);
workerPool.WaitAllAnd(queue.EmptyQueueWaitHandle);
// Something else can arrive at the queue, so we want to stop all of them
workerPool.StopAll();
}
if (slicerResult != null)
{
var errors = slicerResult.GetErrors();
if (errors.Any())
{
foreach (var errMessage in errors)
{
output.WriteLine(errMessage);
}
errorCode = errors.Count();
}
}
output.WriteLine("Total analysis time: {0}", DateTime.Now - start);
var returnValue = errorCode;
if (failedRegressions != null && clousot.options.IsRegression && errorCode >= 0)
{
returnValue = failedRegressions.Where(pair => pair.Value != 0).Select(pair => Math.Abs(pair.Value)).Sum();
}
#if DEBUG
if (clousot.options.IsRegression)
{
Console.WriteLine("[Regression] Returned value {0}", returnValue);
}
#endif
return(returnValue);
}
// TODO: make sure that for the tilted prints no brim or skirt are printed
public string Slice(string stlFilePath, string configFile, SlicingOptions options = null)
{
throw new System.NotImplementedException();
}
// Entry point
public static int Main(string[] args, ISimpleLineWriter output)
{
var start = DateTime.Now;
var slicingOptions = new SlicingOptions(args);
// we use it to parse the clousot options, and to split the work
var clousot = new NewCCI2Driver(slicingOptions.remainingArgs.ToArray(), output);
if (!clousot.CheckOptions())
{
return -1;
}
var errorCode = 0;
WorkerPool workerPool = null;
IQueue queue = null;
IWorkerFactory localWorkerFactory = null;
List<IWorkerFactory> remoteWorkerFactories = null;
Func<SliceDefinition, ISliceWorkResult> workOnSlice = null;
Dictionary<ISliceId, int> failedRegressions = null;
var localWorkers = slicingOptions.LocalWorkers;
var remoteWorkers = slicingOptions.RemoteWorkers;
var workers = localWorkers + remoteWorkers;
// If we have workers, we create a Worker factory
if (workers > 0)
{
IDB db;
if (remoteWorkers > 0 || clousot.options.useCache)
{
// Use Clousot Database
db = new StdDB(clousot.options);
}
else
{
// In-memory database
db = new MemorySingletonDB();
}
workerPool = new WorkerPool("Worker", canCancel: false);
queue = new FIFOQueue(workerPool, db);
// When a job is done, it choses which jobs to put in the queue (e.g., analyze the dependencies)
var scheduler =
clousot.options.InferObjectInvariantsOnlyForReadonlyFields?
new LazySchedulerForObjectInvariants(queue, db) : // use LazySchedulerForObjectInvariants for global fixpoint computation including object invariants
new LazyScheduler(queue, db); // use LazyScheduler for the global fixpoint computation
//var scheduler = new NoloopScheduler(queue);
// Usual cache
var clousotDB = slicingOptions.useDB ? db : null;
var argsForWorker = clousot.argsForWorker.ToFList();
if (localWorkers > 0)
{
if (slicingOptions.cci1)
localWorkerFactory = new Clousot1WorkerFactory(scheduler, argsForWorker, output, clousotDB);
else
localWorkerFactory = new Clousot2WorkerFactory(scheduler, argsForWorker, output, clousotDB);
// TODO: use a lighter version of ClousotMain since options are already checked here
}
if (remoteWorkers > 0) // so far 1 factory per remote worker but we can do better
{
// TODO: specifiy, for each address the number of workers
// We have a list, because we can have several addresses
remoteWorkerFactories = slicingOptions.serviceAddress.Select(addr => new Clousot2SWorkerFactory(scheduler, argsForWorker, output, addr)).ToList<IWorkerFactory>();
}
if (clousot.options.IsRegression)
{
failedRegressions = new Dictionary<ISliceId, int>();
}
// fail if any work fails
scheduler.OnWorkDone += (sliceId, returnCode) =>
{
if (errorCode >= 0)
{
if (returnCode < 0) // special error code, keep only one
{
errorCode = returnCode;
}
else
{
int prevValue;
if (clousot.options.IsRegression)
{
lock (failedRegressions)
{
Contract.Assume(failedRegressions != null);
if (failedRegressions.TryGetValue(sliceId, out prevValue))
{
output.WriteLine("[Regression] We already analyzed {0} with outcome {1}. Now we update the outcome to {2}", sliceId.Dll, prevValue, returnCode);
}
failedRegressions[sliceId] = returnCode;
}
}
errorCode += returnCode; // regression error count, additive
}
}
};
// What we do for each slice. Two things:
// 1. Register the slice in the db
// 2. Add to the queue (via the scheduler, who decides how to do it)
workOnSlice = sliceDef =>
{
var sliceId = db.RegisterSlice(sliceDef);
scheduler.FeedQueue(new ISliceId[] { sliceId });
return null;
};
}
ISlicerResult slicerResult = null;
if (slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time: {0}", DateTime.Now - start);
}
if (workerPool != null)
{
if (localWorkerFactory != null)
for (var i = 0; i < localWorkers; i++)
workerPool.CreateWorker(localWorkerFactory);
if (remoteWorkerFactories != null)
foreach (var factory in remoteWorkerFactories)
workerPool.CreateWorker(factory);
}
if (!slicingOptions.sliceFirst)
{
slicerResult = clousot.SplitWork(workOnSlice);
output.WriteLine("Slicing time and thread creation time : {0}", DateTime.Now - start);
}
if (workerPool != null)
{
// workerPool != null ==> queue != null
Contract.Assume(queue != null);
workerPool.WaitAllAnd(queue.EmptyQueueWaitHandle);
// Something else can arrive at the queue, so we want to stop all of them
workerPool.StopAll();
}
if (slicerResult != null)
{
var errors = slicerResult.GetErrors();
if (errors.Any())
{
foreach (var errMessage in errors)
output.WriteLine(errMessage);
errorCode = errors.Count();
}
}
output.WriteLine("Total analysis time: {0}", DateTime.Now - start);
var returnValue = errorCode;
if (failedRegressions != null && clousot.options.IsRegression && errorCode >= 0)
{
returnValue = failedRegressions.Where(pair => pair.Value != 0).Select(pair => Math.Abs(pair.Value)).Sum();
}
#if DEBUG
if (clousot.options.IsRegression)
{
Console.WriteLine("[Regression] Returned value {0}", returnValue);
}
#endif
return returnValue;
}
public async Task DoWorkAsync(CancellationToken cancellationToken)
{
CloudQueueMessage retrievedMessage;
DateTime startTime;
try
{
// Get the next message
retrievedMessage = await WorkQueue.GetMessageAsync(TimeSpan.FromHours(5), null, null, cancellationToken).ConfigureAwait(false);
if (retrievedMessage == null)
{
return;
}
startTime = DateTime.UtcNow;
}
catch
{
return;
}
var messageContents = retrievedMessage.AsString;
SlicingOptions slicingOptions = null;
try
{
// Get the message
slicingOptions = JsonConvert.DeserializeObject <SlicingOptions>(messageContents);
// Make some fresh directories
CreateDirectories(slicingOptions);
// Populate input file locations
slicingOptions.Obj = Path.Combine(inputPath, slicingOptions.Obj);
slicingOptions.Texture = Path.Combine(inputPath, slicingOptions.Texture);
// Track work started
var workTrackingEntity = TrackWorkStarted(startTime, slicingOptions);
// ** Prep
Trace.TraceInformation("Syncing data");
await VerifySourceDataAsync(slicingOptions, cancellationToken);
// ** Run
Trace.TraceInformation("Starting Processing");
CubeManager manager = new CubeManager(slicingOptions);
if (!string.IsNullOrEmpty(slicingOptions.Texture))
{
slicingOptions.TextureInstance = new Texture(manager.ObjInstance, slicingOptions.Texture);
}
var vertexCounts = await manager.GenerateCubesForTextureTileAsync(outputPath, slicingOptions.TextureTile, slicingOptions, cancellationToken).ConfigureAwait(false);
// Track work completed
TrackWorkCompleted(workTrackingEntity, vertexCounts);
// ** Check if set is complete
CheckForComplete(slicingOptions, manager);
// ** Cleanup
slicingOptions.TextureInstance?.Dispose();
Trace.TraceInformation("Writing Results");
UploadResultData(slicingOptions);
WorkQueue.DeleteMessage(retrievedMessage);
}
catch (Exception ex)
{
Trace.TraceError(ex.ToString());
// Release texure if we have one
slicingOptions?.TextureInstance?.Dispose();
// Either delete this message or make it visible again for retry
if (retrievedMessage.DequeueCount > 3)
{
if (slicingOptions != null)
{
Trace.TraceError($"Maximum Dequeue count hit for message \"{retrievedMessage.AsString}\". Failing set {slicingOptions.SetKey}");
StorageUtilities.UpdateSetFailed(TableClient, slicingOptions.SetKey, ex.ToString());
}
WorkQueue.DeleteMessage(retrievedMessage);
}
else
{
WorkQueue.UpdateMessage(retrievedMessage, TimeSpan.FromSeconds(10), MessageUpdateFields.Visibility);
}
}
finally
{
if (Directory.Exists(outputPath))
{
Directory.Delete(outputPath, true);
}
if (Directory.Exists(inputPath))
{
Directory.Delete(inputPath, true);
}
}
}
static void Main(string[] args)
{
// Get storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(Settings.Default.StorageConnectionString);
// Create the clients
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
BlobClient = storageAccount.CreateCloudBlobClient();
TableClient = storageAccount.CreateCloudTableClient();
// Retrieve a reference to a queue
WorkQueue = queueClient.GetQueueReference(Settings.Default.Queue);
// Create the queue if it doesn't already exist
WorkQueue.CreateIfNotExists();
CloudOptions opt;
try
{
opt = CliParser.Parse <CloudOptions>(args);
if (opt.ForceCubical)
{
int longestGridSide = Math.Max(Math.Max(opt.XSize, opt.YSize), opt.ZSize);
opt.XSize = opt.YSize = opt.ZSize = longestGridSide;
Console.WriteLine("Due to -ForceCubical grid size is now {0},{0},{0}", longestGridSide);
}
var options = new SlicingOptions
{
OverrideMtl = opt.MtlOverride,
GenerateEbo = opt.Ebo,
GenerateOpenCtm = opt.OpenCtm,
Debug = opt.Debug,
GenerateObj = opt.Obj,
Texture = Path.GetFileName(opt.Texture),
Obj = Path.GetFileName(opt.Input.First()),
WriteMtl = opt.WriteMtl,
TextureScale = opt.ScaleTexture,
TextureSliceX = opt.TextureXSize,
TextureSliceY = opt.TextureYSize,
ForceCubicalCubes = opt.ForceCubical,
CubeGrid = new Vector3 {
X = opt.XSize, Y = opt.YSize, Z = opt.ZSize
}
};
string objPath;
if (opt.Input.First().StartsWith("http"))
{
objPath = opt.Input.First();
}
else
{
objPath = StorageUtilities.UploadBlob(BlobClient, opt.Input.First(), Guid.NewGuid().ToString(), "processingdata");
}
string texPath;
if (opt.Texture.StartsWith("http"))
{
texPath = opt.Texture;
}
else
{
texPath = StorageUtilities.UploadBlob(BlobClient, opt.Texture, Guid.NewGuid().ToString(), "processingdata");
}
options.CloudObjPath = objPath;
options.CloudTexturePath = texPath;
options.CloudResultContainer = opt.OutputContainer;
options.CloudResultPath = opt.OutputPath;
// Get texture set size
Vector2 setSize;
if (!string.IsNullOrEmpty(options.Texture) && (options.TextureSliceX + options.TextureSliceY) > 2)
{
setSize = new Vector2(options.TextureSliceX, options.TextureSliceY);
}
else
{
setSize = new Vector2(1, 1);
}
// Queue work
var setEntity = new SetEntity("Set", DateTime.UtcNow)
{
ResultPath = options.CloudResultPath,
ResultContainer = options.CloudResultContainer,
TextureTilesX = setSize.X,
TextureTilesY = setSize.Y
};
options.SetKey = setEntity.RowKey;
StorageUtilities.InsertSetMetadata(TableClient, setEntity);
SpatialUtilities.EnumerateSpace(setSize, (x, y) =>
{
options.TextureTile = new Vector2(x, y);
string message = JsonConvert.SerializeObject(options);
WorkQueue.AddMessage(new CloudQueueMessage(message));
});
}
catch (ParserExit)
{
return;
}
catch (ParseException)
{
Console.WriteLine("usage: PyriteCli --help");
}
}
