public void Execute(AutoResetEvent block)
{
const int filesToCreate = 10000;
var filesProcessed = 0;
// This setup will emit files on another thread
new ObservingProducer <int>()
.Consumes(new FileConsumer <int>())
.Produces(Observable.Range(1, filesToCreate))
.Start();
// This setup will output the contents of loaded files to the console
var producer = new FileProducer <int>();
var logger = new DelegatingConsumer <int>(i =>
{
filesProcessed++;
Console.WriteLine(filesProcessed);
if (filesProcessed >= filesToCreate)
{
block.Set();
}
});
producer.Attach(logger);
producer.Start();
}
/// <summary>Load all tables from the given directory.</summary>
public void LoadFromDirectory(string directory)
{
DirectoryTable.LoadFromFile(directory, s_directoryTableFileName);
FileTable.LoadFromFile(directory, s_fileTableFileName);
PathTable.LoadFromFile(directory, s_pathTableFileName);
PipTable.LoadFromFile(directory, s_pipTableFileName);
PipExecutionTable.LoadFromFile(directory, s_pipExecutionTableFileName);
ProcessExecutionTable.LoadFromFile(directory, s_processExecutionTableFileName);
ProcessPipExecutionTable.LoadFromFile(directory, s_processPipExecutionTableFileName);
StringTable.LoadFromFile(directory, s_stringTableFileName);
WorkerTable.LoadFromFile(directory, s_workerTableFileName);
ConstructRelationTables();
if (File.Exists(Path.Combine(directory, s_fileProducerFileName)))
{
FileProducer.LoadFromFile(directory, s_fileProducerFileName);
}
loadRelationTableIfExists(directory, s_consumedFilesFileName, ConsumedFiles);
loadRelationTableIfExists(directory, s_declaredInputDirectoriesFileName, DeclaredInputDirectories);
loadRelationTableIfExists(directory, s_declaredInputFilesFileName, DeclaredInputFiles);
loadRelationTableIfExists(directory, s_directoryContentsFileName, DirectoryContents);
loadRelationTableIfExists(directory, s_pipDependenciesFileName, PipDependencies);
void loadRelationTableIfExists <TFromId, TToId>(string directory, string fileName, RelationTable <TFromId, TToId> relation)
where TFromId : unmanaged, Id <TFromId>
where TToId : unmanaged, Id <TToId>
{
if (File.Exists(Path.Combine(directory, fileName)))
{
relation.LoadFromFile(directory, fileName);
}
}
}
private void Capture(int num)
{
string imageName = _camera.name + "_" + num;
Dictionary <string, int> labelInstances = new Dictionary <string, int>();
// Call Screen Capture
var screen = CaptureCamera.Capture(_camera, request =>
{
string path = screenCapturePath + "/" + imageName + ".jpg";
// Convert the screen capture to a byte array
Array image = CaptureImageEncoder.Encode(request.data.colorBuffer as Array, 640, 480, GraphicsFormat.R8G8B8A8_UNorm,
CaptureImageEncoder.ImageFormat.Jpg, true);
// Write the screen capture to a file
var result = FileProducer.Write(path, image);
// Wait for Async screen capture request to return and then log data point
if (result)
{
labelInstances.Add("Cube", 100);
labelInstances.Add("Sphere", 111);
labelInstances.Add("Cylinder", 131);
string temp = JsonConvert.SerializeObject(labelInstances);
InstanceCount instanceCount = new InstanceCount(imageName, temp);
// Log data point to file
dataLogger.Log(instanceCount);
return(AsyncRequest.Result.Completed);
}
return(AsyncRequest.Result.Error);
});
}
private void FlushCountersToDisk()
{
Log.I("Flushing counters to disk");
string json = null;
Metadata metadata;
CountersData counters;
lock (_mutex)
{
metadata = new Metadata(
Configuration.Instance.GetInstanceId(),
Configuration.Instance.GetAttemptId(),
AddMetaData?.Invoke()
);
counters = new CountersData(_counters, metadata);
}
json = JsonConvert.SerializeObject(counters, new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore
});
Log.I("Writing the GameSim Counters files..");
if (json != null)
{
var filename = "counters_" + _countersSequence + ".json";
FileProducer.Write(Path.Combine(Manager.Instance.GetDirectoryFor("GameSim"), filename), Encoding.ASCII.GetBytes(json));
_countersSequence++;
}
}
internal void ResetAndFlushCounterToDisk(string counter, Action <string> consumer = null, bool resetCounter = true)
{
var asyncWriteRequest = Manager.Instance.CreateRequest <AsyncRequest <String> >();
Int64 currentCount = -1;
lock (_mutex)
{
if (!_counters.ContainsKey(counter))
{
return;
}
asyncWriteRequest.data = JsonUtility.ToJson(_counters[counter]);
currentCount = _counters[counter]._count++;
if (resetCounter)
{
_counters[counter].Reset();
}
}
asyncWriteRequest.Start((request) =>
{
var filePath = Path.Combine(Manager.Instance.GetDirectoryFor("GameSim"),
counter + "_" + currentCount + ".json");
FileProducer.Write(
filePath,
Encoding.ASCII.GetBytes(request.data));
consumer?.Invoke(filePath);
return(AsyncRequest.Result.Completed);
});
}
private void Capture(int num)
{
string imageName = _camera.name + "_" + num;
// Define Data point object outside async call
DataPoint dataPoint = new DataPoint(_cube.name, _cube.transform.rotation, simElapsed, imageName);
// Call Screen Capture
var screen = CaptureCamera.Capture(_camera, request =>
{
string path = screenCapturePath + "/" + imageName + ".jpg";
// Convert the screen capture to a byte array
Array image = CaptureImageEncoder.Encode(request.data.colorBuffer as Array, 640, 480, GraphicsFormat.R8G8B8A8_UNorm,
CaptureImageEncoder.ImageFormat.Jpg, true);
// Write the screen capture to a file
var result = FileProducer.Write(path, image);
// Wait for Async screen capture request to return and then log data point
if (result)
{
// Log data point to file
dataLogger.Log(dataPoint);
return(AsyncRequest.Result.Completed);
}
return(AsyncRequest.Result.Error);
});
}
private AsyncRequestWrapper CaptureFrameWithLog(
Camera camera,
Unity.Simulation.Logger logger,
string screenCapturePath,
string frameFileNameRoot,
int frameIndex
)
{
// Construct the output file name for the image.
string frameFileBaseName = $"{frameFileNameRoot}_{frameIndex}";
string frameFilePath = $"{screenCapturePath}{Path.DirectorySeparatorChar}{frameFileBaseName}.jpg";
void LogData()
{
logger.Log(new CaptureFrameWithLogDataPoint(frameFileBaseName));
}
// Write the frame entry to the log. Write the log line outside the request callback when the
// execution context is threaded since threaded requests will be executed asynchronously.
if (IsExecutionContextThreaded())
{
LogData();
}
var req = CaptureCamera.Capture(
camera,
request =>
{
// Write the frame entry to the log. We can write the log line within the request callback when
// the execution context is *not* threaded since they will be executed sequentially.
if (!IsExecutionContextThreaded())
{
LogData();
}
// Get the color buffer data and convert it to an image file.
byte[] imgColorData = (byte[])request.data.colorBuffer;
byte[] imgFileData = (byte[])CaptureImageEncoder.EncodeArray(
imgColorData,
32,
32,
GraphicsFormat.R8G8B8A8_UNorm,
CaptureImageEncoder.ImageFormat.Jpg
);
// Attempt to write the image file to disk.
bool fileWritten = FileProducer.Write(frameFilePath, imgFileData);
return((fileWritten) ? AsyncRequest.Result.Completed : AsyncRequest.Result.Error);
},
flipY: false
);
return(new AsyncRequestWrapper(req, frameIndex));
}
static void Main(string[] args)
{
if (args.Length <= 0)
{
Console.WriteLine("usage: ./VeracityTest <input file path> <time to run (ms)> <delay between producer writes (ms)>");
Console.ReadLine();
return;
}
FileInfo inputFile = new FileInfo(args[0]);
if (!inputFile.Exists)
{
Console.WriteLine($"File {inputFile.FullName} doesn't exist!");
Console.ReadLine();
return;
}
Console.WriteLine("control+c to quit early");
int runLength = 10000;
int delay = 500;
if (args.Length > 1 && int.TryParse(args[1], out int val))
{
runLength = val;
}
if (args.Length > 2 && int.TryParse(args[2], out int val2))
{
delay = val2;
}
IDataQueue dataQueue = new DataQueue();
IProducer producer = new FileProducer(dataQueue, inputFile);
IConsumer consoleConsumer = new ConsoleConsumer(dataQueue);
IConsumer fileConsumer = new FileConsumer(dataQueue, new FileInfo(@".\output.txt"));
dataQueue.RegisterListener(ConsumerType.CONSOLE, consoleConsumer);
dataQueue.RegisterListener(ConsumerType.FILE, fileConsumer);
producer.StartProducer(delay);
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
while (stopWatch.ElapsedMilliseconds < runLength)
{
Thread.Sleep(500);
}
stopWatch.Stop();
producer.StopProducer();
}
/// <summary>Save all tables to the given directory.</summary>
public void SaveToDirectory(string directory)
{
DirectoryTable.SaveToFile(directory, s_directoryTableFileName);
FileTable.SaveToFile(directory, s_fileTableFileName);
PathTable.SaveToFile(directory, s_pathTableFileName);
PipTable.SaveToFile(directory, s_pipTableFileName);
PipExecutionTable.SaveToFile(directory, s_pipExecutionTableFileName);
ProcessExecutionTable.SaveToFile(directory, s_processExecutionTableFileName);
ProcessPipExecutionTable.SaveToFile(directory, s_processPipExecutionTableFileName);
StringTable.SaveToFile(directory, s_stringTableFileName);
WorkerTable.SaveToFile(directory, s_workerTableFileName);
FileProducer?.SaveToFile(directory, s_fileProducerFileName);
ConsumedFiles?.SaveToFile(directory, s_consumedFilesFileName);
DeclaredInputDirectories?.SaveToFile(directory, s_declaredInputDirectoriesFileName);
DeclaredInputFiles?.SaveToFile(directory, s_declaredInputFilesFileName);
DirectoryContents?.SaveToFile(directory, s_directoryContentsFileName);
PipDependencies?.SaveToFile(directory, s_pipDependenciesFileName);
}
void CaptureRgbData(Camera cam)
{
Profiler.BeginSample("CaptureDataFromLastFrame");
if (!captureRgbImages)
{
return;
}
var captureFilename = Path.Combine(Manager.Instance.GetDirectoryFor(RgbDirectory), $"{s_RgbFilePrefix}{Time.frameCount}.png");
var dxRootPath = Path.Combine(RgbDirectory, $"{s_RgbFilePrefix}{Time.frameCount}.png");
SensorHandle.ReportCapture(dxRootPath, SensorSpatialData.FromGameObjects(m_EgoMarker == null ? null : m_EgoMarker.gameObject, gameObject), m_PersistentSensorData.Select(kvp => (kvp.Key, kvp.Value)).ToArray());
Func <AsyncRequest <CaptureCamera.CaptureState>, AsyncRequest.Result> colorFunctor;
var width = cam.pixelWidth;
var height = cam.pixelHeight;
var flipY = ShouldFlipY(cam);
colorFunctor = r =>
{
using (s_WriteFrame.Auto())
{
var dataColorBuffer = (byte[])r.data.colorBuffer;
if (flipY)
{
FlipImageY(dataColorBuffer, height);
}
byte[] encodedData;
using (s_EncodeAndSave.Auto())
{
encodedData = ImageConversion.EncodeArrayToPNG(dataColorBuffer, GraphicsFormat.R8G8B8A8_UNorm, (uint)width, (uint)height);
}
return(!FileProducer.Write(captureFilename, encodedData) ? AsyncRequest.Result.Error : AsyncRequest.Result.Completed);
}
};
CaptureCamera.Capture(cam, colorFunctor);
Profiler.EndSample();
}
public IEnumerator ChunkedStream_AppendsBytesToBuffer_FlushesToFileSystem()
{
string path = Path.Combine(Configuration.Instance.GetStoragePathOther(), "log_0.txt");
if (File.Exists(path))
{
File.Delete(path);
}
ChunkedStream producer = new ChunkedStream(8, 1, functor: (AsyncRequest <object> request) =>
{
FileProducer.Write(path, request.data as Array, false);
return(AsyncRequest.Result.Completed);
});
producer.Append(Encoding.ASCII.GetBytes("Test"));
producer.Append(Encoding.ASCII.GetBytes("Unit"));
while (!System.IO.File.Exists(path))
{
yield return(null);
}
Assert.True(System.IO.File.ReadAllText(path) == "TestUnit");
}
public void Execute(AutoResetEvent block)
{
const int filesToCreate = 10000;
var filesProcessed = 0;
// This setup will emit files on another thread
new ObservingProducer<int>()
.Consumes(new FileConsumer<int>())
.Produces(Observable.Range(1, filesToCreate))
.Start();
// This setup will output the contents of loaded files to the console
var producer = new FileProducer<int>();
var logger = new DelegatingConsumer<int>(i =>
{
filesProcessed++;
Console.WriteLine(filesProcessed);
if (filesProcessed >= filesToCreate)
{
block.Set();
}
});
producer.Attach(logger);
producer.Start();
}
