public async Task <CallbackModelWrapper> PopulateImagesAsync(CallbackModelWrapper model)
{
Init(model);
var offset_origin_image = model.offset_origin_image;
var offset_origin_label = model.offset_origin_label;
List <Task> tasks = new List <Task>();
Action <CallbackModelWrapper, CancellationTokenSource> Populate_AddImageAsyncFunc = (x, y) => { tasks.Add(Populate_AddImageAsync(x, y)); };
var cts = new CancellationTokenSource();
Task result = null;
try
{
//the cancellation token will cause maybe 10003 to 10005 to cast taskcancellationexception
//what happens to 10001 and 10002? they might enter into the code block at the same time
//so the cancellation from 10001 might not affect 10002
//if I specify a index 10005 here, the cancellation will yield 3 times TaskCancellation exception here, following code block is not used, since we don't really know how many images are saved within the file, use while(true) instead
//Enumerable.Range(0, 10005).ToList().ForEach((x) =>
//{
// Populate_AddImageAsyncFunc(model, cts);
//});
//after changing to why true, no TaskCanceledException is thrown simply because no new task will
//spin up when the cacellation is invoked, the ones that got canceled at the same time, they just
//do a peaceful return
while (!cts.IsCancellationRequested)
{
Populate_AddImageAsyncFunc(model, cts);
}
result = Task.WhenAll(tasks);
await result;
}
catch (TaskCanceledException tce)
{
Console.WriteLine("is canceld");
}
catch (Exception e)
{
Console.WriteLine("Task IsFaulted: " + result.IsFaulted);
foreach (var inEx in result.Exception.InnerExceptions)
{
Console.WriteLine("Task Inner Exception: " + inEx.Message);
}
}
cts = null;
return(model);
}
public async Task ImageLoader_Test()
{
var imagePath = @"..\..\Files\t10k-images.idx3-ubyte";
var labelPath = @"..\..\Files\t10k-labels.idx1-ubyte";
var loader = new ImageLoader();
var tmp = new CallbackModelWrapper(imagePath, labelPath);
var model = loader.PopulateImages(tmp);
var tmp2 = new CallbackModelWrapper(imagePath, labelPath);
var model2 = await loader.PopulateImagesAsync(tmp2);
foreach (var image in model.Images)
{
var image2 = model2.Images.FirstOrDefault(t => t.orderInCollection == image.orderInCollection);
Assert.IsNotNull(image2, "sync image can't be found from async collection");
//assert label
Assert.IsTrue(image.label.Equals(image2.label));
//assert pixel data
Enumerable.Range(0, image.pixels.Length - 1).ToList().ForEach(index =>
{
//await Task.Delay(20);
Enumerable.Range(0, image.pixels[index].Length - 1).ToList().ForEach(index1 =>
{
Assert.IsTrue(image.pixels[index][index1].Equals(image2.pixels[index][index1]), string.Format("byte is different, index {0}/{1} contains different data => sync: {2} - async: {3}, image label => {4}", index, index1, image.pixels[index][index1], image2.pixels[index][index1], image.label));
});
});
}
}
public CallbackModelWrapper PopulateImages(CallbackModelWrapper model)
{
Init(model);
var pixels = model.GenerateNewPixelMatrices();
int amount = 0;
try
{
while (true)
{
for (int i = 0; i < 28; ++i)
{
for (int j = 0; j < 28; ++j)
{
byte b = brImages.ReadByte();
pixels[i][j] = b;
}
}
byte lbl = brLabels.ReadByte();
var image = new DigitImage(pixels, lbl, ++amount);
model.Images.Add(image);
}
}
catch (Exception e)
{
//will get exception here
//BinaryReader cannot read past end of file error
Console.Write(e.ToString());
}
brImages.Close();
brLabels.Close();
return(model);
}
private void Init(CallbackModelWrapper model)
{
ifsImages = new FileStream(model.imagePath,
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true);
ifsLabels = new FileStream(model.labelPath,
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true);
brLabels = new BinaryReader(ifsLabels);
brImages = new BinaryReader(ifsImages);
int magic1 = brImages.ReadInt32(); // discard
model.numImages = brImages.ReadInt32();
model.numRows = brImages.ReadInt32();
model.numCols = brImages.ReadInt32();
int magic2 = brLabels.ReadInt32();
model.numLabels = brLabels.ReadInt32();
model.offset_origin_image = Convert.ToInt32(ifsImages.Position);
model.offset_origin_label = Convert.ToInt32(ifsLabels.Position);
}
static void Main(string[] args)
{
try
{
var loader = new ImageLoader();
var imagePath = @"..\..\Files\t10k-images.idx3-ubyte";
var labelPath = @"..\..\Files\t10k-labels.idx1-ubyte";
var model = new CallbackModelWrapper(imagePath, labelPath);
Func <CallbackModelWrapper, Task <CallbackModelWrapper> > coreFunc = (x) =>
{
return(loader.PopulateImagesAsync(x));
};
Action finallyAction = () => { };
model = GenericCallbacks.ExecuteSafe(coreFunc, model, FinallyFun: finallyAction).Result;
foreach (var image in model.Images)
{
Console.WriteLine(image.ToString());
Console.ReadLine();
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
Console.WriteLine("\nEnd\n");
Console.ReadLine();
}
public async Task Sync_Async_SameResult_Test_0()
{
//sync
var timer = new Stopwatch();
timer.Start();
long t1 = 0;
long t2 = 0;
const int amount = 10000;
//sync
model = PopulateImages(model, amount);
timer.Stop();
t1 = timer.ElapsedMilliseconds;
//async
timer.Reset();
timer.Restart();
model2 = await PopulateImagesAsync(model2, amount);
timer.Stop();
t2 = timer.ElapsedMilliseconds;
foreach (var image in model.Images)
{
var image2 = model2.Images.FirstOrDefault(t => t.orderInCollection == image.orderInCollection);
Assert.IsNotNull(image2, "sync image can't be found from async collection");
//assert label
Assert.IsTrue(image.label.Equals(image2.label));
//assert pixel data
Enumerable.Range(0, image.pixels.Length - 1).ToList().ForEach(index =>
{
//await Task.Delay(20);
Enumerable.Range(0, image.pixels[index].Length - 1).ToList().ForEach(index1 =>
{
Assert.IsTrue(image.pixels[index][index1].Equals(image2.pixels[index][index1]), string.Format("byte is different, index {0}/{1} contains different data => sync: {2} - async: {3}, image label => {4}", index, index1, image.pixels[index][index1], image2.pixels[index][index1], image.label));
});
});
}
//really 10 times slower
//Assert.IsTrue(t1 < t2);
}
private async Task <CallbackModelWrapper> PopulateImagesAsync(CallbackModelWrapper model, int amount)
{
var offset_origin_image = model.offset_origin_image;
var offset_origin_label = model.offset_origin_label;
var tasks_image = Enumerable.Range(0, amount).Select(index => ReadOneImageAsync(model2, 28, offset_origin_image + index * 28 * 28, offset_origin_label + index, index + 1));
var result = await Task.WhenAll(tasks_image);
foreach (var image in result)
{
model2.Images.Add(image);
}
return(model);
}
public void Init()
{
var imagePath = @"..\..\Files\t10k-images.idx3-ubyte";
var labelPath = @"..\..\Files\t10k-labels.idx1-ubyte";
model = new CallbackModelWrapper(imagePath, labelPath);
model2 = new CallbackModelWrapper(imagePath, labelPath);
ifsImages = new FileStream(imagePath,
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true);
ifsLabels = new FileStream(labelPath,
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true);
brLabels = new BinaryReader(ifsLabels);
brImages = new BinaryReader(ifsImages);
int magic1 = brImages.ReadInt32(); // discard
int numImages = brImages.ReadInt32();
int numRows = brImages.ReadInt32();
int numCols = brImages.ReadInt32();
int magic2 = brLabels.ReadInt32();
int numLabels = brLabels.ReadInt32();
var offset_origin_image = Convert.ToInt32(ifsImages.Position);
var offset_origin_label = Convert.ToInt32(ifsLabels.Position);
/////////////////////
model.numImages = numImages;
model.numRows = numRows;
model.numCols = numCols;
model.numLabels = numLabels;
model.offset_origin_image = offset_origin_image;
model.offset_origin_label = offset_origin_label;
///////////
model2.numImages = numImages;
model2.numRows = numRows;
model2.numCols = numCols;
model2.numLabels = numLabels;
model2.offset_origin_image = offset_origin_image;
model2.offset_origin_label = offset_origin_label;
}
private async Task <DigitImage> ReadOneImageAsync(CallbackModelWrapper model, int imageSize, int offset_image, int offset_label, int orderInCollection)
{
int bufferSize = imageSize * imageSize;
byte[] image_bytes = new byte[bufferSize];
byte[] label_bytes = new byte[1];
using (FileStream fsi = new FileStream(@"..\..\Files\t10k-images.idx3-ubyte",
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true))
using (FileStream fsl = new FileStream(@"..\..\Files\t10k-labels.idx1-ubyte",
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true))
{
//lock (fsi)
//{
// fsi.copy
//}
fsi.Seek(offset_image, SeekOrigin.Begin);
fsl.Seek(offset_label, SeekOrigin.Begin);
var tmp_label = fsl.Read(label_bytes, 0, 1);
var tmp_image = await fsi.ReadAsync(image_bytes, 0, bufferSize);
//var tmp_image = fsi.Read(image_bytes, 0, bufferSize);
}
var pixels = model.GenerateNewPixelMatrices();
for (int i = 0; i < imageSize; ++i)
{
for (int j = 0; j < imageSize; ++j)
{
pixels[i][j] = image_bytes[i * imageSize + j];
}
}
DigitImage image = new DigitImage(pixels, label_bytes[0], orderInCollection);
return(image);
}
private CallbackModelWrapper PopulateImages(CallbackModelWrapper model, int amount)
{
var pixels = model.GenerateNewPixelMatrices();
for (int di = 0; di < amount; ++di)
{
for (int i = 0; i < 28; ++i)
{
for (int j = 0; j < 28; ++j)
{
byte b = brImages.ReadByte();
pixels[i][j] = b;
}
}
byte lbl = brLabels.ReadByte();
var image = new DigitImage(pixels, lbl, di + 1);
model.Images.Add(image);
}
brImages.Close();
brLabels.Close();
return(model);
}
private async Task Populate_AddImageAsync(CallbackModelWrapper model, CancellationTokenSource cts)
{
var index = 0;
lock (model)
{
index = model.index++;
}
int bufferSize = model.ImageSize * model.ImageSize;
var offset_image = model.offset_origin_image + index * bufferSize;
var offset_label = model.offset_origin_label + index;
var orderInCollection = index + 1;
byte[] image_bytes = new byte[bufferSize];
byte[] label_bytes = new byte[1];
//
using (FileStream fsi = new FileStream(model.imagePath,
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true))
using (FileStream fsl = new FileStream(model.labelPath,
FileMode.Open, FileAccess.Read, FileShare.Read,
bufferSize: 4096, useAsync: true))
{
var tmp_label = 0;
var tmp_image = 0;
try
{
fsi.Seek(offset_image, SeekOrigin.Begin);
fsl.Seek(offset_label, SeekOrigin.Begin);
tmp_label = fsl.Read(label_bytes, 0, 1);
tmp_image = await fsi.ReadAsync(image_bytes, 0, bufferSize, cts.Token);
//var tmp_image = fsi.Read(image_bytes, 0, bufferSize);
}
catch (TaskCanceledException e)
{
throw e;
}
//throw exception here, no byte can be read
if (tmp_label == 0 || tmp_image == 0)
{
cts.Cancel();
return;
//cancellation token only yields exception for later threads
//to prevent the current thread continue, either throw an exception or return
//the behavior observed is that the taskcancellation exception can't be caught if
//I throw an exception here, so just do a return
//throw new Exception("reach the end of stream, use token to cancel all tasks");
}
}
//var t = System.Threading.Thread.CurrentThread.ManagedThreadId;
var pixels = model.GenerateNewPixelMatrices();
for (int i = 0; i < model.ImageSize; ++i)
{
for (int j = 0; j < model.ImageSize; ++j)
{
pixels[i][j] = image_bytes[i * model.ImageSize + j];
}
}
DigitImage image = new DigitImage(pixels, label_bytes[0], orderInCollection);
model.Images.Add(image);
}