/// <summary>
/// The DataTransformer constructor.
/// </summary>
/// <param name="log">Specifies the Log used for output.</param>
/// <param name="p">Specifies the TransformationParameter used to create the DataTransformer.</param>
/// <param name="phase">Specifies the Phase under which the DataTransformer is run.</param>
/// <param name="nC">Specifies the channels.</param>
/// <param name="nH">Specifies the height.</param>
/// <param name="nW">Specifies the width.</param>
/// <param name="imgMean">Optionally, specifies the image mean to use.</param>
public DataTransformer(Log log, TransformationParameter p, Phase phase, int nC, int nH, int nW, SimpleDatum imgMean = null)
{
m_log = log;
if (p.mean_file != null)
{
m_protoMean = loadProtoMean(p.mean_file);
}
int nDataSize = nC * nH * nW;
if (imgMean != null)
{
nDataSize = imgMean.Channels * imgMean.Height * imgMean.Width;
}
m_rgTransformedData = new T[nDataSize];
m_param = p;
m_phase = phase;
InitRand();
if (p.use_imagedb_mean)
{
if (m_protoMean == null)
{
m_imgMean = imgMean;
if (m_imgMean != null)
{
m_rgMeanData = m_imgMean.GetData <double>();
}
}
else
{
if (m_protoMean.data.Count > 0)
{
m_rgMeanData = new double[m_protoMean.data.Count];
Array.Copy(m_protoMean.data.ToArray(), m_rgMeanData, m_rgMeanData.Length);
}
else
{
m_rgMeanData = m_protoMean.double_data.ToArray();
}
}
}
if (p.mean_value.Count > 0)
{
m_log.CHECK(p.use_imagedb_mean == false, "Cannot specify use_image_mean and mean_value at the same time.");
for (int c = 0; c < p.mean_value.Count; c++)
{
m_rgMeanValues.Add(p.mean_value[c]);
}
}
}
/// <summary>
/// Resync the transformer with changes in its parameter.
/// </summary>
public void Update(int nDataSize = 0, SimpleDatum imgMean = null)
{
TransformationParameter p = m_param;
if (imgMean != null)
{
nDataSize = imgMean.Channels * imgMean.Height * imgMean.Width;
}
if (nDataSize > 0 || (m_rgfTransformedData != null && nDataSize != m_rgfTransformedData.Length))
{
m_rgTransformedData = new T[nDataSize];
}
if (p.mean_file != null)
{
m_protoMean = loadProtoMean(p.mean_file);
}
if (p.use_imagedb_mean)
{
if (m_protoMean == null)
{
m_imgMean = imgMean;
if (m_imgMean != null)
{
m_rgMeanData = m_imgMean.GetData <double>();
}
}
else
{
if (m_protoMean.data.Count > 0)
{
m_rgMeanData = new double[m_protoMean.data.Count];
Array.Copy(m_protoMean.data.ToArray(), m_rgMeanData, m_rgMeanData.Length);
}
else
{
m_rgMeanData = m_protoMean.double_data.ToArray();
}
}
}
if (p.mean_value.Count > 0)
{
m_log.CHECK(p.use_imagedb_mean == false, "Cannot specify use_image_mean and mean_value at the same time.");
for (int c = 0; c < p.mean_value.Count; c++)
{
m_rgMeanValues.Add(p.mean_value[c]);
}
}
}
/// <summary>
/// Return the string representation of the state.
/// </summary>
/// <returns>The string representation is returned.</returns>
public override string ToString()
{
double[] rgData = m_data.GetData <double>();
string str = "{";
for (int i = 0; i < rgData.Length && i < 5; i++)
{
str += rgData[i].ToString("N4") + ",";
}
str = str.TrimEnd(',');
str += "}";
return("State = " + str + " Done = " + m_bDone.ToString());
}
/// <summary>
/// Returns the data int a 3D form compatible with CV2.
/// </summary>
/// <param name="bGrayscale">Optionally, specifies to return gray scale data (one channel).</param>
/// <param name="dfScale">Optionally, specifies the scale to apply to each item.</param>
/// <param name="sd">Optionally, specifies the data.</param>
/// <returns>The data is returned as a multi-dimensional array.</returns>
public List <List <List <double> > > GetDataAs3D(bool bGrayscale = false, double dfScale = 1, SimpleDatum sd = null)
{
double[] rgData1 = (sd == null) ? Data.ToArray() : sd.GetData <double>();
int nChannels = (sd == null) ? m_state.Item1.Channels : sd.Channels;
int nHeight = (sd == null) ? m_state.Item1.Height : sd.Height;
int nWidth = (sd == null) ? m_state.Item1.Width : sd.Width;
List <List <List <double> > > rgrgrgData = new List <List <List <double> > >();
for (int h = 0; h < nHeight; h++)
{
List <List <double> > rgrgData = new List <List <double> >();
for (int w = 0; w < nWidth; w++)
{
List <double> rgData = new List <double>();
if (bGrayscale)
{
double dfSum = 0;
for (int c = 0; c < nChannels; c++)
{
int nIdx = (c * nHeight * nWidth) + (h * nWidth) + w;
dfSum += rgData1[nIdx];
}
rgData.Add((dfSum / nChannels) * dfScale);
}
else
{
for (int c = 0; c < nChannels; c++)
{
int nIdx = (c * nHeight * nWidth) + (h * nWidth) + w;
double dfVal = rgData1[nIdx];
rgData.Add(dfVal * dfScale);
}
}
rgrgData.Add(rgData);
}
rgrgrgData.Add(rgrgData);
}
return(rgrgrgData);
}
/// <summary>
/// Steps the gym one or more steps with a given action.
/// </summary>
/// <param name="nAction">Specifies the action to run.</param>
/// <param name="nSteps">Specifies the number of steps to run the action.</param>
/// <returns>A tuple containing a double[] with the data, a double with the reward and a bool with the terminal state is returned.</returns>
public CurrentState Step(int nAction, int nSteps = 1)
{
if (m_igym == null)
{
throw new Exception("You must call 'Initialize' first!");
}
for (int i = 0; i < nSteps - 1; i++)
{
m_igym.Step(nAction);
}
Tuple <State, double, bool> state = m_igym.Step(nAction);
bool bIsOpen = (m_nUiId >= 0) ? true : false;
Tuple <Bitmap, SimpleDatum> data = m_igym.Render(bIsOpen, 512, 512, true);
int nDataLen = 0;
SimpleDatum sd = state.Item1.GetData(false, out nDataLen);
Observation obs = new Observation(data.Item1, ImageData.GetImage(data.Item2), m_igym.RequiresDisplayImage, sd.GetData <double>(), state.Item2, state.Item3);
if (bIsOpen)
{
if (m_rgrgActionDistributions != null)
{
overlay(obs.ImageDisplay, m_rgrgActionDistributions);
}
m_gymui.Render(m_nUiId, obs);
Thread.Sleep(m_igym.UiDelay);
}
if (m_igym.SelectedDataType == DATA_TYPE.BLOB)
{
sd = data.Item2;
}
else
{
sd.Clip(nDataLen, null, nDataLen, null);
}
m_state = new Tuple <SimpleDatum, double, bool>(sd, state.Item2, state.Item3);
return(new CurrentState(m_state.Item1.GetData <double>(), m_state.Item2, m_state.Item3));
}
private SimpleDatum preprocess(SimpleDatum sd, bool bGrayscale, double dfScale)
{
if (!bGrayscale && dfScale == 1)
{
return(sd);
}
double[] rgData = sd.GetData <double>();
int nCount = sd.Height * sd.Width;
int nChannels = sd.Channels;
bool bIsReal = sd.IsRealData;
byte[] rgByteData = null;
double[] rgRealData = null;
if (bIsReal && !bGrayscale)
{
nCount *= sd.Channels;
rgRealData = new double[nCount];
}
else
{
bIsReal = false;
nChannels = 1;
rgByteData = new byte[nCount];
}
for (int h = 0; h < sd.Height; h++)
{
for (int w = 0; w < sd.Width; w++)
{
int nIdx = (h * sd.Width) + w;
int nIdxSrc = nIdx * sd.Channels;
if (rgRealData != null)
{
for (int c = 0; c < sd.Channels; c++)
{
double dfVal = rgData[nIdxSrc + c] * dfScale;
rgRealData[nIdxSrc + c] = dfVal;
}
}
else
{
double dfSum = 0;
for (int c = 0; c < sd.Channels; c++)
{
dfSum += rgData[nIdxSrc + c];
}
double dfVal = ((dfSum / sd.Channels) * dfScale);
if (dfVal > 255)
{
dfVal = 255;
}
if (dfVal < 0)
{
dfVal = 0;
}
rgByteData[nIdx] = (byte)dfVal;
}
}
}
SimpleDatum sdResult;
if (rgRealData != null)
{
sdResult = new SimpleDatum(bIsReal, nChannels, sd.Width, sd.Height, sd.Label, sd.TimeStamp, rgRealData, sd.Boost, sd.AutoLabeled, sd.Index);
}
else
{
sdResult = new SimpleDatum(bIsReal, nChannels, sd.Width, sd.Height, sd.Label, sd.TimeStamp, rgByteData, sd.Boost, sd.AutoLabeled, sd.Index);
}
sdResult.Tag = sd.Tag;
return(sdResult);
}
private Datum createNoisyData(int[] rgTopShape, SimpleDatum datum)
{
Blob <T> blobNoise = new Blob <T>(m_cuda, m_log, rgTopShape);
try
{
Filler <T> filler = Filler <T> .Create(m_cuda, m_log, m_param.data_param.data_noise_param.noise_filler);
filler.Fill(blobNoise);
if (m_param.data_param.data_noise_param.use_noisy_mean)
{
SimpleDatum sdMean = m_transformer.ImageMean;
if (sdMean == null)
{
if (m_imgdb == null)
{
m_log.FAIL("No 'mean' image is loaded, yet the MyCaffe Image Database = null, and it is required to get the mean image.");
}
sdMean = m_imgdb.GetImageMean(m_src.ID);
}
if (sdMean == null)
{
m_log.FAIL("The data source '" + m_src.Name + "' does not have a mean image!");
}
if (sdMean.IsRealData)
{
blobNoise.mutable_cpu_diff = sdMean.GetData <T>();
}
else
{
double dfMin = blobNoise.min_data;
double dfMax = blobNoise.max_data;
if (dfMin < -1.0 || dfMax > 1.0)
{
m_log.WriteLine("WARNING! The noise filler is producing numbers outside of the range [-1,1] which may cause a saturated final noise data image.");
}
blobNoise.mutable_cpu_diff = sdMean.GetData <T>();
}
m_cuda.mul(blobNoise.count(), blobNoise.gpu_diff, blobNoise.gpu_data, blobNoise.mutable_gpu_data);
}
Datum datumNoise;
if (typeof(T) == typeof(double))
{
double[] rgdf = convertD(blobNoise.update_cpu_data());
if (datum.IsRealData)
{
List <double> rgdf1 = new List <double>(rgdf);
datumNoise = new Datum(datum.IsRealData, datum.Channels, datum.Width, datum.Height, m_param.data_param.data_noise_param.noise_data_label, DateTime.MinValue, rgdf1, 1, false, -1);
}
else
{
List <byte> rgb = rgdf.Select(p => Math.Min((byte)p, (byte)255)).ToList();
datumNoise = new Datum(datum.IsRealData, datum.Channels, datum.Width, datum.Height, m_param.data_param.data_noise_param.noise_data_label, DateTime.MinValue, rgb, 1, false, -1);
}
}
else
{
float[] rgf = convertF(blobNoise.update_cpu_data());
if (datum.IsRealData)
{
List <float> rgf1 = new List <float>(rgf);
datumNoise = new Datum(datum.IsRealData, datum.Channels, datum.Width, datum.Height, m_param.data_param.data_noise_param.noise_data_label, DateTime.MinValue, rgf1, 1, false, -1);
}
else
{
List <byte> rgb = rgf.Select(p => Math.Min((byte)p, (byte)255)).ToList();
datumNoise = new Datum(datum.IsRealData, datum.Channels, datum.Width, datum.Height, m_param.data_param.data_noise_param.noise_data_label, DateTime.MinValue, rgb, 1, false, -1);
}
}
if (!string.IsNullOrEmpty(m_param.data_param.data_noise_param.noisy_save_path))
{
if (!Directory.Exists(m_param.data_param.data_noise_param.noisy_save_path))
{
m_log.FAIL("The noisy save path '" + m_param.data_param.data_noise_param.noisy_save_path + "' does not exist!");
}
string strPath = m_param.data_param.data_noise_param.noisy_save_path.TrimEnd('\\');
Bitmap bmp = ImageData.GetImage(datumNoise);
bmp.Save(m_param.data_param.data_noise_param.noisy_save_path + "\\noisy_data.png");
bmp.Dispose();
}
return(datumNoise);
}
finally
{
blobNoise.Dispose();
}
}
