private void disposeArray(ref MWNumericArray m)
{
MWNumericArray.DisposeArray(m);
m = null;
}
//this will process chunks froom the samples, using a blocks of data using a divider
//this version works better than FULL but still needs a 64bit version of the exe or else there will be out of memory errors.
// will throw out of memory in several locations. use optimized.
//32bit, dont use more than [100x190000]
public void ProcessChunksFullMem()
{
try
{
_oproblem.labels = new double[_iproblem.labels.Length - K + 1];
_oproblem.samples = new SortedDictionary <int, double> [_iproblem.samples.Length - K + 1];
//_oproblem.samples = new SortedDictionary<int, double>[1];
int[] _indices = getIndices();
//get inverse model once
MWArray[] invModel = new MWArray[1];
invModel = Preferences.Instance.glm.getInverseModel(1, (MWArray)K);
MWNumericArray invmodel = (MWNumericArray)invModel[0];
double[,] invModelDoubleArray = (double[, ])invmodel.ToArray();
//get maximum size of a dictionary inside the array
int maxIndexDictionary = 0;
int _size = 0;
foreach (SortedDictionary <int, double> d in _iproblem.samples)
{
_size = d.Keys.Max();
if (_size > maxIndexDictionary)
{
maxIndexDictionary = _size;
}
}
int MatrixBufSize = Preferences.Instance.glmBufferDivider; //matrix[100,196000]
int fromSample = 0;
int toSample = MatrixBufSize - 1;
// if less than buffer size then we just need to process what there is. set lower limit on samples count
if (toSample > _iproblem.samples.Length)
{
toSample = _iproblem.samples.Length - 1;
}
while (fromSample < _iproblem.samples.Length - 1 - K)
{
//copy each dictionary to the matrix that is going inside the beta calculation function
float[,] matrix = new float[toSample - fromSample + 1, maxIndexDictionary];
/*int mx = 0;
* foreach (SortedDictionary<int, double> d in _iproblem.samples)
* {
* foreach (int key in d.Keys)
* {
* matrix[mx, key - 1] = (float)d[key];
* }
* mx++;
* }*/
for (int i = fromSample; i <= toSample; i++)
{
SortedDictionary <int, double> d = _iproblem.samples[i];
foreach (int key in d.Keys)
{
matrix[i - fromSample, key - 1] = (float)d[key];
}
}
//beta function is calculating for all the samples matrix
MWArray[] Betas = new MWArray[1];
//calculate betas
//Preferences.Instance.glm = new glmDll.MatlabGLM();
Betas = Preferences.Instance.glm.Beta_calculation_indices(1, (MWNumericArray)_indices, (MWNumericArray)matrix, (MWNumericArray)K, (MWNumericArray)Preferences.Instance.glm.getInverseModel(1, (MWArray)K)[0]);
matrix = null;
MWNumericArray betas = (MWNumericArray)Betas[0];
//Betas = null;
double[,] betasArray = (double[, ])betas.ToArray();
MWArray.DisposeArray(Betas);
Betas = null;
MWNumericArray.DisposeArray(betas);
betas = null;
GC.Collect();
//Double[] betasArray = (Double[])betas.ToVector(MWArrayComponent.Real);
//double[] betasArray = (double[])betas.ToArray();
//stitch back betas and labels
for (int i = fromSample + K - 1; i <= toSample; i++)
//for (int i = 0; i < _iproblem.samples.Length - K + 1; i++)
{
double[] vec = new double[_indices.Length];
int key = 0;
for (int j = 0; j < _indices.Length; j++)
{
vec[key] = betasArray[i - (fromSample + K - 1), j];
key++;
}
//_oproblem.samples[i].Keys = (SortedDictionary<int,double>.KeyCollection)_indices;
//convert to dictionary: zip array '0' indices and new beta values. and create a sorted dictionary
//SortedDictionary<int, double> dic = new SortedDictionary<int, double>(_indices.Zip(vec, (k, v) => new { k, v }).ToDictionary(x => x.k, x => x.v));
SortedDictionary <int, double> dic = new SortedDictionary <int, double>(_indices.Zip(vec, (k, v) => new { k, v }).ToDictionary(x => x.k, x => x.v));
//#endif
_oproblem.samples[i - K + 1] = dic;
//GuiPreferences.Instance.setLog("resized from: " + _oproblem.samples.Length.ToString() + " to: " + (_oproblem.samples.Length + 1).ToString() + " " +PublicMethods.getRam().ToString());
//Array.Resize(ref _oproblem.samples, _oproblem.samples.Length + 1);
double _label = _iproblem.labels[i];
int iBackward = i;
while (_label == 1)
{
iBackward--;
_label = _iproblem.labels[iBackward];
}
_oproblem.labels[i - K + 1] = _label; //!!! if baseline take the event that happened before
}
betasArray = null;
fromSample = toSample + 1 - K + 1;
toSample += MatrixBufSize;
GuiPreferences.Instance.setLog("resized finished");
// if less than buffer size then we just need to process what there is. set lower limit on samples count
if (toSample > _iproblem.samples.Length)
{
toSample = _iproblem.samples.Length - 1;
}
GC.Collect();
}
_indices = null;
_oproblem.UpdateMaximumIndex();
}
catch (Exception e)
{
GuiPreferences.Instance.setLog(e.ToString());
}
GuiPreferences.Instance.setLog("finished");
}