/// <summary>
/// This method tries to load as much as possible from the cursor into the buffer until the memorySize is reached.
/// </summary>
public void LoadAsMuchAsPossible()
{
_instValues.ResetIndexing();
_instIndices.ResetIndexing();
_instanceProperties.Clear();
while (_cursorMoveNext)
{
var featureValues = _cursor.Features.GetValues();
int featureCount = featureValues.Length;
// If the instance has no feature, ignore it!
if (featureCount == 0)
{
_cursorMoveNext = _cursor.MoveNext();
continue;
}
// We assume that cursor.Features.values are represented by float and cursor.Features.indices are represented by int
// We conservatively assume that an instance is sparse and therefore, it has an array of Floats and ints for values and indices
int perNonZeroInBytes = sizeof(float) + sizeof(int);
if (featureCount > _trainer.AcceleratedMemoryBudgetBytes / perNonZeroInBytes)
{
// Hopefully this never happens. But the memorySize must >= perNonZeroInBytes * length(the longest instance).
throw _ch.Except("Acceleration memory budget is too small! Need at least {0} MB for at least one of the instances",
featureCount * perNonZeroInBytes / (1024 * 1024));
}
bool couldLoad = true;
if (!_cursor.Features.IsDense)
{
// If it is a sparse instance, load its indices to instIndices buffer
couldLoad = _instIndices.AddToStorage(_cursor.Features.GetIndices());
}
// Load values of an instance into instValues
if (couldLoad)
{
couldLoad = _instValues.AddToStorage(featureValues);
}
// If the load was successful, load the instance properties to instanceProperties
if (couldLoad)
{
float label = _cursor.Label;
InstanceProperties prop = new InstanceProperties(featureCount, label, _cursor.Features.IsDense);
_instanceProperties.Add(prop);
_cursorMoveNext = _cursor.MoveNext();
if (_instanceProperties.Count > (1 << 30))
{
// If it happened to be the case that we have so much memory that we were able to load (1<<30) instances,
// break. This is because in such a case _instanceProperties can only be addressed by int32 and (1<<30) is
// getting close to the limits. This should rarely happen!
_isFullyLoaded = false;
break;
}
}
else
{
// If couldLoad fails at any point (which is becuase of memorySize), isFullyLoaded becomes false forever
_isFullyLoaded = false;
break;
}
}
}
