/// <summary> /// This method tries to load as much as possible from the cursor into the buffer until the memorySize is reached. /// </summary> public void LoadAsMuchAsPossible() { _instValues.ResetIndexing(); _instIndices.ResetIndexing(); _instanceProperties.Clear(); while (_cursorMoveNext) { var featureValues = _cursor.Features.GetValues(); int featureCount = featureValues.Length; // If the instance has no feature, ignore it! if (featureCount == 0) { _cursorMoveNext = _cursor.MoveNext(); continue; } // We assume that cursor.Features.values are represented by float and cursor.Features.indices are represented by int // We conservatively assume that an instance is sparse and therefore, it has an array of Floats and ints for values and indices int perNonZeroInBytes = sizeof(float) + sizeof(int); if (featureCount > _trainer.AcceleratedMemoryBudgetBytes / perNonZeroInBytes) { // Hopefully this never happens. But the memorySize must >= perNonZeroInBytes * length(the longest instance). throw _ch.Except("Acceleration memory budget is too small! Need at least {0} MB for at least one of the instances", featureCount * perNonZeroInBytes / (1024 * 1024)); } bool couldLoad = true; if (!_cursor.Features.IsDense) { // If it is a sparse instance, load its indices to instIndices buffer couldLoad = _instIndices.AddToStorage(_cursor.Features.GetIndices()); } // Load values of an instance into instValues if (couldLoad) { couldLoad = _instValues.AddToStorage(featureValues); } // If the load was successful, load the instance properties to instanceProperties if (couldLoad) { float label = _cursor.Label; InstanceProperties prop = new InstanceProperties(featureCount, label, _cursor.Features.IsDense); _instanceProperties.Add(prop); _cursorMoveNext = _cursor.MoveNext(); if (_instanceProperties.Count > (1 << 30)) { // If it happened to be the case that we have so much memory that we were able to load (1<<30) instances, // break. This is because in such a case _instanceProperties can only be addressed by int32 and (1<<30) is // getting close to the limits. This should rarely happen! _isFullyLoaded = false; break; } } else { // If couldLoad fails at any point (which is becuase of memorySize), isFullyLoaded becomes false forever _isFullyLoaded = false; break; } } }