private int LoadData(Select select, Action <DataTable> onChunkLoaded, string selectExpression = null)
{
select.CheckArgumentNull("selectQuery");
onChunkLoaded.CheckArgumentNull("onChunkLoaded");
int chunkSize = _config.ChunkSize;
int maxRecordsCount = _config.MaxRecordsCount;
int minRecordsCount = _config.MinRecordsCount;
QueryColumnExpression idExpression = GetSelectIdColumnExpression(select, selectExpression);
var lastValueParameter = new QueryParameter("IdLastValue", null);
var pageableSelectCondition = new PageableSelectCondition {
LastValueParameter = lastValueParameter,
LeftExpression = idExpression,
OrderByItem = new OrderByItem(idExpression, OrderDirectionStrict.Ascending)
};
var options = new PageableSelectOptions(pageableSelectCondition, chunkSize, PageableSelectDirection.First);
Select pageableSelect = select.ToPageable(options);
bool hasRecords = true;
int totalRecordsCount = 0;
bool useMaxRecordsLimit = maxRecordsCount > 0;
using (DBExecutor dbExecutor = _userConnection.EnsureDBConnection(QueryKind.Limited)) {
while (hasRecords && (!useMaxRecordsLimit || totalRecordsCount < maxRecordsCount))
{
hasRecords = false;
using (IDataReader dr = pageableSelect.ExecuteReader(dbExecutor)) {
var dataTable = new DataTable();
dataTable.Load(dr);
int recordsCount = dataTable.Rows.Count;
totalRecordsCount += recordsCount;
if (totalRecordsCount < minRecordsCount && recordsCount < chunkSize)
{
throw new NotEnoughDataForTrainingException(
$"There's not enough data for upload. Minimal number: {minRecordsCount}," +
$" Current: {totalRecordsCount}");
}
if (recordsCount == 0)
{
continue;
}
hasRecords = true;
if (useMaxRecordsLimit && totalRecordsCount >= maxRecordsCount)
{
SendNotification("LocalizableStrings.RecordCountExceedsLimitMessage.Value");
}
DataRow lastRow = dataTable.Rows[recordsCount - 1];
string rowId = lastRow["Id"]?.ToString();
if (string.IsNullOrEmpty(rowId))
{
throw new Exception("Rowset can't contain a record with empty Id");
}
onChunkLoaded(dataTable);
lastValueParameter.Value = new Guid(rowId);
options.Direction = PageableSelectDirection.Next;
AddOrderByItemsConditions(select, options, lastRow);
pageableSelect = select.ToPageable(options);
}
}
}
return(totalRecordsCount);
}
private PageableSelectOptions GetPageableOptions(ReadingOptions readingOptions)
{
var idExpression = new QueryColumnExpression(_portalMessageFileAlias, "Id");
idExpression.Alias = _portalMessageFileIdAlias;
var lastValueParameter = new QueryParameter("IdLastValue", readingOptions.LastValue);
var pageableSelectCondition = new PageableSelectCondition {
LastValueParameter = lastValueParameter,
LeftExpression = idExpression,
OrderByItem = new OrderByItem(idExpression, OrderDirectionStrict.Ascending)
};
return(new PageableSelectOptions(pageableSelectCondition, readingOptions.RowCount, PageableSelectDirection.Next));
}
