public DataTable ReadResult(NormalizeInfo dbStructure, long queryId, bool readServiceColumns = false)
{
const long sessionId = 1L;
string modSql = null;
string strSQL = null;
IEnumerable <SQLiteParameter> parameters = null;
DataTable dataTableResult = null;
if (dbStructure != null)
{
Common.Table table = Common.Table.GetTable(
this.Connection,
dbStructure.TableName
);
if (table != null)
{
CommonRowFiller.GetQueryModifier(
out modSql,
out parameters,
queryId,
sessionId
);
strSQL = dbStructure.GetSelectQuery(modSql, readServiceColumns)
+ " ORDER BY " + CommonRowFiller.RowOrderFieldName;
ReadTableCommand command = new ReadTableCommand(
this.Connection,
strSQL,
parameters.ToArray()
);
command.Execute(100);
if (!readServiceColumns)
{
CommonRowFiller.RemoveServiceColumns(command.Result);
}
dataTableResult = command.Result;
}
}
return(dataTableResult);
}
public void PrepareTables(NormalizeInfo normalizeInfo, bool mainTable = true)
{
if (normalizeInfo != null)
{
TableDefinition tableDefinition = normalizeInfo.GetTableDefinition();
// Log.DebugFormat("PrepareTables:BeginLock");
lock (this._tablesLock)
{
if (!this._tables.ContainsKey(normalizeInfo))
{
if (mainTable)
{
CommonRowFiller.ModifyTableDefinition(tableDefinition);
if (this.FileName.Equals("file::memory:"))
{
CommonRowFiller.ModifyTableDefinitionAddOptionField(
tableDefinition,
CommonRowFiller.RowHashFieldName + "Old",
SqlDbType.NVarChar,
true
);
}
}
if (tableDefinition.Fields.Any())
{
ReportTable table = new ReportTable(this.Connection, tableDefinition);
table.UpdateScheme();
this._tables.Add(normalizeInfo, table);
}
foreach (NormalizeInfo childDirectory in normalizeInfo.ChildDirectories)
{
this.PrepareTables(childDirectory, false);
}
}
}
// Log.DebugFormat("PrepareTables:EndLock");
}
}
protected virtual long?SaveQueryData(
long queryId,
NormalizeInfo dbStructure,
DataTable queryData,
long sessionId
)
{
DataTable currentTable = this.ReadResult(dbStructure, queryId, true);
Dictionary <string, int> currentDict = new Dictionary <string, int>();
Dictionary <string, int> insertDict = new Dictionary <string, int>();
Dictionary <int, int> rowOrderDict = new Dictionary <int, int>();
HashSet <string> duplicateRows = new HashSet <string>();
HashSet <string> updatedRows = new HashSet <string>();
List <ITableRow> rows = new List <ITableRow>();
int currentRowOrder = 0;
List <string> rowsToDelete = new List <string>();
bool needInsertCommand = false;
ReportTable table = null;
long rowsUnchanged = 0L;
long rowsInserted = 0L;
long rowsUpdated = 0L;
long rowsDeleted = 0L;
long rowsDuplicatedDeleted = 0L;
List <RowUpdateOrderInput> rowsToUpdateOrder = new List <RowUpdateOrderInput>();
List <RowUpdateInput> rowsToUpdate = new List <RowUpdateInput>();
if (currentTable != null && currentTable.Rows != null)
{
for (int i = 0; i < currentTable.Rows.Count; i++)
{
DataRow row = currentTable.Rows[i];
string hash = row[CommonRowFiller.RowHashFieldName].ToString();
if (!currentDict.ContainsKey(hash))
{
currentDict.Add(hash, i);
}
else
{
duplicateRows.Add(hash);
}
}
}
this.PrepareTables(dbStructure);
// Log.DebugFormat(
// @"SaveQueryData:BeginLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
lock (this._tablesLock)
{
if (!this._tables.TryGetValue(dbStructure, out table))
{
Log.DebugFormat("SaveQueryData:EndLock(TryGetValue):QueryId:'{0}';sessionId:'{1}'",
queryId,
sessionId
);
return(null);
}
}
// Log.DebugFormat(
// @"SaveQueryData:EndLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
int queryRowOrder = 0;
foreach (DataRow queryRow in queryData.Rows)
{
string hash = GetHashFromDataRow(queryRow);
bool hashExists = currentDict.ContainsKey(hash);
if (hashExists && !duplicateRows.Contains(hash))
{
int storedOrder = currentDict[hash];
if (storedOrder == queryRowOrder)
{
rowsUnchanged++;
}
else
{
rowsToUpdateOrder.Add(new RowUpdateOrderInput(currentRowOrder, hash));
}
updatedRows.Add(hash);
currentDict.Remove(hash);
currentRowOrder++;
}
else
{
bool needInsert = !insertDict.ContainsKey(hash) && !updatedRows.Contains(hash);
if (needInsert)
{
insertDict.Add(hash, currentRowOrder);
rowOrderDict.Add(currentRowOrder, queryRowOrder);
currentRowOrder++;
}
}
queryRowOrder++;
}
foreach (string hash in currentDict.Keys)
{
if (!updatedRows.Contains(hash) && !duplicateRows.Contains(hash))
{
rowsToDelete.Add(hash);
}
}
// Log.DebugFormat(
// @"SaveQueryData:BeginLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
lock (this._tablesLock)
{
foreach (KeyValuePair <string, int> keyPair in insertDict)
{
string hash = keyPair.Key;
int rowOrder = keyPair.Value;
DataRow row = queryData.Rows[rowOrderDict[rowOrder]];
ITableRow tableRow;
this.ProcessRowForWrite(row, dbStructure, out tableRow, false);
if (rowsToDelete.Count > 0)
{
string hashOld = rowsToDelete[0];
CommonRowFiller.ModifyTableRow(tableRow, queryId, sessionId, rowOrder, hash);
rowsToUpdate.Add(new RowUpdateInput(hashOld, tableRow));
rowsToDelete.RemoveAt(0);
}
else
{
needInsertCommand = true;
CommonRowFiller.ModifyTableRow(tableRow, queryId, sessionId, rowOrder, hash);
rows.Add(tableRow);
}
}
if (duplicateRows.Any())
{
Log.ErrorFormat("SaveQueryData. Duplicated rows found. QueryId:'{0}';sessionId:'{1}';count:{2}",
queryId,
sessionId,
duplicateRows.Count
);
rowsDuplicatedDeleted += table.DeleteRows(queryId, sessionId, duplicateRows.ToList());
}
rowsUpdated += table.UpdateRowsOrderByHash(queryId, sessionId, rowsToUpdateOrder);
rowsUpdated += table.UpdateRowsByHash(queryId, sessionId, rowsToUpdate);
if (rowsToDelete.Count > 0)
{
rowsDeleted += table.DeleteRows(queryId, sessionId, rowsToDelete);
}
if (needInsertCommand)
{
rowsInserted += table.ReplaceRowsTrans(rows);
}
}
// Log.DebugFormat(
// @"SaveQueryData:EndLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
Log.DebugFormat("rowsUnchanged:{0};rowsInserted:{1};rowsUpdated:{2};rowsDeleted:{3};rowsDuplicatedDeleted:{4}",
rowsUnchanged,
rowsInserted,
rowsUpdated,
rowsDeleted,
rowsDuplicatedDeleted
);
return(rowsInserted + rowsUnchanged + rowsUpdated + rowsDuplicatedDeleted);
}
private long?SaveMemoryQueryData(
long queryId,
NormalizeInfo dbStructure,
DataTable queryData,
long sessionId,
DataTable currentTable
)
{
if (!this._isMemoryStorage)
{
return(0L);
}
ReportTable table = null;
Dictionary <int, int> rowOrderDict = new Dictionary <int, int>();
Dictionary <string, int> currentDict = new Dictionary <string, int>();
Dictionary <string, int> insertDict = new Dictionary <string, int>();
Dictionary <string, int> updateDict = new Dictionary <string, int>();
HashSet <string> duplicateRows = new HashSet <string>();
HashSet <string> updatedRows = new HashSet <string>();
//List<RowUpdateInput> rowsToUpdate = new List<RowUpdateInput>();
//List<RowUpdateOrderInput> rowsToUpdateOrder = new List<RowUpdateOrderInput>();
List <ITableRow> rows = new List <ITableRow>();
List <ITableRow> rowsToUpdateList = new List <ITableRow>();
List <string> rowsToDelete = new List <string>();
bool needInsertCommand = false;
int currentRowOrder = 0;
int queryRowOrder = 0;
//long rowsDeleted = 0L;
//long rowsDuplicatedDeleted = 0L;
long rowsInserted = 0L;
//long rowsUnchanged = 0L;
long rowsUpdated = 0L;
if (currentTable != null && currentTable.Rows != null)
{
for (int i = 0; i < currentTable.Rows.Count; i++)
{
DataRow row = currentTable.Rows[i];
string hash = row[CommonRowFiller.RowHashFieldName].ToString();
if (!currentDict.ContainsKey(hash))
{
currentDict.Add(hash, i);
}
else
{
duplicateRows.Add(hash);
}
}
}
this.PrepareTables(dbStructure);
// Log.DebugFormat(
// @"SaveQueryData:BeginLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
lock (this._tablesLock)
{
if (!this._tables.TryGetValue(dbStructure, out table))
{
Log.DebugFormat("SaveQueryData:EndLock(TryGetValue):QueryId:'{0}';sessionId:'{1}'",
queryId,
sessionId
);
return(null);
}
}
// Log.DebugFormat(
// @"SaveQueryData:EndLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
foreach (DataRow queryRow in queryData.Rows)
{
string hash = GetHashFromDataRow(queryRow);
bool hashExists = currentDict.ContainsKey(hash);
if (hashExists && !duplicateRows.Contains(hash))
{
updateDict.Add(hash, currentRowOrder);
updatedRows.Add(hash);
currentDict.Remove(hash);
rowOrderDict.Add(currentRowOrder, queryRowOrder);
currentRowOrder++;
}
else
{
bool needInsert = !insertDict.ContainsKey(hash) && !updatedRows.Contains(hash);
if (needInsert)
{
insertDict.Add(hash, currentRowOrder);
rowOrderDict.Add(currentRowOrder, queryRowOrder);
currentRowOrder++;
}
}
queryRowOrder++;
}
foreach (string hash in currentDict.Keys)
{
if (!updatedRows.Contains(hash) && !duplicateRows.Contains(hash))
{
rowsToDelete.Add(hash);
}
}
// Log.DebugFormat(
// @"SaveQueryData:BeginLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
lock (this._tablesLock)
{
foreach (KeyValuePair <string, int> keyPair in insertDict)
{
string hash = keyPair.Key;
int rowOrder = keyPair.Value;
DataRow row = queryData.Rows[rowOrderDict[rowOrder]];
ITableRow tableRow;
this.ProcessRowForWrite(row, dbStructure, out tableRow, false);
if (rowsToDelete.Count > 0L)
{
string hashOld = rowsToDelete[0];
CommonRowFiller.ModifyTableRow(tableRow, queryId, sessionId, rowOrder, hash);
tableRow.Values[CommonRowFiller.RowHashFieldName + "Old"] = hashOld;
rowsToUpdateList.Add(tableRow);
rowsToDelete.RemoveAt(0);
}
else
{
needInsertCommand = true;
CommonRowFiller.ModifyTableRow(tableRow, queryId, sessionId, rowOrder, hash);
tableRow.Values[CommonRowFiller.RowHashFieldName + "Old"] = -1;
rows.Add(tableRow);
}
}
foreach (KeyValuePair <string, int> keyPair in updateDict)
{
string hash = keyPair.Key;
int rowOrder = keyPair.Value;
DataRow row = queryData.Rows[rowOrderDict[rowOrder]];
ITableRow tableRow;
this.ProcessRowForWrite(row, dbStructure, out tableRow, false);
CommonRowFiller.ModifyTableRow(tableRow, queryId, sessionId, rowOrder, hash);
tableRow.Values[CommonRowFiller.RowHashFieldName + "Old"] = hash;
rowsToUpdateList.Add(tableRow);
}
rowsUpdated += table.ReplaceRowsTrans(rowsToUpdateList);
if (needInsertCommand)
{
rowsInserted += table.ReplaceRowsTrans(rows);
}
}
// Log.DebugFormat(
// @"SaveQueryData:EndLock:QueryId:'{0}';sessionId:'{1}'",
// queryId,
// sessionId
// );
Log.DebugFormat("rowsInserted:{0};rowsUpdated:{1}",
rowsInserted,
rowsUpdated
);
return(rowsInserted + rowsUpdated);
}
