public string GetHeaders(ListStr lHeaders)
{
if (conf.overrideHeaders)
{
return(conf.lCustomHeaders.ToStr(conf.separator));
}
return(lHeaders.ToStr(conf.separator));
}
/// <summary>
/// Get the number of comments for a given document or multiple documents
/// at once if a list of ids is provided
/// </summary>
/// <param name="client"></param>
/// <param name="docid"></param>
private void Count(EngineClient client, string docid)
{
ListStr docids = Method.JsonRequest.ValueListStr("docids");
if (docids != null && docids.Count > 0)
{
var sep = dynamicSeparator("=", docids.ToStr(' ')); // Since document ids potentially contains ';', we have to look for a custom separator dynamically
var sql = $"SELECT DISTRIBUTION('docid,SEPARATOR=\"{sep}\"') AS ids FROM {indexname} WHERE {Sql.In("docid", docids)} LIMIT 1";
Cursor cursor = client.ExecCursor(sql);
if (cursor != null)
{
// Write JSON
var counts = Json.NewObject();
var pairs = cursor.GetAttribute("ids").Split(new [] { sep }, StringSplitOptions.None);
foreach (var pair in pairs)
{
int i = pair.LastIndexOf(';');
counts.Set(pair.Substring(0, i), int.Parse(pair.Substring(i + 1)));
}
JsonResponse.Set("counts", counts);
}
else
{
throw new Exception("Cursor is null!");
}
}
else
{
var sql = $"SELECT id FROM {indexname} WHERE docid={Str.SqlValue(docid)} ORDER BY modified LIMIT 1";
Cursor cursor = client.ExecCursor(sql);
if (cursor != null)
{
// Write JSON
JsonResponse.Set("count", cursor.TotalRowCount);
}
else
{
throw new Exception("Cursor is null!");
}
}
}
public override Return OnExecute()
{
string indexes;
HashSet <string> hsMD5 = new HashSet <string>();
int emptyLinesCount = 0;
int duplicatesLinesCount = 0;
int writtenLinesCount = 0;
int rowProcessed = 0;
ListStr lHeaders = new ListStr();
EngineClient _client = null;
Sinequa.Engine.Client.Cursor _cursor = null;
//do not use StreamWriter in a using statement. In simulate mode the file is not created so this will trigger an exception.
StreamWriter sw = null;
try
{
_client = EngineClientsPool.FromPoolByIndexList(conf.listIndexes, out indexes, conf.engine, false);
}
catch (Exception ex)
{
Sys.LogError("Cannot get Engine Client from pool for indexes [" + conf.listIndexes + "]");
Sys.LogError(ex);
return(Return.Error);
}
Sys.Log("Using Engine client [" + _client.Name + "]");
try
{
Sys.Log("Execute query [" + conf.GetSQL() + "]");
_cursor = _client.ExecCursor(conf.GetSQL());
if (_cursor == null)
{
Sys.LogError("Cannot get cursor for query [" + conf.GetSQL() + "]");
return(Return.Error);
}
DocExportIndexToCsv doc = new DocExportIndexToCsv(this, _cursor, conf.dColumnColumnAlias);
var context = new IDocContext {
Doc = doc
};
Sys.Log("Query processingtime [" + _cursor.GetAttribute("processingtime") + "]");
Sys.Log("Query row count [" + _cursor.TotalRowCount + "]");
int globalTimer = Sys.TimerStart();
if (!conf.simulate)
{
sw = new StreamWriter(conf.destinationFilePath, false, Encoding.UTF8);
}
int localTimer = Sys.TimerStart();
while (!_cursor.End())
{
rowProcessed++;
if (rowProcessed % logStatsEveryNRows == 0)
{
Sys.Log("----------------------------------------------------");
Sys.Log("Number of rows processed [" + rowProcessed + "] ");
Sys.Log("Number of lines exported [" + writtenLinesCount + "] ");
Sys.Log("Number of empty lines removed [" + emptyLinesCount + "] ");
Sys.Log("Number of duplicated lines removed [" + duplicatesLinesCount + "] ");
Sys.Log("Processing [" + logStatsEveryNRows + "] rows in [", Sys.TimerGetText(Sys.TickDuration(localTimer)), "]");
localTimer = Sys.TimerStart();
}
ListStr l = new ListStr();
bool isEmpty = true;
for (int i = 0; i < _cursor.ColumnCount; i++)
{
if (conf.addHeaders && rowProcessed == 1) //headers
{
string header = _cursor.GetColumnName(i);
if (conf.useDblQuote)
{
header = "\"" + header + "\"";
}
lHeaders.Add(header);
}
string colValue = Str.Empty;
//cursor column match column mapping column name ?
if (conf.lColumnMapping.Exists(x => Str.EQNC(x.columnName, _cursor.GetColumnName(i))))
{
//get all matching column mapping for current column
List <ColumnMapping> lColumnMapping = conf.lColumnMapping.FindAll(x => Str.EQNC(x.columnName, _cursor.GetColumnName(i)));
foreach (ColumnMapping columnMapping in lColumnMapping)
{
if (columnMapping.slectionQuery.IsSelected(context, doc)) //match selection query ? if so, apply value pattern
{
Sys.Log2(40, "Column [" + columnMapping.columnName + "] match selection query [" + columnMapping.slectionQuery.Sql + "]");
colValue = IDocHelper.GetValue(context, doc, columnMapping.valueExpression);
Sys.Log2(40, "Column [" + columnMapping.columnName + "] use value pattern [" + columnMapping.valueExpression + "] = [" + colValue + "]");
break; //stop mapping when selection query match
}
else
{
Sys.Log2(40, "Column [" + columnMapping.columnName + "] don't match selection query [" + columnMapping.slectionQuery.Sql + "]");
continue; //go to next expression
}
}
}
//no column mapping, get value from cursor
else
{
colValue = _cursor.GetColumn(i);
}
if (!Str.IsEmpty(colValue))
{
isEmpty = false;
}
if (conf.useReplaceSeparator)
{
colValue = colValue.Replace(conf.separator, conf.replaceSeparator); //replace separator in values
}
if (conf.useDblQuote)
{
colValue = "\"" + colValue + "\""; //use double quote
}
l.Add(colValue);
}
string line = l.ToStr(conf.separator);
if (conf.removeDuplicates) //remove duplicates
{
string MD5 = Str.Md5(line);
if (!hsMD5.Add(MD5))
{
duplicatesLinesCount++;
_cursor.MoveNext();
continue;
}
}
if (conf.removeEmptyLines && isEmpty) //remove empty lines
{
emptyLinesCount++;
_cursor.MoveNext();
continue;
}
writtenLinesCount++;
if (conf.simulate) //simulate, add headers and line into logs
{
if (conf.addHeaders && rowProcessed == 1)
{
Sys.Log(GetHeaders(lHeaders)); // write headers
}
Sys.Log(line); //write line
if (writtenLinesCount >= conf.simulateCount)
{
break;
}
}
else
{
if (conf.addHeaders && rowProcessed == 1)
{
sw.WriteLine(GetHeaders(lHeaders)); // write headers
}
sw.WriteLine(line); //write line
}
_cursor.MoveNext();
}
if (sw != null)
{
sw.Close(); //dispose stream writer
}
Sys.Log("----------------------------------------------------");
if (conf.removeEmptyLines)
{
Sys.Log("Number of empty lines removed [" + emptyLinesCount + "]");
}
if (conf.removeDuplicates)
{
Sys.Log("Number of duplicated lines removed [" + duplicatesLinesCount + "]");
}
Sys.Log("[" + writtenLinesCount + "] lines exported into file [" + conf.destinationFilePath + "]");
Sys.Log("Processing [" + rowProcessed + "] rows in [", Sys.TimerGetText(Sys.TickDuration(globalTimer)), "]");
}
catch (Exception ex)
{
Sys.LogError("Select index Cursor error : ", ex);
try
{
if (_client != null)
{
EngineClientsPool.ToPool(_client);
}
}
catch (Exception exe)
{
Sys.LogError("EngineClientsPool ToPool : ", exe);
Sys.Log(exe.StackTrace);
}
}
finally
{
try
{
if (_cursor != null)
{
_cursor.Close();
}
}
catch (Exception ex)
{
Sys.LogError("Close cursor error : ", ex);
Sys.Log(ex.StackTrace);
}
EngineClientsPool.ToPool(_client);
}
return(base.OnExecute());
}