private void UpdateRelevance(List <string> pages, Dictionary <string, string> url2id, Dictionary <string, int> url2idx, double[] values, double min, double max)
{
CCIndex index = CC.Current.Indexes[indexname];
EngineClient client = null;
try
{
client = EngineClientsPool.FromPool(index);
foreach (string page in pages)
{
int idx = url2idx[page];
string id = url2id[page];
double value = 0.50 * Math.Sqrt((values[idx] - min) / (max - min)); // Value will be between 0 and 50%
String sqlquery = String.Format("UPDATE {3} SET {2} = '{0}' WHERE id = '{1}';", value, id.Replace("'", "''"), column_score, indexname);
//Sys.Log("processing query for ", sqlquery);
client.Exec(sqlquery);
}
}
finally
{
EngineClientsPool.ToPool(client);
}
}
public void GetPages(List <string> pages, Dictionary <string, string> url2id, Dictionary <string, int> url2idx)
{
CCIndex index = CC.Current.Indexes[indexname];
EngineClient client = null;
try
{
client = EngineClientsPool.FromPool(index);
String sqlquery = Str.And("select id,", column_id, " from ", indexname, " where collection = '", collection, "' and ", column_id, "<> ''");
Sys.Log("processing query for ", sqlquery);
Cursor cursor = client.ExecCursor(sqlquery);
if (cursor != null)
{
Sys.Log("Number of rows: ", cursor.CursorRowCount);
int duplicates = 0;
for (int i = 0; i < cursor.CursorRowCount; i++)
{
//Sys.Log("doc " + i);
string docid = cursor.GetColumn("id");
string pagerankid = cursor.GetColumn(column_id);
if (!url2id.ContainsKey(pagerankid)) // Duplicates id are possible...
{
pages.Add(pagerankid);
url2id[pagerankid] = docid;
url2idx[pagerankid] = i - duplicates;
}
else
{
duplicates++;
}
//Sys.Log("Added doc " + doc.Id);
cursor.MoveNext();
}
}
else
{
Sys.Log("No doc");
}
}
finally
{
EngineClientsPool.ToPool(client);
}
}
public void GetLinks(Dictionary <string, int> url2idx, ArrayList matrix)
{
CCIndex index = CC.Current.Indexes[indexname];
EngineClient client = null;
try
{
client = EngineClientsPool.FromPool(index);
String sqlquery = Str.And("select ", column_id, ",", column_links, " from ", indexname, " where collection = '", collection, "' and ", column_id, "<> ''");
Sys.Log("processing query for ", sqlquery);
Cursor cursor = client.ExecCursor(sqlquery);
if (cursor != null)
{
Sys.Log("Number of rows: ", cursor.CursorRowCount);
for (int i = 0; i < cursor.CursorRowCount; i++)
{
string pagerankid = cursor.GetColumn(column_id);
string links = cursor.GetColumn(column_links);
//Sys.Log("doc ", i, " ", pagerankid, " ", url2idx[pagerankid]);
List <int> doc = matrix[url2idx[pagerankid]] as List <int>;
if (links != null && Str.NEQ(links, ""))
{
foreach (string link in links.Split(';'))
{
if (url2idx.ContainsKey(link))
{
doc.Add(url2idx[link]);
}
}
}
//Sys.Log("Added doc " + url2idx[pagerankid]);
cursor.MoveNext();
}
}
else
{
Sys.Log("No doc");
}
}
finally
{
EngineClientsPool.ToPool(client);
}
}
//return a client to the pool
public static bool EngineClientToPool(EngineClient client)
{
try
{
EngineClientsPool.ToPool(client);
}
catch (Exception exe)
{
Sys.LogError($"Cannot return EngineClient to pool for [{client.Name}]");
Sys.LogError(exe);
return(false);
}
return(true);
}
//Get engine client
//please make sure client is return to the pool
public static EngineClient EngineClientFromPool(string engineName)
{
EngineClient client = null;
try
{
client = EngineClientsPool.FromPool(engineName);
}
catch (Exception ex)
{
Sys.LogError($"Cannot get EngineClient for engine [{engineName}]");
Sys.LogError(ex);
//exit execution, return client to pool
EngineClientToPool(client);
return(null);
}
return(client);
}
public override Return OnExecute()
{
string indexes;
HashSet <string> hsMD5 = new HashSet <string>();
int emptyLinesCount = 0;
int duplicatesLinesCount = 0;
int writtenLinesCount = 0;
int rowProcessed = 0;
ListStr lHeaders = new ListStr();
EngineClient _client = null;
Sinequa.Engine.Client.Cursor _cursor = null;
//do not use StreamWriter in a using statement. In simulate mode the file is not created so this will trigger an exception.
StreamWriter sw = null;
try
{
_client = EngineClientsPool.FromPoolByIndexList(conf.listIndexes, out indexes, conf.engine, false);
}
catch (Exception ex)
{
Sys.LogError("Cannot get Engine Client from pool for indexes [" + conf.listIndexes + "]");
Sys.LogError(ex);
return(Return.Error);
}
Sys.Log("Using Engine client [" + _client.Name + "]");
try
{
Sys.Log("Execute query [" + conf.GetSQL() + "]");
_cursor = _client.ExecCursor(conf.GetSQL());
if (_cursor == null)
{
Sys.LogError("Cannot get cursor for query [" + conf.GetSQL() + "]");
return(Return.Error);
}
DocExportIndexToCsv doc = new DocExportIndexToCsv(this, _cursor, conf.dColumnColumnAlias);
var context = new IDocContext {
Doc = doc
};
Sys.Log("Query processingtime [" + _cursor.GetAttribute("processingtime") + "]");
Sys.Log("Query row count [" + _cursor.TotalRowCount + "]");
int globalTimer = Sys.TimerStart();
if (!conf.simulate)
{
sw = new StreamWriter(conf.destinationFilePath, false, Encoding.UTF8);
}
int localTimer = Sys.TimerStart();
while (!_cursor.End())
{
rowProcessed++;
if (rowProcessed % logStatsEveryNRows == 0)
{
Sys.Log("----------------------------------------------------");
Sys.Log("Number of rows processed [" + rowProcessed + "] ");
Sys.Log("Number of lines exported [" + writtenLinesCount + "] ");
Sys.Log("Number of empty lines removed [" + emptyLinesCount + "] ");
Sys.Log("Number of duplicated lines removed [" + duplicatesLinesCount + "] ");
Sys.Log("Processing [" + logStatsEveryNRows + "] rows in [", Sys.TimerGetText(Sys.TickDuration(localTimer)), "]");
localTimer = Sys.TimerStart();
}
ListStr l = new ListStr();
bool isEmpty = true;
for (int i = 0; i < _cursor.ColumnCount; i++)
{
if (conf.addHeaders && rowProcessed == 1) //headers
{
string header = _cursor.GetColumnName(i);
if (conf.useDblQuote)
{
header = "\"" + header + "\"";
}
lHeaders.Add(header);
}
string colValue = Str.Empty;
//cursor column match column mapping column name ?
if (conf.lColumnMapping.Exists(x => Str.EQNC(x.columnName, _cursor.GetColumnName(i))))
{
//get all matching column mapping for current column
List <ColumnMapping> lColumnMapping = conf.lColumnMapping.FindAll(x => Str.EQNC(x.columnName, _cursor.GetColumnName(i)));
foreach (ColumnMapping columnMapping in lColumnMapping)
{
if (columnMapping.slectionQuery.IsSelected(context, doc)) //match selection query ? if so, apply value pattern
{
Sys.Log2(40, "Column [" + columnMapping.columnName + "] match selection query [" + columnMapping.slectionQuery.Sql + "]");
colValue = IDocHelper.GetValue(context, doc, columnMapping.valueExpression);
Sys.Log2(40, "Column [" + columnMapping.columnName + "] use value pattern [" + columnMapping.valueExpression + "] = [" + colValue + "]");
break; //stop mapping when selection query match
}
else
{
Sys.Log2(40, "Column [" + columnMapping.columnName + "] don't match selection query [" + columnMapping.slectionQuery.Sql + "]");
continue; //go to next expression
}
}
}
//no column mapping, get value from cursor
else
{
colValue = _cursor.GetColumn(i);
}
if (!Str.IsEmpty(colValue))
{
isEmpty = false;
}
if (conf.useReplaceSeparator)
{
colValue = colValue.Replace(conf.separator, conf.replaceSeparator); //replace separator in values
}
if (conf.useDblQuote)
{
colValue = "\"" + colValue + "\""; //use double quote
}
l.Add(colValue);
}
string line = l.ToStr(conf.separator);
if (conf.removeDuplicates) //remove duplicates
{
string MD5 = Str.Md5(line);
if (!hsMD5.Add(MD5))
{
duplicatesLinesCount++;
_cursor.MoveNext();
continue;
}
}
if (conf.removeEmptyLines && isEmpty) //remove empty lines
{
emptyLinesCount++;
_cursor.MoveNext();
continue;
}
writtenLinesCount++;
if (conf.simulate) //simulate, add headers and line into logs
{
if (conf.addHeaders && rowProcessed == 1)
{
Sys.Log(GetHeaders(lHeaders)); // write headers
}
Sys.Log(line); //write line
if (writtenLinesCount >= conf.simulateCount)
{
break;
}
}
else
{
if (conf.addHeaders && rowProcessed == 1)
{
sw.WriteLine(GetHeaders(lHeaders)); // write headers
}
sw.WriteLine(line); //write line
}
_cursor.MoveNext();
}
if (sw != null)
{
sw.Close(); //dispose stream writer
}
Sys.Log("----------------------------------------------------");
if (conf.removeEmptyLines)
{
Sys.Log("Number of empty lines removed [" + emptyLinesCount + "]");
}
if (conf.removeDuplicates)
{
Sys.Log("Number of duplicated lines removed [" + duplicatesLinesCount + "]");
}
Sys.Log("[" + writtenLinesCount + "] lines exported into file [" + conf.destinationFilePath + "]");
Sys.Log("Processing [" + rowProcessed + "] rows in [", Sys.TimerGetText(Sys.TickDuration(globalTimer)), "]");
}
catch (Exception ex)
{
Sys.LogError("Select index Cursor error : ", ex);
try
{
if (_client != null)
{
EngineClientsPool.ToPool(_client);
}
}
catch (Exception exe)
{
Sys.LogError("EngineClientsPool ToPool : ", exe);
Sys.Log(exe.StackTrace);
}
}
finally
{
try
{
if (_cursor != null)
{
_cursor.Close();
}
}
catch (Exception ex)
{
Sys.LogError("Close cursor error : ", ex);
Sys.Log(ex.StackTrace);
}
EngineClientsPool.ToPool(_client);
}
return(base.OnExecute());
}
/// <summary>
/// Main entry point of the web service. This method simply retrieves the "action"
/// input from the request, and calls the corresponding method accordingly (among which
/// Create(), Read(), Update(), Delete(), Like()).
/// </summary>
public override void OnPluginMethod()
{
//Sys.Log("Comments Web Service Plugin Start");
string action = ensureStrInput("action");
string docid = ensureStrInput("docid");
if (docid == null || action == null)
{
return;
}
if (!Method.Session.HasAccessToDocId(docid))
{
SetError(403, "You do not have access to this document");
return;
}
CCIndex index = CC.Current.Indexes[indexname];
EngineClient client = null;
try
{
client = EngineClientsPool.FromPool(index);
switch (action.ToLower())
{
case "create":
Create(client, docid);
break;
case "read":
Read(client, docid);
break;
case "update":
Update(client, docid);
break;
case "delete":
Delete(client, docid);
break;
case "like":
Like(client, docid);
break;
default:
SetError(400, "Invalid 'action' input. Possible actions: 'create','read','update','delete','like'");
break;
}
}
catch (Exception ex)
{
JsonResponse.SetValue("error", ex.StackTrace);
SetError(400, ex.Message);
}
finally
{
EngineClientsPool.ToPool(client);
}
//Sys.Log("Comments Web Service Plugin End");
}
