/// <summary>
/// Index all messages of a channel
/// </summary>
/// <param name="channelName">Channel name for logging</param>
/// <param name="channel">Json object which contains messages</param>
/// <returns></returns>
public bool IndexChannelMessages(string channelName, JsonObject channel)
{
Sys.Log("Index Messages of '" + channelName + "'");
bool ok = true;
SlackMessage doc = new SlackMessage();
doc.cntr = this;
if (Connector.IsModeRealTime())
{
Sys.Log("Index messages from date: " + Connector.RealTimeReferenceDate);
}
string oldest = Connector.IsModeRealTime() ? Sys.ToStr(Dat.ToUnixTimestamp(Connector.RealTimeReferenceDate)) : "";
string latest = "";
bool has_more = false;
do
{
//iterate on all messages by page (has_more = true)
has_more = false;
JsonObject response = slackGet("conversations.history?channel=" + channel.ValueStr("id") + "&latest=" + latest + "&oldest=" + oldest) as JsonObject;
if (response == null || (response != null && !response.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(response)); return(false);
}
if (response != null)
{
has_more = response.ValueBoo("has_more", false);
JsonArray messages = response.GetAsArray("messages");
if (messages != null)
{
for (int i = 0; i < messages.EltCount(); i++)
{
//clear and reuse the doc instance
doc.Clear();
//current message
JsonObject message = messages.Elt(i) as JsonObject;
//set custom to connectorDoc (slackDoc)
doc.channel = channel;
doc.message = message;
//set mandatory fields: id, fileext & version
doc.Id = message.GetValue("user") + "-" + message.GetValue("ts");
doc.Version = message.GetValue("ts");
doc.FileExt = "htm";
//set latest for Slack API Pagination
latest = doc.Version;
//Process Doc (send to Indexer)
ok &= Connector.ProcessConnectorDoc(doc);
}
}
}
}while (has_more);
return(ok);
}
/// <summary>
/// Starting point for the generic connector in partition mode)
/// </summary>
/// <param name="partition">Connector instance</param>
/// <returns></returns>
public override bool OnGenericLoadPartition(ConnectorPartition partition)
{
//Sinequa object for a user/group
ConnectorPrincipal p = new ConnectorPrincipal();
//index users
JsonObject response = slackGet("users.list") as JsonObject;
if (response == null || (response != null && !response.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(response)); return(false);
}
if (response != null)
{
JsonArray members = response.GetAsArray("members");
if (members != null)
{
for (int i = 0; i < members.EltCount(); i++)
{
//clear and reuse the principal instance
p.Clear();
JsonDocProperties user = new JsonDocProperties(members.Elt(i));
//is a user
p.IsUser = true;
//principal fields
p.Id = user.GetValue("id");
p.Name = user.GetValue("name");
p.Fullname = user.GetValue("real_name");
p.Email = user.GetValue("profile.email");
Sys.Log2(10, "User: "******"usergroups.list") as JsonObject;
if (response == null || (response != null && !response.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(response)); return(false);
}
if (response != null)
{
JsonArray members = response.GetAsArray("usergroups");
if (members != null)
{
for (int i = 0; i < members.EltCount(); i++)
{
//clear and reuse the principal instance
p.Clear();
JsonDocProperties group = new JsonDocProperties(members.Elt(i));
//is a group
p.IsGroup = true;
//principal fields
p.Id = group.GetValue("id");
p.Name = group.GetValue("name");
JsonObject response2 = slackGet("usergroups.users.list?usergroup=" + p.Id) as JsonObject;
if (response2 != null)
{
//add memmbers of this group
p.Member = response.Get("users")?.ToListStr();
}
Sys.Log2(10, "group: ", p.Name);
group.LogProperties();
//do partition mappings
Connector.PartitionMappings?.Apply(Ctxt, p, group);
//add principal in the partition
Connector.Partition.Write(p);
}
}
}
//index channels, see as groups
response = slackGet("conversations.list") as JsonObject;
if (response == null || (response != null && !response.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(response)); return(false);
}
if (response != null)
{
JsonArray members = response.GetAsArray("channels");
if (members != null)
{
for (int i = 0; i < members.EltCount(); i++)
{
//clear and reuse the principal instance
p.Clear();
JsonDocProperties channel = new JsonDocProperties(members.Elt(i));
//is a group
p.IsGroup = true;
//principal fields
p.Id = channel.GetValue("id");
p.Name = channel.GetValue("name");
//Get more details for having members (otherwise the list is truncated)
JsonObject responseInfo = slackGet("conversations.info?channel=" + p.Id) as JsonObject;
if (responseInfo == null || (responseInfo != null && !responseInfo.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(responseInfo)); return(false);
}
if (responseInfo != null)
{
JsonObject channelinfo = responseInfo.GetAsObject("channel");
//add memmbers of this group
p.Member = channelinfo.Get("members")?.ToListStr();
}
Sys.Log2(10, "group (channel): ", p.Name);
channel.LogProperties();
//do partition mappings
Connector.PartitionMappings?.Apply(Ctxt, p, channel);
//add principal in the partition
Connector.Partition.Write(p);
}
}
}
return(true);
}
/// <summary>
/// Index all files of a channel
/// </summary>
/// <param name="channelName">Channel name for logging</param>
/// <param name="channel">Json object which contains messages</param>
/// <returns></returns>
public bool IndexChannelFiles(string channelName, JsonObject channel)
{
Sys.Log("Index Files of '" + channelName + "'");
bool ok = true;
SlackFile doc = new SlackFile();
doc.cntr = this;
if (Connector.IsModeRealTime())
{
Sys.Log("Index files from date: " + Connector.RealTimeReferenceDate);
}
string latest = Connector.IsModeRealTime() ? Sys.ToStr(Dat.ToUnixTimestamp(Connector.RealTimeReferenceDate)) : "";
bool has_more = false;
int page = 0;
int pages = 1;
do
{
//iterate on all files by page (has_more = true)
has_more = false;
page++;
JsonObject response = slackGet("files.list?channel=" + channel.ValueStr("id") + "&ts_from=" + latest + "&page=" + page) as JsonObject;
if (response == null || (response != null && !response.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(response)); return(false);
}
if (response != null)
{
pages = response.ValueInt("paging.pages");
has_more = response.ValueBoo("has_more", false);
JsonArray files = response.GetAsArray("files");
if (files != null)
{
for (int i = 0; i < files.EltCount(); i++)
{
//clear and reuse the doc instance
doc.Clear();
//current message
JsonObject file = files.Elt(i) as JsonObject;
//set custom to connectorDoc (slackDoc)
doc.channel = channel;
doc.file = file;
//set mandatory fields: id, fileext & version
doc.Id = file.GetValue("id");
doc.Version = file.GetValue("timestamp");
string mimetype = file.GetValue("mimetype");
if (Str.EQ(mimetype, "text/html"))
{
doc.FileExt = "htm";
}
else
{
doc.FileExt = Str.PathGetFileExt(file.GetValue("name"));
}
//set latest for Slack API Pagination
latest = doc.Version;
//Process Doc (send to Indexer)
ok &= Connector.ProcessConnectorDoc(doc);
}
}
}
}while (page < pages);
return(ok);
}
/// <summary>
/// Starting point of the Generic Connector for the collection
/// </summary>
/// <returns>true if succeded otherwise false</returns>
public override bool OnGenericIndexCollection()
{
bool ok = true;
users = new MapOf <JsonObject>();
//put all users info in cache
JsonObject response = slackGet("users.list") as JsonObject;
if (response == null || (response != null && !response.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(response)); return(false);
}
if (response != null)
{
JsonArray members = response.GetAsArray("members");
if (members != null)
{
for (int i = 0; i < members.EltCount(); i++)
{
JsonObject user = members.Elt(i) as JsonObject;
//Add user object in cache
users.Add(user.GetValue("id"), user);
}
}
}
//index channels
response = slackGet("conversations.list") as JsonObject;
if (response == null || (response != null && !response.ValueBoo("ok")))
{
Sys.LogError(Json.Serialize(response)); return(false);
}
if (response != null)
{
JsonArray channels = response.GetAsArray("channels");
if (channels != null)
{
for (int i = 0; i < channels.EltCount(); i++)
{
JsonObject channel = channels.Elt(i) as JsonObject;
string Name = channel.GetValue("name");
//check if included/excluded from conf
if (channelMatcher.IsExcluded(Name))
{
continue;
}
//Index the Channel
if (indexMessages)
{
ok &= IndexChannelMessages(Name, channel);
}
if (indexFiles)
{
ok &= IndexChannelFiles(Name, channel);
}
}
}
}
return(ok);
}