Esempio n. 1
0
        /// <summary>
        /// Index all messages of a channel
        /// </summary>
        /// <param name="channelName">Channel name for logging</param>
        /// <param name="channel">Json object which contains messages</param>
        /// <returns></returns>
        public bool IndexChannelMessages(string channelName, JsonObject channel)
        {
            Sys.Log("Index Messages of '" + channelName + "'");
            bool         ok  = true;
            SlackMessage doc = new SlackMessage();

            doc.cntr = this;
            if (Connector.IsModeRealTime())
            {
                Sys.Log("Index messages from date: " + Connector.RealTimeReferenceDate);
            }
            string oldest   = Connector.IsModeRealTime() ? Sys.ToStr(Dat.ToUnixTimestamp(Connector.RealTimeReferenceDate)) : "";
            string latest   = "";
            bool   has_more = false;

            do
            {
                //iterate on all messages by page (has_more = true)
                has_more = false;
                JsonObject response = slackGet("conversations.history?channel=" + channel.ValueStr("id") + "&latest=" + latest + "&oldest=" + oldest) as JsonObject;
                if (response == null || (response != null && !response.ValueBoo("ok")))
                {
                    Sys.LogError(Json.Serialize(response)); return(false);
                }
                if (response != null)
                {
                    has_more = response.ValueBoo("has_more", false);
                    JsonArray messages = response.GetAsArray("messages");
                    if (messages != null)
                    {
                        for (int i = 0; i < messages.EltCount(); i++)
                        {
                            //clear and reuse the doc instance
                            doc.Clear();

                            //current message
                            JsonObject message = messages.Elt(i) as JsonObject;

                            //set custom to connectorDoc (slackDoc)
                            doc.channel = channel;
                            doc.message = message;

                            //set mandatory fields: id, fileext & version
                            doc.Id      = message.GetValue("user") + "-" + message.GetValue("ts");
                            doc.Version = message.GetValue("ts");
                            doc.FileExt = "htm";


                            //set latest for Slack API Pagination
                            latest = doc.Version;

                            //Process Doc (send to Indexer)
                            ok &= Connector.ProcessConnectorDoc(doc);
                        }
                    }
                }
            }while (has_more);
            return(ok);
        }
Esempio n. 2
0
        /// <summary>
        /// Starting point for the generic connector in partition mode)
        /// </summary>
        /// <param name="partition">Connector instance</param>
        /// <returns></returns>
        public override bool OnGenericLoadPartition(ConnectorPartition partition)
        {
            //Sinequa object for a user/group
            ConnectorPrincipal p = new ConnectorPrincipal();

            //index users
            JsonObject response = slackGet("users.list") as JsonObject;

            if (response == null || (response != null && !response.ValueBoo("ok")))
            {
                Sys.LogError(Json.Serialize(response)); return(false);
            }
            if (response != null)
            {
                JsonArray members = response.GetAsArray("members");
                if (members != null)
                {
                    for (int i = 0; i < members.EltCount(); i++)
                    {
                        //clear and reuse the principal instance
                        p.Clear();

                        JsonDocProperties user = new JsonDocProperties(members.Elt(i));

                        //is a user
                        p.IsUser = true;


                        //principal fields
                        p.Id       = user.GetValue("id");
                        p.Name     = user.GetValue("name");
                        p.Fullname = user.GetValue("real_name");
                        p.Email    = user.GetValue("profile.email");

                        Sys.Log2(10, "User: "******"usergroups.list") as JsonObject;
            if (response == null || (response != null && !response.ValueBoo("ok")))
            {
                Sys.LogError(Json.Serialize(response)); return(false);
            }
            if (response != null)
            {
                JsonArray members = response.GetAsArray("usergroups");
                if (members != null)
                {
                    for (int i = 0; i < members.EltCount(); i++)
                    {
                        //clear and reuse the principal instance
                        p.Clear();

                        JsonDocProperties group = new JsonDocProperties(members.Elt(i));

                        //is a group
                        p.IsGroup = true;

                        //principal fields
                        p.Id   = group.GetValue("id");
                        p.Name = group.GetValue("name");
                        JsonObject response2 = slackGet("usergroups.users.list?usergroup=" + p.Id) as JsonObject;
                        if (response2 != null)
                        {
                            //add memmbers of this group
                            p.Member = response.Get("users")?.ToListStr();
                        }

                        Sys.Log2(10, "group: ", p.Name);
                        group.LogProperties();
                        //do partition mappings
                        Connector.PartitionMappings?.Apply(Ctxt, p, group);

                        //add principal in the partition
                        Connector.Partition.Write(p);
                    }
                }
            }

            //index channels, see as groups
            response = slackGet("conversations.list") as JsonObject;
            if (response == null || (response != null && !response.ValueBoo("ok")))
            {
                Sys.LogError(Json.Serialize(response)); return(false);
            }
            if (response != null)
            {
                JsonArray members = response.GetAsArray("channels");
                if (members != null)
                {
                    for (int i = 0; i < members.EltCount(); i++)
                    {
                        //clear and reuse the principal instance
                        p.Clear();

                        JsonDocProperties channel = new JsonDocProperties(members.Elt(i));

                        //is a group
                        p.IsGroup = true;

                        //principal fields
                        p.Id   = channel.GetValue("id");
                        p.Name = channel.GetValue("name");

                        //Get more details for having members (otherwise the list is truncated)
                        JsonObject responseInfo = slackGet("conversations.info?channel=" + p.Id) as JsonObject;
                        if (responseInfo == null || (responseInfo != null && !responseInfo.ValueBoo("ok")))
                        {
                            Sys.LogError(Json.Serialize(responseInfo)); return(false);
                        }
                        if (responseInfo != null)
                        {
                            JsonObject channelinfo = responseInfo.GetAsObject("channel");
                            //add memmbers of this group
                            p.Member = channelinfo.Get("members")?.ToListStr();
                        }

                        Sys.Log2(10, "group (channel): ", p.Name);
                        channel.LogProperties();
                        //do partition mappings
                        Connector.PartitionMappings?.Apply(Ctxt, p, channel);

                        //add principal in the partition
                        Connector.Partition.Write(p);
                    }
                }
            }

            return(true);
        }
Esempio n. 3
0
        /// <summary>
        ///          Index all files of a channel
        /// </summary>
        /// <param name="channelName">Channel name for logging</param>
        /// <param name="channel">Json object which contains messages</param>
        /// <returns></returns>
        public bool IndexChannelFiles(string channelName, JsonObject channel)
        {
            Sys.Log("Index Files of '" + channelName + "'");
            bool      ok  = true;
            SlackFile doc = new SlackFile();

            doc.cntr = this;
            if (Connector.IsModeRealTime())
            {
                Sys.Log("Index files from date: " + Connector.RealTimeReferenceDate);
            }
            string latest = Connector.IsModeRealTime() ? Sys.ToStr(Dat.ToUnixTimestamp(Connector.RealTimeReferenceDate)) : "";

            bool has_more = false;
            int  page     = 0;
            int  pages    = 1;

            do
            {
                //iterate on all files by page (has_more = true)
                has_more = false;
                page++;
                JsonObject response = slackGet("files.list?channel=" + channel.ValueStr("id") + "&ts_from=" + latest + "&page=" + page) as JsonObject;
                if (response == null || (response != null && !response.ValueBoo("ok")))
                {
                    Sys.LogError(Json.Serialize(response)); return(false);
                }
                if (response != null)
                {
                    pages    = response.ValueInt("paging.pages");
                    has_more = response.ValueBoo("has_more", false);
                    JsonArray files = response.GetAsArray("files");
                    if (files != null)
                    {
                        for (int i = 0; i < files.EltCount(); i++)
                        {
                            //clear and reuse the doc instance
                            doc.Clear();

                            //current message
                            JsonObject file = files.Elt(i) as JsonObject;

                            //set custom to connectorDoc (slackDoc)
                            doc.channel = channel;
                            doc.file    = file;

                            //set mandatory fields: id, fileext & version
                            doc.Id      = file.GetValue("id");
                            doc.Version = file.GetValue("timestamp");
                            string mimetype = file.GetValue("mimetype");
                            if (Str.EQ(mimetype, "text/html"))
                            {
                                doc.FileExt = "htm";
                            }
                            else
                            {
                                doc.FileExt = Str.PathGetFileExt(file.GetValue("name"));
                            }


                            //set latest for Slack API Pagination
                            latest = doc.Version;

                            //Process Doc (send to Indexer)
                            ok &= Connector.ProcessConnectorDoc(doc);
                        }
                    }
                }
            }while (page < pages);
            return(ok);
        }
Esempio n. 4
0
        /// <summary>
        /// Starting point of the Generic Connector for the collection
        /// </summary>
        /// <returns>true if succeded otherwise false</returns>
        public override bool OnGenericIndexCollection()
        {
            bool ok = true;

            users = new MapOf <JsonObject>();
            //put all users info in cache
            JsonObject response = slackGet("users.list") as JsonObject;

            if (response == null || (response != null && !response.ValueBoo("ok")))
            {
                Sys.LogError(Json.Serialize(response)); return(false);
            }
            if (response != null)
            {
                JsonArray members = response.GetAsArray("members");
                if (members != null)
                {
                    for (int i = 0; i < members.EltCount(); i++)
                    {
                        JsonObject user = members.Elt(i) as JsonObject;
                        //Add user object in cache
                        users.Add(user.GetValue("id"), user);
                    }
                }
            }


            //index channels
            response = slackGet("conversations.list") as JsonObject;
            if (response == null || (response != null && !response.ValueBoo("ok")))
            {
                Sys.LogError(Json.Serialize(response)); return(false);
            }
            if (response != null)
            {
                JsonArray channels = response.GetAsArray("channels");
                if (channels != null)
                {
                    for (int i = 0; i < channels.EltCount(); i++)
                    {
                        JsonObject channel = channels.Elt(i) as JsonObject;
                        string     Name    = channel.GetValue("name");
                        //check if included/excluded from conf
                        if (channelMatcher.IsExcluded(Name))
                        {
                            continue;
                        }

                        //Index the Channel
                        if (indexMessages)
                        {
                            ok &= IndexChannelMessages(Name, channel);
                        }
                        if (indexFiles)
                        {
                            ok &= IndexChannelFiles(Name, channel);
                        }
                    }
                }
            }
            return(ok);
        }