Пример #1
0
 public void CreateIndexes(ESConnection conn, ESIndexCmd._CheckIndexFlags flags)
 {
     for (int i = 0; i < list.Count; i++)
     {
         list[i].Create(conn, flags);
     }
 }
Пример #2
0
        private void setNumberOfReplicas(ESConnection conn, String name, int replicas)
        {
            JObject obj = new JObject();

            obj.WriteToken("settings.number_of_replicas", replicas);
            conn.Put(name + "/_settings", obj).ThrowIfError();
        }
Пример #3
0
 public void PrepareClose(ESConnection conn)
 {
     for (int i = 0; i < list.Count; i++)
     {
         list[i].PrepareClose(conn);
     }
 }
Пример #4
0
 public void OptionalRename(ESConnection conn, int defaultGenerationsToKeep = 2)
 {
     for (int i = 0; i < list.Count; i++)
     {
         list[i].OptionalRename(conn);
     }
 }
Пример #5
0
 public void OptionalOptimize(ESConnection conn)
 {
     for (int i = 0; i < list.Count; i++)
     {
         list[i].OptionalOptimize(conn);
     }
 }
Пример #6
0
 public void Flush(ESConnection conn)
 {
     for (int i = 0; i < list.Count; i++)
     {
         list[i].Flush(conn);
     }
 }
Пример #7
0
        public void Flush(ESConnection conn)
        {
            if (!IsOpen)
            {
                return;
            }
            ESIndexCmd cmd = createIndexCmd(conn);

            cmd.Flush(IndexName);
        }
Пример #8
0
            public void OnPrepareRequest(ESConnection conn, HttpWebRequest req)
            {
                WebStreamProvider ufe = elt as WebStreamProvider;

                if (ufe == null)
                {
                    return;
                }
                ufe.PrepareRequest(req);
            }
Пример #9
0
 public ESDataEndpoint(ESEndpoint endpoint, ESIndexDocType doctype)
     : base(endpoint)
 {
     this.Connection = endpoint.Connection;
     this.DocType    = doctype;
     this.cacheSize  = endpoint.CacheSize;
     if (endpoint.MaxParallel > 0)
     {
         asyncQ = AsyncRequestQueue.Create(endpoint.MaxParallel);
     }
 }
Пример #10
0
        public JObject LoadByKey(ESConnection conn, String key)
        {
            if (!IndexExists)
            {
                goto NOT_EXIST;
            }

            JObject        record = null;
            ESMainResponse resp;

            if (KeyFieldName == null)
            {
                goto NOT_EXIST;
            }
            ;
            if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase))
            {
                String url = UrlPart + "/" + HttpUtility.UrlEncode(key);
                resp = conn.Get(url);
                if (resp.StatusCode == HttpStatusCode.NotFound)
                {
                    goto NOT_EXIST;
                }
                resp.ThrowIfError();
                if (!resp.JObject.ReadBool("found"))
                {
                    goto NOT_EXIST;
                }
                record = resp.JObject;
            }
            else
            {
                var req = createMatchQueryRequest(KeyFieldName, key);
                resp = conn.Post(UrlPart + "/_search", req);
                if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
                {
                    goto NOT_EXIST;
                }
                resp.ThrowIfError();
                JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false);
                if (hits == null || hits.Count == 0)
                {
                    goto NOT_EXIST;
                }

                record = (JObject)hits[0];
            }
            Logs.DebugLog.Log("Key={0}: Record={1}", key, record);
            return((JObject)record.SelectToken("_source", false));

NOT_EXIST:
            return(null);
        }
Пример #11
0
        public void OptionalOptimize(ESConnection conn)
        {
            if (OptimizeToSegments <= 0)
            {
                return;
            }
            if (!IsOpen)
            {
                return;
            }
            ESIndexCmd cmd = createIndexCmd(conn);

            cmd.Optimize(IndexName, OptimizeToSegments, OptimizeWait);
        }
Пример #12
0
        public bool DeleteByKey(ESConnection conn, String key)
        {
            if (!IndexExists)
            {
                goto NOT_EXIST;
            }

            ESMainResponse resp;

            if (KeyFieldName == null)
            {
                goto NOT_EXIST;
            }
            ;
            Logs.DebugLog.Log("DeleteByKey ({0})", key);
            if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase))
            {
                return(deleteById(conn, key));
            }
            var req = createMatchQueryRequest(KeyFieldName, key);

            resp = conn.Post(UrlPart + "/_search", req);
            if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
            {
                goto NOT_EXIST;
            }
            resp.ThrowIfError();

            JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false);

            if (hits == null || hits.Count == 0)
            {
                goto NOT_EXIST;
            }

            String id = (String)((JObject)hits[0])["_id"];

            if (id == null)
            {
                goto NOT_EXIST;
            }

            return(deleteById(conn, id));

NOT_EXIST:
            return(false);
        }
Пример #13
0
        public void PrepareClose(ESConnection conn)
        {
            if (!IsOpen)
            {
                return;
            }
            ESIndexCmd cmd = createIndexCmd(conn);

            if (savedRefreshInterval != null)
            {
                JObject curSettings = new JObject();
                curSettings["refresh_interval"] = savedRefreshInterval;
                cmd.PutSettings(curSettings);
                conn.Logger.Log("-- RefreshInterval restored to {0}", savedRefreshInterval);
            }
            cmd.Flush(IndexName);
        }
Пример #14
0
        private bool deleteById(ESConnection conn, String key)
        {
            if (!IndexExists)
            {
                return(false);
            }

            String         url  = UrlPart + "/" + HttpUtility.UrlEncode(key);
            ESMainResponse resp = conn.Delete(url);

            Logs.DebugLog.Log("DeleteById ({0}) stat={1}", key, resp.StatusCode);
            if (resp.StatusCode == HttpStatusCode.NotFound)
            {
                return(false);
            }
            resp.ThrowIfError();
            return(true);
        }
Пример #15
0
 public static ESConnection SetLogging(PipelineContext ctx, ESConnection conn)
 {
     ESConnection._Logging flags = 0;
     if (ctx.Switches.IsOn("es.log"))
     {
         flags |= ESConnection._Logging.LogRequest | ESConnection._Logging.LogResponse;
     }
     if (ctx.Switches.IsOn("es.logrequest"))
     {
         flags |= ESConnection._Logging.LogRequest;
     }
     if (ctx.Switches.IsOn("es.logresponse"))
     {
         flags |= ESConnection._Logging.LogResponse;
     }
     conn.Logging = flags;
     return(conn);
 }
Пример #16
0
        public ESEndpoint(ImportEngine engine, XmlNode node)
            : base(engine, node)
        {
            Connection  = new ESConnection(node.ReadStr("@url"));
            CacheSize   = node.ReadInt("@cache", -1);
            MaxParallel = node.ReadInt("@maxparallel", 0);
            ReadOnly    = node.ReadBool("@readonly", false);
            XmlNode root = node.SelectSingleNode("indexes");

            if (root == null)
            {
                root = node;
            }

            Indexes = new ESIndexDefinitions(engine.Xml, root, _loadConfig);
            if (Indexes.Count == 0)
            {
                throw new BMNodeException(node, "At least 1 index+type is required!");
            }

            WaitFor           = ESHealthCmd.SplitRequestedClusterStatus(node.ReadStr("waitfor/@status", "Green | Yellow"), out WaitForAlt);
            WaitForTimeout    = node.ReadInt("waitfor/@timeout", 30);
            WaitForMustExcept = node.ReadBool("waitfor/@except", true);
        }
Пример #17
0
 public ESCall(ILogger <ESCall> logger, IConfiguration config, ESClient client)
 {
     _logger   = logger;
     _client   = client;
     _settings = config.GetSection("AppSettings:ESConnection").Get <ESConnection>();
 }
Пример #18
0
 public ESCmdEndPoint(ESEndpoint endpoint, ESIndexDocType doctype)
     : base(endpoint)
 {
     this.Connection = endpoint.Connection;
     this.Doctype    = doctype;
 }
Пример #19
0
        public ExistState Exists(ESConnection conn, String key, DateTime?timeStamp = null)
        {
            JObject        record = null;
            ESMainResponse resp;
            bool           dateNeeded = DateFieldName != null && timeStamp != null;

            if (KeyFieldName == null)
            {
                goto NOT_EXIST;
            }
            ;
            if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase))
            {
                String url = UrlPart + "/" + HttpUtility.UrlEncode(key);
                if (dateNeeded)
                {
                    url += "?fields=" + HttpUtility.UrlEncode(DateFieldName);
                }
                resp = conn.Get(url);
                if (resp.StatusCode == HttpStatusCode.NotFound)
                {
                    goto NOT_EXIST;
                }
                resp.ThrowIfError();
                if (!resp.JObject.ReadBool("found"))
                {
                    goto NOT_EXIST;
                }
                if (!dateNeeded)
                {
                    return(ExistState.Exist);
                }
                record = resp.JObject;
            }
            else
            {
                JObject req = createMatchQueryRequest(KeyFieldName, key);
                req.AddArray("_source").Add(DateFieldName);
                resp = conn.Post(UrlPart + "/_search", req);
                if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
                {
                    goto NOT_EXIST;
                }
                resp.ThrowIfError();
                JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false);
                if (hits == null || hits.Count == 0)
                {
                    goto NOT_EXIST;
                }

                if (!dateNeeded)
                {
                    return(ExistState.Exist);
                }
                record = (JObject)hits[0];
            }
            DateTime dt = record.ReadDate("_source." + DateFieldName, DateTime.MinValue);

            if (dt == DateTime.MinValue)
            {
                if (conn.Logger != null)
                {
                    conn.Logger.Log(_LogType.ltWarning, "Exists: Record without field [" + DateFieldName + "] returned.");
                }
                return(ExistState.Exist);
            }
            //Logs.DebugLog.Log("Record=" + record);
            //Logs.DebugLog.Log("RecDate={0}, Timestamp={1}, cmp={2}", date, timeStamp, Comparer<DateTime>.Default.Compare((DateTime)date, (DateTime)timeStamp));
            if (dt < timeStamp)
            {
                return(ExistState.ExistOlder);
            }
            if (dt > timeStamp)
            {
                return(ExistState.ExistNewer);
            }
            return(ExistState.ExistSame);

NOT_EXIST:
            return(ExistState.NotExist);
        }
Пример #20
0
        private void patchConfig(ESConnection conn, JObject config)
        {
            if (config == null)
            {
                return;
            }

            bool   v5 = conn.Version.FormattedVersion.Major >= 5;
            JToken FALSE, TRUE;
            String stringType;

            if (v5)
            {
                FALSE      = false;
                TRUE       = true;
                stringType = "text";
            }
            else
            {
                FALSE      = "no";
                TRUE       = "not_analyzed";
                stringType = "string";
            }

            const String ERRORS   = "errors_";
            const String ADMIN    = "admin_";
            JObject      mappings = (JObject)config["mappings"];

            if (mappings == null)
            {
                return;
            }

            Logger logger = Logs.CreateLogger("import", "indexdef");

            if (checkProperties(mappings, ERRORS))
            {
                logger.Log(_LogType.ltInfo, "ES config: mapping created for [{0}].", ERRORS);
            }
            if (checkProperties(mappings, ADMIN))
            {
                logger.Log(_LogType.ltInfo, "ES config: mapping created for [{0}].", ADMIN);
            }

            foreach (var kvp in mappings)
            {
                String  key = kvp.Key;
                JObject o   = kvp.Value as JObject;
                if (o == null)
                {
                    continue;
                }
                if (disableAll(o))
                {
                    logger.Log(_LogType.ltInfo, "ES config: disabled _all for type [{0}]. If you don't want this, add '_all: {{enabled: true}}'. ", key);
                }

                var props = (JObject)o["properties"];
                if (props == null)
                {
                    continue;
                }
                switch (kvp.Key)
                {
                case ERRORS:
                    addIfNotFound(props, "err_ds", stringType, FALSE);
                    addIfNotFound(props, "err_date", "date", TRUE);
                    addIfNotFound(props, "err_key", stringType, FALSE);
                    addIfNotFound(props, "err_text", stringType, FALSE);
                    addIfNotFound(props, "err_stack", stringType, FALSE);
                    continue;

                case ADMIN:
                    addIfNotFound(props, "adm_ds", stringType, FALSE);
                    addIfNotFound(props, "adm_date", "date", TRUE);
                    addIfNotFound(props, "adm_flags", stringType, FALSE);
                    addIfNotFound(props, "adm_state", stringType, FALSE);
                    addIfNotFound(props, "adm_added", "long", FALSE);
                    addIfNotFound(props, "adm_deleted", "long", FALSE);
                    addIfNotFound(props, "adm_emitted", "long", FALSE);
                    addIfNotFound(props, "adm_errors", "long", FALSE);
                    addIfNotFound(props, "adm_skipped", "long", FALSE);
                    continue;
                }
            }
        }
Пример #21
0
        public bool Create(ESConnection conn, ESIndexCmd._CheckIndexFlags flags)
        {
            if (!Active)
            {
                return(false);
            }
            if (IsOpen)
            {
                return(false);
            }

            //Switch off append flags if there's nothing to append
            if (IndexDateTimeFormat == null || Generations == 0)
            {
                flags &= ~ESIndexCmd._CheckIndexFlags.AppendDate;
            }

            if (ReadOnly)
            {
                flags |= ESIndexCmd._CheckIndexFlags.DontCreate;
            }

            DocMappings = null;
            DateTime configDate;
            JObject  configJson = onLoadConfig(this, ConfigFile, out configDate);

            patchConfig(conn, configJson);

            ESIndexCmd cmd = createIndexCmd(conn);

            cmd.OnCreate = overrideShardsOnCreate;
            bool isNew;

            IndexName    = cmd.CheckIndexFromFile(AliasName, configJson, configDate, flags, out isNew);//PW naar kijken!
            IndexExists  = IndexName != null;
            IsNewIndex   = isNew;
            RenameNeeded = (flags & ESIndexCmd._CheckIndexFlags.AppendDate) != 0 && isNew;

            //Get possible mappings and determine default doctype
            conn.Logger.Log("");
            conn.Logger.Log("get mappings");
            DocMappings = new List <String>();

            if (IndexName != null) //Might be null if the index was not found (if Readonly==true)
            {
                ESGetMappingResponse resp = cmd.GetIndexMappings(IndexName);
                conn.Logger.Log("Index[0]={0}", resp[0].Name);
                foreach (var mapping in resp[0])
                {
                    conn.Logger.Log("-- Mapping={0}", mapping.Name);
                    DocMappings.Add(mapping.Name);
                }
            }

            if (RefreshIntervalDuringImport != null && !ReadOnly)
            {
                JObject curSettings = cmd.GetSettings();
                this.savedRefreshInterval = curSettings.ReadStr("refresh_interval", "1s");
                curSettings = new JObject();
                curSettings["refresh_interval"] = RefreshIntervalDuringImport;
                cmd.PutSettings(curSettings);
                conn.Logger.Log("-- RefreshInterval changed from={0} into {1}", savedRefreshInterval, RefreshIntervalDuringImport);
            }
            IsOpen = true;
            return(isNew);
        }
Пример #22
0
 private ESIndexCmd createIndexCmd(ESConnection conn)
 {
     return(new ESIndexCmd(conn, this.IndexDateTimeFormat));
 }
Пример #23
0
        public void OptionalRename(ESConnection conn)
        {
            if (!IsOpen || !Active)
            {
                return;
            }
            IsOpen = false;
            if (!RenameNeeded)
            {
                return;
            }

            Logger logger = conn.Logger.Clone(GetType().Name);

            logger.Log("Optional rename name={0}, alias={1}, gen={2}", IndexName, AliasName, Generations);
            ESIndexCmd cmd = createIndexCmd(conn);

            String existingIndexName;

            cmd.GetIndexMappingsAndRealName(AliasName, out existingIndexName);

            logger.Log("Optional rename alias={0}, existing={1}", AliasName, existingIndexName);

            //Check if the current index was created without a timestamp. If so, we will just remove it
            //This is more or less a backward compat. issue
            if (String.Compare(AliasName, existingIndexName, StringComparison.InvariantCultureIgnoreCase) == 0)
            {
                logger.Log("Removing index {0}", existingIndexName);
                cmd.RemoveIndex(existingIndexName);
            }

            cmd.MoveAllAliases(AliasName, IndexName, existingIndexName);
            if (ExtraAlias != null)
            {
                cmd.CreateAliases(ExtraAlias.SplitStandard(), IndexName);
            }

            //Changing the #replica's if requested.
            if (this.NumReplicasAfterIndexed > 0)
            {
                logger.Log("Setting the #replicas for {0} to {1}", IndexName, NumReplicasAfterIndexed);
                setNumberOfReplicas(conn, IndexName, NumReplicasAfterIndexed);
            }

            if (Generations <= 0)
            {
                return;
            }
            List <String> indexes = cmd.GetIndexes(AliasName, true);

            logger.Log("Found {0} indexes starting with {1}:", indexes.Count, AliasName);
            int existingIdx = -1;
            int currentIdx  = -1;

            for (int i = 0; i < indexes.Count; i++)
            {
                String s    = indexes[i];
                String what = String.Empty;
                if (String.Equals(s, existingIndexName, StringComparison.InvariantCultureIgnoreCase))
                {
                    existingIdx = i;
                    what        = " [existing alias]";
                }
                else if (String.Equals(s, IndexName, StringComparison.InvariantCultureIgnoreCase))
                {
                    currentIdx = i;
                    what       = " [current index]";
                }
                logger.Log("__ " + s + what);
            }

            //Check if the returned list is OK (our index should be in, as well as the current alias)
            if (currentIdx < 0)
            {
                throwNotFound(IndexName);
            }
            if (existingIdx < 0 && existingIndexName != null)
            {
                String msg = notFoundMsg(existingIndexName);
                Logs.ErrorLog.Log(msg);
                logger.Log(_LogType.ltError, msg);
            }

            //Remove all indexes between the current one and an existing alias. These are artifacts of temp. or crashed imports
            logger.Log("Removing indexes between existing ({0}) and current ({1})...", existingIdx, currentIdx);
            for (int i = existingIdx + 1; i < currentIdx; i++)
            {
                logger.Log("Removing index {0}", indexes[i]);
                cmd.RemoveIndex(indexes[i]);
            }

            //Remove all indexes between start-of-time and the existing alias, keeping 'generationsToKeep' generations
            logger.Log("Removing indexes between epoch and existing ({0}), keeping {1} generations...", existingIdx, Generations);
            for (int i = 0; i <= existingIdx - Generations + 1; i++)
            {
                logger.Log("Removing index {0}", indexes[i]);
                cmd.RemoveIndex(indexes[i]);
            }
        }
Пример #24
0
        private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt)
        {
            int maxParallel = elt.ContextNode.ReadInt("@maxparallel", this.maxParallel);
            int splitUntil  = elt.ContextNode.ReadInt("@splituntil", this.splitUntil);

            if (splitUntil < 0)
            {
                splitUntil = int.MaxValue;
            }
            bool scan = elt.ContextNode.ReadBool("@scan", this.scan);

            String url = elt.ToString();

            ctx.SendItemStart(elt);
            String  command = elt.ContextNode.ReadStr("@command", null);
            String  index   = command != null ? null : elt.ContextNode.ReadStr("@index"); //mutual exclusive with command
            String  reqBody = elt.ContextNode.ReadStr("request", this.requestBody);
            JObject req     = null;

            if (reqBody != null)
            {
                req = JObject.Parse(reqBody);
            }
            ctx.DebugLog.Log("Request scan={1}, body={0}", reqBody, scan);
            try
            {
                Uri             uri  = new Uri(url);
                ESConnection    conn = ESHelper.CreateConnection(ctx, url);
                ContextCallback cb   = new ContextCallback(ctx, this, elt);
                conn.Timeout          = timeoutInMs; //Same timeout as what we send to ES
                conn.OnPrepareRequest = cb.OnPrepareRequest;
                if (command != null)
                {
                    var resp = conn.SendCmd("POST", command, reqBody);
                    resp.ThrowIfError();
                    Pipeline.EmitToken(ctx, sink, resp.JObject, "response", splitUntil);
                }
                else
                {
                    ESRecordEnum e = new ESRecordEnum(conn, index, req, numRecords, timeout, scan);
                    if (maxParallel > 0)
                    {
                        e.Async = true;
                    }
                    ctx.ImportLog.Log("Starting scan of {0} records. Index={1}, connection={2}, async={3}, buffersize={4} requestbody={5}, splituntil={6}, scan={7}.", e.Count, index, url, e.Async, numRecords, req != null, splitUntil, scan);
                    foreach (var doc in e)
                    {
                        ctx.IncrementEmitted();
                        sink.HandleValue(ctx, "record/_sort", doc.Sort);
                        sink.HandleValue(ctx, "record/_type", doc.Type);
                        if (splitUntil != 0)
                        {
                            foreach (var kvp in doc)
                            {
                                String pfx = "record/" + kvp.Key;
                                if (splitUntil == 1)
                                {
                                    sink.HandleValue(ctx, pfx, kvp.Value);
                                    continue;
                                }
                                Pipeline.EmitToken(ctx, sink, kvp.Value, pfx, splitUntil - 1);
                            }
                        }
                        sink.HandleValue(ctx, "record", doc);
                    }
                }
                ctx.SendItemStop();
            }
            catch (Exception e)
            {
                ctx.HandleException(e);
            }
        }