public void CreateIndexes(ESConnection conn, ESIndexCmd._CheckIndexFlags flags) { for (int i = 0; i < list.Count; i++) { list[i].Create(conn, flags); } }
private void setNumberOfReplicas(ESConnection conn, String name, int replicas) { JObject obj = new JObject(); obj.WriteToken("settings.number_of_replicas", replicas); conn.Put(name + "/_settings", obj).ThrowIfError(); }
public void PrepareClose(ESConnection conn) { for (int i = 0; i < list.Count; i++) { list[i].PrepareClose(conn); } }
public void OptionalRename(ESConnection conn, int defaultGenerationsToKeep = 2) { for (int i = 0; i < list.Count; i++) { list[i].OptionalRename(conn); } }
public void OptionalOptimize(ESConnection conn) { for (int i = 0; i < list.Count; i++) { list[i].OptionalOptimize(conn); } }
public void Flush(ESConnection conn) { for (int i = 0; i < list.Count; i++) { list[i].Flush(conn); } }
public void Flush(ESConnection conn) { if (!IsOpen) { return; } ESIndexCmd cmd = createIndexCmd(conn); cmd.Flush(IndexName); }
public void OnPrepareRequest(ESConnection conn, HttpWebRequest req) { WebStreamProvider ufe = elt as WebStreamProvider; if (ufe == null) { return; } ufe.PrepareRequest(req); }
public ESDataEndpoint(ESEndpoint endpoint, ESIndexDocType doctype) : base(endpoint) { this.Connection = endpoint.Connection; this.DocType = doctype; this.cacheSize = endpoint.CacheSize; if (endpoint.MaxParallel > 0) { asyncQ = AsyncRequestQueue.Create(endpoint.MaxParallel); } }
public JObject LoadByKey(ESConnection conn, String key) { if (!IndexExists) { goto NOT_EXIST; } JObject record = null; ESMainResponse resp; if (KeyFieldName == null) { goto NOT_EXIST; } ; if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase)) { String url = UrlPart + "/" + HttpUtility.UrlEncode(key); resp = conn.Get(url); if (resp.StatusCode == HttpStatusCode.NotFound) { goto NOT_EXIST; } resp.ThrowIfError(); if (!resp.JObject.ReadBool("found")) { goto NOT_EXIST; } record = resp.JObject; } else { var req = createMatchQueryRequest(KeyFieldName, key); resp = conn.Post(UrlPart + "/_search", req); if (resp.StatusCode == System.Net.HttpStatusCode.NotFound) { goto NOT_EXIST; } resp.ThrowIfError(); JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false); if (hits == null || hits.Count == 0) { goto NOT_EXIST; } record = (JObject)hits[0]; } Logs.DebugLog.Log("Key={0}: Record={1}", key, record); return((JObject)record.SelectToken("_source", false)); NOT_EXIST: return(null); }
public void OptionalOptimize(ESConnection conn) { if (OptimizeToSegments <= 0) { return; } if (!IsOpen) { return; } ESIndexCmd cmd = createIndexCmd(conn); cmd.Optimize(IndexName, OptimizeToSegments, OptimizeWait); }
public bool DeleteByKey(ESConnection conn, String key) { if (!IndexExists) { goto NOT_EXIST; } ESMainResponse resp; if (KeyFieldName == null) { goto NOT_EXIST; } ; Logs.DebugLog.Log("DeleteByKey ({0})", key); if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase)) { return(deleteById(conn, key)); } var req = createMatchQueryRequest(KeyFieldName, key); resp = conn.Post(UrlPart + "/_search", req); if (resp.StatusCode == System.Net.HttpStatusCode.NotFound) { goto NOT_EXIST; } resp.ThrowIfError(); JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false); if (hits == null || hits.Count == 0) { goto NOT_EXIST; } String id = (String)((JObject)hits[0])["_id"]; if (id == null) { goto NOT_EXIST; } return(deleteById(conn, id)); NOT_EXIST: return(false); }
public void PrepareClose(ESConnection conn) { if (!IsOpen) { return; } ESIndexCmd cmd = createIndexCmd(conn); if (savedRefreshInterval != null) { JObject curSettings = new JObject(); curSettings["refresh_interval"] = savedRefreshInterval; cmd.PutSettings(curSettings); conn.Logger.Log("-- RefreshInterval restored to {0}", savedRefreshInterval); } cmd.Flush(IndexName); }
private bool deleteById(ESConnection conn, String key) { if (!IndexExists) { return(false); } String url = UrlPart + "/" + HttpUtility.UrlEncode(key); ESMainResponse resp = conn.Delete(url); Logs.DebugLog.Log("DeleteById ({0}) stat={1}", key, resp.StatusCode); if (resp.StatusCode == HttpStatusCode.NotFound) { return(false); } resp.ThrowIfError(); return(true); }
public static ESConnection SetLogging(PipelineContext ctx, ESConnection conn) { ESConnection._Logging flags = 0; if (ctx.Switches.IsOn("es.log")) { flags |= ESConnection._Logging.LogRequest | ESConnection._Logging.LogResponse; } if (ctx.Switches.IsOn("es.logrequest")) { flags |= ESConnection._Logging.LogRequest; } if (ctx.Switches.IsOn("es.logresponse")) { flags |= ESConnection._Logging.LogResponse; } conn.Logging = flags; return(conn); }
public ESEndpoint(ImportEngine engine, XmlNode node) : base(engine, node) { Connection = new ESConnection(node.ReadStr("@url")); CacheSize = node.ReadInt("@cache", -1); MaxParallel = node.ReadInt("@maxparallel", 0); ReadOnly = node.ReadBool("@readonly", false); XmlNode root = node.SelectSingleNode("indexes"); if (root == null) { root = node; } Indexes = new ESIndexDefinitions(engine.Xml, root, _loadConfig); if (Indexes.Count == 0) { throw new BMNodeException(node, "At least 1 index+type is required!"); } WaitFor = ESHealthCmd.SplitRequestedClusterStatus(node.ReadStr("waitfor/@status", "Green | Yellow"), out WaitForAlt); WaitForTimeout = node.ReadInt("waitfor/@timeout", 30); WaitForMustExcept = node.ReadBool("waitfor/@except", true); }
public ESCall(ILogger <ESCall> logger, IConfiguration config, ESClient client) { _logger = logger; _client = client; _settings = config.GetSection("AppSettings:ESConnection").Get <ESConnection>(); }
public ESCmdEndPoint(ESEndpoint endpoint, ESIndexDocType doctype) : base(endpoint) { this.Connection = endpoint.Connection; this.Doctype = doctype; }
public ExistState Exists(ESConnection conn, String key, DateTime?timeStamp = null) { JObject record = null; ESMainResponse resp; bool dateNeeded = DateFieldName != null && timeStamp != null; if (KeyFieldName == null) { goto NOT_EXIST; } ; if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase)) { String url = UrlPart + "/" + HttpUtility.UrlEncode(key); if (dateNeeded) { url += "?fields=" + HttpUtility.UrlEncode(DateFieldName); } resp = conn.Get(url); if (resp.StatusCode == HttpStatusCode.NotFound) { goto NOT_EXIST; } resp.ThrowIfError(); if (!resp.JObject.ReadBool("found")) { goto NOT_EXIST; } if (!dateNeeded) { return(ExistState.Exist); } record = resp.JObject; } else { JObject req = createMatchQueryRequest(KeyFieldName, key); req.AddArray("_source").Add(DateFieldName); resp = conn.Post(UrlPart + "/_search", req); if (resp.StatusCode == System.Net.HttpStatusCode.NotFound) { goto NOT_EXIST; } resp.ThrowIfError(); JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false); if (hits == null || hits.Count == 0) { goto NOT_EXIST; } if (!dateNeeded) { return(ExistState.Exist); } record = (JObject)hits[0]; } DateTime dt = record.ReadDate("_source." + DateFieldName, DateTime.MinValue); if (dt == DateTime.MinValue) { if (conn.Logger != null) { conn.Logger.Log(_LogType.ltWarning, "Exists: Record without field [" + DateFieldName + "] returned."); } return(ExistState.Exist); } //Logs.DebugLog.Log("Record=" + record); //Logs.DebugLog.Log("RecDate={0}, Timestamp={1}, cmp={2}", date, timeStamp, Comparer<DateTime>.Default.Compare((DateTime)date, (DateTime)timeStamp)); if (dt < timeStamp) { return(ExistState.ExistOlder); } if (dt > timeStamp) { return(ExistState.ExistNewer); } return(ExistState.ExistSame); NOT_EXIST: return(ExistState.NotExist); }
private void patchConfig(ESConnection conn, JObject config) { if (config == null) { return; } bool v5 = conn.Version.FormattedVersion.Major >= 5; JToken FALSE, TRUE; String stringType; if (v5) { FALSE = false; TRUE = true; stringType = "text"; } else { FALSE = "no"; TRUE = "not_analyzed"; stringType = "string"; } const String ERRORS = "errors_"; const String ADMIN = "admin_"; JObject mappings = (JObject)config["mappings"]; if (mappings == null) { return; } Logger logger = Logs.CreateLogger("import", "indexdef"); if (checkProperties(mappings, ERRORS)) { logger.Log(_LogType.ltInfo, "ES config: mapping created for [{0}].", ERRORS); } if (checkProperties(mappings, ADMIN)) { logger.Log(_LogType.ltInfo, "ES config: mapping created for [{0}].", ADMIN); } foreach (var kvp in mappings) { String key = kvp.Key; JObject o = kvp.Value as JObject; if (o == null) { continue; } if (disableAll(o)) { logger.Log(_LogType.ltInfo, "ES config: disabled _all for type [{0}]. If you don't want this, add '_all: {{enabled: true}}'. ", key); } var props = (JObject)o["properties"]; if (props == null) { continue; } switch (kvp.Key) { case ERRORS: addIfNotFound(props, "err_ds", stringType, FALSE); addIfNotFound(props, "err_date", "date", TRUE); addIfNotFound(props, "err_key", stringType, FALSE); addIfNotFound(props, "err_text", stringType, FALSE); addIfNotFound(props, "err_stack", stringType, FALSE); continue; case ADMIN: addIfNotFound(props, "adm_ds", stringType, FALSE); addIfNotFound(props, "adm_date", "date", TRUE); addIfNotFound(props, "adm_flags", stringType, FALSE); addIfNotFound(props, "adm_state", stringType, FALSE); addIfNotFound(props, "adm_added", "long", FALSE); addIfNotFound(props, "adm_deleted", "long", FALSE); addIfNotFound(props, "adm_emitted", "long", FALSE); addIfNotFound(props, "adm_errors", "long", FALSE); addIfNotFound(props, "adm_skipped", "long", FALSE); continue; } } }
public bool Create(ESConnection conn, ESIndexCmd._CheckIndexFlags flags) { if (!Active) { return(false); } if (IsOpen) { return(false); } //Switch off append flags if there's nothing to append if (IndexDateTimeFormat == null || Generations == 0) { flags &= ~ESIndexCmd._CheckIndexFlags.AppendDate; } if (ReadOnly) { flags |= ESIndexCmd._CheckIndexFlags.DontCreate; } DocMappings = null; DateTime configDate; JObject configJson = onLoadConfig(this, ConfigFile, out configDate); patchConfig(conn, configJson); ESIndexCmd cmd = createIndexCmd(conn); cmd.OnCreate = overrideShardsOnCreate; bool isNew; IndexName = cmd.CheckIndexFromFile(AliasName, configJson, configDate, flags, out isNew);//PW naar kijken! IndexExists = IndexName != null; IsNewIndex = isNew; RenameNeeded = (flags & ESIndexCmd._CheckIndexFlags.AppendDate) != 0 && isNew; //Get possible mappings and determine default doctype conn.Logger.Log(""); conn.Logger.Log("get mappings"); DocMappings = new List <String>(); if (IndexName != null) //Might be null if the index was not found (if Readonly==true) { ESGetMappingResponse resp = cmd.GetIndexMappings(IndexName); conn.Logger.Log("Index[0]={0}", resp[0].Name); foreach (var mapping in resp[0]) { conn.Logger.Log("-- Mapping={0}", mapping.Name); DocMappings.Add(mapping.Name); } } if (RefreshIntervalDuringImport != null && !ReadOnly) { JObject curSettings = cmd.GetSettings(); this.savedRefreshInterval = curSettings.ReadStr("refresh_interval", "1s"); curSettings = new JObject(); curSettings["refresh_interval"] = RefreshIntervalDuringImport; cmd.PutSettings(curSettings); conn.Logger.Log("-- RefreshInterval changed from={0} into {1}", savedRefreshInterval, RefreshIntervalDuringImport); } IsOpen = true; return(isNew); }
private ESIndexCmd createIndexCmd(ESConnection conn) { return(new ESIndexCmd(conn, this.IndexDateTimeFormat)); }
public void OptionalRename(ESConnection conn) { if (!IsOpen || !Active) { return; } IsOpen = false; if (!RenameNeeded) { return; } Logger logger = conn.Logger.Clone(GetType().Name); logger.Log("Optional rename name={0}, alias={1}, gen={2}", IndexName, AliasName, Generations); ESIndexCmd cmd = createIndexCmd(conn); String existingIndexName; cmd.GetIndexMappingsAndRealName(AliasName, out existingIndexName); logger.Log("Optional rename alias={0}, existing={1}", AliasName, existingIndexName); //Check if the current index was created without a timestamp. If so, we will just remove it //This is more or less a backward compat. issue if (String.Compare(AliasName, existingIndexName, StringComparison.InvariantCultureIgnoreCase) == 0) { logger.Log("Removing index {0}", existingIndexName); cmd.RemoveIndex(existingIndexName); } cmd.MoveAllAliases(AliasName, IndexName, existingIndexName); if (ExtraAlias != null) { cmd.CreateAliases(ExtraAlias.SplitStandard(), IndexName); } //Changing the #replica's if requested. if (this.NumReplicasAfterIndexed > 0) { logger.Log("Setting the #replicas for {0} to {1}", IndexName, NumReplicasAfterIndexed); setNumberOfReplicas(conn, IndexName, NumReplicasAfterIndexed); } if (Generations <= 0) { return; } List <String> indexes = cmd.GetIndexes(AliasName, true); logger.Log("Found {0} indexes starting with {1}:", indexes.Count, AliasName); int existingIdx = -1; int currentIdx = -1; for (int i = 0; i < indexes.Count; i++) { String s = indexes[i]; String what = String.Empty; if (String.Equals(s, existingIndexName, StringComparison.InvariantCultureIgnoreCase)) { existingIdx = i; what = " [existing alias]"; } else if (String.Equals(s, IndexName, StringComparison.InvariantCultureIgnoreCase)) { currentIdx = i; what = " [current index]"; } logger.Log("__ " + s + what); } //Check if the returned list is OK (our index should be in, as well as the current alias) if (currentIdx < 0) { throwNotFound(IndexName); } if (existingIdx < 0 && existingIndexName != null) { String msg = notFoundMsg(existingIndexName); Logs.ErrorLog.Log(msg); logger.Log(_LogType.ltError, msg); } //Remove all indexes between the current one and an existing alias. These are artifacts of temp. or crashed imports logger.Log("Removing indexes between existing ({0}) and current ({1})...", existingIdx, currentIdx); for (int i = existingIdx + 1; i < currentIdx; i++) { logger.Log("Removing index {0}", indexes[i]); cmd.RemoveIndex(indexes[i]); } //Remove all indexes between start-of-time and the existing alias, keeping 'generationsToKeep' generations logger.Log("Removing indexes between epoch and existing ({0}), keeping {1} generations...", existingIdx, Generations); for (int i = 0; i <= existingIdx - Generations + 1; i++) { logger.Log("Removing index {0}", indexes[i]); cmd.RemoveIndex(indexes[i]); } }
private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt) { int maxParallel = elt.ContextNode.ReadInt("@maxparallel", this.maxParallel); int splitUntil = elt.ContextNode.ReadInt("@splituntil", this.splitUntil); if (splitUntil < 0) { splitUntil = int.MaxValue; } bool scan = elt.ContextNode.ReadBool("@scan", this.scan); String url = elt.ToString(); ctx.SendItemStart(elt); String command = elt.ContextNode.ReadStr("@command", null); String index = command != null ? null : elt.ContextNode.ReadStr("@index"); //mutual exclusive with command String reqBody = elt.ContextNode.ReadStr("request", this.requestBody); JObject req = null; if (reqBody != null) { req = JObject.Parse(reqBody); } ctx.DebugLog.Log("Request scan={1}, body={0}", reqBody, scan); try { Uri uri = new Uri(url); ESConnection conn = ESHelper.CreateConnection(ctx, url); ContextCallback cb = new ContextCallback(ctx, this, elt); conn.Timeout = timeoutInMs; //Same timeout as what we send to ES conn.OnPrepareRequest = cb.OnPrepareRequest; if (command != null) { var resp = conn.SendCmd("POST", command, reqBody); resp.ThrowIfError(); Pipeline.EmitToken(ctx, sink, resp.JObject, "response", splitUntil); } else { ESRecordEnum e = new ESRecordEnum(conn, index, req, numRecords, timeout, scan); if (maxParallel > 0) { e.Async = true; } ctx.ImportLog.Log("Starting scan of {0} records. Index={1}, connection={2}, async={3}, buffersize={4} requestbody={5}, splituntil={6}, scan={7}.", e.Count, index, url, e.Async, numRecords, req != null, splitUntil, scan); foreach (var doc in e) { ctx.IncrementEmitted(); sink.HandleValue(ctx, "record/_sort", doc.Sort); sink.HandleValue(ctx, "record/_type", doc.Type); if (splitUntil != 0) { foreach (var kvp in doc) { String pfx = "record/" + kvp.Key; if (splitUntil == 1) { sink.HandleValue(ctx, pfx, kvp.Value); continue; } Pipeline.EmitToken(ctx, sink, kvp.Value, pfx, splitUntil - 1); } } sink.HandleValue(ctx, "record", doc); } } ctx.SendItemStop(); } catch (Exception e) { ctx.HandleException(e); } }