public void CreateIndexes(ESConnection conn, ESIndexCmd._CheckIndexFlags flags)
{
for (int i = 0; i < list.Count; i++)
{
list[i].Create(conn, flags);
}
}
private void setNumberOfReplicas(ESConnection conn, String name, int replicas)
{
JObject obj = new JObject();
obj.WriteToken("settings.number_of_replicas", replicas);
conn.Put(name + "/_settings", obj).ThrowIfError();
}
public void PrepareClose(ESConnection conn)
{
for (int i = 0; i < list.Count; i++)
{
list[i].PrepareClose(conn);
}
}
public void OptionalRename(ESConnection conn, int defaultGenerationsToKeep = 2)
{
for (int i = 0; i < list.Count; i++)
{
list[i].OptionalRename(conn);
}
}
public void OptionalOptimize(ESConnection conn)
{
for (int i = 0; i < list.Count; i++)
{
list[i].OptionalOptimize(conn);
}
}
public void Flush(ESConnection conn)
{
for (int i = 0; i < list.Count; i++)
{
list[i].Flush(conn);
}
}
public void Flush(ESConnection conn)
{
if (!IsOpen)
{
return;
}
ESIndexCmd cmd = createIndexCmd(conn);
cmd.Flush(IndexName);
}
public void OnPrepareRequest(ESConnection conn, HttpWebRequest req)
{
WebStreamProvider ufe = elt as WebStreamProvider;
if (ufe == null)
{
return;
}
ufe.PrepareRequest(req);
}
public ESDataEndpoint(ESEndpoint endpoint, ESIndexDocType doctype)
: base(endpoint)
{
this.Connection = endpoint.Connection;
this.DocType = doctype;
this.cacheSize = endpoint.CacheSize;
if (endpoint.MaxParallel > 0)
{
asyncQ = AsyncRequestQueue.Create(endpoint.MaxParallel);
}
}
public JObject LoadByKey(ESConnection conn, String key)
{
if (!IndexExists)
{
goto NOT_EXIST;
}
JObject record = null;
ESMainResponse resp;
if (KeyFieldName == null)
{
goto NOT_EXIST;
}
;
if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase))
{
String url = UrlPart + "/" + HttpUtility.UrlEncode(key);
resp = conn.Get(url);
if (resp.StatusCode == HttpStatusCode.NotFound)
{
goto NOT_EXIST;
}
resp.ThrowIfError();
if (!resp.JObject.ReadBool("found"))
{
goto NOT_EXIST;
}
record = resp.JObject;
}
else
{
var req = createMatchQueryRequest(KeyFieldName, key);
resp = conn.Post(UrlPart + "/_search", req);
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
{
goto NOT_EXIST;
}
resp.ThrowIfError();
JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false);
if (hits == null || hits.Count == 0)
{
goto NOT_EXIST;
}
record = (JObject)hits[0];
}
Logs.DebugLog.Log("Key={0}: Record={1}", key, record);
return((JObject)record.SelectToken("_source", false));
NOT_EXIST:
return(null);
}
public void OptionalOptimize(ESConnection conn)
{
if (OptimizeToSegments <= 0)
{
return;
}
if (!IsOpen)
{
return;
}
ESIndexCmd cmd = createIndexCmd(conn);
cmd.Optimize(IndexName, OptimizeToSegments, OptimizeWait);
}
public bool DeleteByKey(ESConnection conn, String key)
{
if (!IndexExists)
{
goto NOT_EXIST;
}
ESMainResponse resp;
if (KeyFieldName == null)
{
goto NOT_EXIST;
}
;
Logs.DebugLog.Log("DeleteByKey ({0})", key);
if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase))
{
return(deleteById(conn, key));
}
var req = createMatchQueryRequest(KeyFieldName, key);
resp = conn.Post(UrlPart + "/_search", req);
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
{
goto NOT_EXIST;
}
resp.ThrowIfError();
JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false);
if (hits == null || hits.Count == 0)
{
goto NOT_EXIST;
}
String id = (String)((JObject)hits[0])["_id"];
if (id == null)
{
goto NOT_EXIST;
}
return(deleteById(conn, id));
NOT_EXIST:
return(false);
}
public void PrepareClose(ESConnection conn)
{
if (!IsOpen)
{
return;
}
ESIndexCmd cmd = createIndexCmd(conn);
if (savedRefreshInterval != null)
{
JObject curSettings = new JObject();
curSettings["refresh_interval"] = savedRefreshInterval;
cmd.PutSettings(curSettings);
conn.Logger.Log("-- RefreshInterval restored to {0}", savedRefreshInterval);
}
cmd.Flush(IndexName);
}
private bool deleteById(ESConnection conn, String key)
{
if (!IndexExists)
{
return(false);
}
String url = UrlPart + "/" + HttpUtility.UrlEncode(key);
ESMainResponse resp = conn.Delete(url);
Logs.DebugLog.Log("DeleteById ({0}) stat={1}", key, resp.StatusCode);
if (resp.StatusCode == HttpStatusCode.NotFound)
{
return(false);
}
resp.ThrowIfError();
return(true);
}
public static ESConnection SetLogging(PipelineContext ctx, ESConnection conn)
{
ESConnection._Logging flags = 0;
if (ctx.Switches.IsOn("es.log"))
{
flags |= ESConnection._Logging.LogRequest | ESConnection._Logging.LogResponse;
}
if (ctx.Switches.IsOn("es.logrequest"))
{
flags |= ESConnection._Logging.LogRequest;
}
if (ctx.Switches.IsOn("es.logresponse"))
{
flags |= ESConnection._Logging.LogResponse;
}
conn.Logging = flags;
return(conn);
}
public ESEndpoint(ImportEngine engine, XmlNode node)
: base(engine, node)
{
Connection = new ESConnection(node.ReadStr("@url"));
CacheSize = node.ReadInt("@cache", -1);
MaxParallel = node.ReadInt("@maxparallel", 0);
ReadOnly = node.ReadBool("@readonly", false);
XmlNode root = node.SelectSingleNode("indexes");
if (root == null)
{
root = node;
}
Indexes = new ESIndexDefinitions(engine.Xml, root, _loadConfig);
if (Indexes.Count == 0)
{
throw new BMNodeException(node, "At least 1 index+type is required!");
}
WaitFor = ESHealthCmd.SplitRequestedClusterStatus(node.ReadStr("waitfor/@status", "Green | Yellow"), out WaitForAlt);
WaitForTimeout = node.ReadInt("waitfor/@timeout", 30);
WaitForMustExcept = node.ReadBool("waitfor/@except", true);
}
public ESCall(ILogger <ESCall> logger, IConfiguration config, ESClient client)
{
_logger = logger;
_client = client;
_settings = config.GetSection("AppSettings:ESConnection").Get <ESConnection>();
}
public ESCmdEndPoint(ESEndpoint endpoint, ESIndexDocType doctype)
: base(endpoint)
{
this.Connection = endpoint.Connection;
this.Doctype = doctype;
}
public ExistState Exists(ESConnection conn, String key, DateTime?timeStamp = null)
{
JObject record = null;
ESMainResponse resp;
bool dateNeeded = DateFieldName != null && timeStamp != null;
if (KeyFieldName == null)
{
goto NOT_EXIST;
}
;
if (KeyFieldName.Equals("_id", StringComparison.InvariantCultureIgnoreCase))
{
String url = UrlPart + "/" + HttpUtility.UrlEncode(key);
if (dateNeeded)
{
url += "?fields=" + HttpUtility.UrlEncode(DateFieldName);
}
resp = conn.Get(url);
if (resp.StatusCode == HttpStatusCode.NotFound)
{
goto NOT_EXIST;
}
resp.ThrowIfError();
if (!resp.JObject.ReadBool("found"))
{
goto NOT_EXIST;
}
if (!dateNeeded)
{
return(ExistState.Exist);
}
record = resp.JObject;
}
else
{
JObject req = createMatchQueryRequest(KeyFieldName, key);
req.AddArray("_source").Add(DateFieldName);
resp = conn.Post(UrlPart + "/_search", req);
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
{
goto NOT_EXIST;
}
resp.ThrowIfError();
JArray hits = (JArray)resp.JObject.SelectToken("hits.hits", false);
if (hits == null || hits.Count == 0)
{
goto NOT_EXIST;
}
if (!dateNeeded)
{
return(ExistState.Exist);
}
record = (JObject)hits[0];
}
DateTime dt = record.ReadDate("_source." + DateFieldName, DateTime.MinValue);
if (dt == DateTime.MinValue)
{
if (conn.Logger != null)
{
conn.Logger.Log(_LogType.ltWarning, "Exists: Record without field [" + DateFieldName + "] returned.");
}
return(ExistState.Exist);
}
//Logs.DebugLog.Log("Record=" + record);
//Logs.DebugLog.Log("RecDate={0}, Timestamp={1}, cmp={2}", date, timeStamp, Comparer<DateTime>.Default.Compare((DateTime)date, (DateTime)timeStamp));
if (dt < timeStamp)
{
return(ExistState.ExistOlder);
}
if (dt > timeStamp)
{
return(ExistState.ExistNewer);
}
return(ExistState.ExistSame);
NOT_EXIST:
return(ExistState.NotExist);
}
private void patchConfig(ESConnection conn, JObject config)
{
if (config == null)
{
return;
}
bool v5 = conn.Version.FormattedVersion.Major >= 5;
JToken FALSE, TRUE;
String stringType;
if (v5)
{
FALSE = false;
TRUE = true;
stringType = "text";
}
else
{
FALSE = "no";
TRUE = "not_analyzed";
stringType = "string";
}
const String ERRORS = "errors_";
const String ADMIN = "admin_";
JObject mappings = (JObject)config["mappings"];
if (mappings == null)
{
return;
}
Logger logger = Logs.CreateLogger("import", "indexdef");
if (checkProperties(mappings, ERRORS))
{
logger.Log(_LogType.ltInfo, "ES config: mapping created for [{0}].", ERRORS);
}
if (checkProperties(mappings, ADMIN))
{
logger.Log(_LogType.ltInfo, "ES config: mapping created for [{0}].", ADMIN);
}
foreach (var kvp in mappings)
{
String key = kvp.Key;
JObject o = kvp.Value as JObject;
if (o == null)
{
continue;
}
if (disableAll(o))
{
logger.Log(_LogType.ltInfo, "ES config: disabled _all for type [{0}]. If you don't want this, add '_all: {{enabled: true}}'. ", key);
}
var props = (JObject)o["properties"];
if (props == null)
{
continue;
}
switch (kvp.Key)
{
case ERRORS:
addIfNotFound(props, "err_ds", stringType, FALSE);
addIfNotFound(props, "err_date", "date", TRUE);
addIfNotFound(props, "err_key", stringType, FALSE);
addIfNotFound(props, "err_text", stringType, FALSE);
addIfNotFound(props, "err_stack", stringType, FALSE);
continue;
case ADMIN:
addIfNotFound(props, "adm_ds", stringType, FALSE);
addIfNotFound(props, "adm_date", "date", TRUE);
addIfNotFound(props, "adm_flags", stringType, FALSE);
addIfNotFound(props, "adm_state", stringType, FALSE);
addIfNotFound(props, "adm_added", "long", FALSE);
addIfNotFound(props, "adm_deleted", "long", FALSE);
addIfNotFound(props, "adm_emitted", "long", FALSE);
addIfNotFound(props, "adm_errors", "long", FALSE);
addIfNotFound(props, "adm_skipped", "long", FALSE);
continue;
}
}
}
public bool Create(ESConnection conn, ESIndexCmd._CheckIndexFlags flags)
{
if (!Active)
{
return(false);
}
if (IsOpen)
{
return(false);
}
//Switch off append flags if there's nothing to append
if (IndexDateTimeFormat == null || Generations == 0)
{
flags &= ~ESIndexCmd._CheckIndexFlags.AppendDate;
}
if (ReadOnly)
{
flags |= ESIndexCmd._CheckIndexFlags.DontCreate;
}
DocMappings = null;
DateTime configDate;
JObject configJson = onLoadConfig(this, ConfigFile, out configDate);
patchConfig(conn, configJson);
ESIndexCmd cmd = createIndexCmd(conn);
cmd.OnCreate = overrideShardsOnCreate;
bool isNew;
IndexName = cmd.CheckIndexFromFile(AliasName, configJson, configDate, flags, out isNew);//PW naar kijken!
IndexExists = IndexName != null;
IsNewIndex = isNew;
RenameNeeded = (flags & ESIndexCmd._CheckIndexFlags.AppendDate) != 0 && isNew;
//Get possible mappings and determine default doctype
conn.Logger.Log("");
conn.Logger.Log("get mappings");
DocMappings = new List <String>();
if (IndexName != null) //Might be null if the index was not found (if Readonly==true)
{
ESGetMappingResponse resp = cmd.GetIndexMappings(IndexName);
conn.Logger.Log("Index[0]={0}", resp[0].Name);
foreach (var mapping in resp[0])
{
conn.Logger.Log("-- Mapping={0}", mapping.Name);
DocMappings.Add(mapping.Name);
}
}
if (RefreshIntervalDuringImport != null && !ReadOnly)
{
JObject curSettings = cmd.GetSettings();
this.savedRefreshInterval = curSettings.ReadStr("refresh_interval", "1s");
curSettings = new JObject();
curSettings["refresh_interval"] = RefreshIntervalDuringImport;
cmd.PutSettings(curSettings);
conn.Logger.Log("-- RefreshInterval changed from={0} into {1}", savedRefreshInterval, RefreshIntervalDuringImport);
}
IsOpen = true;
return(isNew);
}
private ESIndexCmd createIndexCmd(ESConnection conn)
{
return(new ESIndexCmd(conn, this.IndexDateTimeFormat));
}
public void OptionalRename(ESConnection conn)
{
if (!IsOpen || !Active)
{
return;
}
IsOpen = false;
if (!RenameNeeded)
{
return;
}
Logger logger = conn.Logger.Clone(GetType().Name);
logger.Log("Optional rename name={0}, alias={1}, gen={2}", IndexName, AliasName, Generations);
ESIndexCmd cmd = createIndexCmd(conn);
String existingIndexName;
cmd.GetIndexMappingsAndRealName(AliasName, out existingIndexName);
logger.Log("Optional rename alias={0}, existing={1}", AliasName, existingIndexName);
//Check if the current index was created without a timestamp. If so, we will just remove it
//This is more or less a backward compat. issue
if (String.Compare(AliasName, existingIndexName, StringComparison.InvariantCultureIgnoreCase) == 0)
{
logger.Log("Removing index {0}", existingIndexName);
cmd.RemoveIndex(existingIndexName);
}
cmd.MoveAllAliases(AliasName, IndexName, existingIndexName);
if (ExtraAlias != null)
{
cmd.CreateAliases(ExtraAlias.SplitStandard(), IndexName);
}
//Changing the #replica's if requested.
if (this.NumReplicasAfterIndexed > 0)
{
logger.Log("Setting the #replicas for {0} to {1}", IndexName, NumReplicasAfterIndexed);
setNumberOfReplicas(conn, IndexName, NumReplicasAfterIndexed);
}
if (Generations <= 0)
{
return;
}
List <String> indexes = cmd.GetIndexes(AliasName, true);
logger.Log("Found {0} indexes starting with {1}:", indexes.Count, AliasName);
int existingIdx = -1;
int currentIdx = -1;
for (int i = 0; i < indexes.Count; i++)
{
String s = indexes[i];
String what = String.Empty;
if (String.Equals(s, existingIndexName, StringComparison.InvariantCultureIgnoreCase))
{
existingIdx = i;
what = " [existing alias]";
}
else if (String.Equals(s, IndexName, StringComparison.InvariantCultureIgnoreCase))
{
currentIdx = i;
what = " [current index]";
}
logger.Log("__ " + s + what);
}
//Check if the returned list is OK (our index should be in, as well as the current alias)
if (currentIdx < 0)
{
throwNotFound(IndexName);
}
if (existingIdx < 0 && existingIndexName != null)
{
String msg = notFoundMsg(existingIndexName);
Logs.ErrorLog.Log(msg);
logger.Log(_LogType.ltError, msg);
}
//Remove all indexes between the current one and an existing alias. These are artifacts of temp. or crashed imports
logger.Log("Removing indexes between existing ({0}) and current ({1})...", existingIdx, currentIdx);
for (int i = existingIdx + 1; i < currentIdx; i++)
{
logger.Log("Removing index {0}", indexes[i]);
cmd.RemoveIndex(indexes[i]);
}
//Remove all indexes between start-of-time and the existing alias, keeping 'generationsToKeep' generations
logger.Log("Removing indexes between epoch and existing ({0}), keeping {1} generations...", existingIdx, Generations);
for (int i = 0; i <= existingIdx - Generations + 1; i++)
{
logger.Log("Removing index {0}", indexes[i]);
cmd.RemoveIndex(indexes[i]);
}
}
private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt)
{
int maxParallel = elt.ContextNode.ReadInt("@maxparallel", this.maxParallel);
int splitUntil = elt.ContextNode.ReadInt("@splituntil", this.splitUntil);
if (splitUntil < 0)
{
splitUntil = int.MaxValue;
}
bool scan = elt.ContextNode.ReadBool("@scan", this.scan);
String url = elt.ToString();
ctx.SendItemStart(elt);
String command = elt.ContextNode.ReadStr("@command", null);
String index = command != null ? null : elt.ContextNode.ReadStr("@index"); //mutual exclusive with command
String reqBody = elt.ContextNode.ReadStr("request", this.requestBody);
JObject req = null;
if (reqBody != null)
{
req = JObject.Parse(reqBody);
}
ctx.DebugLog.Log("Request scan={1}, body={0}", reqBody, scan);
try
{
Uri uri = new Uri(url);
ESConnection conn = ESHelper.CreateConnection(ctx, url);
ContextCallback cb = new ContextCallback(ctx, this, elt);
conn.Timeout = timeoutInMs; //Same timeout as what we send to ES
conn.OnPrepareRequest = cb.OnPrepareRequest;
if (command != null)
{
var resp = conn.SendCmd("POST", command, reqBody);
resp.ThrowIfError();
Pipeline.EmitToken(ctx, sink, resp.JObject, "response", splitUntil);
}
else
{
ESRecordEnum e = new ESRecordEnum(conn, index, req, numRecords, timeout, scan);
if (maxParallel > 0)
{
e.Async = true;
}
ctx.ImportLog.Log("Starting scan of {0} records. Index={1}, connection={2}, async={3}, buffersize={4} requestbody={5}, splituntil={6}, scan={7}.", e.Count, index, url, e.Async, numRecords, req != null, splitUntil, scan);
foreach (var doc in e)
{
ctx.IncrementEmitted();
sink.HandleValue(ctx, "record/_sort", doc.Sort);
sink.HandleValue(ctx, "record/_type", doc.Type);
if (splitUntil != 0)
{
foreach (var kvp in doc)
{
String pfx = "record/" + kvp.Key;
if (splitUntil == 1)
{
sink.HandleValue(ctx, pfx, kvp.Value);
continue;
}
Pipeline.EmitToken(ctx, sink, kvp.Value, pfx, splitUntil - 1);
}
}
sink.HandleValue(ctx, "record", doc);
}
}
ctx.SendItemStop();
}
catch (Exception e)
{
ctx.HandleException(e);
}
}
