internal void OpenIndex(PipelineContext ctx, ESIndexDefinition index)
{
if (index.IsOpen)
{
return;
}
ESHelper.SetLogging(ctx, Connection);
ESIndexCmd._CheckIndexFlags flags = ESIndexCmd._CheckIndexFlags.AppendDate;
if (ReadOnly)
{
flags |= ESIndexCmd._CheckIndexFlags.DontCreate;
}
if ((ctx.ImportFlags & _ImportFlags.ImportFull) != 0)
{
flags |= ESIndexCmd._CheckIndexFlags.ForceCreate;
}
index.Create(Connection, flags);
WaitForStatus();
var adminEp = this.GetAdminEndpoint(ctx);
if (adminEp != null)
{
int oldCount = ctx.RunAdministrations.Count;
ctx.RunAdministrations.Merge(adminEp.LoadAdministration(ctx));
if (ctx.RunAdministrations.Count != oldCount)
{
ctx.ImportLog.Log("-- merged {0} run-administrations from endpoint {1}. Now contains {2} runs.", ctx.RunAdministrations.Count - oldCount, this.Name, ctx.RunAdministrations.Count);
}
}
}
protected override void Open(PipelineContext ctx)
{
base.Open(ctx);
ESHelper.SetLogging(ctx, Connection);
ctx.ImportLog.Log("ESEndpoint '{0}' [cache={1}, maxparallel={2}, readonly={3}, url={4}]", Name, CacheSize, MaxParallel, ReadOnly, Connection.BaseUri);
}
private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt)
{
int maxParallel = elt.ContextNode.ReadInt("@maxparallel", this.maxParallel);
int splitUntil = elt.ContextNode.ReadInt("@splituntil", this.splitUntil);
if (splitUntil < 0)
{
splitUntil = int.MaxValue;
}
bool scan = elt.ContextNode.ReadBool("@scan", this.scan);
String url = elt.ToString();
ctx.SendItemStart(elt);
String command = elt.ContextNode.ReadStr("@command", null);
String index = command != null ? null : elt.ContextNode.ReadStr("@index"); //mutual exclusive with command
String reqBody = elt.ContextNode.ReadStr("request", this.requestBody);
JObject req = null;
if (reqBody != null)
{
req = JObject.Parse(reqBody);
}
ctx.DebugLog.Log("Request scan={1}, body={0}", reqBody, scan);
try
{
Uri uri = new Uri(url);
ESConnection conn = ESHelper.CreateConnection(ctx, url);
ContextCallback cb = new ContextCallback(ctx, this, elt);
conn.Timeout = timeoutInMs; //Same timeout as what we send to ES
conn.OnPrepareRequest = cb.OnPrepareRequest;
if (command != null)
{
var resp = conn.SendCmd("POST", command, reqBody);
resp.ThrowIfError();
Pipeline.EmitToken(ctx, sink, resp.JObject, "response", splitUntil);
}
else
{
ESRecordEnum e = new ESRecordEnum(conn, index, req, numRecords, timeout, scan);
if (maxParallel > 0)
{
e.Async = true;
}
ctx.ImportLog.Log("Starting scan of {0} records. Index={1}, connection={2}, async={3}, buffersize={4} requestbody={5}, splituntil={6}, scan={7}.", e.Count, index, url, e.Async, numRecords, req != null, splitUntil, scan);
foreach (var doc in e)
{
ctx.IncrementEmitted();
sink.HandleValue(ctx, "record/_sort", doc.Sort);
sink.HandleValue(ctx, "record/_type", doc.Type);
if (splitUntil != 0)
{
foreach (var kvp in doc)
{
String pfx = "record/" + kvp.Key;
if (splitUntil == 1)
{
sink.HandleValue(ctx, pfx, kvp.Value);
continue;
}
Pipeline.EmitToken(ctx, sink, kvp.Value, pfx, splitUntil - 1);
}
}
sink.HandleValue(ctx, "record", doc);
}
}
ctx.SendItemStop();
}
catch (Exception e)
{
ctx.HandleException(e);
}
}
