Exemple #1
0
        public override void EmitRecord(PipelineContext ctx, String recordKey, String recordField, IDatasourceSink sink, String eventKey, int maxLevel)
        {
            JObject obj = DocType.LoadByKey(Connection, recordKey);

            if (obj == null)
            {
                return;
            }
            JToken token = (recordField == null) ? obj : obj.GetValue(recordField, StringComparison.InvariantCultureIgnoreCase);

            if (token != null)
            {
                Pipeline.EmitToken(ctx, sink, token, eventKey, maxLevel);
            }
        }
        private void importRecord(PipelineContext ctx, IDatasourceSink sink, Stream strm, int splitUntil)
        {
            JsonTextReader rdr = new JsonTextReader(new StreamReader(strm, true));
            JToken         jt  = JObject.ReadFrom(rdr);

            rdr.Close();
            strm.Close();

            if (jt.Type != JTokenType.Array)
            {
                Pipeline.EmitToken(ctx, sink, jt, "record", splitUntil);
                ctx.IncrementEmitted();
            }
            else
            {
                foreach (var item in (JArray)jt)
                {
                    Pipeline.EmitToken(ctx, sink, item, "record", splitUntil);
                    ctx.IncrementEmitted();
                }
            }
        }
Exemple #3
0
        private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt)
        {
            int maxParallel = elt.ContextNode.ReadInt("@maxparallel", this.maxParallel);
            int splitUntil  = elt.ContextNode.ReadInt("@splituntil", this.splitUntil);

            if (splitUntil < 0)
            {
                splitUntil = int.MaxValue;
            }
            bool scan = elt.ContextNode.ReadBool("@scan", this.scan);

            String url = elt.ToString();

            ctx.SendItemStart(elt);
            String  command = elt.ContextNode.ReadStr("@command", null);
            String  index   = command != null ? null : elt.ContextNode.ReadStr("@index"); //mutual exclusive with command
            String  reqBody = elt.ContextNode.ReadStr("request", this.requestBody);
            JObject req     = null;

            if (reqBody != null)
            {
                req = JObject.Parse(reqBody);
            }
            ctx.DebugLog.Log("Request scan={1}, body={0}", reqBody, scan);
            try
            {
                Uri             uri  = new Uri(url);
                ESConnection    conn = ESHelper.CreateConnection(ctx, url);
                ContextCallback cb   = new ContextCallback(ctx, this, elt);
                conn.Timeout          = timeoutInMs; //Same timeout as what we send to ES
                conn.OnPrepareRequest = cb.OnPrepareRequest;
                if (command != null)
                {
                    var resp = conn.SendCmd("POST", command, reqBody);
                    resp.ThrowIfError();
                    Pipeline.EmitToken(ctx, sink, resp.JObject, "response", splitUntil);
                }
                else
                {
                    ESRecordEnum e = new ESRecordEnum(conn, index, req, numRecords, timeout, scan);
                    if (maxParallel > 0)
                    {
                        e.Async = true;
                    }
                    ctx.ImportLog.Log("Starting scan of {0} records. Index={1}, connection={2}, async={3}, buffersize={4} requestbody={5}, splituntil={6}, scan={7}.", e.Count, index, url, e.Async, numRecords, req != null, splitUntil, scan);
                    foreach (var doc in e)
                    {
                        ctx.IncrementEmitted();
                        sink.HandleValue(ctx, "record/_sort", doc.Sort);
                        sink.HandleValue(ctx, "record/_type", doc.Type);
                        if (splitUntil != 0)
                        {
                            foreach (var kvp in doc)
                            {
                                String pfx = "record/" + kvp.Key;
                                if (splitUntil == 1)
                                {
                                    sink.HandleValue(ctx, pfx, kvp.Value);
                                    continue;
                                }
                                Pipeline.EmitToken(ctx, sink, kvp.Value, pfx, splitUntil - 1);
                            }
                        }
                        sink.HandleValue(ctx, "record", doc);
                    }
                }
                ctx.SendItemStop();
            }
            catch (Exception e)
            {
                ctx.HandleException(e);
            }
        }