示例#1
0
        public FolderReader(IConnectionContext input, IRowFactory rowFactory)
        {
            var readers      = new List <IRead>();
            var searchOption = (SearchOption)Enum.Parse(typeof(SearchOption), input.Connection.SearchOption, true);

            input.Debug(() => $"Searching folder: {input.Connection.Folder}");
            var files = new DirectoryInfo(input.Connection.Folder).GetFiles(input.Connection.SearchPattern, searchOption).OrderBy(f => f.CreationTime).ToArray();

            input.Debug(() => $"Found {files.Length} files.");
            foreach (var file in files)
            {
                input.Debug(() => $"Found file: {file.Name}");

                var context = new PipelineContext(input.Logger, input.Process, input.Entity, input.Field, input.Transform);

                var fileConnection = input.Connection.Clone();
                fileConnection.Provider = "file";
                fileConnection.File     = file.FullName;

                var fileInput = new InputContext(context, new Incrementer(context))
                {
                    Connection = fileConnection
                };

                if (file.Extension.ToLower().Contains("xls"))
                {
                    readers.Add(new ExcelReader(fileInput, rowFactory));
                }
                else
                {
                    readers.Add(new DelimitedFileReader(fileInput, rowFactory));
                }
            }
            _reader = new CompositeReader(readers);
        }
        private string SqlDrop(string tempTable)
        {
            var sql = $"DROP TABLE {_cf.Enclose(tempTable)}";

            _context.Debug(() => sql);
            return(sql);
        }
        string SqlDrop(string tempTable, IConnectionFactory cf)
        {
            var sql = $"DROP TABLE {cf.Enclose(tempTable)}";

            _context.Debug(() => sql);
            return(sql);
        }
        public string Render()
        {
            var p = new Dictionary <string, string>();
            var l = new Cfg.Net.Loggers.MemoryLogger();

            // get template
            _context.Debug(() => $"Reading {_template.File}");
            var templateContent = _templateReader.Read(_template.File, p, l);

            if (l.Errors().Any())
            {
                foreach (var error in l.Errors())
                {
                    _context.Error(error);
                }
                return(string.Empty);
            }

            // get parameters (other than process)
            var parameters = new ExpandoObject();

            foreach (var parameter in _template.Parameters)
            {
                ((IDictionary <string, object>)parameters).Add(parameter.Name, parameter.Value);
            }
            if (p.Any())
            {
                foreach (var parameter in p)
                {
                    ((IDictionary <string, object>)parameters)[parameter.Key] = parameter.Value;
                }
            }

            try {
                _context.Debug(() => $"Compiling {_template.Name}.");
                return(_service.RunCompile(templateContent, _template.Name, typeof(SolrTemplateModel), new SolrTemplateModel {
                    Context = _context, Process = _context.Process, Parameters = parameters
                }));
            } catch (TemplateCompilationException ex) {
                _context.Error($"Error parsing template {_template.Name}.");
                _context.Error($"There are {ex.CompilerErrors.Count} errors.");
                foreach (var error in ex.CompilerErrors)
                {
                    _context.Error(error.ErrorText);
                }
                Utility.CodeToError(_context, ex.SourceCode);
                return(string.Empty);
            }
        }
        static string SqlDrop(IConnectionContext context, string tempTable)
        {
            var sql = string.Format("DROP TABLE #{0};", tempTable);

            context.Debug(sql);
            return(sql);
        }
示例#6
0
        static string SqlDrop(IConnectionContext context, string tempTable)
        {
            var sql = $"DROP TABLE #{tempTable};";

            context.Debug(() => sql);
            return(sql);
        }
示例#7
0
        public void Write(IEnumerable <IRow> rows)
        {
            var l = new Cfg.Net.Loggers.MemoryLogger();

            _output.Debug(() => $"Loading template {_output.Connection.Template}");
            var template = _templateReader.Read(_output.Connection.Template, new Dictionary <string, string>(), l);

            if (l.Errors().Any())
            {
                foreach (var error in l.Errors())
                {
                    _output.Error(error);
                }
            }
            else
            {
                using (var service = RazorEngineService.Create(_config)) {
                    //File.WriteAllText(_output.Connection.File, service.RunCompile(template, _output.Connection.Name, typeof(RazorModel), new RazorModel(_output.Process, _output.Entity, rows)));

                    using (var file = new StreamWriter(_output.Connection.File)) {
                        service.RunCompile(template, _output.Connection.Name, file, typeof(RazorModel), new RazorModel(_output.Process, _output.Entity, rows));
                    }

                    // the template must set Model.Entity.Inserts
                }
            }
        }
示例#8
0
        string SqlInsertTemplate(IConnectionContext context, string tempTable)
        {
            var sql = $"INSERT #{tempTable} VALUES ({string.Join(",", _keys.Select(k => "@" + k.FieldName()))});";

            context.Debug(() => sql);
            return(sql);
        }
        static string SqlInsertTemplate(IConnectionContext context, string tempTable, Field[] keys)
        {
            var sql = string.Format("INSERT #{0} VALUES ({1});", tempTable, string.Join(",", keys.Select(k => "@" + k.FieldName())));

            context.Debug(sql);
            return(sql);
        }
示例#10
0
        static string SqlCreateKeysTable(IConnectionContext context, IConnectionFactory cf, string tempTable)
        {
            var columnsAndDefinitions = string.Join(",", context.Entity.GetPrimaryKey().Select(f => cf.Enclose(f.FieldName()) + " " + cf.SqlDataType(f) + " NOT NULL"));
            var sql = $@"CREATE TABLE #{tempTable}({columnsAndDefinitions})";

            context.Debug(() => sql);
            return(sql);
        }
 static string SqlQuery(Field[] keys, IConnectionContext context, string tempTable, Field hashCode) {
     var names = string.Join(",", keys.Select(f => "k.[" + f.FieldName() + "]"));
     var table = context.Entity.OutputTableName(context.Process.Name);
     var joins = string.Join(" AND ", keys.Select(f => "o.[" + f.FieldName() + "] = k.[" + f.FieldName() + "]"));
     var sql = string.Format("SELECT {0},o.[{1}] FROM #{2} k WITH (NOLOCK) INNER JOIN [{3}] o WITH (NOLOCK) ON ({4})", names, hashCode.FieldName(), tempTable, table, joins);
     context.Debug(sql);
     return sql;
 }
        static string SqlCreateKeysTable(IConnectionContext context, string tempTable)
        {
            var columnsAndDefinitions = string.Join(",", context.Entity.GetPrimaryKey().Select(f => "[" + f.FieldName() + "] " + f.SqlDataType() + " NOT NULL"));
            var sql = string.Format(@"CREATE TABLE #{0}({1})", tempTable, columnsAndDefinitions);

            context.Debug(sql);
            return(sql);
        }
示例#13
0
        public void Write(IEnumerable <IRow> rows)
        {
            var l = new Cfg.Net.Loggers.MemoryLogger();

            _output.Debug(() => $"Loading template {_output.Connection.Template}");
            var template = _templateReader.Read(_output.Connection.Template, new Dictionary <string, string>(), l);

            template = Regex.Replace(template, "^@model .+$", string.Empty, RegexOptions.Multiline);

            if (l.Errors().Any())
            {
                foreach (var error in l.Errors())
                {
                    _output.Error(error);
                }
            }
            else
            {
                var engine = new RazorEngine();
                IRazorEngineCompiledTemplate <RazorEngineTemplateBase <RazorModel> > compiledTemplate;

                try {
                    compiledTemplate = engine.Compile <RazorEngineTemplateBase <RazorModel> >(template, builder => {
                        builder.AddAssemblyReference(typeof(Configuration.Process));
                        builder.AddAssemblyReference(typeof(Cfg.Net.CfgNode));
                        builder.AddAssemblyReferenceByName("System.Collections");
                    });
                    // doesn't appear to be a way to stream output yet (in this library), so will just write to string and then file
                    var output = compiledTemplate.Run(instance => {
                        instance.Model = new RazorModel()
                        {
                            Process = _output.Process,
                            Entity  = _output.Entity,
                            Rows    = rows
                        };
                    });
                    File.WriteAllText(_output.Connection.File, output);
                } catch (RazorEngineCompilationException ex) {
                    foreach (var error in ex.Errors)
                    {
                        var line = error.Location.GetLineSpan();
                        _output.Error($"C# error on line {line.StartLinePosition.Line}, column {line.StartLinePosition.Character}.");
                        _output.Error(error.GetMessage());
                    }
                    _output.Error(ex.Message.Replace("{", "{{").Replace("}", "}}"));
                    Utility.CodeToError(_output, template);
                } catch (System.AggregateException ex) {
                    _output.Error(ex.Message.Replace("{", "{{").Replace("}", "}}"));
                    foreach (var error in ex.InnerExceptions)
                    {
                        _output.Error(error.Message.Replace("{", "{{").Replace("}", "}}"));
                    }
                    Utility.CodeToError(_output, template);
                }

                // the template must set Model.Entity.Inserts
            }
        }
        static string SqlQuery(Field[] keys, IConnectionContext context, string tempTable, Field hashCode)
        {
            var names = string.Join(",", keys.Select(f => "k.[" + f.FieldName() + "]"));
            var table = context.Entity.OutputTableName(context.Process.Name);
            var joins = string.Join(" AND ", keys.Select(f => "o.[" + f.FieldName() + "] = k.[" + f.FieldName() + "]"));
            var sql   = string.Format("SELECT {0},o.[{1}] FROM #{2} k WITH (NOLOCK) INNER JOIN [{3}] o WITH (NOLOCK) ON ({4})", names, hashCode.FieldName(), tempTable, table, joins);

            context.Debug(sql);
            return(sql);
        }
        public IEnumerable <IRow> Read()
        {
            using (var cn = _cf.GetConnection()) {
                cn.Open();
                var cmd = cn.CreateCommand();

                cmd.CommandTimeout = 0;
                cmd.CommandType    = CommandType.Text;
                cmd.CommandText    = $@"
                    SELECT {string.Join(",", _fields.Select(f => _readFrom == ReadFrom.Output ? _cf.Enclose(f.FieldName()) : _cf.Enclose(f.Name)))} 
                    FROM {_schemaPrefix}{_cf.Enclose(_tableOrView)} {(_connection.Provider == "sqlserver" && _context.Entity.NoLock ? "WITH (NOLOCK)" : string.Empty)}
                    {_filter};";
                _context.Debug(() => cmd.CommandText);

                IDataReader reader;
                try {
                    reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess);
                } catch (System.Data.Common.DbException e) {
                    _context.Error($"Error reading data from {_connection.Name}, {_tableOrView}.");
                    _context.Error(e.Message);
                    yield break;
                }

                if (_context.Connection.Buffer)
                {
                    var buffer = new List <IRow>();
                    while (reader.Read())
                    {
                        _rowCount++;
                        buffer.Add(_rowCreator.Create(reader, _fields));
                    }
                    foreach (var row in buffer)
                    {
                        yield return(row);
                    }
                }
                else
                {
                    while (reader.Read())
                    {
                        _rowCount++;
                        yield return(_rowCreator.Create(reader, _fields));
                    }
                }

                _context.Info("{0} from {1}", _rowCount, _connection.Name);
            }
        }
        public void Write(IEnumerable <IRow> rows)
        {
            var l = new Cfg.Net.Loggers.MemoryLogger();

            _output.Debug(() => $"Loading template {_output.Connection.Template}");
            var template = _templateReader.Read(_output.Connection.Template, new Dictionary <string, string>(), l);

            if (l.Errors().Any())
            {
                foreach (var error in l.Errors())
                {
                    _output.Error(error);
                }
            }
            else
            {
                var context = new VelocityContext();
                context.Put("Model", new VelocityModel(_output.Process, _output.Entity, rows));

                using (var file = new StreamWriter(_output.Connection.File)) {
                    NVelocity.App.Velocity.Evaluate(context, file, RuntimeConstants.RUNTIME_LOG_LOGSYSTEM, template);
                }
            }
        }
        public IEnumerable <IRow> Read()
        {
            if (_parent.Entities.Sum(e => e.Inserts + e.Updates + e.Deletes) == 0)
            {
                yield break;
            }
            ;

            var batches    = _parent.Entities.Select(e => e.BatchId).ToArray();
            var minBatchId = batches.Min();
            var maxBatchId = batches.Max();

            _output.Info("Batch Range: {0} to {1}.", minBatchId, maxBatchId);

            var threshold = minBatchId - 1;

            var sql = string.Empty;

            if (_cf.AdoProvider == AdoProvider.SqlCe)
            {
                // because SqlCe doesn't support views, re-construct the parent view's definition

                var ctx     = new PipelineContext(_output.Logger, _parent);
                var master  = _parent.Entities.First(e => e.IsMaster);
                var builder = new StringBuilder();

                builder.AppendLine($"SELECT {string.Join(",", _output.Entity.Fields.Where(f => f.Output).Select(f => _cf.Enclose(f.Source.Split('.')[0]) + "." + _cf.Enclose(f.Source.Split('.')[1])))}");
                foreach (var from in ctx.SqlStarFroms(_cf))
                {
                    builder.AppendLine(@from);
                }
                builder.AppendLine($"WHERE {_cf.Enclose(Utility.GetExcelName(master.Index))}.{_cf.Enclose(master.TflBatchId().FieldName())} > @Threshold;");

                sql = builder.ToString();
            }
            else
            {
                sql = $@"
                SELECT {string.Join(",", _output.Entity.Fields.Where(f => f.Output).Select(f => _cf.Enclose(f.Alias)))} 
                FROM {_cf.Enclose(_output.Process.Star)} {(_cf.AdoProvider == AdoProvider.SqlServer ? "WITH (NOLOCK)" : string.Empty)} 
                WHERE {_cf.Enclose(Constants.TflBatchId)} > @Threshold;";
            }

            _output.Debug(() => sql);

            using (var cn = _cf.GetConnection()) {
                cn.Open();

                var cmd = cn.CreateCommand();

                cmd.CommandTimeout = 0;
                cmd.CommandType    = CommandType.Text;
                cmd.CommandText    = sql;

                var min = cmd.CreateParameter();
                min.ParameterName = "@Threshold";
                min.Value         = threshold;
                min.Direction     = ParameterDirection.Input;
                min.DbType        = DbType.Int32;

                cmd.Parameters.Add(min);

                var reader     = cmd.ExecuteReader(CommandBehavior.SequentialAccess);
                var rowCount   = 0;
                var fieldArray = _output.Entity.Fields.ToArray();
                while (reader.Read())
                {
                    rowCount++;
                    _output.Increment();
                    yield return(_rowCreator.Create(reader, fieldArray));
                }
                _output.Info("{0} from {1}", rowCount, _output.Connection.Name);
            }
        }
 string SqlInsertTemplate(IConnectionContext context, string tempTable) {
     var sql = string.Format("INSERT #{0} VALUES ({1});", tempTable, string.Join(",", _keys.Select(k => "@" + k.FieldName())));
     context.Debug(sql);
     return sql;
 }
 static string SqlCreateKeysTable(IConnectionContext context, string tempTable) {
     var columnsAndDefinitions = string.Join(",", context.Entity.GetPrimaryKey().Select(f => "[" + f.FieldName() + "] " + f.SqlDataType() + " NOT NULL"));
     var sql = string.Format(@"CREATE TABLE #{0}({1})", tempTable, columnsAndDefinitions);
     context.Debug(sql);
     return sql;
 }
 static string SqlDrop(IConnectionContext context, string tempTable) {
     var sql = string.Format("DROP TABLE #{0};", tempTable);
     context.Debug(sql);
     return sql;
 }
        public IEnumerable <IRow> Read()
        {
            ElasticsearchResponse <DynamicResponse> response;
            ElasticsearchDynamicValue hits;

            var    from = 0;
            var    size = 10;
            string body;
            bool   warned = false;

            var scroll = !_context.Entity.IsPageRequest();

            if (!scroll)
            {
                from = (_context.Entity.Page * _context.Entity.Size) - _context.Entity.Size;
                body = WriteQuery(_fields, _readFrom, _context, scroll: false, from: from, size: _context.Entity.Size);
            }
            else
            {
                body     = WriteQuery(_fields, _readFrom, _context, scroll: false, from: 0, size: 0);
                response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);
                if (response.Success)
                {
                    hits = response.Body["hits"] as ElasticsearchDynamicValue;
                    if (hits != null && hits.HasValue)
                    {
                        var total = hits["total"];

                        try {
                            if (_version.Major >= 7) // version 7 changed total to an object with "value" and "relation" properties
                            {
                                size = Convert.ToInt32(total["value"].Value);
                            }
                            else
                            {
                                size = Convert.ToInt32(total.Value);
                            }
                        } catch (Exception ex) {
                            warned = true;
                            _context.Debug(() => total);
                            _context.Warn($"Could not get total number of matching documents from the elasticsearch response.  Are you sure you using version {_version}?");
                            _context.Error(ex, ex.Message);
                        }
                        body = WriteQuery(_fields, _readFrom, _context, scroll: true, from: 0, size: size > ElasticsearchDefaultSizeLimit ? DefaultSize : size);
                    }
                }
            }

            _context.Debug(() => body);
            _context.Entity.Query = body;

            response = scroll
            ? _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body, p => p.AddQueryString("scroll", _context.Connection.Scroll))
            : _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);

            if (!response.Success)
            {
                LogError(response);
                yield break;
            }

            try {
                if (_version.Major >= 7) // version 7 changed total to an object with "value" and "relation" properties
                {
                    _context.Entity.Hits = Convert.ToInt32(response.Body["hits"]["total"]["value"].Value);
                }
                else
                {
                    _context.Entity.Hits = Convert.ToInt32(response.Body["hits"]["total"].Value);
                }
            } catch (Exception ex) {
                if (!warned)
                {
                    _context.Debug(() => response.Body["hits"]);
                    _context.Warn($"Could not get total number of matching documents from the elasticsearch response.  Are you sure you using version {_version}?");
                    _context.Error(ex.Message);
                }
            }

            hits = response.Body["hits"]["hits"] as ElasticsearchDynamicValue;

            if (hits == null || !hits.HasValue)
            {
                _context.Warn("No hits from elasticsearch");
                yield break;
            }

            var docs = hits.Value as IList <object>;

            if (docs == null)
            {
                _context.Error("No documents returned from elasticsearch!");
                yield break;
            }

            // if any of the fields do not exist, yield break
            if (docs.Count > 0)
            {
                var doc    = docs.First() as IDictionary <string, object>;
                var source = doc?["_source"] as IDictionary <string, object>;
                if (source == null)
                {
                    _context.Error("Missing _source from elasticsearch response!");
                    yield break;
                }

                for (var i = 0; i < _fields.Length; i++)
                {
                    if (source.ContainsKey(_fieldNames[i]))
                    {
                        continue;
                    }

                    _context.Error($"Field {_fieldNames[i]} does not exist!");
                    yield break;
                }
            }

            var count = 0;

            foreach (var d in docs)
            {
                var doc    = (IDictionary <string, object>)d;
                var row    = _rowFactory.Create();
                var source = (IDictionary <string, object>)doc["_source"];
                for (var i = 0; i < _fields.Length; i++)
                {
                    row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                }
                yield return(row);
            }
            count += docs.Count;

            // get this from first search response (maybe), unless you have to aggregate it from all...
            foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet" && !string.IsNullOrEmpty(f.Map)))
            {
                var map     = _context.Process.Maps.First(m => m.Name == filter.Map);
                var buckets = response.Body["aggregations"][filter.Key]["buckets"] as ElasticsearchDynamicValue;
                if (buckets == null || !buckets.HasValue)
                {
                    continue;
                }

                var items = buckets.Value as IEnumerable <object>;

                if (items == null)
                {
                    continue;
                }

                foreach (var item in items.OfType <IDictionary <string, object> >())
                {
                    map.Items.Add(new MapItem {
                        From = $"{item["key"]} ({item["doc_count"]})", To = item["key"]
                    });
                }
            }

            if (!response.Body.ContainsKey("_scroll_id"))
            {
                yield break;
            }

            if (size == count)
            {
                _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = response.Body["_scroll_id"].Value }));
                yield break;
            }

            var scrolls = new HashSet <string>();

            do
            {
                var scrollId = response.Body["_scroll_id"].Value;
                scrolls.Add(scrollId);
                response = _client.Scroll <DynamicResponse>(new PostData <object>(new { scroll = _context.Connection.Scroll, scroll_id = scrollId }));
                if (response.Success)
                {
                    docs = (IList <object>)response.Body["hits"]["hits"].Value;
                    foreach (var d in docs)
                    {
                        var doc    = (IDictionary <string, object>)d;
                        var row    = _rowFactory.Create();
                        var source = (IDictionary <string, object>)doc["_source"];
                        for (var i = 0; i < _fields.Length; i++)
                        {
                            row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                        }
                        yield return(row);
                    }
                    count += docs.Count;
                }
                else
                {
                    LogError(response);
                }
            } while (response.Success && count < size);

            _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = scrolls.ToArray() }));
        }
示例#22
0
        public IEnumerable <IRow> Read()
        {
            ElasticsearchResponse <DynamicResponse> response;

            string body;

            if (_context.Entity.IsPageRequest())
            {
                var from = (_context.Entity.Page * _context.Entity.PageSize) - _context.Entity.PageSize;
                body = WriteQuery(_fields, _readFrom, _context, from, _context.Entity.PageSize);
            }
            else
            {
                body     = WriteQuery(_fields, _readFrom, _context, 0, 0);
                response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);
                if (response.Success)
                {
                    var hits = response.Body["hits"] as ElasticsearchDynamicValue;
                    if (hits != null && hits.HasValue)
                    {
                        var properties = hits.Value as IDictionary <string, object>;
                        if (properties != null && properties.ContainsKey("total"))
                        {
                            var size = Convert.ToInt32(properties["total"]) + 1;
                            body = WriteQuery(_fields, _readFrom, _context, 0, size);
                        }
                    }
                }
            }
            _context.Debug(() => body);
            _context.Entity.Query = body;

            response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);

            if (response.Success)
            {
                _context.Entity.Hits = Convert.ToInt32((response.Body["hits"]["total"] as ElasticsearchDynamicValue).Value);
                var hits = response.Body["hits"]["hits"] as ElasticsearchDynamicValue;
                if (hits != null && hits.HasValue)
                {
                    var docs = hits.Value as IEnumerable <object>;
                    if (docs != null)
                    {
                        foreach (var doc in docs.OfType <IDictionary <string, object> >())
                        {
                            var row = _rowFactory.Create();
                            if (doc != null && doc.ContainsKey("_source"))
                            {
                                var source = doc["_source"] as IDictionary <string, object>;
                                if (source != null)
                                {
                                    for (var i = 0; i < _fields.Length; i++)
                                    {
                                        var field = _fields[i];
                                        row[field] = field.Convert(source[_fieldNames[i]]);
                                    }
                                }
                            }
                            _context.Increment();
                            yield return(row);
                        }
                    }
                }

                foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet" && !string.IsNullOrEmpty(f.Map)))
                {
                    var map     = _context.Process.Maps.First(m => m.Name == filter.Map);
                    var buckets = response.Body["aggregations"][filter.Key]["buckets"] as ElasticsearchDynamicValue;
                    if (buckets == null || !buckets.HasValue)
                    {
                        continue;
                    }

                    var items = buckets.Value as IEnumerable <object>;

                    if (items == null)
                    {
                        continue;
                    }

                    foreach (var item in items.OfType <IDictionary <string, object> >())
                    {
                        map.Items.Add(new MapItem {
                            From = $"{item["key"]} ({item["doc_count"]})", To = item["key"]
                        }.WithDefaults());
                    }
                }
            }
            else
            {
                _context.Error(response.DebugInformation);
            }
        }
示例#23
0
        public IEnumerable <IRow> Read()
        {
            ElasticsearchResponse <DynamicResponse> response;
            ElasticsearchDynamicValue hits;
            var    from = 1;
            var    size = 10;
            string body;

            if (_context.Entity.IsPageRequest())
            {
                from = (_context.Entity.Page * _context.Entity.PageSize) - _context.Entity.PageSize;
                body = WriteQuery(_fields, _readFrom, _context, from, _context.Entity.PageSize);
            }
            else
            {
                body     = WriteQuery(_fields, _readFrom, _context, 0, 0);
                response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);
                if (response.Success)
                {
                    hits = response.Body["hits"] as ElasticsearchDynamicValue;
                    if (hits != null && hits.HasValue)
                    {
                        var properties = hits.Value as IDictionary <string, object>;
                        if (properties != null && properties.ContainsKey("total"))
                        {
                            size = Convert.ToInt32(properties["total"]);
                            body = WriteQuery(_fields, _readFrom, _context, 0, size > 10000 ? 10000 : size);
                        }
                    }
                }
            }

            _context.Debug(() => body);
            _context.Entity.Query = body;

            // move 10000 to configurable limit
            response = from + size > 10000
                ? _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body, p => p.Scroll(TimeSpan.FromMinutes(1.0)))
                : _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);

            if (!response.Success)
            {
                LogError(response);
                yield break;
            }

            _context.Entity.Hits = Convert.ToInt32((response.Body["hits"]["total"] as ElasticsearchDynamicValue).Value);
            hits = response.Body["hits"]["hits"] as ElasticsearchDynamicValue;


            if (hits == null || !hits.HasValue)
            {
                _context.Warn("No hits from elasticsearch");
                yield break;
            }

            var docs = hits.Value as IList <object>;

            if (docs == null)
            {
                _context.Error("No documents returned from elasticsearch!");
                yield break;
            }

            // if any of the fields do not exist, yield break
            if (docs.Count > 0)
            {
                var doc    = docs.First() as IDictionary <string, object>;
                var source = doc?["_source"] as IDictionary <string, object>;
                if (source == null)
                {
                    _context.Error("Missing _source from elasticsearch response!");
                    yield break;
                }

                for (var i = 0; i < _fields.Length; i++)
                {
                    if (source.ContainsKey(_fieldNames[i]))
                    {
                        continue;
                    }

                    _context.Error($"Field {_fieldNames[i]} does not exist!");
                    yield break;
                }
            }

            var count = 0;

            foreach (var d in docs)
            {
                var doc    = (IDictionary <string, object>)d;
                var row    = _rowFactory.Create();
                var source = (IDictionary <string, object>)doc["_source"];
                for (var i = 0; i < _fields.Length; i++)
                {
                    row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                }
                _context.Increment();
                yield return(row);
            }
            count += docs.Count;

            // get this from first search response (maybe), unless you have to aggregate it from all...
            foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet" && !string.IsNullOrEmpty(f.Map)))
            {
                var map     = _context.Process.Maps.First(m => m.Name == filter.Map);
                var buckets = response.Body["aggregations"][filter.Key]["buckets"] as ElasticsearchDynamicValue;
                if (buckets == null || !buckets.HasValue)
                {
                    continue;
                }

                var items = buckets.Value as IEnumerable <object>;

                if (items == null)
                {
                    continue;
                }

                foreach (var item in items.OfType <IDictionary <string, object> >())
                {
                    map.Items.Add(new MapItem {
                        From = $"{item["key"]} ({item["doc_count"]})", To = item["key"]
                    });
                }
            }

            if (!response.Body.ContainsKey("_scroll_id"))
            {
                yield break;
            }

            if (size == count)
            {
                _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = response.Body["_scroll_id"].Value }));
                yield break;
            }

            var scrolls = new HashSet <string>();

            do
            {
                var scrollId = response.Body["_scroll_id"].Value;
                scrolls.Add(scrollId);
                response = _client.Scroll <DynamicResponse>(new PostData <object>(new { scroll = "1m", scroll_id = scrollId }));
                if (response.Success)
                {
                    docs = (IList <object>)response.Body["hits"]["hits"].Value;
                    foreach (var d in docs)
                    {
                        var doc    = (IDictionary <string, object>)d;
                        var row    = _rowFactory.Create();
                        var source = (IDictionary <string, object>)doc["_source"];
                        for (var i = 0; i < _fields.Length; i++)
                        {
                            row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                        }
                        _context.Increment();
                        yield return(row);
                    }
                    count += docs.Count;
                }
                else
                {
                    LogError(response);
                }
            } while (response.Success && count < size);

            _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = scrolls.ToArray() }));
        }