public IEnumerable <IRow> Read()
        {
            var encoding = Encoding.GetEncoding(_context.Connection.Encoding);
            var lineNo   = 0;

            if (System.IO.Path.GetExtension(_context.Connection.File) == ".xml")
            {
                var row = _rowFactory.Create();
                row[_field] = System.IO.File.ReadAllText(_context.Connection.File, encoding);
                yield return(row);
            }
            else
            {
                foreach (var line in System.IO.File.ReadLines(_context.Connection.File, encoding))
                {
                    ++lineNo;
                    if (lineNo < _context.Connection.Start)
                    {
                        continue;
                    }
                    var row = _rowFactory.Create();
                    row[_field] = line;
                    yield return(row);
                }
            }
        }
Exemple #2
0
        public IRow Create(IDataReader reader, Field[] fields)
        {
            var fieldCount = Math.Min(reader.FieldCount, fields.Length);
            var row        = _rowFactory.Create();

            for (var i = 0; i < fieldCount; i++)
            {
                var field = fields[i];
                if (field.Type == "string")
                {
                    if (reader.GetFieldType(i) == typeof(string))
                    {
                        row[field] = reader.IsDBNull(i) ? null : reader.GetString(i);
                    }
                    else
                    {
                        TypeMismatch(field, reader, i);
                        var value = reader.GetValue(i);
                        row[field] = value == DBNull.Value ? null : value;
                    }
                }
                else
                {
                    var value = reader.GetValue(i);
                    row[field] = value == DBNull.Value ? null : value;
                }
            }
            return(row);
        }
        public IEnumerable <IRow> Read()
        {
            var fields = _input.Entity.Fields.Where(f => f.Input).ToArray();
            var rows   = new List <IRow>();

            foreach (var row in _input.Entity.Rows)
            {
                var typed = _rowFactory.Create();
                foreach (var field in fields)
                {
                    if (row.Map.ContainsKey(field.Name))
                    {
                        typed[field] = field.Convert(row[field.Name]);
                    }
                    else
                    {
                        if (_missing.Add(field.Name))
                        {
                            _input.Warn($"An internal row in {_input.Entity.Alias} is missing the field {field.Name}.");
                        }
                    }
                }
                _input.Increment();
                rows.Add(typed);
            }
            _input.Entity.Hits = rows.Count;
            return(rows);
        }
        private IEnumerable <IRow> PreRead()
        {
            var rows = new List <IRow>();

            foreach (var row in _input.Entity.Rows)
            {
                var stringRow = _rowFactory.Create();
                foreach (var field in _fields)
                {
                    if (row.Map.ContainsKey(field.Name))
                    {
                        stringRow[field] = row[field.Name];
                    }
                    else
                    {
                        if (_missing.Add(field.Name))
                        {
                            _input.Warn($"An internal row in {_input.Entity.Alias} is missing the field {field.Name}.");
                        }
                    }
                }
                rows.Add(stringRow);
            }
            _input.Entity.Hits = rows.Count;
            return(rows);
        }
Exemple #5
0
        public IEnumerable <IRow> Read()
        {
            if (!_run)
            {
                yield break;
            }

            do
            {
                var response = _client.DescribeLogGroupsAsync().Result;
                foreach (var logGroup in response.LogGroups)
                {
                    var row = _rowFactory.Create();

                    foreach (var field in _context.InputFields)
                    {
                        var name = field.Name.ToLower();
                        switch (name)
                        {
                        case "arn":
                            row[field] = logGroup.Arn ?? string.Empty;
                            break;

                        case "creationtime":
                            row[field] = logGroup.CreationTime;
                            break;

                        case "kmskeyid":
                            row[field] = logGroup.KmsKeyId ?? string.Empty;
                            break;

                        case "loggroupname":
                            row[field] = logGroup.LogGroupName ?? string.Empty;
                            break;

                        case "metricfiltercount":
                            row[field] = logGroup.MetricFilterCount;
                            break;

                        case "retentionindays":
                            row[field] = logGroup.RetentionInDays.GetValueOrDefault();
                            break;

                        case "storedbytes":
                            row[field] = logGroup.StoredBytes;
                            break;

                        default:
                            break;
                        }
                    }

                    yield return(row);
                }
                _request.NextToken = response.NextToken;
            } while (!string.IsNullOrEmpty(_request.NextToken));
        }
        public IEnumerable <IRow> Read()
        {
            if (_inputField == null)
            {
                _input.Error("You must have one input field for console provider input.");
                yield break;
            }

            // Start the child process.
            using (var p = new Process {
                StartInfo =
                {
                    UseShellExecute        = false,
                    RedirectStandardOutput = true,
                    FileName               = _input.Connection.Command,
                    Arguments              = _input.Connection.Arguments
                }
            }) {
                if (_input.Connection.Folder != string.Empty)
                {
                    p.StartInfo.WorkingDirectory = _input.Connection.Folder;
                }

                // Redirect the output stream of the child process.
                p.Start();

                // Do not wait for the child process to exit before
                // reading to the end of its redirected stream.
                // p.WaitForExit();
                // Read the output stream first and then wait.
                var output = p.StandardOutput.ReadToEnd();

                var lineNumber = 1;
                foreach (var line in new LineReader(output).Read())
                {
                    if (line == string.Empty || lineNumber < _input.Connection.Start)
                    {
                        lineNumber++;
                        continue;
                    }

                    if (_input.Connection.End > 0 && lineNumber > _input.Connection.End)
                    {
                        yield break;
                    }

                    var row = _rowFactory.Create();
                    row[_inputField] = line;
                    lineNumber++;
                    yield return(row);
                }


                p.WaitForExit();
            };
        }
Exemple #7
0
        public IEnumerable <IRow> Read()
        {
            var types = Constants.TypeDefaults();
            var row   = _rowFactory.Create();

            foreach (var field in _context.Entity.GetAllFields())
            {
                row[field] = field.Convert(field.Default == Constants.DefaultSetting ? types[field.Type] : field.Convert(field.Default));
            }
            yield return(row);
        }
        public IEnumerable <IRow> Read()
        {
            _context.Debug(() => "Reading file stream.");

            var start = _context.Connection.Start;
            var end   = 0;

            if (_context.Entity.IsPageRequest())
            {
                start += (_context.Entity.Page * _context.Entity.Size) - _context.Entity.Size;
                end    = start + _context.Entity.Size;
            }

            var current = _context.Connection.Start;

            var engine = FileHelpersEngineFactory.Create(_context);

            using (engine.BeginReadStream(new StreamReader(_stream))) {
                foreach (var record in engine)
                {
                    if (end == 0 || current.Between(start, end))
                    {
                        var values = engine.LastRecordValues;
                        var row    = _rowFactory.Create();
                        for (var i = 0; i < _context.InputFields.Length; i++)
                        {
                            var field = _context.InputFields[i];
                            if (field.Type == "string")
                            {
                                row[field] = values[i] as string;
                            }
                            else
                            {
                                row[field] = field.Convert(values[i]);
                            }
                        }
                        yield return(row);
                    }
                    ++current;
                    if (current == end)
                    {
                        break;
                    }
                }
            }

            if (engine.ErrorManager.HasErrors)
            {
                foreach (var error in engine.ErrorManager.Errors)
                {
                    _context.Error(error.ExceptionInfo.Message);
                }
            }
        }
        public IEnumerable <IRow> Read()
        {
            var searchOption = (SearchOption)Enum.Parse(typeof(SearchOption), _input.Connection.SearchOption, true);

            _input.Info($"Searching folder: {_input.Connection.Folder}");

            var files = new DirectoryInfo(_input.Connection.Folder).GetFiles(_input.Connection.SearchPattern, searchOption);

            _input.Info($"Found {files.Length} files.");

            var names = _input.InputFields.Select(f => f.Name.ToLower()).ToArray();

            foreach (var file in files)
            {
                var row = _rowFactory.Create();
                for (var i = 0; i < _input.InputFields.Length; i++)
                {
                    var field = _input.InputFields[i];
                    switch (names[i])
                    {
                    case "creationtimeutc":
                        row[field] = file.CreationTimeUtc;
                        break;

                    case "directoryname":
                        row[field] = file.DirectoryName;
                        break;

                    case "extension":
                        row[field] = file.Extension;
                        break;

                    case "fullname":
                        row[field] = file.FullName;
                        break;

                    case "lastwritetimeutc":
                        row[field] = file.LastWriteTimeUtc;
                        break;

                    case "length":
                        row[field] = file.Length;
                        break;

                    case "name":
                        row[field] = file.Name;
                        break;
                    }
                }

                yield return(row);
            }
        }
        public IEnumerable <IRow> Read()
        {
            var row    = _rowFactory.Create();
            var lookup = new Dictionary <string, Parameter>();

            foreach (var parameter in _input.Process.Parameters)
            {
                lookup[parameter.Name] = parameter;
            }
            foreach (var field in _fields)
            {
                var fallBack = field.DefaultValue();
                if (lookup.ContainsKey(field.Name))
                {
                    var p = lookup[field.Name];
                    if (_parameters.ContainsKey(field.Name))
                    {
                        if (_parameters[field.Name] == null)
                        {
                            row[field] = fallBack;
                        }
                        else
                        {
                            row[field] = Convert(p, _parameters[field.Name]);
                        }
                        _parameters.Remove(field.Name); // since parameter may be transformed and placed in value, we discard the original
                    }
                    else
                    {
                        if (p.Value == null)
                        {
                            row[field] = fallBack;
                        }
                        else
                        {
                            row[field] = Convert(p, p.Value);
                        }
                    }
                }
                else
                {
                    row[field] = fallBack;
                }
            }

            _input.Entity.Hits = 1;

            yield return(row);

            yield break;
        }
Exemple #11
0
      public IEnumerable<IRow> Read() {

         if (!_run) {
            yield break;
         }

         do {
            var response = _client.ListInstancesAsync(_request).Result;
            foreach (var inst in response.InstanceSummaryList) {
               var row = _rowFactory.Create();

               foreach (var field in _context.InputFields) {
                  var name = field.Name.ToLower();
                  switch (name) {
                     case "arn":
                        row[field] = inst.Arn ?? string.Empty;
                        break;
                     case "createdtime":
                        row[field] = inst.CreatedTime;
                        break;
                     case "id":
                        row[field] = inst.Id;
                        break;
                     case "identitymanagementtype":
                        row[field] = inst.IdentityManagementType.Value ?? string.Empty;
                        break;
                     case "inboundcallsenabled":
                        row[field] = inst.InboundCallsEnabled;
                        break;
                     case "instancealias":
                        row[field] = inst.InstanceAlias ?? string.Empty;
                        break;
                     case "instancestatus":
                        row[field] = inst.InstanceStatus.Value ?? string.Empty;
                        break;
                     case "outboundcallsenabled":
                        row[field] = inst.OutboundCallsEnabled;
                        break;
                     case "servicerole":
                        row[field] = inst.ServiceRole ?? string.Empty;
                        break;
                     default:
                        break;
                  }
               }

               yield return row;
            }
            _request.NextToken = response.NextToken;
         } while (!string.IsNullOrEmpty(_request.NextToken));
      }
        public IEnumerable <IRow> Read()
        {
            var row = _rowFactory.Create();

            try {
                row[_field] = _client.DownloadString(_context.Connection.Url);
            } catch (Exception ex) {
                _context.Error(ex.Message);
                _context.Debug(() => ex.StackTrace);
                yield break;
            }

            yield return(row);
        }
Exemple #13
0
        public IEnumerable <IRow> Read()
        {
            ErrorMode errorMode;

            Enum.TryParse(_context.Connection.ErrorMode, true, out errorMode);

            var engine = new FileHelperAsyncEngine(_builder.CreateRecordClass());

            engine.ErrorManager.ErrorMode  = errorMode;
            engine.ErrorManager.ErrorLimit = _context.Connection.ErrorLimit;

            _context.Debug(() => $"Reading {_fileInfo.Name}.");

            using (engine.BeginReadFile(_fileInfo.FullName)) {
                foreach (var record in engine)
                {
                    var values = engine.LastRecordValues;
                    var row    = _rowFactory.Create();
                    for (var i = 0; i < _context.InputFields.Length; i++)
                    {
                        var field = _context.InputFields[i];
                        if (field.Type == "string")
                        {
                            row[field] = values[i] as string;
                        }
                        else
                        {
                            row[field] = field.Convert(values[i]);
                        }
                    }
                    if (_rowCondition.Eval(row))
                    {
                        yield return(row);
                    }
                }

                if (engine.ErrorManager.HasErrors)
                {
                    foreach (var error in engine.ErrorManager.Errors)
                    {
                        _context.Error(error.ExceptionInfo.Message);
                    }
                }
            }
        }
Exemple #14
0
        public IRow Create(IDataReader reader, Field[] fields)
        {
            var fieldCount = Math.Min(reader.FieldCount, fields.Length);
            var row        = _rowFactory.Create();

            if (_errors == null)    // check types
            {
                _errors = new bool[fields.Length];
                for (var i = 0; i < fieldCount; i++)
                {
                    _errors[i] = reader.GetFieldType(i) != _typeMap[fields[i].Type];
                    if (!_errors[i])
                    {
                        continue;
                    }

                    if (fields[i].Type != "char")
                    {
                        if (_context.Connection.Provider != "sqlite")
                        {
                            _context.Warn("Type mismatch for {0}. Expected {1}, but read {2}.", fields[i].Name, fields[i].Type, reader.GetFieldType(i));
                        }
                    }
                }
            }

            for (var i = 0; i < fieldCount; i++)
            {
                if (reader.IsDBNull(i))
                {
                    continue;
                }
                if (_errors[i])
                {
                    row[fields[i]] = fields[i].Convert(reader.GetValue(i));
                }
                else
                {
                    row[fields[i]] = reader.GetValue(i);
                }
            }
            return(row);
        }
Exemple #15
0
        public IEnumerable <IRow> Read()
        {
            if (!_run)
            {
                yield break;
            }

            do
            {
                var response = _client.ListUsersAsync(_request).Result;
                foreach (var inst in response.UserSummaryList)
                {
                    var row = _rowFactory.Create();

                    foreach (var field in _context.InputFields)
                    {
                        var name = field.Name.ToLower();
                        switch (name)
                        {
                        case "arn":
                            row[field] = inst.Arn ?? string.Empty;
                            break;

                        case "id":
                            row[field] = inst.Id ?? string.Empty;
                            break;

                        case "username":
                            row[field] = inst.Username ?? string.Empty;
                            break;

                        default:
                            break;
                        }
                    }

                    yield return(row);
                }
                _request.NextToken = response.NextToken;
            } while (!string.IsNullOrEmpty(_request.NextToken));
        }
Exemple #16
0
        public IEnumerable <IRow> Read()
        {
            if (_inputField == null)
            {
                _input.Error("You must have one input field for console provider input.");
                yield break;
            }

            if (!System.Console.IsInputRedirected)
            {
                yield break;
            }


            string line;
            var    lineNumber = 1;

            while ((line = System.Console.In.ReadLine()) != null)
            {
                if (line == string.Empty || lineNumber < _input.Connection.Start)
                {
                    lineNumber++;
                    continue;
                }

                if (_input.Connection.End > 0 && lineNumber > _input.Connection.End)
                {
                    yield break;
                }

                var row = _rowFactory.Create();
                row[_inputField] = line;
                lineNumber++;
                yield return(row);
            }
        }
Exemple #17
0
        public IEnumerable <IRow> Read()
        {
            ElasticsearchResponse <DynamicResponse> response;
            ElasticsearchDynamicValue hits;
            var    from = 1;
            var    size = 10;
            string body;

            if (_context.Entity.IsPageRequest())
            {
                from = (_context.Entity.Page * _context.Entity.PageSize) - _context.Entity.PageSize;
                body = WriteQuery(_fields, _readFrom, _context, from, _context.Entity.PageSize);
            }
            else
            {
                body     = WriteQuery(_fields, _readFrom, _context, 0, 0);
                response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);
                if (response.Success)
                {
                    hits = response.Body["hits"] as ElasticsearchDynamicValue;
                    if (hits != null && hits.HasValue)
                    {
                        var properties = hits.Value as IDictionary <string, object>;
                        if (properties != null && properties.ContainsKey("total"))
                        {
                            size = Convert.ToInt32(properties["total"]);
                            body = WriteQuery(_fields, _readFrom, _context, 0, size > 10000 ? 10000 : size);
                        }
                    }
                }
            }

            _context.Debug(() => body);
            _context.Entity.Query = body;

            // move 10000 to configurable limit
            response = from + size > 10000
                ? _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body, p => p.Scroll(TimeSpan.FromMinutes(1.0)))
                : _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);

            if (!response.Success)
            {
                LogError(response);
                yield break;
            }

            _context.Entity.Hits = Convert.ToInt32((response.Body["hits"]["total"] as ElasticsearchDynamicValue).Value);
            hits = response.Body["hits"]["hits"] as ElasticsearchDynamicValue;


            if (hits == null || !hits.HasValue)
            {
                _context.Warn("No hits from elasticsearch");
                yield break;
            }

            var docs = hits.Value as IList <object>;

            if (docs == null)
            {
                _context.Error("No documents returned from elasticsearch!");
                yield break;
            }

            // if any of the fields do not exist, yield break
            if (docs.Count > 0)
            {
                var doc    = docs.First() as IDictionary <string, object>;
                var source = doc?["_source"] as IDictionary <string, object>;
                if (source == null)
                {
                    _context.Error("Missing _source from elasticsearch response!");
                    yield break;
                }

                for (var i = 0; i < _fields.Length; i++)
                {
                    if (source.ContainsKey(_fieldNames[i]))
                    {
                        continue;
                    }

                    _context.Error($"Field {_fieldNames[i]} does not exist!");
                    yield break;
                }
            }

            var count = 0;

            foreach (var d in docs)
            {
                var doc    = (IDictionary <string, object>)d;
                var row    = _rowFactory.Create();
                var source = (IDictionary <string, object>)doc["_source"];
                for (var i = 0; i < _fields.Length; i++)
                {
                    row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                }
                _context.Increment();
                yield return(row);
            }
            count += docs.Count;

            // get this from first search response (maybe), unless you have to aggregate it from all...
            foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet" && !string.IsNullOrEmpty(f.Map)))
            {
                var map     = _context.Process.Maps.First(m => m.Name == filter.Map);
                var buckets = response.Body["aggregations"][filter.Key]["buckets"] as ElasticsearchDynamicValue;
                if (buckets == null || !buckets.HasValue)
                {
                    continue;
                }

                var items = buckets.Value as IEnumerable <object>;

                if (items == null)
                {
                    continue;
                }

                foreach (var item in items.OfType <IDictionary <string, object> >())
                {
                    map.Items.Add(new MapItem {
                        From = $"{item["key"]} ({item["doc_count"]})", To = item["key"]
                    });
                }
            }

            if (!response.Body.ContainsKey("_scroll_id"))
            {
                yield break;
            }

            if (size == count)
            {
                _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = response.Body["_scroll_id"].Value }));
                yield break;
            }

            var scrolls = new HashSet <string>();

            do
            {
                var scrollId = response.Body["_scroll_id"].Value;
                scrolls.Add(scrollId);
                response = _client.Scroll <DynamicResponse>(new PostData <object>(new { scroll = "1m", scroll_id = scrollId }));
                if (response.Success)
                {
                    docs = (IList <object>)response.Body["hits"]["hits"].Value;
                    foreach (var d in docs)
                    {
                        var doc    = (IDictionary <string, object>)d;
                        var row    = _rowFactory.Create();
                        var source = (IDictionary <string, object>)doc["_source"];
                        for (var i = 0; i < _fields.Length; i++)
                        {
                            row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                        }
                        _context.Increment();
                        yield return(row);
                    }
                    count += docs.Count;
                }
                else
                {
                    LogError(response);
                }
            } while (response.Success && count < size);

            _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = scrolls.ToArray() }));
        }
Exemple #18
0
        public IEnumerable <IRow> Read()
        {
            _context.Debug(() => $"Reading {_fileInfo.Name}.");

            var start = _context.Connection.Start;
            var end   = 0;

            if (_context.Entity.IsPageRequest())
            {
                start += (_context.Entity.Page * _context.Entity.Size) - _context.Entity.Size;
                end    = start + _context.Entity.Size;
            }

            var current = _context.Connection.Start;
            var engine  = FileHelpersEngineFactory.Create(_context);

            IDisposable reader;

            try {
                reader = engine.BeginReadFile(_fileInfo.FullName);
            } catch (Exception ex) {
                _context.Error(ex.Message);
                yield break;
            }

            using (reader) {
                foreach (var record in engine)
                {
                    if (end == 0 || current.Between(start, end))
                    {
                        var values = engine.LastRecordValues;
                        var row    = _rowFactory.Create();
                        for (var i = 0; i < _context.InputFields.Length; i++)
                        {
                            var field = _context.InputFields[i];
                            row[field] = values[i];
                        }

                        if (_fileField != null)
                        {
                            row[_fileField] = _context.Connection.File;
                        }
                        yield return(row);
                    }
                    ++current;
                    if (current == end)
                    {
                        break;
                    }
                }
            }

            if (engine.ErrorManager.HasErrors)
            {
                foreach (var error in engine.ErrorManager.Errors)
                {
                    _context.Error(error.ExceptionInfo.Message);
                    _context.Error($"Error processing line {error.LineNumber} in {_context.Connection.File}.");
                    _context.Warn(error.RecordString.Replace("{", "{{").Replace("}", "}}"));
                }
            }
        }
        public IEnumerable <IRow> Read()
        {
            ElasticsearchResponse <DynamicResponse> response;

            string body;

            if (_context.Entity.IsPageRequest())
            {
                var from = (_context.Entity.Page * _context.Entity.PageSize) - _context.Entity.PageSize;
                body = WriteQuery(_fields, _readFrom, _context, from, _context.Entity.PageSize);
            }
            else
            {
                body     = WriteQuery(_fields, _readFrom, _context, 0, 0);
                response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);
                if (response.Success)
                {
                    var hits = response.Body["hits"] as ElasticsearchDynamicValue;
                    if (hits != null && hits.HasValue)
                    {
                        var properties = hits.Value as IDictionary <string, object>;
                        if (properties != null && properties.ContainsKey("total"))
                        {
                            var size = Convert.ToInt32(properties["total"]) + 1;
                            body = WriteQuery(_fields, _readFrom, _context, 0, size);
                        }
                    }
                }
            }
            _context.Debug(() => body);
            _context.Entity.Query = body;

            response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);

            if (response.Success)
            {
                _context.Entity.Hits = Convert.ToInt32((response.Body["hits"]["total"] as ElasticsearchDynamicValue).Value);
                var hits = response.Body["hits"]["hits"] as ElasticsearchDynamicValue;
                if (hits != null && hits.HasValue)
                {
                    var docs = hits.Value as IEnumerable <object>;
                    if (docs != null)
                    {
                        foreach (var doc in docs.OfType <IDictionary <string, object> >())
                        {
                            var row = _rowFactory.Create();
                            if (doc != null && doc.ContainsKey("_source"))
                            {
                                var source = doc["_source"] as IDictionary <string, object>;
                                if (source != null)
                                {
                                    for (var i = 0; i < _fields.Length; i++)
                                    {
                                        var field = _fields[i];
                                        row[field] = field.Convert(source[_fieldNames[i]]);
                                    }
                                }
                            }
                            _context.Increment();
                            yield return(row);
                        }
                    }
                }

                foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet" && !string.IsNullOrEmpty(f.Map)))
                {
                    var map     = _context.Process.Maps.First(m => m.Name == filter.Map);
                    var buckets = response.Body["aggregations"][filter.Key]["buckets"] as ElasticsearchDynamicValue;
                    if (buckets == null || !buckets.HasValue)
                    {
                        continue;
                    }

                    var items = buckets.Value as IEnumerable <object>;

                    if (items == null)
                    {
                        continue;
                    }

                    foreach (var item in items.OfType <IDictionary <string, object> >())
                    {
                        map.Items.Add(new MapItem {
                            From = $"{item["key"]} ({item["doc_count"]})", To = item["key"]
                        }.WithDefaults());
                    }
                }
            }
            else
            {
                _context.Error(response.DebugInformation);
            }
        }
        public IEnumerable <IRow> Read()
        {
            AbstractSolrQuery query = SolrQuery.All;
            var filterQueries       = new Collection <ISolrQuery>();
            var facetQueries        = new Collection <ISolrFacetQuery>();

            if (_context.Entity.Filter.Any())
            {
                var queries = new Collection <ISolrQuery>();

                foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "search" && f.Value != "*"))
                {
                    if (filter.Field == string.Empty)
                    {
                        queries.Add(new SolrQuery(filter.Expression));
                    }
                    else
                    {
                        foreach (var term in Terms(filter.Value))
                        {
                            queries.Add(new SolrQueryByField(filter.Field, term)
                            {
                                Quoted = false
                            });
                        }
                    }
                }

                query = queries.Any() ? new SolrMultipleCriteriaQuery(queries, "AND") : SolrQuery.All;

                foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "filter"))
                {
                    if (filter.Field == string.Empty)
                    {
                        filterQueries.Add(new SolrQuery(filter.Expression));
                    }
                    else
                    {
                        if (filter.Value != "*")
                        {
                            foreach (var term in Terms(filter.Value))
                            {
                                queries.Add(new SolrQueryByField(filter.Field, term)
                                {
                                    Quoted = false
                                });
                            }
                        }
                    }
                }

                foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet"))
                {
                    facetQueries.Add(new SolrFacetFieldQuery(filter.Field)
                    {
                        MinCount = filter.Min,
                        Limit    = filter.Size
                    });
                    if (filter.Value != "*")
                    {
                        if (filter.Value.IndexOf(',') > 0)
                        {
                            filterQueries.Add(new SolrQueryInList(filter.Field, filter.Value.Split(new[] { ',' })));
                        }
                        else
                        {
                            filterQueries.Add(new SolrQueryByField(filter.Field, filter.Value));
                        }
                    }
                }
            }

            int           rows;
            StartOrCursor startOrCursor;

            if (_context.Entity.IsPageRequest())
            {
                rows          = _context.Entity.PageSize;
                startOrCursor = new StartOrCursor.Start((_context.Entity.Page * _context.Entity.PageSize) - _context.Entity.PageSize);
            }
            else
            {
                rows = _context.Entity.ReadSize > 0 ? _context.Entity.ReadSize : _solr.Query(query, new QueryOptions {
                    StartOrCursor = new StartOrCursor.Start(0), Rows = 0
                }).NumFound;
                startOrCursor = _context.Entity.ReadSize == 0 ? (StartOrCursor) new StartOrCursor.Start(0) : StartOrCursor.Cursor.Start;
            }

            var sortOrder = new Collection <SortOrder>();

            foreach (var orderBy in _context.Entity.Order)
            {
                Field field;
                if (_context.Entity.TryGetField(orderBy.Field, out field))
                {
                    var name = field.SortField.ToLower();
                    sortOrder.Add(new SortOrder(name, orderBy.Sort == "asc" ? SolrNet.Order.ASC : SolrNet.Order.DESC));
                }
            }
            sortOrder.Add(new SortOrder("score", SolrNet.Order.DESC));

            var result = _solr.Query(
                query,
                new QueryOptions {
                StartOrCursor = startOrCursor,
                Rows          = rows,
                Fields        = _fieldNames,
                OrderBy       = sortOrder,
                FilterQueries = filterQueries,
                Facet         = new FacetParameters {
                    Queries = facetQueries, Sort = false
                }
            }
                );

            foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet"))
            {
                if (result.FacetFields.ContainsKey(filter.Field))
                {
                    var facet = result.FacetFields[filter.Field];
                    var map   = _context.Process.Maps.First(m => m.Name == filter.Map);
                    foreach (var f in facet)
                    {
                        map.Items.Add(new MapItem {
                            From = $"{f.Key} ({f.Value})", To = f.Key
                        });
                    }
                }
            }

            if (result.NumFound <= 0)
            {
                yield break;
            }

            _context.Entity.Hits = result.NumFound;

            foreach (var row in result.Select(x => DocToRow(_rowFactory.Create(), _fields, x)))
            {
                _context.Increment();
                ++_localCount;
                yield return(row);
            }

            // using cursor, solr 4.7+ (un-tested)
            while (result.NextCursorMark != null)
            {
                result = _solr.Query(
                    query,
                    new QueryOptions {
                    StartOrCursor = result.NextCursorMark,
                    Rows          = _context.Entity.ReadSize,
                    Fields        = _fieldNames,
                    OrderBy       = sortOrder,
                    FilterQueries = filterQueries,
                    Facet         = new FacetParameters {
                        Queries = facetQueries, Sort = false
                    }
                }
                    );

                foreach (var row in result.Select(r => DocToRow(_rowFactory.Create(), _fields, r)))
                {
                    _context.Increment();
                    ++_localCount;
                    yield return(row);
                }
            }

            // traditional paging
            if (_context.Entity.ReadSize == 0 || _localCount >= result.NumFound)
            {
                yield break;
            }

            var pages = result.NumFound / _context.Entity.ReadSize;

            for (var page = 1; page <= pages; page++)
            {
                result = _solr.Query(
                    query,
                    new QueryOptions {
                    StartOrCursor = new StartOrCursor.Start(page * _context.Entity.ReadSize),
                    Rows          = _context.Entity.ReadSize,
                    Fields        = _fieldNames,
                    OrderBy       = sortOrder,
                    FilterQueries = filterQueries,
                    Facet         = new FacetParameters {
                        Queries = facetQueries, Sort = false
                    }
                }
                    );

                foreach (var row in result.Select(r => DocToRow(_rowFactory.Create(), _fields, r)))
                {
                    ++_localCount;
                    _context.Increment();
                    yield return(row);
                }
            }
        }
        private IEnumerable <IRow> Flatten(string key, object obj, LinkedList <IRow> results = null)
        {
            IField field = _fields.ContainsKey(key) ? _fields[key] : null;

            // in the beginning, create an empty result
            if (results == null)
            {
                results = new LinkedList <IRow>();
            }

            if (obj == null)
            {
                return(results);
            }

            var dict = obj as IDictionary <string, object>;

            if (dict == null)
            {
                var list = obj as IList <object>;
                if (list != null)
                {
                    foreach (var item in list)
                    {
                        results.AddLast(_rowFactory.Create());
                        Flatten(key, item, results);
                    }
                }
            }
            else
            {
                if (dict.Count == 1 && dict.ContainsKey("value"))
                {
                    if (field == null)
                    {
                        _missing.Add(key);
                    }
                    else
                    {
                        // set a field value
                        var value = dict["value"];
                        if (results.Last.Value[field] == null)
                        {
                            results.Last.Value[field] = value;
                        }
                        else
                        {
                            // this must be a total
                            var total = _rowFactory.Create();
                            total[field] = value;
                            results.AddLast(total);
                        }
                    }
                }
                else
                {
                    if (dict.ContainsKey("buckets"))
                    {
                        Flatten(key, dict["buckets"], results);
                    }
                    else
                    {
                        if (dict.ContainsKey("key") && dict.ContainsKey("doc_count"))
                        {
                            foreach (var pair in dict)
                            {
                                if (pair.Key == "key")
                                {
                                    if (field == null)
                                    {
                                        _missing.Add(key);
                                    }
                                    else
                                    {
                                        // set a key field value
                                        results.Last()[field] = pair.Value;
                                    }
                                }
                                else
                                {
                                    if (pair.Key != "doc_count")
                                    {
                                        Flatten(pair.Key, pair.Value, results);
                                    }
                                }
                            }
                        }
                        else
                        {
                            foreach (var pair in dict)
                            {
                                Flatten(pair.Key, pair.Value, results);
                            }
                        }
                    }
                }
            }

            if (!_missing.Any())
            {
                return(results);
            }

            foreach (var missing in _missing)
            {
                _context.Warn($"The query returns field {missing}, but you do not have that field defined in {_context.Entity.Alias}.");
            }
            return(results);
        }
        public override IEnumerable <IRow> Transform(IEnumerable <IRow> rows)
        {
            foreach (var row in rows)
            {
                var outerRow = row;
                var innerRow = _rowFactory.Create();
                foreach (var field in _fields)
                {
                    innerRow[field] = field.Default == Constants.DefaultSetting ? _typeDefaults[field.Type] : field.Convert(field.Default);
                }

                var    innerRows = new List <IRow>();
                string startKey  = null;

                var xml = row[_input] as string;

                if (!string.IsNullOrEmpty(xml))
                {
                    xml = xml.Trim();
                    using (var reader = XmlReader.Create(new StringReader(xml), Settings)) {
                        if (_findRoot)
                        {
                            do
                            {
                                reader.Read();
                            } while (reader.Name != _root);
                        }
                        else
                        {
                            reader.Read();
                        }

                        do
                        {
                            if (_nameMap.ContainsKey(reader.Name))
                            {
                                // must while here because reader.Read*Xml advances the reader
                                while (_nameMap.ContainsKey(reader.Name) && reader.IsStartElement())
                                {
                                    InnerRow(ref startKey, reader.Name, ref innerRow, ref outerRow, ref innerRows);

                                    var field = _nameMap[reader.Name];
                                    var value = field.ReadInnerXml ? reader.ReadInnerXml() : reader.ReadOuterXml();
                                    if (value != string.Empty)
                                    {
                                        innerRow[field] = field.Convert(value);
                                    }
                                }
                            }
                            else if (_searchAttributes && reader.HasAttributes)
                            {
                                for (var i = 0; i < reader.AttributeCount; i++)
                                {
                                    reader.MoveToNextAttribute();
                                    if (!_nameMap.ContainsKey(reader.Name))
                                    {
                                        continue;
                                    }

                                    InnerRow(ref startKey, reader.Name, ref innerRow, ref outerRow, ref innerRows);

                                    var field = _nameMap[reader.Name];
                                    if (!string.IsNullOrEmpty(reader.Value))
                                    {
                                        innerRow[field] = field.Convert(reader.Value);
                                    }
                                }
                            }
                            if (_findRoot && !reader.IsStartElement() && reader.Name == _root)
                            {
                                break;
                            }
                        } while (reader.Read());
                    }
                }
                AddInnerRow(ref innerRow, ref outerRow, ref innerRows);
                foreach (var r in innerRows)
                {
                    yield return(r);
                }
            }
        }
        public IEnumerable <IRow> Read()
        {
            IRow  row     = null;
            Field field   = null;
            var   types   = Constants.TypeSystem();
            var   current = 0;
            var   start   = 0;
            var   end     = 0;

            if (_context.Entity.IsPageRequest())
            {
                start += (_context.Entity.Page * _context.Entity.Size) - _context.Entity.Size;
                end    = start + _context.Entity.Size;
            }

            var textReader = new StreamReader(_stream);
            var reader     = new JsonTextReader(textReader);

            var textWriter = new StringWriter();
            var jsonWriter = new JsonTextWriter(textWriter);

            while (reader.Read())
            {
                switch (reader.TokenType)
                {
                case JsonToken.StartObject:
                    if (reader.Depth == 1)
                    {
                        row = _rowFactory.Create();
                    }
                    if (reader.Depth > 1)
                    {
                        jsonWriter.WriteStartObject();
                    }
                    break;

                case JsonToken.EndObject:
                    if (reader.Depth == 1)
                    {
                        if (end == 0 || current.Between(start, end))
                        {
                            yield return(row);
                        }
                        ++current;
                        if (current == end)
                        {
                            yield break;
                        }
                    }
                    if (reader.Depth > 1 && field != null)
                    {
                        jsonWriter.WriteEndObject();
                        jsonWriter.Flush();
                        row[field] = textWriter.ToString();
                        field      = null;
                        textWriter = new StringWriter();
                        jsonWriter = new JsonTextWriter(textWriter);
                    }
                    break;

                case JsonToken.StartArray:
                    if (reader.Depth > 0)
                    {
                        jsonWriter.WriteStartArray();
                    }
                    break;

                case JsonToken.EndArray:
                    if (reader.Depth > 0)
                    {
                        jsonWriter.WriteEndArray();
                    }
                    break;

                case JsonToken.PropertyName:
                    var name = (string)reader.Value;
                    if (_fieldLookup.ContainsKey(name))
                    {
                        if (reader.Depth == 2)
                        {
                            field = _fieldLookup[name];
                        }
                    }
                    if (reader.Depth > 2)
                    {
                        jsonWriter.WritePropertyName(name);
                    }
                    break;

                default:
                    if (reader.Depth == 2 && reader.Value != null && field != null)
                    {
                        if (types[field.Type] == reader.ValueType)
                        {
                            row[field] = reader.Value;
                        }
                        else
                        {
                            row[field] = field.Convert(reader.Value);
                        }
                        field = null;
                    }
                    if (reader.Depth > 2 && reader.Value != null)
                    {
                        jsonWriter.WriteValue(reader.Value);
                    }
                    break;
                }
            }

            _context.Entity.Hits = current;
        }
Exemple #24
0
        public IEnumerable <IRow> Read()
        {
            // TODO: fix major duplication here

            using (var fileStream = File.Open(_fileInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
                var isBinary = _fileInfo.Extension.ToLower() == ".xls";
                using (var reader = isBinary ? ExcelReaderFactory.CreateBinaryReader(fileStream) : ExcelReaderFactory.CreateOpenXmlReader(fileStream)) {
                    int index = 0;
                    if (reader == null)
                    {
                        yield break;
                    }

                    var start = _context.Connection.Start;
                    var end   = 0;
                    if (_context.Entity.IsPageRequest())
                    {
                        start += (_context.Entity.Page * _context.Entity.PageSize) - _context.Entity.PageSize;
                        end    = start + _context.Entity.PageSize;
                    }

                    for (var i = 0; i < start; i++)
                    {
                        reader.Read();
                        ++index;
                    }

                    var readerHasData = false;
                    var emptyDetector = new StringBuilder();

                    if (reader.Read())
                    {
                        ++index;
                        readerHasData = true;

                        if (end > 0 && end <= index)
                        {
                            _context.Entity.Hits++;
                        }
                        else
                        {
                            emptyDetector.Clear();
                            var row = _rowFactory.Create();
                            for (var i = 0; i < _context.InputFields.Length; i++)
                            {
                                var field = _context.InputFields[i];

                                var expected = Constants.TypeSystem()[field.Type];
                                var actual   = reader.IsDBNull(i) ? null : reader.GetValue(i);

                                if (_context.Entity.DataTypeWarnings)
                                {
                                    if (actual != null && expected != actual.GetType())
                                    {
                                        _context.Warn($"The {field.Alias} field in {_context.Entity.Alias} expects a {expected}, but is reading a ({actual.GetType().Name}){actual}.");
                                    }
                                }

                                if (field.Type == "string")
                                {
                                    row[field] = reader.IsDBNull(i) ? null : reader.GetValue(i).ToString();
                                }
                                else
                                {
                                    row[field] = reader.IsDBNull(i) ? null : field.Convert(reader.GetValue(i));
                                }
                                emptyDetector.Append(row[field]);
                            }
                            emptyDetector.Trim(" ");
                            if (!emptyDetector.ToString().Equals(string.Empty))
                            {
                                _context.Entity.Hits++;
                                yield return(row);
                            }
                        }
                    }

                    if (readerHasData)
                    {
                        while (reader.Read())
                        {
                            ++index;
                            if (end > 0 && end <= index)
                            {
                                _context.Entity.Hits++;
                            }
                            else
                            {
                                emptyDetector.Clear();
                                var row = _rowFactory.Create();
                                for (var i = 0; i < _context.InputFields.Length; i++)
                                {
                                    var field = _context.InputFields[i];
                                    if (field.Type == "string")
                                    {
                                        row[field] = reader.IsDBNull(i) ? null : reader.GetString(i);
                                    }
                                    else
                                    {
                                        row[field] = reader.IsDBNull(i) ? null : field.Convert(reader.GetValue(i));
                                    }
                                    emptyDetector.Append(row[field]);
                                }
                                emptyDetector.Trim(" ");
                                if (!emptyDetector.ToString().Equals(string.Empty))
                                {
                                    _context.Entity.Hits++;
                                    yield return(row);
                                }
                            }
                        }
                    }
                    else
                    {
                        // try read from data set
                        var dr = reader.AsDataSet().CreateDataReader();

                        for (var i = 0; i < start; i++)
                        {
                            dr.Read();
                            ++index;
                        }

                        // check type on first read
                        if (dr.Read())
                        {
                            ++index;
                            if (end <= index)
                            {
                                _context.Entity.Hits++;
                            }
                            else
                            {
                                emptyDetector.Clear();
                                var row = _rowFactory.Create();
                                for (var i = 0; i < _context.InputFields.Length; i++)
                                {
                                    var field = _context.InputFields[i];

                                    var expected = Constants.TypeSystem()[field.Type];
                                    var actual   = dr.GetValue(i).GetType();
                                    if (expected != actual)
                                    {
                                        _context.Warn($"The {field.Alias} field in {_context.Entity.Alias} expects a {expected}, but is reading a {actual}.");
                                    }

                                    if (field.Type == "string")
                                    {
                                        row[field] = dr.GetValue(i) as string;
                                    }
                                    else
                                    {
                                        row[field] = field.Convert(dr.GetValue(i));
                                    }
                                    emptyDetector.Append(row[field]);
                                }
                                emptyDetector.Trim(" ");
                                if (!emptyDetector.ToString().Equals(string.Empty))
                                {
                                    _context.Entity.Hits++;
                                    yield return(row);
                                }
                            }
                        }

                        while (dr.Read())
                        {
                            emptyDetector.Clear();
                            var row = _rowFactory.Create();
                            for (var i = 0; i < _context.InputFields.Length; i++)
                            {
                                var field = _context.InputFields[i];
                                if (field.Type == "string")
                                {
                                    row[field] = dr.GetString(i);
                                }
                                else
                                {
                                    row[field] = field.Convert(dr.GetValue(i));
                                }
                                emptyDetector.Append(row[field]);
                            }
                            emptyDetector.Trim(" ");
                            if (!emptyDetector.ToString().Equals(string.Empty))
                            {
                                _context.Entity.Hits++;
                                yield return(row);
                            }
                        }
                    }
                }
            }
        }
Exemple #25
0
        public IEnumerable <IRow> Read()
        {
            var stream = _client.OpenRead(_context.Connection.Url);

            if (stream == null)
            {
                _context.Error("Could not open {0}.", _context.Connection.Url);
                yield break;
            }

            var start         = _context.Connection.Start;
            var end           = _context.Connection.End;
            var isPageRequest = _context.Entity.IsPageRequest();

            if (isPageRequest)
            {
                start += ((_context.Entity.Page * _context.Entity.PageSize) - _context.Entity.PageSize);
                end    = start + _context.Entity.PageSize;
            }

            using (var reader = new StreamReader(stream)) {
                string line;

                _context.Entity.Hits = 1;

                if (start > 1)
                {
                    for (var i = 1; i < start; i++)
                    {
                        reader.ReadLine();
                        _context.Entity.Hits++;
                    }
                }

                while ((line = reader.ReadLine()) != null)
                {
                    if (end > 0 && _context.Entity.Hits >= end)
                    {
                        if (isPageRequest)
                        {
                            _context.Entity.Hits++;
                            continue;
                        }

                        yield break;
                    }

                    _context.Entity.Hits++;
                    var tokens = _regex.Split(line.Trim('"'));
                    if (tokens.Length > 0)
                    {
                        var row = _rowFactory.Create();
                        for (var i = 0; i < _context.InputFields.Length && i < tokens.Length; i++)
                        {
                            var field = _context.InputFields[i];
                            row[field] = field.Convert(tokens[i]);
                        }
                        yield return(row);
                    }
                }

                if (isPageRequest && start > 1)
                {
                    _context.Entity.Hits -= (start - 1);
                }
            }
        }
        private IEnumerable <IRow> PreRead()
        {
            _context.Debug(() => "Reading file stream.");

            var ignoreFirstLines = _context.Connection.Start > 1 ? _context.Connection.Start - 1 : _context.Connection.Start;

            var start = _context.Connection.Start;
            var end   = 0;

            if (_context.Entity.IsPageRequest())
            {
                start += (_context.Entity.Page * _context.Entity.Size) - _context.Entity.Size;
                end    = start + _context.Entity.Size;
            }

            var current = _context.Connection.Start;

            var configuration = new CsvConfiguration(CultureInfo.InvariantCulture)
            {
                IgnoreBlankLines = true,
                Delimiter        = string.IsNullOrEmpty(_context.Connection.Delimiter) ? "," : _context.Connection.Delimiter,
                Encoding         = Encoding.GetEncoding(_context.Connection.Encoding)
            };

            if (_context.Connection.ErrorMode.Equals("IgnoreAndContinue", System.StringComparison.OrdinalIgnoreCase))
            {
                configuration.BadDataFound = null; // skip the record
            }

            if (_context.Connection.TextQualifier != string.Empty)
            {
                configuration.Escape = _context.Connection.TextQualifier[0];
                configuration.Quote  = _context.Connection.TextQualifier[0];
            }

            using (var csv = new CsvReader(_streamReader, configuration)) {
                while (csv.Read())
                {
                    if (csv.Parser.RawRow <= ignoreFirstLines)
                    {
                        continue;
                    }

                    if (end == 0 || current.Between(start, end))
                    {
                        var row = _rowFactory.Create();
                        for (int i = 0; i < _context.InputFields.Length; i++)
                        {
                            var data  = csv.GetField(i);
                            var field = _context.InputFields[i];
                            row[field] = data;
                        }
                        yield return(row);

                        ++_context.Entity.Hits;
                    }
                    ++current;
                    if (current == end)
                    {
                        break;
                    }
                }
            }

            _streamReader.Close();
        }
Exemple #27
0
        public IEnumerable <IRow> Read()
        {
            var encoding = Encoding.GetEncoding(_context.Connection.Encoding);
            var lineNo   = 0;

            if (_fileInfo.Extension == ".xml")
            {
                var row = _rowFactory.Create();
                row[_field] = System.IO.File.ReadAllText(_fileInfo.FullName, encoding);
                yield return(row);
            }
            else
            {
                if (_context.Connection.LinePattern != string.Empty)
                {
                    var regex    = new Regex(_context.Connection.LinePattern, RegexOptions.Compiled);
                    var prevLine = string.Empty;

                    foreach (var line in ReadLines(_fileInfo.FullName, encoding))
                    {
                        ++lineNo;

                        if (_linesToKeep.Contains(lineNo))
                        {
                            _context.Connection.Lines[lineNo] = line;
                        }

                        if (lineNo < _context.Connection.Start)
                        {
                            continue;
                        }

                        if (regex.IsMatch(line)) // CURRENT LINE PASSES

                        {
                            if (regex.IsMatch(prevLine)) // PREVIOUS LINE PASSES
                            {
                                var row = _rowFactory.Create();
                                row[_field] = string.Copy(prevLine);
                                prevLine    = line;
                                yield return(row);
                            }
                            else // PREVIOUS LINE FAILS
                            {
                                prevLine = line;
                            }
                        }
                        else // CURRENT LINE FAILS
                        {
                            var combined = prevLine + " " + line;

                            if (regex.IsMatch(prevLine))
                            {
                                if (regex.IsMatch(combined)) // IF COMBINED THEY STILL PASS, COMBINE AND CONTINUE
                                {
                                    prevLine = combined;
                                }
                                else // IF COMBINED THEY FAIL, LET THE VALID PREVIOUS LINE THROUGH AND PUT LINE IN PREV LINE IN HOPES SUBSEQUENT LINES WILL MAKE IT PASS
                                {
                                    var row = _rowFactory.Create();
                                    row[_field] = string.Copy(prevLine);
                                    prevLine    = line;
                                    yield return(row);
                                }
                            }
                            else
                            {
                                prevLine = combined;
                            }
                        }
                    }

                    if (regex.IsMatch(prevLine))
                    {
                        var row = _rowFactory.Create();
                        row[_field] = prevLine;
                        yield return(row);
                    }
                }
                else
                {
                    foreach (var line in ReadLines(_fileInfo.FullName, encoding))
                    {
                        ++lineNo;

                        if (_linesToKeep.Contains(lineNo))
                        {
                            _context.Connection.Lines[lineNo] = line;
                        }

                        if (lineNo < _context.Connection.Start)
                        {
                            continue;
                        }

                        var row = _rowFactory.Create();
                        row[_field] = line;
                        yield return(row);
                    }
                }
            }
        }
        public IEnumerable <IRow> Read()
        {
            ElasticsearchResponse <DynamicResponse> response;
            ElasticsearchDynamicValue hits;

            var    from = 0;
            var    size = 10;
            string body;
            bool   warned = false;

            var scroll = !_context.Entity.IsPageRequest();

            if (!scroll)
            {
                from = (_context.Entity.Page * _context.Entity.Size) - _context.Entity.Size;
                body = WriteQuery(_fields, _readFrom, _context, scroll: false, from: from, size: _context.Entity.Size);
            }
            else
            {
                body     = WriteQuery(_fields, _readFrom, _context, scroll: false, from: 0, size: 0);
                response = _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);
                if (response.Success)
                {
                    hits = response.Body["hits"] as ElasticsearchDynamicValue;
                    if (hits != null && hits.HasValue)
                    {
                        var total = hits["total"];

                        try {
                            if (_version.Major >= 7) // version 7 changed total to an object with "value" and "relation" properties
                            {
                                size = Convert.ToInt32(total["value"].Value);
                            }
                            else
                            {
                                size = Convert.ToInt32(total.Value);
                            }
                        } catch (Exception ex) {
                            warned = true;
                            _context.Debug(() => total);
                            _context.Warn($"Could not get total number of matching documents from the elasticsearch response.  Are you sure you using version {_version}?");
                            _context.Error(ex, ex.Message);
                        }
                        body = WriteQuery(_fields, _readFrom, _context, scroll: true, from: 0, size: size > ElasticsearchDefaultSizeLimit ? DefaultSize : size);
                    }
                }
            }

            _context.Debug(() => body);
            _context.Entity.Query = body;

            response = scroll
            ? _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body, p => p.AddQueryString("scroll", _context.Connection.Scroll))
            : _client.Search <DynamicResponse>(_context.Connection.Index, _typeName, body);

            if (!response.Success)
            {
                LogError(response);
                yield break;
            }

            try {
                if (_version.Major >= 7) // version 7 changed total to an object with "value" and "relation" properties
                {
                    _context.Entity.Hits = Convert.ToInt32(response.Body["hits"]["total"]["value"].Value);
                }
                else
                {
                    _context.Entity.Hits = Convert.ToInt32(response.Body["hits"]["total"].Value);
                }
            } catch (Exception ex) {
                if (!warned)
                {
                    _context.Debug(() => response.Body["hits"]);
                    _context.Warn($"Could not get total number of matching documents from the elasticsearch response.  Are you sure you using version {_version}?");
                    _context.Error(ex.Message);
                }
            }

            hits = response.Body["hits"]["hits"] as ElasticsearchDynamicValue;

            if (hits == null || !hits.HasValue)
            {
                _context.Warn("No hits from elasticsearch");
                yield break;
            }

            var docs = hits.Value as IList <object>;

            if (docs == null)
            {
                _context.Error("No documents returned from elasticsearch!");
                yield break;
            }

            // if any of the fields do not exist, yield break
            if (docs.Count > 0)
            {
                var doc    = docs.First() as IDictionary <string, object>;
                var source = doc?["_source"] as IDictionary <string, object>;
                if (source == null)
                {
                    _context.Error("Missing _source from elasticsearch response!");
                    yield break;
                }

                for (var i = 0; i < _fields.Length; i++)
                {
                    if (source.ContainsKey(_fieldNames[i]))
                    {
                        continue;
                    }

                    _context.Error($"Field {_fieldNames[i]} does not exist!");
                    yield break;
                }
            }

            var count = 0;

            foreach (var d in docs)
            {
                var doc    = (IDictionary <string, object>)d;
                var row    = _rowFactory.Create();
                var source = (IDictionary <string, object>)doc["_source"];
                for (var i = 0; i < _fields.Length; i++)
                {
                    row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                }
                yield return(row);
            }
            count += docs.Count;

            // get this from first search response (maybe), unless you have to aggregate it from all...
            foreach (var filter in _context.Entity.Filter.Where(f => f.Type == "facet" && !string.IsNullOrEmpty(f.Map)))
            {
                var map     = _context.Process.Maps.First(m => m.Name == filter.Map);
                var buckets = response.Body["aggregations"][filter.Key]["buckets"] as ElasticsearchDynamicValue;
                if (buckets == null || !buckets.HasValue)
                {
                    continue;
                }

                var items = buckets.Value as IEnumerable <object>;

                if (items == null)
                {
                    continue;
                }

                foreach (var item in items.OfType <IDictionary <string, object> >())
                {
                    map.Items.Add(new MapItem {
                        From = $"{item["key"]} ({item["doc_count"]})", To = item["key"]
                    });
                }
            }

            if (!response.Body.ContainsKey("_scroll_id"))
            {
                yield break;
            }

            if (size == count)
            {
                _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = response.Body["_scroll_id"].Value }));
                yield break;
            }

            var scrolls = new HashSet <string>();

            do
            {
                var scrollId = response.Body["_scroll_id"].Value;
                scrolls.Add(scrollId);
                response = _client.Scroll <DynamicResponse>(new PostData <object>(new { scroll = _context.Connection.Scroll, scroll_id = scrollId }));
                if (response.Success)
                {
                    docs = (IList <object>)response.Body["hits"]["hits"].Value;
                    foreach (var d in docs)
                    {
                        var doc    = (IDictionary <string, object>)d;
                        var row    = _rowFactory.Create();
                        var source = (IDictionary <string, object>)doc["_source"];
                        for (var i = 0; i < _fields.Length; i++)
                        {
                            row[_fields[i]] = _fields[i].Convert(source[_fieldNames[i]]);
                        }
                        yield return(row);
                    }
                    count += docs.Count;
                }
                else
                {
                    LogError(response);
                }
            } while (response.Success && count < size);

            _client.ClearScroll <DynamicResponse>(new PostData <object>(new { scroll_id = scrolls.ToArray() }));
        }
        public IRow Create(IDataReader reader, Field[] fields)
        {
            var row = _rowFactory.Create();

            if (_fieldCount == 0)
            {
                _fieldCount  = Math.Min(reader.FieldCount, fields.Length);
                _conversions = new List <Func <object, object> >(_fieldCount);
                for (var i = 0; i < _fieldCount; i++)
                {
                    _conversions.Add(null);
                }
                _errors = new bool[fields.Length];
                for (var i = 0; i < _fieldCount; i++)
                {
                    var inputType = reader.GetFieldType(i);
                    _errors[i] = inputType != _typeMap[fields[i].Type];

                    if (_errors[i])
                    {
                        if (fields[i].Transforms.Any() && fields[i].Transforms.First().Method == "convert")
                        {
                            _conversions[i] = o => o;  // the user has set a conversion
                        }
                        else
                        {
                            _conversions[i] = fields[i].Convert;
                            _context.Warn("Type mismatch for {0}. Expected {1}, but read {2}.  Change type or add conversion.", fields[i].Name, fields[i].Type, inputType);
                        }
                    }
                    else
                    {
                        _conversions[i] = o => o;
                    }
                }

                for (var i = 0; i < _fieldCount; i++)
                {
                    if (reader.IsDBNull(i))
                    {
                        continue;
                    }
                    if (_errors[i])
                    {
                        var value = reader.GetValue(i);
                        try {
                            row[fields[i]] = fields[i].Type == "object" ? value : _conversions[i](value);
                        } catch (FormatException) {
                            _context.Error($"Could not convert value {value} in field {fields[i].Alias} to {fields[i].Type}");
                        }
                    }
                    else
                    {
                        row[fields[i]] = reader.GetValue(i);
                    }
                }
            }
            else
            {
                for (var i = 0; i < _fieldCount; i++)
                {
                    if (reader.IsDBNull(i))
                    {
                        continue;
                    }
                    if (_errors[i])
                    {
                        row[fields[i]] = fields[i].Type == "object" ? reader.GetValue(i) : _conversions[i](reader.GetValue(i));
                    }
                    else
                    {
                        row[fields[i]] = reader.GetValue(i);
                    }
                }
            }

            return(row);
        }
        public IEnumerable <IRow> Read()
        {
            var searchOption = (SearchOption)Enum.Parse(typeof(SearchOption), _input.Connection.SearchOption, true);

            _input.Info($"Searching folder: {_input.Connection.Folder}");

            var files = new DirectoryInfo(_input.Connection.Folder).GetFiles(_input.Connection.SearchPattern, searchOption);

            _input.Info($"Found {files.Length} files.");

            var names = _input.InputFields.Select(f => f.Name.ToLower()).ToArray();

            foreach (var file in files)
            {
                var row = _rowFactory.Create();
                for (var i = 0; i < _input.InputFields.Length; i++)
                {
                    var field = _input.InputFields[i];
                    switch (names[i])
                    {
                    case "creation":
                    case "created":
                    case "creationtime":
                    case "creationtimeutc":
                        row[field] = file.CreationTimeUtc;
                        break;

                    case "folder":
                    case "foldername":
                    case "directory":
                    case "directoryname":
                        row[field] = file.DirectoryName ?? string.Empty;
                        break;

                    case "ext":
                    case "extension":
                        row[field] = file.Extension;
                        break;

                    case "fullname":
                        row[field] = file.FullName;
                        break;

                    case "lastwrite":
                    case "updated":
                    case "lastwritetime":
                    case "lastwritetimeutc":
                        row[field] = file.LastWriteTimeUtc;
                        break;

                    case "lastaccess":
                    case "accessed":
                    case "lastaccesstime":
                    case "lastaccesstimeutc":
                        row[field] = file.LastAccessTimeUtc;
                        break;

                    case "size":
                    case "bytes":
                    case "length":
                        if (field.Type != "long" && field.Type != "int64")
                        {
                            _input.Warn($"The {names[i]} input is type long, but field is setup as {field.Type}.");
                        }
                        row[field] = file.Length;
                        break;

                    case "name":
                        row[field] = file.Name;
                        break;

                    case "readonly":
                    case "isreadonly":
                        row[field] = file.IsReadOnly;
                        break;
                    }
                }

                yield return(row);
            }
        }