protected AbstractLuceneLoad(string folder)
        {
            _luceneWriter = new LuceneWriter(folder);

            _counters.Add("None", 0);
            _counters.Add("Create", 0);
            _counters.Add("Update", 0);

            Schema = new Dictionary<string, LuceneFieldSettings>();
        }
Example #2
0
        protected AbstractLuceneLoad(string folder)
        {
            _luceneWriter = new LuceneWriter(folder);

            _counters.Add("None", 0);
            _counters.Add("Create", 0);
            _counters.Add("Update", 0);

            Schema = new Dictionary <string, LuceneFieldSettings>();
        }
        protected override void Initialize()
        {
            var config = (SqloogleBotConfiguration)ConfigurationManager.GetSection("sqloogleBot");
            var miaFolder = Path.Combine(config.SearchIndexPath, "MIA");

            if (!Directory.Exists(miaFolder))
                Directory.CreateDirectory(miaFolder);

            using (var writer = new LuceneWriter(miaFolder)) {
                writer.Clean();
            }

            Register(new ParallelUnionAllOperation(config.ServerMiaCrawlProcesses()));
            Register(new MissingIndexTransform());
            RegisterLast(new LuceneLoad(miaFolder));
        }
Example #4
0
        protected override void Initialize()
        {
            var config    = (SqloogleBotConfiguration)ConfigurationManager.GetSection("sqloogleBot");
            var miaFolder = Path.Combine(config.SearchIndexPath, "MIA");

            if (!Directory.Exists(miaFolder))
            {
                Directory.CreateDirectory(miaFolder);
            }

            using (var writer = new LuceneWriter(miaFolder)) {
                writer.Clean();
            }

            Register(new ParallelUnionAllOperation(config.ServerMiaCrawlProcesses()));
            Register(new MissingIndexTransform());
            RegisterLast(new LuceneLoad(miaFolder));
        }
Example #5
0
        public override IEnumerable <Row> Execute(IEnumerable <Row> rows)
        {
            PrepareSchema();

            using (var writer = new LuceneWriter(_folder)) {
                foreach (var row in rows)
                {
                    if (row["action"] == null)
                    {
                        throw new InvalidOperationException("There is no action column.  A valid action is None, Create, or Update!");
                    }

                    var action = row["action"].ToString();
                    row.Remove("action");

                    _counters[action] += 1;

                    switch (action)
                    {
                    case "None":
                        break;

                    case "Create":
                        writer.Add(RowToDoc(row));
                        break;

                    case "Delete":
                        writer.Delete(row["id"].ToString());
                        break;

                    case "Update":
                        writer.Update(row["id"].ToString(), RowToDoc(row));
                        break;
                    }
                }

                writer.Commit();
                writer.Optimize();
            }

            Info("Lucene Create: {0}, Update: {1}, and None: {2}.", _counters["Create"], _counters["Update"], _counters["None"]);
            yield break;
        }