Exemplo n.º 1
0
        public MapReduceProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
        {
            maxNullIndex = node.ReadInt("@max_null_index", -1);
            fanOut       = node.ReadInt("@fan_out", 100);
            if (fanOut <= 0)
            {
                throw new BMNodeException(node, "Count should be > 0.");
            }

            if (node.ReadInt("write/@maxparallel", 1) > 0)
            {
                bufferSize = node.ReadInt("write/@buffer", 100);
            }
            readMaxParallel = node.ReadInt("read/@maxparallel", 1);

            directory = node.ReadStr("dir/@name", null);
            if (directory != null)
            {
                directory = engine.Xml.CombinePath(directory);
                keepFiles = node.ReadBool("dir/@keepfiles", false);
                compress  = node.ReadBool("dir/@compress", true);
            }
            List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);

            sorter = JComparer.Create(list);

            XmlNode hashNode = node.SelectSingleNode("hasher");

            if (hashNode == null)
            {
                hasher = sorter;
            }
            else
            {
                hasher = sorter.Clone(hashNode.ReadStr("@from_sort", null));
                if (hasher == null)
                {
                    list   = KeyAndType.CreateKeyList(hashNode, "key", true);
                    hasher = JComparer.Create(list);
                }
            }

            XmlNode undupNode = node.SelectSingleNode("undupper");

            if (undupNode != null)
            {
                undupper = sorter.Clone(undupNode.ReadStr("@from_sort", null));
                if (undupper == null)
                {
                    list     = KeyAndType.CreateKeyList(undupNode, "key", true);
                    undupper = JComparer.Create(list);
                }

                XmlNode actionsNode = undupNode.SelectSingleNode("actions");
                if (actionsNode != null)
                {
                    undupActions = new UndupActions(engine, this, actionsNode);
                }
            }
        }
Exemplo n.º 2
0
 protected UndupActions(PipelineContext ctx, UndupActions other)
 {
     actions = new List <IUndupAction>(other.actions.Count);
     foreach (var a in other.actions)
     {
         actions.Add(a.Clone(ctx));
     }
 }
Exemplo n.º 3
0
        public SortProcessor(PipelineContext ctx, SortProcessor other, IDataEndpoint epOrnextProcessor)
            : base(other, epOrnextProcessor)
        {
            Sorter   = other.Sorter;
            Undupper = other.Undupper;
            if (other.undupActions != null)
            {
                undupActions = other.undupActions.Clone(ctx);
            }

            afterSort  = other.afterSort;
            beforeSort = other.beforeSort;
        }
Exemplo n.º 4
0
        public SortProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
        {
            List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);

            Sorter = JComparer.Create(list);

            //Interpret undupper
            XmlNode undupNode = node.SelectSingleNode("undupper");

            if (undupNode != null)
            {
                Undupper = Sorter.Clone(undupNode.ReadStr("@from_sort", null));
                if (Undupper == null)
                {
                    list     = KeyAndType.CreateKeyList(undupNode, "key", true);
                    Undupper = JComparer.Create(list);
                }

                XmlNode actionsNode = undupNode.SelectSingleNode("actions");
                if (actionsNode != null)
                {
                    undupActions = new UndupActions(engine, this, actionsNode);
                }
            }

            //Interpret sort scripts
            beforeSort = new DelayedScriptHolder(engine, node.SelectSingleNode("beforesort"), Name);
            if (beforeSort.ScriptName == null)
            {
                beforeSort = null;
            }
            afterSort = new DelayedScriptHolder(engine, node.SelectSingleNode("aftersort"), Name);
            if (afterSort.ScriptName == null)
            {
                afterSort = null;
            }
        }
Exemplo n.º 5
0
 public MapReduceProcessor(PipelineContext ctx, MapReduceProcessor other, IDataEndpoint epOrnextProcessor)
     : base(other, epOrnextProcessor)
 {
     directory       = other.directory;
     hasher          = other.hasher;
     sorter          = other.sorter;
     undupper        = other.undupper;
     keepFiles       = other.keepFiles;
     fanOut          = other.fanOut;
     compress        = other.compress;
     maxNullIndex    = other.maxNullIndex;
     bufferSize      = other.bufferSize;
     readMaxParallel = other.readMaxParallel;
     if (other.undupActions != null)
     {
         undupActions = other.undupActions.Clone(ctx);
     }
     if (bufferSize > 0)
     {
         buffer = new List <JObject>(bufferSize);
         asyncQ = AsyncRequestQueue.Create(1);
     }
     ctx.ImportLog.Log("Postprocessor [{0}]: mapping to {1}. Fan-out={2}.", Name, directory == null ? "<memory>" : directory, fanOut);
 }