示例#1
0
        public MapReduceProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
        {
            maxNullIndex = node.ReadInt("@max_null_index", -1);
            fanOut       = node.ReadInt("@fan_out", 100);
            if (fanOut <= 0)
            {
                throw new BMNodeException(node, "Count should be > 0.");
            }

            if (node.ReadInt("write/@maxparallel", 1) > 0)
            {
                bufferSize = node.ReadInt("write/@buffer", 100);
            }
            readMaxParallel = node.ReadInt("read/@maxparallel", 1);

            directory = node.ReadStr("dir/@name", null);
            if (directory != null)
            {
                directory = engine.Xml.CombinePath(directory);
                keepFiles = node.ReadBool("dir/@keepfiles", false);
                compress  = node.ReadBool("dir/@compress", true);
            }
            List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);

            sorter = JComparer.Create(list);

            XmlNode hashNode = node.SelectSingleNode("hasher");

            if (hashNode == null)
            {
                hasher = sorter;
            }
            else
            {
                hasher = sorter.Clone(hashNode.ReadStr("@from_sort", null));
                if (hasher == null)
                {
                    list   = KeyAndType.CreateKeyList(hashNode, "key", true);
                    hasher = JComparer.Create(list);
                }
            }

            XmlNode undupNode = node.SelectSingleNode("undupper");

            if (undupNode != null)
            {
                undupper = sorter.Clone(undupNode.ReadStr("@from_sort", null));
                if (undupper == null)
                {
                    list     = KeyAndType.CreateKeyList(undupNode, "key", true);
                    undupper = JComparer.Create(list);
                }

                XmlNode actionsNode = undupNode.SelectSingleNode("actions");
                if (actionsNode != null)
                {
                    undupActions = new UndupActions(engine, this, actionsNode);
                }
            }
        }
示例#2
0
 public TopProcessor(PipelineContext ctx, TopProcessor other, IDataEndpoint epOrnextProcessor)
     : base(other, epOrnextProcessor)
 {
     sorter   = other.sorter;
     reverse  = other.reverse;
     topCount = other.topCount;
     prique   = new FixedPriorityQueue <JObject>(topCount, ComparisonWrappers.Create(sorter, reverse));
 }
示例#3
0
        public TopProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
        {
            List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", true);

            sorter    = JComparer.Create(list);
            topCount  = node.ReadInt("@count");
            reverse   = node.ReadBool("@reverse", false);
            sortAfter = node.ReadEnum("@sortafter", _SortAfter.True);
        }
示例#4
0
 public MemoryBasedMapperWriters(JComparer hasher, JComparer comparer, int cnt)
 {
     this.hasher   = hasher;
     this.comparer = comparer;
     writers       = new List <JObject> [cnt];
     if (cnt != 1 && hasher == null)
     {
         throw new BMException("Cannot create MemoryBasedMapperWriters without a hasher if fanout={0}.", cnt);
     }
     createLists();
 }
示例#5
0
 public SortedObjectEnumerator(StreamReader rdr, String filename, int index, JComparer sorter)
     : base(rdr, filename, index)
 {
     this.sorter = sorter;
     buffer      = base.GetAll();
     if (sorter != null)
     {
         buffer.Sort(sorter);
     }
     bufferLast  = buffer.Count - 1;
     bufferIndex = -1;
 }
示例#6
0
        public SortProcessor(PipelineContext ctx, SortProcessor other, IDataEndpoint epOrnextProcessor)
            : base(other, epOrnextProcessor)
        {
            Sorter   = other.Sorter;
            Undupper = other.Undupper;
            if (other.undupActions != null)
            {
                undupActions = other.undupActions.Clone(ctx);
            }

            afterSort  = other.afterSort;
            beforeSort = other.beforeSort;
        }
示例#7
0
        public FileBasedMapperWriters(JComparer hasher, JComparer comparer, String dir, String id, int cnt, bool compress, bool keepFiles = false)
        {
            this.hasher    = hasher;
            this.comparer  = comparer;
            this.compress  = compress;
            this.keepFiles = keepFiles;
            fileNames      = new String[cnt];
            writers        = new StreamWriter[cnt];

            IOUtils.ForceDirectories(dir, false);
            String part1 = Path.GetFullPath(dir + "\\" + id);

            dir = IOUtils.DelSlash(part1);
            for (int i = 0; i < cnt; i++)
            {
                fileNames[i] = String.Format("{0}_{1}.tmp", part1, i);
                writers[i]   = createWriter(fileNames[i], compress, keepFiles);
            }
        }
示例#8
0
        public SortProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
        {
            List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);

            Sorter = JComparer.Create(list);

            //Interpret undupper
            XmlNode undupNode = node.SelectSingleNode("undupper");

            if (undupNode != null)
            {
                Undupper = Sorter.Clone(undupNode.ReadStr("@from_sort", null));
                if (Undupper == null)
                {
                    list     = KeyAndType.CreateKeyList(undupNode, "key", true);
                    Undupper = JComparer.Create(list);
                }

                XmlNode actionsNode = undupNode.SelectSingleNode("actions");
                if (actionsNode != null)
                {
                    undupActions = new UndupActions(engine, this, actionsNode);
                }
            }

            //Interpret sort scripts
            beforeSort = new DelayedScriptHolder(engine, node.SelectSingleNode("beforesort"), Name);
            if (beforeSort.ScriptName == null)
            {
                beforeSort = null;
            }
            afterSort = new DelayedScriptHolder(engine, node.SelectSingleNode("aftersort"), Name);
            if (afterSort.ScriptName == null)
            {
                afterSort = null;
            }
        }
示例#9
0
 public MapReduceProcessor(PipelineContext ctx, MapReduceProcessor other, IDataEndpoint epOrnextProcessor)
     : base(other, epOrnextProcessor)
 {
     directory       = other.directory;
     hasher          = other.hasher;
     sorter          = other.sorter;
     undupper        = other.undupper;
     keepFiles       = other.keepFiles;
     fanOut          = other.fanOut;
     compress        = other.compress;
     maxNullIndex    = other.maxNullIndex;
     bufferSize      = other.bufferSize;
     readMaxParallel = other.readMaxParallel;
     if (other.undupActions != null)
     {
         undupActions = other.undupActions.Clone(ctx);
     }
     if (bufferSize > 0)
     {
         buffer = new List <JObject>(bufferSize);
         asyncQ = AsyncRequestQueue.Create(1);
     }
     ctx.ImportLog.Log("Postprocessor [{0}]: mapping to {1}. Fan-out={2}.", Name, directory == null ? "<memory>" : directory, fanOut);
 }
示例#10
0
 public KeysComparer(JComparer comparer)
 {
     this.comparer = comparer;
 }
示例#11
0
 public UniqueProcessor(PipelineContext ctx, UniqueProcessor other, IDataEndpoint epOrnextProcessor)
     : base(other, epOrnextProcessor)
 {
     undupper = other.undupper;
 }
示例#12
0
        public UniqueProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
        {
            List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("undupper"), "key", true);

            undupper = JComparer.Create(list);
        }