public MapReduceProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
maxNullIndex = node.ReadInt("@max_null_index", -1);
fanOut = node.ReadInt("@fan_out", 100);
if (fanOut <= 0)
{
throw new BMNodeException(node, "Count should be > 0.");
}
if (node.ReadInt("write/@maxparallel", 1) > 0)
{
bufferSize = node.ReadInt("write/@buffer", 100);
}
readMaxParallel = node.ReadInt("read/@maxparallel", 1);
directory = node.ReadStr("dir/@name", null);
if (directory != null)
{
directory = engine.Xml.CombinePath(directory);
keepFiles = node.ReadBool("dir/@keepfiles", false);
compress = node.ReadBool("dir/@compress", true);
}
List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);
sorter = JComparer.Create(list);
XmlNode hashNode = node.SelectSingleNode("hasher");
if (hashNode == null)
{
hasher = sorter;
}
else
{
hasher = sorter.Clone(hashNode.ReadStr("@from_sort", null));
if (hasher == null)
{
list = KeyAndType.CreateKeyList(hashNode, "key", true);
hasher = JComparer.Create(list);
}
}
XmlNode undupNode = node.SelectSingleNode("undupper");
if (undupNode != null)
{
undupper = sorter.Clone(undupNode.ReadStr("@from_sort", null));
if (undupper == null)
{
list = KeyAndType.CreateKeyList(undupNode, "key", true);
undupper = JComparer.Create(list);
}
XmlNode actionsNode = undupNode.SelectSingleNode("actions");
if (actionsNode != null)
{
undupActions = new UndupActions(engine, this, actionsNode);
}
}
}
public TopProcessor(PipelineContext ctx, TopProcessor other, IDataEndpoint epOrnextProcessor)
: base(other, epOrnextProcessor)
{
sorter = other.sorter;
reverse = other.reverse;
topCount = other.topCount;
prique = new FixedPriorityQueue <JObject>(topCount, ComparisonWrappers.Create(sorter, reverse));
}
public TopProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", true);
sorter = JComparer.Create(list);
topCount = node.ReadInt("@count");
reverse = node.ReadBool("@reverse", false);
sortAfter = node.ReadEnum("@sortafter", _SortAfter.True);
}
public MemoryBasedMapperWriters(JComparer hasher, JComparer comparer, int cnt)
{
this.hasher = hasher;
this.comparer = comparer;
writers = new List <JObject> [cnt];
if (cnt != 1 && hasher == null)
{
throw new BMException("Cannot create MemoryBasedMapperWriters without a hasher if fanout={0}.", cnt);
}
createLists();
}
public SortedObjectEnumerator(StreamReader rdr, String filename, int index, JComparer sorter)
: base(rdr, filename, index)
{
this.sorter = sorter;
buffer = base.GetAll();
if (sorter != null)
{
buffer.Sort(sorter);
}
bufferLast = buffer.Count - 1;
bufferIndex = -1;
}
public SortProcessor(PipelineContext ctx, SortProcessor other, IDataEndpoint epOrnextProcessor)
: base(other, epOrnextProcessor)
{
Sorter = other.Sorter;
Undupper = other.Undupper;
if (other.undupActions != null)
{
undupActions = other.undupActions.Clone(ctx);
}
afterSort = other.afterSort;
beforeSort = other.beforeSort;
}
public FileBasedMapperWriters(JComparer hasher, JComparer comparer, String dir, String id, int cnt, bool compress, bool keepFiles = false)
{
this.hasher = hasher;
this.comparer = comparer;
this.compress = compress;
this.keepFiles = keepFiles;
fileNames = new String[cnt];
writers = new StreamWriter[cnt];
IOUtils.ForceDirectories(dir, false);
String part1 = Path.GetFullPath(dir + "\\" + id);
dir = IOUtils.DelSlash(part1);
for (int i = 0; i < cnt; i++)
{
fileNames[i] = String.Format("{0}_{1}.tmp", part1, i);
writers[i] = createWriter(fileNames[i], compress, keepFiles);
}
}
public SortProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("sorter"), "key", false);
Sorter = JComparer.Create(list);
//Interpret undupper
XmlNode undupNode = node.SelectSingleNode("undupper");
if (undupNode != null)
{
Undupper = Sorter.Clone(undupNode.ReadStr("@from_sort", null));
if (Undupper == null)
{
list = KeyAndType.CreateKeyList(undupNode, "key", true);
Undupper = JComparer.Create(list);
}
XmlNode actionsNode = undupNode.SelectSingleNode("actions");
if (actionsNode != null)
{
undupActions = new UndupActions(engine, this, actionsNode);
}
}
//Interpret sort scripts
beforeSort = new DelayedScriptHolder(engine, node.SelectSingleNode("beforesort"), Name);
if (beforeSort.ScriptName == null)
{
beforeSort = null;
}
afterSort = new DelayedScriptHolder(engine, node.SelectSingleNode("aftersort"), Name);
if (afterSort.ScriptName == null)
{
afterSort = null;
}
}
public MapReduceProcessor(PipelineContext ctx, MapReduceProcessor other, IDataEndpoint epOrnextProcessor)
: base(other, epOrnextProcessor)
{
directory = other.directory;
hasher = other.hasher;
sorter = other.sorter;
undupper = other.undupper;
keepFiles = other.keepFiles;
fanOut = other.fanOut;
compress = other.compress;
maxNullIndex = other.maxNullIndex;
bufferSize = other.bufferSize;
readMaxParallel = other.readMaxParallel;
if (other.undupActions != null)
{
undupActions = other.undupActions.Clone(ctx);
}
if (bufferSize > 0)
{
buffer = new List <JObject>(bufferSize);
asyncQ = AsyncRequestQueue.Create(1);
}
ctx.ImportLog.Log("Postprocessor [{0}]: mapping to {1}. Fan-out={2}.", Name, directory == null ? "<memory>" : directory, fanOut);
}
public KeysComparer(JComparer comparer)
{
this.comparer = comparer;
}
public UniqueProcessor(PipelineContext ctx, UniqueProcessor other, IDataEndpoint epOrnextProcessor)
: base(other, epOrnextProcessor)
{
undupper = other.undupper;
}
public UniqueProcessor(ImportEngine engine, XmlNode node) : base(engine, node)
{
List <KeyAndType> list = KeyAndType.CreateKeyList(node.SelectMandatoryNode("undupper"), "key", true);
undupper = JComparer.Create(list);
}