protected virtual void GetDGlobals() { foreach (SlaveInfo slave in dslaves) { slave.nstm.WriteByte((byte)'O'); int len = 0; buf = XContent.ReceiveXBytes(slave.nstm, out len, buf); DGlobalsM.FromBytes(buf, len, true); } }
public void LocalExec(string code, string[] usings, string methodname) { string suings = ""; if (usings != null) { foreach (string nm in usings) { suings = suings + "using " + nm + ";" + Environment.NewLine; } } RemoteExecFullSource( @"using System; using System.Collections.Generic; using System.Linq; using System.Text; using MySpace.DataMining.DistributedObjects; " + suings + @" namespace RemoteExec { public class UserRExec : MySpace.DataMining.DistributedObjects.IRemote { public int GetOutputFileCount(int n, IList<long> appendsizes) { return 0; } public void OnRemote() { StaticGlobals.DSpace_SlaveIP = DSpace_SlaveIP; StaticGlobals.DSpace_SlaveHost = DSpace_SlaveHost; StaticGlobals.DSpace_BlocksTotalCount = DSpace_BlocksTotalCount; StaticGlobals.DSpace_BlockID = DSpace_BlockID; StaticGlobals.ExecutionContext = ExecutionContextType.LOCAL; StaticGlobals.DSpace_Hosts = new string[]{" + ExpandListCode(StaticGlobals.DSpace_Hosts).Replace('`', '"') + @"}; StaticGlobals.DSpace_MaxDGlobals = " + StaticGlobals.DSpace_MaxDGlobals.ToString() + @"; " + DGlobalsM.ToCode() + @" " + methodname + @"(); } " + code + @" } }", "UserRExec"); GetDGlobals(); }
// Returns number of files written. // if outputbasefilesize>0 writes to multiple files (-1 for no max) // outputfilename contains %n to be replaced with current file number, starting at 0. // If multiple slaves, returns total files of all slaves. // inputdfsnodes must be accessible by the remote machine. Can be star-delimited failover names (starnames). // outputbasepath can contain %n for multiple outputs. // outputdfsdirs is a list of network directories of where to put the output files; each chunk is written to: outputdfsdirs[i % outputdfsdirs.Count] public void RemoteExec(IList <string> inputdfsnodes, IList <string> outputdfsdirs, IList <string> outputfilenames, long outputbasefilesize, string code, string[] usings, List <List <long> > outputsizeses, List <List <string> > outputdfsnodeses, List <string> inputdfsfilenames, List <int> inputdfsnodesoffsets) { if (outputbasefilesize <= 0) { outputbasefilesize = long.MaxValue; } StringBuilder infs = new StringBuilder(); { for (int i = 0; i < inputdfsnodes.Count; i++) { if (i != 0) { infs.Append(", "); if (0 == (i % 100)) { infs.Append(Environment.NewLine); } } infs.Append("@`"); infs.Append(inputdfsnodes[i]); infs.Append("`"); } } StringBuilder infilenames = new StringBuilder(); StringBuilder nodesoffsets = new StringBuilder(); StringBuilder sbinputrecordlengths = new StringBuilder(); if (inputdfsfilenames != null) { for (int oi = 0; oi < inputdfsfilenames.Count; oi++) { if (oi != 0) { infilenames.Append(", "); nodesoffsets.Append(", "); sbinputrecordlengths.Append(", "); if (0 == (oi % 100)) { infilenames.Append(Environment.NewLine); nodesoffsets.Append(Environment.NewLine); sbinputrecordlengths.Append(Environment.NewLine); } } infilenames.Append("@`"); infilenames.Append(inputdfsfilenames[oi]); infilenames.Append("`"); nodesoffsets.Append(inputdfsnodesoffsets[oi]); sbinputrecordlengths.Append(InputRecordLengths[oi]); } } StringBuilder outds = new StringBuilder(); if (null != outputdfsdirs) { for (int i = 0; i < outputdfsdirs.Count; i++) { if (i != 0) { outds.Append(", "); if (0 == (i % 100)) { outds.Append(Environment.NewLine); } } outds.Append("@`"); outds.Append(outputdfsdirs[i]); outds.Append("`"); } } StringBuilder outfs = new StringBuilder(); if (null != outputfilenames) { for (int i = 0; i < outputfilenames.Count; i++) { if (i != 0) { outfs.Append(", "); if (0 == (i % 100)) { outfs.Append(Environment.NewLine); } } outfs.Append("@`"); outfs.Append(outputfilenames[i]); outfs.Append("`"); } } StringBuilder outputreclens = new StringBuilder(); for (int i = 0; i < OutputRecordLengths.Count; i++) { if (0 != i) { outputreclens.Append(","); } outputreclens.Append(OutputRecordLengths[i]); } string susings = ""; if (usings != null) { foreach (string nm in usings) { susings += "using " + nm + ";" + Environment.NewLine; } } #if DEBUG if (inputdfsfilenames != null) { //System.Diagnostics.Debugger.Launch(); } #endif RemoteExecFullSource( (@"using System; using System.Collections.Generic; using System.Linq; using System.Text; using MySpace.DataMining.DistributedObjects; " + susings + @" namespace RemoteExec { public class RemoteInputStream : System.IO.Stream, MySpace.DataMining.CollaborativeFilteringObjects3.ISequenceInput { string[] infs = new string [] { " + infs.ToString() + @" }; string[] infilenames = new string [] { " + infilenames.ToString() + @" }; int[] nodesoffsets = new int [] { " + nodesoffsets.ToString() + @" }; int[] inputrecordlengths = new int [] { " + sbinputrecordlengths.ToString() + @" }; const int _CookTimeout = " + CookTimeout.ToString() + @"; const int _CookRetries = " + CookRetries.ToString() + @"; int InputRecordLength = " + InputRecordLength.ToString() + @"; public RemoteInputStream(UserRExec r) { this.r = r; StaticGlobals.DSpace_InputRecordLength = " + InputRecordLength.ToString() + @"; if(infilenames.Length > 0) { StaticGlobals.DSpace_InputFileName = infilenames[0]; } if(0 == infs.Length) { whichseqreader = 0; } } public IList<string> GetSequenceInputs() { return infs; } public void CreateSequenceFile(string name) { using(System.IO.StreamWriter sf = System.IO.File.CreateText(name)) { sf.WriteLine(`*sequence*`); for(int i = 0; i < infs.Length; i++) { int istar = infs[i].IndexOf('*'); if(-1 != istar) { sf.WriteLine(infs[i].Substring(0, istar)); } else { sf.WriteLine(infs[i]); } } } } public bool EndOfStream { get { return seqend; } } public bool ReadRecordAppend(List<byte> record) { if(EndOfStream) { return false; } if(InputRecordLength < 1 && InputRecordLength != -2) { throw new Exception(`Cannot read records; not in binary mode`); } int remain = 0; if(InputRecordLength != -2) { remain = InputRecordLength; } else { remain = Entry.BytesToRecordLength(this); if(remain == 0) { return false; } } while(remain > 0) { int ib = this.ReadByte(); if(-1 == ib) { return false; } record.Add((byte)ib); remain--; } return true; } public bool ReadRecordAppend(System.IO.Stream record) { if(EndOfStream) { return false; } if(InputRecordLength < 1 && InputRecordLength != -2) { throw new Exception(`Cannot read records; not in record rectangular binary mode`); } int remain = 0; if(InputRecordLength != -2) { remain = InputRecordLength; } else { remain = Entry.BytesToRecordLength(this); if(remain == 0) { return false; } } while(remain > 0) { int ib = this.ReadByte(); if(-1 == ib) { return false; } record.WriteByte((byte)ib); remain--; } return true; } public bool ReadLineAppend(List<byte> line) { if(EndOfStream) { return false; } if(InputRecordLength > 0) { throw new Exception(`Cannot read-line in fixed-length record rectangular binary mode`); } bool readany = false; for(;;) { int ib = this.ReadByte(); if(-1 == ib) { return readany; } readany = true; if((int)'\n' == ib) { break; } if((int)'\r' != ib) { line.Add((byte)ib); } } return true; } public bool ReadBinary(ref Blob blob) { StringBuilder sb = new StringBuilder(); bool readany = ReadLineAppend(sb); if(!readany) { return false; } blob = Blob.FromDfsLine(sb.ToString()); return true; } public bool ReadLineAppend(StringBuilder line) { if(EndOfStream) { return false; } if(InputRecordLength > 0) { throw new Exception(`Cannot read-line in fixed-length record rectangular binary mode`); } List<byte> buf = new List<byte>(); bool readany = false; for(;;) { int ib = this.ReadByte(); if(-1 == ib) { break; } readany = true; if((int)'\n' == ib) { break; } if((int)'\r' != ib) { buf.Add((byte)ib); } } if(buf.Count > 0) { line.Append(System.Text.Encoding.UTF8.GetString(buf.ToArray())); } return readany; } public bool ReadLineAppend(System.IO.Stream line) { if(EndOfStream) { return false; } if(InputRecordLength > 0) { throw new Exception(`Cannot read-line in fixed-length record rectangular binary mode`); } bool readany = false; for(;;) { int ib = this.ReadByte(); if(-1 == ib) { return readany; } readany = true; if((int)'\n' == ib) { break; } if((int)'\r' != ib) { line.WriteByte((byte)ib); } } return true; } public override bool CanRead { get { return true; } } public override bool CanSeek { get { return false; } } public override bool CanWrite { get { return false; } } public override void Flush() { } public override long Length { get { throw new NotImplementedException(); } } public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } } // From 0 to NumberOfParts-1 public long CurrentPart { get { if(whichseqreader < 0) { return 0; } if(whichseqreader >= infs.Length) { return infs.Length - 1; } return whichseqreader; } } // From 0 to NumberOfParts-1 // partposition is the position within the part, defaults to 0. // partposition does not include header. // Note: partposition can seek to the middle of a record or line. public void SeekToPart(long partindex, long partposition) { if(partindex >= infs.Length || partindex < 0) { throw new IndexOutOfRangeException(`Part index out of range`); } _seekseq((int)partindex); if(0 != partposition) { curpartpos = curseqfile.Seek((long)curpartheadersize + partposition, System.IO.SeekOrigin.Begin); } } public void SeekToPart(long partindex) { SeekToPart(partindex, 0); } public long NumberOfParts { get { return infs.Length; } } // Does not include header. public long CurrentPartPosition { get { if(whichseqreader < 0) { return 0; } if(whichseqreader >= infs.Length) { return curpartfulllength - curpartheadersize; } //return curseqfile.Position - curpartheadersize; #if DEBUGmore if(curseqfile.Position != curpartpos) { throw new Exception(`DEBUG: CurrentPartPosition: (curseqfile.Position != curpartpos)`); } #endif return curpartpos - curpartheadersize; } } // Does not include header. public long CurrentPartLength { get { return curpartfulllength - curpartheadersize; } } public override int Read(byte[] array, int offset, int count) { int rd = 0; try { if(-1 == whichseqreader) { _seqinit(); } while (rd < count) { if (seqend) { /*if (0 == rd) { throw new EndOfStreamException(`End of file; end of all files in sequences has been reached`); }*/ break; } int onerd = curseqfile.Read(array, offset + rd, count - rd); curpartpos += onerd; rd += onerd; if (0 != rd) { prevbyte = array[offset + rd - 1]; } if (rd < count) { _movenextseq(); /*if (lmode) { if ('\n' != prevbyte) { prevbyte = (byte)'\n'; array[offset + rd] = (byte)'\n'; rd++; } }*/ continue; } break; } } catch { // Suppress error message. To-do: fix. } #if DEBUGmore if(null != curseqfile) { if(curpartpos != curseqfile.Position) { throw new Exception(`DEBUG: Read: (curpartpos{` + curpartpos + `} != curseqfile.Position{` + curseqfile.Position + `})`); } if(curpartfulllength != curseqfile.Length) { throw new Exception(`DEBUG: Read: (curpartfulllength != curseqfile.Length)`); } } if(curpartpos > curpartfulllength || curpartpos < 0) { throw new Exception(`DEBUG: Read: (curpartpos > curpartfulllength || curpartpos < 0)`); } #endif if(curpartpos == curpartfulllength) { // Skip to the next file right now to load info about it (record length) _movenextseq(); } return rd; } byte[] _rb = null; public override int ReadByte() { if(null == _rb) { _rb = new byte[1]; } if(1 != this.Read(_rb, 0, 1)) { return -1; } return _rb[0]; } public override long Seek(long offset, System.IO.SeekOrigin origin) { throw new NotImplementedException(); } public override void SetLength(long value) { throw new NotImplementedException(); } public override void Write(byte[] buffer, int offset, int count) { throw new NotImplementedException(); } UserRExec r; System.IO.Stream curseqfile = null; string curfilename; int whichseqreader = -1; byte prevbyte = 0; int bufsz = " + FILE_BUFFER_SIZE.ToString() + @"; // Buffer size. static byte compressenumout = " + CompressFileOutput.ToString() + @"; int curpartheadersize = 0; long curpartfulllength = 0; // Includes header size. long curpartpos = 0; // Includes header size. void _seqinit() { if (0 != infs.Length) { whichseqreader = 0; curfilename = infs[0]; _setinputfilename(0); curseqfile = GetStreamFromSeqFileName(curfilename, true); } } void _skipseq() { if (null != curseqfile) { curseqfile = null; if (whichseqreader < infs.Length) { whichseqreader++; if(whichseqreader < infs.Length) { curfilename = infs[whichseqreader]; _setinputfilename(whichseqreader); curseqfile = GetStreamFromSeqFileName(infs[whichseqreader], true); } //return true; } //return false; } } // Note: this can be called before _seqinit. void _seekseq(int seektopart) { if (null != curseqfile) { curseqfile.Close(); curseqfile = null; } whichseqreader = seektopart; curfilename = infs[whichseqreader]; _fixinputfilename(whichseqreader); curseqfile = GetStreamFromSeqFileName(infs[whichseqreader], true); prevbyte = 0; } int curoffset = 0; int curfi = 0; void _setinputfilename(int offset) { if(offset == curoffset) { StaticGlobals.DSpace_InputFileName = infilenames[curfi]; InputRecordLength = inputrecordlengths[curfi]; StaticGlobals.DSpace_InputRecordLength = InputRecordLength; curfi++; if(curfi < nodesoffsets.Length) { curoffset = nodesoffsets[curfi]; } } } void _fixinputfilename(int offset) { /*if(offset == 0) { curoffset = 0; curfi = 0; _setinputfilename(0); return; }*/ for(curfi = 0; curfi < nodesoffsets.Length; curfi++) { curoffset = nodesoffsets[curfi]; if(nodesoffsets[curfi] > offset) { break; } StaticGlobals.DSpace_InputFileName = infilenames[curfi]; InputRecordLength = inputrecordlengths[curfi]; StaticGlobals.DSpace_InputRecordLength = InputRecordLength; } } void _movenextseq() { if (null != curseqfile) { curseqfile.Close(); _skipseq(); } } protected System.IO.Stream GetStreamFromSeqFileName(string fn, bool skipdfschunkheader) { System.IO.Stream stm; bool anydata = false; if (bufsz > 0) { //stm = new System.IO.FileStream(fn, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, bufsz); stm = new MySpace.DataMining.AELight.DfsFileNodeStream(fn, true, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, bufsz); } else { stm = new MySpace.DataMining.AELight.DfsFileNodeStream(fn, true, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, " + FILE_BUFFER_SIZE.ToString() + @"); } anydata = 0 != stm.Length; if(1 == compressenumout) { stm = new System.IO.Compression.GZipStream(stm, System.IO.Compression.CompressionMode.Decompress); } curpartheadersize = 0; curpartpos = 0; if(skipdfschunkheader && anydata) { byte[] buf = new byte[32]; _StreamReadExact(stm, buf, 4); int hlen = Entry.BytesToInt(buf); if(hlen > 4) { int hremain = hlen - 4; if(hremain > buf.Length) { buf = new byte[hremain]; } _StreamReadExact(stm, buf, hremain); } curpartheadersize = hlen; } curpartfulllength = stm.Length; curpartpos = curpartheadersize; return stm; } static void _StreamReadExact(System.IO.Stream stm, byte[] buf, int len) { int sofar = 0; while (sofar < len) { int xread = stm.Read(buf, sofar, len - sofar); if (xread <= 0) { throw new System.IO.IOException(`Unable to read from stream`); } sofar += xread; } } bool seqend { get { return whichseqreader >= infs.Length; } } } public class RemoteOutputStream : System.IO.Stream { public RemoteOutputStream(UserRExec r) { this.r = r; StaticGlobals.DSpace_OutputRecordLength = " + OutputRecordLength.ToString() + @"; } public RemoteOutputStream(UserRExec r, string outputfilename, int outputrecordlength): this(r) { this._basefilename = outputfilename; this.OutputRecordLength = outputrecordlength; this._WriteSamples = outputrecordlength < 1; } UserRExec r; const int _CookTimeout = " + CookTimeout.ToString() + @"; const int _CookRetries = " + CookRetries.ToString() + @"; int OutputRecordLength = -1; bool _WriteSamples = false; public void WriteRecord(IList<byte> record) { if(OutputRecordLength == -1) { throw new Exception(`Cannot write records; not in binary mode`); } int recordCount = record.Count; if(OutputRecordLength != -2) { if(recordCount != OutputRecordLength) // && OutputRecordLength > 0) { throw new Exception(`Record length mismatch; got length of ` + recordCount.ToString() + ` when expecting length of ` + OutputRecordLength.ToString()); } } Begin(); if(OutputRecordLength == -2) { WriteRecordLength(recordCount); } for(int i = 0; i < recordCount; i++) { this.WriteByte(record[i]); } End(); } void WriteRecordLength(int reclen) { Entry.RecordLengthToBytes(reclen, this); } public void WriteLine(IList<byte> line) { if(OutputRecordLength > 0) { throw new Exception(`Cannot write-line in fixed-length record rectangular binary mode`); } int lc = line.Count; for(int i = 0; i < lc; i++) { this.WriteByte(line[i]); } string nl = Environment.NewLine; for(int i = 0; i < nl.Length; i++) { this.WriteByte((byte)nl[i]); } } public void WriteLine(StringBuilder line) { if(OutputRecordLength > 0) { throw new Exception(`Cannot write-line in fixed-length record rectangular binary mode`); } WriteLine(line.ToString()); } public void WriteLine(string line) { if(OutputRecordLength > 0) { throw new Exception(`Cannot write-line in fixed-length record rectangular binary mode`); } byte[] buf = System.Text.Encoding.UTF8.GetBytes(line); foreach(byte b in buf) { this.WriteByte(b); } string nl = Environment.NewLine; for(int i = 0; i < nl.Length; i++) { this.WriteByte((byte)nl[i]); } } public void WriteBinary(Blob blob) { WriteLine(blob.ToDfsLine()); } public override bool CanRead { get { return false; } } public override bool CanSeek { get { return false; } } public override bool CanWrite { get { return true; } } public override void Flush() { if(null != _outstm) { _outstm.Flush(); if(_WriteSamples) { _outsamps.Flush(); } } } public override long Length { get { throw new NotImplementedException(); } } public override long Position { get { throw new NotImplementedException(); } set { throw new NotImplementedException(); } } public override int Read(byte[] buffer, int offset, int count) { throw new NotImplementedException(); } public override long Seek(long offset, System.IO.SeekOrigin origin) { throw new NotImplementedException(); } public override void SetLength(long value) { throw new NotImplementedException(); } void _checkstream() { if(-1 == _filenum || _cursize >= _basefilesize) { if(_samechunk == 0 || -1 == _filenum) { _nextfile(); } } } public void Begin() { _samechunk++; } public void End() { if(0 == _samechunk) { throw new Exception(`RemoteOutputStream: mismatched Begin() and End()`); } _samechunk--; if(0 == _samechunk) { _checkstream(); } } byte[] chunkline = new byte[0x400 * 0x40]; int chunklinelen = 0; void _WriteToChunk(byte[] buffer, int offset, int count) { _outstm.Write(buffer, offset, count); _cursize += count; _totalsize += count; for(int i = 0; i < count; i++) { byte b = buffer[offset + i]; if(chunklinelen < chunkline.Length) { chunkline[chunklinelen++] = b; } if('\n' == b) { if(_WriteSamples) { if(0 != chunklinelen) { if(_cursize >= _nextsamplepos) { _outsamps.Write(chunkline, 0, chunklinelen); _nextsamplepos += DfsSampleDistance; } chunklinelen = 0; } } } } } // Begin() forces writes to be in the same chunk. public override void Write(byte[] buffer, int offset, int count) { another_write: //if(-1 == _filenum || _donextfile) if(_donextfile) { _nextfile(); } if(0 == _samechunk) { int xc = 0; for(; xc < count; xc++) { if((byte)'\n' == buffer[offset + xc]) { if(xc + 1 + _cursize >= _basefilesize) { xc++; // Include the \n. _WriteToChunk(buffer, offset, xc); _donextfile = true; if(xc >= count) { return; } offset += xc; count -= xc; goto another_write; } } } } _WriteToChunk(buffer, offset, count); } byte[] _wb = null; public override void WriteByte(byte x) { if(null == _wb) { _wb = new byte[1]; } _wb[0] = x; Write(_wb, 0, 1); } private void WriteByteRecord(byte x) { if(null == _wb) { _wb = new byte[1]; } _wb[0] = x; _WriteToChunk(_wb, 0, 1); } internal void _done() { if(null != _outstm) { _outstm.Close(); //_outstm = null; _filesizes.Add(HEADERSIZE + _cursize); if(_WriteSamples) { _outsamps.Close(); } } } internal int _getoutputfilecount(IList<long> appendsizes) { if(null != appendsizes) { for(int i = 0; i < _filenum + 1; i++) { string fn = _basefilename.Replace(`%n`, i.ToString()); long x = _filesizes[i]; if(x >= HEADERSIZE) { x -= HEADERSIZE; } appendsizes.Add(x); } } return _filenum + 1; } void _nextfile() { _filenum++; string fn = _basefilename.Replace(`%n`, _filenum.ToString()); if(null != _outstm) { _filesizes.Add(HEADERSIZE + _cursize); _outstm.Close(); if(_WriteSamples) { _outsamps.Close(); } } if(0 == outds.Length) { throw new Exception(`RemoteOutputStream error: outputdfsdirs is empty`); } string fp = outds[(OutputStartingPoint + _filenum) % outds.Length] + @`\` + fn; _outstm = new System.IO.FileStream(fp, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Read, " + FILE_BUFFER_SIZE.ToString() + @"); if(_WriteSamples) { _outsamps = new System.IO.FileStream(fp + `.zsa`, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.Read, " + FILE_BUFFER_SIZE.ToString() + @"); _nextsamplepos = 0; } if(1 == compressenumout) { _outstm = new System.IO.Compression.GZipStream(_outstm, System.IO.Compression.CompressionMode.Compress); } // Write header... Entry.ToBytes(HEADERSIZE, _smallbuf, 0); _bytebuf.Clear(); Entry.ToBytesAppend64(_totalsize, _bytebuf); _smallbuf[4] = _bytebuf[0]; _smallbuf[5] = _bytebuf[1]; _smallbuf[6] = _bytebuf[2]; _smallbuf[7] = _bytebuf[3]; _smallbuf[8] = _bytebuf[4]; _smallbuf[9] = _bytebuf[5]; _smallbuf[10] = _bytebuf[6]; _smallbuf[11] = _bytebuf[7]; _outstm.Write(_smallbuf, 0, HEADERSIZE); _donextfile = false; _cursize = 0; } public RemoteOutputStream GetOutputByIndex(int index) { if(parentlist == null) { throw new Exception(`RemoteOutputStream.GetOutputByIndex parentlist is null.`); } if(index < 0 || index > parentlist.Length - 1) { throw new Exception(`RemoteOutputStream.GetOutputByIndex index is out of range.`); } return parentlist[index]; } internal string _basefilename; internal RemoteOutputStream[] parentlist; const long _basefilesize = " + outputbasefilesize.ToString() + @"; const int OutputStartingPoint = " + OutputStartingPoint.ToString() + @"; static byte compressenumout = " + CompressFileOutput.ToString() + @"; int _filenum = -1; System.IO.Stream _outstm = null; System.IO.Stream _outsamps = null; long _cursize = 0; long _totalsize = 0; byte[] _smallbuf = new byte[4 + 8]; List<byte> _bytebuf = new List<byte>(); int _samechunk = 0; bool _donextfile = true; // Start! List<long> _filesizes = new List<long>(); string[] outds = new string [] { " + outds.ToString() + @" }; const int HEADERSIZE = 4 + 8; long _nextsamplepos = -1; const long DfsSampleDistance = " + DfsSampleDistance.ToString() + @"; } public class UserRExec : MySpace.DataMining.DistributedObjects.IRemote { RemoteInputStream rinput; RemoteOutputStream routput; RemoteOutputStream[] routputs; public int GetOutputFileCount(int n, IList<long> appendsizes) { return routputs[n]._getoutputfilecount(appendsizes); } public void OnRemote() { rinput = new RemoteInputStream(this); string[] outputfilenames = new string[]{" + outfs.ToString() + @"}; int[] outputreclens = new int[] {" + outputreclens.ToString() + @"}; routputs = new RemoteOutputStream[outputfilenames.Length]; for(int oi = 0; oi < outputfilenames.Length; oi++) { routputs[oi] = new RemoteOutputStream(this, outputfilenames[oi], outputreclens[oi]); routputs[oi].parentlist = routputs; } routput = routputs[0]; StaticGlobals.DSpace_SlaveIP = DSpace_SlaveIP; StaticGlobals.DSpace_SlaveHost = DSpace_SlaveHost; StaticGlobals.DSpace_BlocksTotalCount = DSpace_BlocksTotalCount; StaticGlobals.DSpace_BlockID = DSpace_BlockID; StaticGlobals.ExecutionContext = ExecutionContextType.REMOTE; StaticGlobals.DSpace_Hosts = new string[]{" + ExpandListCode(StaticGlobals.DSpace_Hosts) + @"}; StaticGlobals.DSpace_OutputDirection = `" + StaticGlobals.DSpace_OutputDirection + @"`; StaticGlobals.DSpace_OutputDirection_ascending = " + (StaticGlobals.DSpace_OutputDirection_ascending ? "true" : "false") + @"; StaticGlobals.DSpace_MaxDGlobals = " + StaticGlobals.DSpace_MaxDGlobals.ToString() + @"; ").Replace('`', '"') + DGlobalsM.ToCode() + @" Remote(rinput, routput); for(int oi = 0; oi < routputs.Length; oi++) { routputs[oi]._done(); } } " + code + (@" } }"), "UserRExec"); GetRemoteOutputFilesCreated(outputfilenames, outputsizeses, outputdfsnodeses); }
protected override void ProcessCommand(System.Net.Sockets.NetworkStream nstm, char tag) { int len; switch (tag) { case 'R': // Remote! { string classname = XContent.ReceiveXString(nstm, buf); string xlibfn = CreateXlibFileName("remote"); { buf = XContent.ReceiveXBytes(nstm, out len, buf); if (0 != len) { System.IO.FileStream stm = System.IO.File.Create(xlibfn); stm.Write(buf, 0, len); stm.Close(); } } string dllfn = CreateDllFileName("remote"); { buf = XContent.ReceiveXBytes(nstm, out len, buf); System.IO.FileStream stm = System.IO.File.Create(dllfn); stm.Write(buf, 0, len); stm.Close(); } if (XLog.logging) { string xclassname = classname; if (null == xclassname) { xclassname = "<null>"; } XLog.log("Loading IRemote plugin named " + xclassname + " for remote: " + dllfn); } rem = LoadRemotePlugin(dllfn, classname); #if DEBUG try { rem.OnRemote(); } catch (Exception e) { throw new UserException(e); } #else rem.OnRemote(); #endif } break; case 'O': //Query DGlobals { int byteCount = DGlobalsM.ToBytes(ref buf); XContent.SendXContent(nstm, buf, byteCount); } break; case 'r': { buf = XContent.ReceiveXBytes(nstm, out len, buf); int n = Entry.BytesToInt(buf); int count = 0; if (null != rem) { List <long> appendsizes = new List <long>(); try { count = rem.GetOutputFileCount(n, appendsizes); } catch (Exception e) { throw new DistributedObjectsSlave.DistObjectAbortException(e); } if (buf.Length < 4 + 8 * appendsizes.Count) { buf = new byte[Entry.Round2Power(4 + 8 * appendsizes.Count)]; } Entry.ToBytes(count, buf, 0); int offset = 4; for (int i = 0; i < appendsizes.Count; i++, offset += 8) { Entry.LongToBytes(appendsizes[i], buf, offset); } XContent.SendXContent(nstm, buf, 4 + 8 * appendsizes.Count); break; // ! } Entry.ToBytes(count, buf, 0); XContent.SendXContent(nstm, buf, 4); } break; default: base.ProcessCommand(nstm, tag); break; } }