/// <summary> /// Decode a Qap1-encoded Sexp /// </summary> /// <param name="data">The byte stream in which the Sexp is encoded</param> /// <param name="start">At which index of data does the Sexp begin?</param> /// <returns>The decoded Sexp.</returns> private static Sexp DecodeSexp(byte[] data, ref long start) { // pull sexp type byte xt = data[start]; // calculate length of payload var lengthBuf = new byte[8]; Array.Copy(data, start + 1, lengthBuf, 0, 3); start += 4; if ((xt & XtLarge) == XtLarge) { Array.Copy(data, start, lengthBuf, 3, 4); start += 4; xt -= XtLarge; } var length = ( long )BitConverter.ToUInt64(lengthBuf, 0); // has attributes? process first SexpTaggedList attrs = null; if ((xt & XtHasAttr) == XtHasAttr) { xt -= XtHasAttr; long oldstart = start; attrs = ( SexpTaggedList )DecodeSexp(data, ref start); length -= start - oldstart; } long end = start + length; Sexp result; switch (xt) { case XtNull: { if (length != 0) { throw new RserveException("Attempting to decode an SexpNull, but it is followed by data when it shouldn't be."); } result = new SexpNull(); } break; case XtSymName: { // keep all characters up to the first null var symnNamBuf = new byte[length]; Array.Copy(data, start, symnNamBuf, 0, length); string res = Encoding.UTF8.GetString(symnNamBuf); result = new SexpSymname(res.Split('\x00')[0]); } break; case XtArrayInt: { var res = new int[length / 4]; var intBuf = new byte[4]; for (long i = 0; i < length; i += 4) { Array.Copy(data, start + i, intBuf, 0, 4); res[i / 4] = BitConverter.ToInt32(intBuf, 0); } // is date or just an integer? if ((attrs != null) && (attrs.ContainsKey("class") && attrs["class"].AsStrings.Contains("Date"))) { result = new SexpArrayDate(res); } else { result = new SexpArrayInt(res); } } break; case XtArrayBool: { if (length < 4) { throw new RserveException("Decoding an SexpArrayBool where data doesn't seem to contain a data length field."); } var boolLengthBuf = new byte[4]; Array.Copy(data, start, boolLengthBuf, 0, 4); var datalength = BitConverter.ToInt32(boolLengthBuf, 0); if (datalength > length - 4) { throw new RserveException("Decoding an SexpArrayBool where transmitted data field too short for number of entries."); } var res = new bool?[datalength]; for (int i = 0; i < datalength; i++) { // R logical is false if 0, true if 1, and NA if 2 switch (data[start + i + 4]) { case 0: res[i] = false; break; case 1: res[i] = true; break; case 2: res[i] = null; break; default: throw new RserveException("Decoding an SexpArrayBool and found an element in the array that is not an R bool: " + data[start + i + 4]); } } result = new SexpArrayBool(res); } break; case XtArrayDouble: { var res = new double[length / 8]; var doubleBuf = new byte[8]; for (long i = 0; i < length; i += 8) { Array.Copy(data, start + i, doubleBuf, 0, 8); res[i / 8] = BitConverter.ToDouble(doubleBuf, 0); } // is date or just a double? if ((attrs != null) && (attrs.ContainsKey("class") && attrs["class"].AsStrings.Contains("Date"))) { result = new SexpArrayDate(res.Select(Convert.ToInt32)); } else { result = new SexpArrayDouble(res); } } break; case XtArrayString: { var res = new List <string>(); long i = 0; for (long j = 0; j < length; j++) { if (data[start + j] != 0) { continue; } if ((j == i + 1) && (data[start + i] == 255)) { res.Add(null); } else { if (data[start + i] == 255) { i++; } var stringBuf = new byte[j - i]; Array.Copy(data, start + i, stringBuf, 0, j - i); res.Add(Encoding.UTF8.GetString(stringBuf)); } i = j + 1; } result = new SexpArrayString(res); } break; case XtListNoTag: case XtLangNoTag: case XtVector: result = new SexpList(); while (start < end) { result.Add(DecodeSexp(data, ref start)); } break; case XtLangTag: case XtListTag: result = new SexpTaggedList(); while (start < end) { Sexp val = DecodeSexp(data, ref start); Sexp key = DecodeSexp(data, ref start); result.Add(key.IsNull ? String.Empty : key.AsString, val); } break; case XtRaw: { var d = new byte[length]; Array.Copy(data, start, d, 0, length); result = new SexpQap1Raw(xt, d); } break; default: throw new RserveException("Cannot decode an Sexp because the type is not recognized: " + xt); } if (start > end) { throw new RserveException("When decoding an Sexp, more data consumed than provided."); } start = end; if (attrs != null) { foreach (var a in attrs.AsSexpDictionary) { result.Attributes.Add(a.Key, a.Value); } } return(result); }
/// <summary> /// Encode a Sexp in Qap1 format /// </summary> /// <param name="s">The Sexp to be encoded</param> /// <returns>QAP4-encoded bit stream</returns> private static IEnumerable<byte> EncodeSexp( Sexp s ) { var t = s.GetType(); var res = new List<byte>(); byte xt; SexpTaggedList attrs = null; if ( s.Attributes.Count > 0 ) { attrs = new SexpTaggedList(); foreach ( var a in s.Attributes ) { attrs.Add( a.Key , a.Value ); } res.AddRange( EncodeSexp( attrs ) ); } if ( t == typeof( SexpNull ) ) { xt = XtNull; } else if ( t == typeof( SexpArrayDouble ) ) { xt = XtArrayDouble; var v = ( ( SexpArrayDouble )s ).Value; foreach ( var t1 in v ) { res.AddRange( BitConverter.GetBytes( t1 ) ); } } else if ( t == typeof( SexpArrayInt ) ) { xt = XtArrayInt; var v = ( ( SexpArrayInt )s ).Value; foreach ( var t1 in v ) { res.AddRange( BitConverter.GetBytes( t1 ) ); } } else if ( t == typeof( SexpArrayDate ) ) { xt = XtArrayInt; var v = ( ( SexpArrayInt )s ).Value; foreach ( var t1 in v ) { res.AddRange( BitConverter.GetBytes( t1 ) ); } } else if ( t == typeof( SexpArrayBool ) ) { xt = XtArrayBool; var v = ( SexpArrayBool )s; res.AddRange( BitConverter.GetBytes( v.Count ) ); // R logical is false if 0, true if 1, and NA if 2 res.AddRange( v.Cast<SexpArrayBool>().Select( x => x.AsByte ) ); // protocol requires us to pad with null while ( res.Count % 4 != 0 ) { res.Add( 0 ); } } else if ( t == typeof( SexpTaggedList ) ) { xt = XtListTag; var v = ( SexpTaggedList )s; foreach ( var a in v.AsSexpDictionary ) { res.AddRange( EncodeSexp( a.Value ) ); res.AddRange( EncodeSexp( new SexpSymname( a.Key ) ) ); } } else if ( t == typeof( SexpList ) ) { xt = XtVector; var v = ( ( SexpList )s ).Value; foreach ( var a in v ) { res.AddRange( EncodeSexp( a ) ); } } else if ( t == typeof( SexpArrayString ) ) { xt = XtArrayString; var v = ( ( SexpArrayString )s ).Value; foreach ( var a in v ) { // Rserve represents NA strings using 0xff (255). if ( a == null ) { res.Add( 255 ); } else { var b = Encoding.UTF8.GetBytes( a ); // If 0xff occurs in the beginning of a string it should be doubled to avoid misrepresentation. if ( ( b.Length > 0 ) && ( b[ 0 ] == 255 ) ) { res.Add( 255 ); } res.AddRange( b ); } res.Add( 0 ); } } else if ( t == typeof( SexpSymname ) ) { xt = XtSymName; var v = ( ( SexpSymname )s ).Value; var b = Encoding.UTF8.GetBytes( v ); res.AddRange( b ); res.Add( 0 ); } else { throw new RserveException( "Cannot encode an unknown Sexp type " + t.GetType().Name ); } if ( attrs != null ) { xt |= XtHasAttr; } // get payload length long len = res.LongCount(); byte[] lenBytes = BitConverter.GetBytes( len ); // populate header (first four bytes) IEnumerable<byte> header = lenBytes.Take( 3 ); // a large dataset is > 16MB, it requires the XtLarge flag and an extra 4 bytes in the header to esablish correct payload size bool isLargeData = len > 0xfffff0; if ( isLargeData ) { xt |= XtLarge; header = lenBytes.Take( 7 ); } // insert header res.InsertRange( 0 , header ); res.Insert( 0 , xt ); return res; }
/// <summary> /// Encode a Sexp in Qap1 format /// </summary> /// <param name="s">The Sexp to be encoded</param> /// <returns>QAP4-encoded bit stream</returns> private static IEnumerable <byte> EncodeSexp(Sexp s) { var t = s.GetType(); var res = new List <byte>(); byte xt; SexpTaggedList attrs = null; if (s.Attributes.Count > 0) { attrs = new SexpTaggedList(); foreach (var a in s.Attributes) { attrs.Add(a.Key, a.Value); } res.AddRange(EncodeSexp(attrs)); } if (t == typeof(SexpNull)) { xt = XtNull; } else if (t == typeof(SexpArrayDouble)) { xt = XtArrayDouble; var v = (( SexpArrayDouble )s).Value; foreach (var t1 in v) { res.AddRange(BitConverter.GetBytes(t1)); } } else if (t == typeof(SexpArrayInt)) { xt = XtArrayInt; var v = (( SexpArrayInt )s).Value; foreach (var t1 in v) { res.AddRange(BitConverter.GetBytes(t1)); } } else if (t == typeof(SexpArrayDate)) { xt = XtArrayInt; var v = (( SexpArrayInt )s).Value; foreach (var t1 in v) { res.AddRange(BitConverter.GetBytes(t1)); } } else if (t == typeof(SexpArrayBool)) { xt = XtArrayBool; var v = ( SexpArrayBool )s; res.AddRange(BitConverter.GetBytes(v.Count)); // R logical is false if 0, true if 1, and NA if 2 res.AddRange(v.Cast <SexpArrayBool>().Select(x => x.AsByte)); // protocol requires us to pad with null while (res.Count % 4 != 0) { res.Add(0); } } else if (t == typeof(SexpTaggedList)) { xt = XtListTag; var v = ( SexpTaggedList )s; foreach (var a in v.AsSexpDictionary) { res.AddRange(EncodeSexp(a.Value)); res.AddRange(EncodeSexp(new SexpSymname(a.Key))); } } else if (t == typeof(SexpList)) { xt = XtVector; var v = (( SexpList )s).Value; foreach (var a in v) { res.AddRange(EncodeSexp(a)); } } else if (t == typeof(SexpArrayString)) { xt = XtArrayString; var v = (( SexpArrayString )s).Value; foreach (var a in v) { // Rserve represents NA strings using 0xff (255). if (a == null) { res.Add(255); } else { var b = Encoding.UTF8.GetBytes(a); // If 0xff occurs in the beginning of a string it should be doubled to avoid misrepresentation. if ((b.Length > 0) && (b[0] == 255)) { res.Add(255); } res.AddRange(b); } res.Add(0); } } else if (t == typeof(SexpSymname)) { xt = XtSymName; var v = (( SexpSymname )s).Value; var b = Encoding.UTF8.GetBytes(v); res.AddRange(b); res.Add(0); } else { throw new RserveException("Cannot encode an unknown Sexp type " + t.GetType().Name); } if (attrs != null) { xt |= XtHasAttr; } // get payload length long len = res.LongCount(); byte[] lenBytes = BitConverter.GetBytes(len); // populate header (first four bytes) IEnumerable <byte> header = lenBytes.Take(3); // a large dataset is > 16MB, it requires the XtLarge flag and an extra 4 bytes in the header to esablish correct payload size bool isLargeData = len > 0xfffff0; if (isLargeData) { xt |= XtLarge; header = lenBytes.Take(7); } // insert header res.InsertRange(0, header); res.Insert(0, xt); return(res); }
/// <summary> /// Decode a Qap1-encoded Sexp /// </summary> /// <param name="data">The byte stream in which the Sexp is encoded</param> /// <param name="start">At which index of data does the Sexp begin?</param> /// <returns>The decoded Sexp.</returns> private static Sexp DecodeSexp( byte[] data , ref long start ) { // pull sexp type byte xt = data[ start ]; // calculate length of payload var lengthBuf = new byte[ 8 ]; Array.Copy( data , start + 1 , lengthBuf , 0 , 3 ); start += 4; if ( ( xt & XtLarge ) == XtLarge ) { Array.Copy( data , start , lengthBuf , 3 , 4 ); start += 4; xt -= XtLarge; } var length = ( long )BitConverter.ToUInt64( lengthBuf , 0 ); // has attributes? process first SexpTaggedList attrs = null; if ( ( xt & XtHasAttr ) == XtHasAttr ) { xt -= XtHasAttr; long oldstart = start; attrs = ( SexpTaggedList )DecodeSexp( data , ref start ); length -= start - oldstart; } long end = start + length; Sexp result; switch ( xt ) { case XtNull: { if ( length != 0 ) { throw new RserveException( "Attempting to decode an SexpNull, but it is followed by data when it shouldn't be." ); } result = new SexpNull(); } break; case XtSymName: { // keep all characters up to the first null var symnNamBuf = new byte[ length ]; Array.Copy( data , start , symnNamBuf , 0 , length ); string res = Encoding.UTF8.GetString( symnNamBuf ); result = new SexpSymname( res.Split( '\x00' )[ 0 ] ); } break; case XtArrayInt: { var res = new int[ length / 4 ]; var intBuf = new byte[ 4 ]; for ( long i = 0 ; i < length ; i += 4 ) { Array.Copy( data , start + i , intBuf , 0 , 4 ); res[ i / 4 ] = BitConverter.ToInt32( intBuf , 0 ); } // is date or just an integer? if ( ( attrs != null ) && ( attrs.ContainsKey( "class" ) && attrs[ "class" ].AsStrings.Contains( "Date" ) ) ) { result = new SexpArrayDate( res ); } else { result = new SexpArrayInt( res ); } } break; case XtArrayBool: { if ( length < 4 ) { throw new RserveException( "Decoding an SexpArrayBool where data doesn't seem to contain a data length field." ); } var boolLengthBuf = new byte[ 4 ]; Array.Copy( data , start , boolLengthBuf , 0 , 4 ); var datalength = BitConverter.ToInt32( boolLengthBuf , 0 ); if ( datalength > length - 4 ) { throw new RserveException( "Decoding an SexpArrayBool where transmitted data field too short for number of entries." ); } var res = new bool?[ datalength ]; for ( int i = 0 ; i < datalength ; i++ ) { // R logical is false if 0, true if 1, and NA if 2 switch ( data[ start + i + 4 ] ) { case 0: res[ i ] = false; break; case 1: res[ i ] = true; break; case 2: res[ i ] = null; break; default: throw new RserveException( "Decoding an SexpArrayBool and found an element in the array that is not an R bool: " + data[ start + i + 4 ] ); } } result = new SexpArrayBool( res ); } break; case XtArrayDouble: { var res = new double[ length / 8 ]; var doubleBuf = new byte[ 8 ]; for ( long i = 0 ; i < length ; i += 8 ) { Array.Copy( data , start + i , doubleBuf , 0 , 8 ); res[ i / 8 ] = BitConverter.ToDouble( doubleBuf , 0 ); } // is date or just a double? if ( ( attrs != null ) && ( attrs.ContainsKey( "class" ) && attrs[ "class" ].AsStrings.Contains( "Date" ) ) ) { result = new SexpArrayDate( res.Select( Convert.ToInt32 ) ); } else { result = new SexpArrayDouble( res ); } } break; case XtArrayString: { var res = new List<string>(); long i = 0; for ( long j = 0 ; j < length ; j++ ) { if ( data[ start + j ] != 0 ) { continue; } if ( ( j == i + 1 ) && ( data[ start + i ] == 255 ) ) { res.Add( null ); } else { if ( data[ start + i ] == 255 ) { i++; } var stringBuf = new byte[ j - i ]; Array.Copy( data , start + i , stringBuf , 0 , j - i ); res.Add( Encoding.UTF8.GetString( stringBuf ) ); } i = j + 1; } result = new SexpArrayString( res ); } break; case XtListNoTag: case XtLangNoTag: case XtVector: result = new SexpList(); while ( start < end ) { result.Add( DecodeSexp( data , ref start ) ); } break; case XtLangTag: case XtListTag: result = new SexpTaggedList(); while ( start < end ) { Sexp val = DecodeSexp( data , ref start ); Sexp key = DecodeSexp( data , ref start ); result.Add( key.IsNull ? String.Empty : key.AsString , val ); } break; case XtRaw: { var d = new byte[ length ]; Array.Copy( data , start , d , 0 , length ); result = new SexpQap1Raw( xt , d ); } break; default: throw new RserveException( "Cannot decode an Sexp because the type is not recognized: " + xt ); } if ( start > end ) { throw new RserveException( "When decoding an Sexp, more data consumed than provided." ); } start = end; if ( attrs != null ) { foreach ( var a in attrs.AsSexpDictionary ) { result.Attributes.Add( a.Key , a.Value ); } } return result; }