public override TypedData Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
switch (data.DataType) {
case TypedDataType.Image:
var rawImageProgress = new MutableProgressTracker();
var uploadProgress = new MutableProgressTracker();
var aggregateProgress = new AggregateProgressTracker(new ProgressTrackerCollection {
{ rawImageProgress, ImageWriter.ProcessTimeEstimate.ProgressWeight },
{ uploadProgress, ProcessTimeEstimate.ProgressWeight },
});
aggregateProgress.BindTo(progress);
progress.Status = "Converting image";
using (var rawImageData = ImageWriter.Process(data, rawImageProgress, cancelToken)) {
progress.Status = "Uploading image";
return Upload(rawImageData, uploadProgress, cancelToken);
}
default:
return null;
}
}
public void ParseNext_WhenBufferContainsUnsignedString_ReturnsExpectedTypedDataTypeAndValue()
{
// arrange
int offset = 0;
// 00010000 11000000 10000110 01000110 11000110
BitArray ba = new BitArray(
new bool[] {
false, false, false, true, false, false, false, false,
true, true, false, false, false, false, false, false,
true, false, false, false, false, true, true, false,
false, true, false, false, false, true, true, false,
true, true, false, false, false, true, true, false
});
byte[] buffer = new byte[5];
ba.CopyTo(buffer, 0);
// act
TypedData result = TypedDataParser.ParseNext(buffer, ref offset);
// assert
Assert.AreEqual(DataType.String, result.Type);
Assert.AreEqual("abc", result.Value);
}
public static object ToObject(this TypedData typedData)
{
switch (typedData.DataCase)
{
case RpcDataType.Bytes:
case RpcDataType.Stream:
return(typedData.Bytes.ToByteArray());
case RpcDataType.String:
return(typedData.String);
case RpcDataType.Json:
return(JsonConvert.DeserializeObject(typedData.Json, _datetimeSerializerSettings));
case RpcDataType.Http:
return(RpcMessageExtensionUtilities.ConvertFromHttpMessageToExpando(typedData.Http));
case RpcDataType.Int:
return(typedData.Int);
case RpcDataType.Double:
return(typedData.Double);
case RpcDataType.None:
return(null);
default:
// TODO better exception
throw new InvalidOperationException("Unknown RpcDataType");
}
}
internal override void Parse(byte[] buffer, ref int offset)
{
while (offset < buffer.Length)
{
VariableInt messageNameLength = VariableInt.DecodeVariableInt(buffer.Skip(offset).ToArray());
offset += messageNameLength.Length;
string messageName = Encoding.ASCII.GetString(buffer, offset, (int)messageNameLength.Value);
offset += messageName.Length;
byte numberOfArgs = buffer.Skip(offset).Take(1).First();
offset++;
var message = new SpoeMessage(messageName);
for (byte i = 0; i < numberOfArgs; i++)
{
VariableInt keyNameLength = VariableInt.DecodeVariableInt(buffer.Skip(offset).ToArray());
offset += keyNameLength.Length;
string keyname = Encoding.ASCII.GetString(buffer, offset, (int)keyNameLength.Value);
offset += keyname.Length;
TypedData data = TypedDataParser.ParseNext(buffer, ref offset);
message.Args.Add(keyname, data);
}
this.Messages.Add(message);
}
}
public void ParseNext_WhenBufferContainsBinary_ReturnsExpectedTypedDataTypeAndValue()
{
// arrange
int offset = 0;
// act
// 10010000 11100000 01100110 11110110 11110110 10111100 01000110 10000110 01001110
BitArray ba = new BitArray(
new bool[] {
true, false, false, true, false, false, false, false,
true, true, true, false, false, false, false, false,
false, true, true, false, false, true, true, false,
true, true, true, true, false, true, true, false,
true, true, true, true, false, true, true, false,
true, false, true, true, true, true, false, false,
false, true, false, false, false, true, true, false,
true, false, false, false, false, true, true, false,
false, true, false, false, true, true, true, false
});
byte[] buffer = new byte[9];
ba.CopyTo(buffer, 0);
TypedData result = TypedDataParser.ParseNext(buffer, ref offset);
// assert
Assert.AreEqual(DataType.Binary, result.Type);
Assert.AreEqual("Zm9vPWJhcg==", result.ToString()); // base64 foo=bar
}
internal static TypedData ToRpcHttp(this HttpResponseData response)
{
var http = new RpcHttp()
{
StatusCode = ((int)response.StatusCode).ToString()
};
if (response.Body != null)
{
http.Body = response.Body.ToRpc();
}
else
{
// TODO: Is this correct? Passing a null body causes the entire
// response to become the body in functions. Need to investigate.
http.Body = string.Empty.ToRpc();
}
if (response.Headers != null)
{
foreach (var pair in response.Headers)
{
// maybe check or validate that the headers make sense?
http.Headers.Add(pair.Key.ToLowerInvariant(), pair.Value.ToString());
}
}
var typedData = new TypedData
{
Http = http
};
return(typedData);
}
public async Task ToRpc_Collection_String_With_Capabilities_Value()
{
var logger = MockNullLoggerFactory.CreateLogger();
var capabilities = new Capabilities(logger);
MapField <string, string> addedCapabilities = new MapField <string, string>
{
{ RpcWorkerConstants.TypedDataCollection, RpcWorkerConstants.TypedDataCollection }
};
capabilities.UpdateCapabilities(addedCapabilities);
string[] arrString = { "element1", "element_2" };
TypedData returned_typedata = await arrString.ToRpc(logger, capabilities);
TypedData typedData = new TypedData();
CollectionString collectionString = new CollectionString();
foreach (string element in arrString)
{
if (!string.IsNullOrEmpty(element))
{
collectionString.String.Add(element);
}
}
typedData.CollectionString = collectionString;
Assert.Equal(typedData.CollectionString, returned_typedata.CollectionString);
Assert.Equal(typedData.CollectionString.String[0], returned_typedata.CollectionString.String[0]);
}
public void TestObjectToTypedDataRpcHttpContentTypeInHeader()
{
var data = "<html></html>";
var input = new HttpResponseContext
{
Body = data,
Headers = new Hashtable {
{ "content-type", "text/html" }
}
};
var expected = new TypedData
{
Http = new RpcHttp
{
StatusCode = "200",
Body = new TypedData {
String = data
},
Headers = { { "content-type", "text/html" } }
}
};
Assert.Equal(expected, input.ToTypedData());
}
public static TypedData ToRpc(this object value)
{
TypedData typedData = new TypedData();
if (value == null)
{
return(typedData);
}
if (value is byte[] arr)
{
typedData.Bytes = ByteString.CopyFrom(arr);
}
else if (value is string str)
{
typedData.String = str;
}
else if (value is HttpResponseData response)
{
typedData = response.ToRpcHttp();
}
else if (value.GetType().IsArray)
{
typedData = ToRpcCollection(value);
}
else
{
typedData = value.ToRpcDefault();
}
return(typedData);
}
public void TestTypedDataToObjectHttpRequestContextWithUrlData()
{
var method = "Get";
var url = "https://example.com";
var key = "foo";
var value = "bar";
var input = new TypedData
{
Http = new RpcHttp
{
Method = method,
Url = url
}
};
input.Http.Headers.Add(key, value);
input.Http.Params.Add(key, value);
input.Http.Query.Add(key, value);
var httpRequestContext = (HttpRequestContext)input.ToObject();
Assert.Equal(httpRequestContext.Method, method);
Assert.Equal(httpRequestContext.Url, url);
Assert.Null(httpRequestContext.Body);
Assert.Null(httpRequestContext.RawBody);
Assert.Single(httpRequestContext.Headers);
Assert.Single(httpRequestContext.Params);
Assert.Single(httpRequestContext.Query);
Assert.Equal(httpRequestContext.Headers[key], value);
Assert.Equal(httpRequestContext.Params[key], value);
Assert.Equal(httpRequestContext.Query[key], value);
}
public async Task ToRpc_Collection_Long_With_Capabilities_Value()
{
var logger = MockNullLoggerFactory.CreateLogger();
var capabilities = new Capabilities(logger);
MapField <string, string> addedCapabilities = new MapField <string, string>
{
{ RpcWorkerConstants.TypedDataCollection, RpcWorkerConstants.TypedDataCollection }
};
capabilities.UpdateCapabilities(addedCapabilities);
long[] arrLong = { 1L, 2L };
TypedData returned_typedata = await arrLong.ToRpc(logger, capabilities);
TypedData typedData = new TypedData();
CollectionSInt64 collectionLong = new CollectionSInt64();
foreach (long element in arrLong)
{
collectionLong.Sint64.Add(element);
}
typedData.CollectionSint64 = collectionLong;
Assert.Equal(typedData.CollectionSint64, returned_typedata.CollectionSint64);
Assert.Equal(typedData.CollectionSint64.Sint64[0], returned_typedata.CollectionSint64.Sint64[0]);
}
public async Task ToRpc_Collection_Double_With_Capabilities_Value()
{
var logger = MockNullLoggerFactory.CreateLogger();
var capabilities = new Capabilities(logger);
MapField <string, string> addedCapabilities = new MapField <string, string>
{
{ RpcWorkerConstants.TypedDataCollection, RpcWorkerConstants.TypedDataCollection }
};
capabilities.UpdateCapabilities(addedCapabilities);
double[] arrDouble = { 1.1, 2.2 };
TypedData returned_typedata = await arrDouble.ToRpc(logger, capabilities);
TypedData typedData = new TypedData();
CollectionDouble collectionDouble = new CollectionDouble();
foreach (double element in arrDouble)
{
collectionDouble.Double.Add(element);
}
typedData.CollectionDouble = collectionDouble;
Assert.Equal(typedData.CollectionDouble, returned_typedata.CollectionDouble);
Assert.Equal(typedData.CollectionDouble.Double[0], returned_typedata.CollectionDouble.Double[0]);
}
public async Task ToRpc_Collection_Byte_With_Capabilities_Value()
{
var logger = MockNullLoggerFactory.CreateLogger();
var capabilities = new Capabilities(logger);
MapField <string, string> addedCapabilities = new MapField <string, string>
{
{ RpcWorkerConstants.TypedDataCollection, RpcWorkerConstants.TypedDataCollection }
};
capabilities.UpdateCapabilities(addedCapabilities);
byte[][] arrBytes = new byte[2][];
arrBytes[0] = new byte[] { 22 };
arrBytes[1] = new byte[] { 11 };
TypedData returned_typedata = await arrBytes.ToRpc(logger, capabilities);
TypedData typedData = new TypedData();
CollectionBytes collectionBytes = new CollectionBytes();
foreach (byte[] element in arrBytes)
{
if (element != null)
{
collectionBytes.Bytes.Add(ByteString.CopyFrom(element));
}
}
typedData.CollectionBytes = collectionBytes;
Assert.Equal(typedData.CollectionBytes, returned_typedata.CollectionBytes);
Assert.Equal(typedData.CollectionBytes.Bytes[0], returned_typedata.CollectionBytes.Bytes[0]);
}
public void TestTypedDataToObjectHttpRequestContextBodyData_DoesDeserializeStringFromJson(string contentType, string stringData)
{
var rpcHttp = contentType == null
? new RpcHttp
{
Body = new TypedData {
String = stringData
}
}
: new RpcHttp
{
Headers = { { "content-type", contentType } },
Body = new TypedData {
String = stringData
}
};
var input = new TypedData
{
Http = rpcHttp
};
var httpRequestContext = (HttpRequestContext)input.ToObject();
var expected = JsonConvert.DeserializeObject <Hashtable>(stringData);
var actual = (Hashtable)httpRequestContext.Body;
Assert.Equal(expected["Key"], actual["Key"]);
}
public void TestObjectToTypedData_PSObjectToString()
{
object input = PSObject.AsPSObject("Hello World");
TypedData output = input.ToTypedData();
Assert.Equal(TypedData.DataOneofCase.String, output.DataCase);
Assert.Equal("Hello World", output.String);
}
public static TypedData ToTypedData(this object value)
{
TypedData typedData = new TypedData();
if (value == null)
{
return(typedData);
}
switch (value)
{
case double d:
typedData.Double = d;
break;
case long l:
typedData.Int = l;
break;
case int i:
typedData.Int = i;
break;
case byte[] arr:
typedData.Bytes = ByteString.CopyFrom(arr);
break;
case Stream s:
typedData.Stream = ByteString.FromStream(s);
break;
case HttpResponseContext http:
typedData.Http = http.ToRpcHttp();
break;
case IDictionary hashtable:
typedData.Json = JsonConvert.SerializeObject(hashtable);
break;
default:
// Handle everything else as a string
var str = value.ToString();
// Attempt to parse the string into json. If it fails,
// fallback to storing as a string
try
{
JsonConvert.DeserializeObject(str);
typedData.Json = str;
}
catch
{
typedData.String = str;
}
break;
}
return(typedData);
}
public void TestObjectToTypedData_PSObjectToBytes()
{
var data = new byte[] { 12, 23, 34 };
object input = PSObject.AsPSObject(data);
TypedData output = input.ToTypedData();
Assert.Equal(TypedData.DataOneofCase.Bytes, output.DataCase);
Assert.Equal(3, output.Bytes.Length);
}
public void SkipSamples(int n)
{
if (n < 0 || Shape[-1] < n)
{
throw new ArgumentOutOfRangeException("n");
}
TypedData.SkipElements(n * _stride);
Shape[-1] -= n;
}
public TypedData Upload(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
try {
var response = HttpRequest.Execute(Uri, GetRequestData(data), RequestMethod, progress, cancelToken);
return GetResponseData(response, data);
} catch (OperationCanceledException e) {
throw CommandCanceledException.Wrap(e, this);
}
}
public void AddSamples(IList <T> list)
{
if (list.Count % _stride != 0)
{
throw new ArgumentException("Length of list should be a multiplication of the dimension of features");
}
TypedData.AddList(list);
Shape[-1] += list.Count / _stride;
}
public void ToString_WhenValueIsBoolean_ReturnsStringValue(bool value, string expectedResult)
{
// arrange
TypedData data = new TypedData(DataType.Boolean, value);
// act
string result = data.ToString();
// assert
Assert.AreEqual(expectedResult, result);
}
public void Bytes_WhenTypeIsIpv4_ReturnsExpectedResult()
{
// arrange
TypedData data = new TypedData(DataType.Ipv4, "1.1.1.1");
// act
byte[] bytes = data.Bytes;
// assert
Assert.AreEqual("0110000010000000100000001000000010000000", ToBitString(bytes));
}
public void TestTypedDataToObjectStream()
{
var data = ByteString.CopyFromUtf8("Hello World");
var input = new TypedData {
Stream = data
};
var expected = data.ToByteArray();
Assert.Equal(expected, (byte[])input.ToObject());
}
public void TestTypedDataToObjectDouble()
{
var data = 2.2;
var input = new TypedData {
Double = data
};
var expected = data;
Assert.Equal(expected, (double)input.ToObject());
}
public void TestTypedDataToObjectInt()
{
long data = 2;
var input = new TypedData {
Int = data
};
var expected = data;
Assert.Equal(expected, (long)input.ToObject());
}
public void TestTypedDataToObjectString()
{
var data = "Hello World";
var input = new TypedData {
String = data
};
var expected = data;
Assert.Equal(expected, (string)input.ToObject());
}
public void Bytes_WhenTypeIsBoolean_ReturnsExpectedResult()
{
// arrange
TypedData data = new TypedData(DataType.Boolean, true);
// act
byte[] bytes = data.Bytes;
// assert
Assert.AreEqual("10001000", ToBitString(bytes));
}
public void Bytes_WhenTypeIsUint32_ReturnsExpectedResult()
{
// arrange
TypedData data = new TypedData(DataType.Uint32, (uint)16380);
// act
byte[] bytes = data.Bytes;
// assert
Assert.AreEqual("11000000001111110000111101100000", ToBitString(bytes));
}
public void TestObjectToTypedData_PSObjectToStream()
{
using (var data = new MemoryStream(new byte[] { 12, 23, 34 }))
{
object input = PSObject.AsPSObject(data);
TypedData output = input.ToTypedData();
Assert.Equal(TypedData.DataOneofCase.Stream, output.DataCase);
Assert.Equal(3, output.Stream.Length);
}
}
public void TestObjectToTypedDataJsonString()
{
var data = "{\"foo\":\"bar\"}";
var input = (object)data;
var expected = new TypedData
{
Json = data
};
Assert.Equal(expected, input.ToTypedData());
}
public void TestObjectToTypedDataBytes()
{
var data = ByteString.CopyFromUtf8("Hello World!").ToByteArray();
var input = (object)data;
var expected = new TypedData
{
Bytes = ByteString.CopyFrom(data)
};
Assert.Equal(expected, input.ToTypedData());
}
public void TestObjectToTypedDataString()
{
var data = "Hello World!";
var input = (object)data;
var expected = new TypedData
{
String = data
};
Assert.Equal(expected, input.ToTypedData());
}
public override TypedData Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
switch (data.DataType) {
case TypedDataType.Text:
progress.Status = "Uploading text";
return Upload(data, progress, cancelToken);
default:
return null;
}
}
public TypedData Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
switch (data.DataType) {
case TypedDataType.Image:
return TypedData.FromStream(Encode((Bitmap) data.Data, progress, cancelToken), data.Name);
case TypedDataType.Stream:
return TypedData.FromImage(Decode((Stream) data.Data, progress, cancelToken), data.Name);
default:
throw new NotSupportedException();
}
}
public TypedData Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
if (data.DataType != TypedDataType.Image) {
throw new ArgumentException("data must be an image", "data");
}
if (!IsCodecValid(Codec)) {
throw new InvalidOperationException("Codec must be non-null and support bitmap encoding");
}
// No dispose
var processedImage = Codec.Process(data, progress, cancelToken);
var name = processedImage.Name;
return new TypedData(processedImage.DataType, processedImage.Data, GetDataName(name));
}
public TypedData Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
// ToList is needed to prevent new MPT's from being
// created for each loop of commandProgressTrackers
var commandProgressTrackers = this.commands.Select((command) => new {
ProgressTracker = new MutableProgressTracker(),
Command = command
}).ToList();
var aggregateProgress = new AggregateProgressTracker(commandProgressTrackers.Select(
(cpt) => new ProgressTrackerCollectionItem(
cpt.ProgressTracker,
cpt.Command.ProcessTimeEstimate.ProgressWeight
)
));
aggregateProgress.BindTo(progress);
bool shouldDisposeData = false;
foreach (var cpt in commandProgressTrackers) {
cancelToken.ThrowIfCancellationRequested();
TypedData newData;
try {
cancelToken.ThrowIfCancellationRequested();
if (!cpt.Command.IsValid()) {
throw new CommandInvalidException(cpt.Command);
}
newData = cpt.Command.Process(data, cpt.ProgressTracker, cancelToken);
cancelToken.ThrowIfCancellationRequested();
} finally {
if (shouldDisposeData && data != null) {
data.Dispose();
}
}
data = newData;
shouldDisposeData = true;
}
return data;
}
public abstract TypedData Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken);
protected virtual MultipartData GetRequestData(TypedData data)
{
return null;
}
protected abstract TypedData GetResponseData(HttpWebResponse response, TypedData originalData);
protected override TypedData GetResponseData(HttpWebResponse response, TypedData originalData)
{
var urlText = HttpRequest.GetResponseText(response);
return TypedData.FromUri(new Uri(urlText, UriKind.Absolute), originalData.Name);
}
TypedData ICommand.Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
Execute(progress, cancelToken);
return null;
}
protected override TypedData GetResponseData(HttpWebResponse response, TypedData originalData)
{
return TypedData.FromUri(response.ResponseUri, originalData.Name);
}
public TypedData Process(TypedData data, IMutableProgressTracker progress, CancellationToken cancelToken)
{
ICommand command;
if (!Routes.TryGetValue(data.DataType, out command)) {
throw new InvalidOperationException("Data type not routed");
}
return command.Process(data, progress, cancelToken);
}
