public void TestMapInputWithControlMessageCodec() { float[] baseMessage = { 0, 1 }; var config = TangFactory.GetTang().NewConfigurationBuilder() .BindImplementation(GenericType <IStreamingCodec <float[]> > .Class, GenericType <FloatArrayStreamingCodec> .Class) .Build(); IStreamingCodec <MapInputWithControlMessage <float[]> > codec = TangFactory.GetTang().NewInjector(config).GetInstance <MapInputWithControlMessageCodec <float[]> >(); MemoryStream stream = new MemoryStream(); IDataWriter writer = new StreamDataWriter(stream); codec.Write(new MapInputWithControlMessage <float[]>(baseMessage, MapControlMessage.AnotherRound), writer); codec.Write(new MapInputWithControlMessage <float[]>(MapControlMessage.Stop), writer); stream.Position = 0; IDataReader reader = new StreamDataReader(stream); var message1 = codec.Read(reader); var message2 = codec.Read(reader); Assert.Equal(message1.Message[0], baseMessage[0]); Assert.Equal(message1.Message[1], baseMessage[1]); Assert.Null(message2.Message); Assert.Equal(message1.ControlMessage, MapControlMessage.AnotherRound); Assert.Equal(message2.ControlMessage, MapControlMessage.Stop); }
public async Task TestCodecToStreamingCodecConfiguration() { var config = CodecToStreamingCodecConfiguration <int> .Conf .Set(CodecToStreamingCodecConfiguration <int> .Codec, GenericType <IntCodec> .Class) .Build(); IStreamingCodec <PipelineMessage <int> > streamingCodec = TangFactory.GetTang().NewInjector(config).GetInstance <IStreamingCodec <PipelineMessage <int> > >(); CancellationToken token = new CancellationToken(); int obj = 5; PipelineMessage <int> message = new PipelineMessage <int>(obj, true); var stream = new MemoryStream(); IDataWriter writer = new StreamDataWriter(stream); streamingCodec.Write(message, writer); PipelineMessage <int> message1 = new PipelineMessage <int>(obj + 1, false); await streamingCodec.WriteAsync(message1, writer, token); stream.Position = 0; IDataReader reader = new StreamDataReader(stream); var res1 = streamingCodec.Read(reader); var res2 = await streamingCodec.ReadAsync(reader, token); Assert.Equal(obj, res1.Data); Assert.Equal(obj + 1, res2.Data); Assert.Equal(true, res1.IsLast); Assert.Equal(false, res2.IsLast); }
public async Task TestCodecToStreamingCodec() { var config = TangFactory.GetTang().NewConfigurationBuilder() .BindImplementation(GenericType <ICodec <int> > .Class, GenericType <IntCodec> .Class) .BindImplementation(GenericType <IStreamingCodec <int> > .Class, GenericType <CodecToStreamingCodec <int> > .Class) .Build(); IStreamingCodec <int> streamingCodec = TangFactory.GetTang().NewInjector(config).GetInstance <IStreamingCodec <int> >(); CancellationToken token = new CancellationToken(); int obj = 5; var stream = new MemoryStream(); IDataWriter writer = new StreamDataWriter(stream); streamingCodec.Write(obj, writer); await streamingCodec.WriteAsync(obj + 1, writer, token); stream.Position = 0; IDataReader reader = new StreamDataReader(stream); int res1 = streamingCodec.Read(reader); int res2 = await streamingCodec.ReadAsync(reader, token); Assert.Equal(obj, res1); Assert.Equal(obj + 1, res2); }
public void TestFileCount() { StreamDataReader streamDataReader = new StreamDataReader(@"C:/Temp/TestData.txt"); ProcessData p = new ProcessData(streamDataReader); Assert.AreEqual(13, p.processDataReader()); }
public override bool Read() { if (skipAll) { return(false); } if (pagination.GetActualRowCount() == null) { return(StreamDataReader.Read()); } return(++rowNumber <= pagination.GetActualRowCount() && StreamDataReader.Read()); }
private bool SkipOffset() { for (int i = 0; i < pagination.GetActualOffset(); i++) { if (!StreamDataReader.Read()) { return(true); } } rowNumber = 0; return(false); }
static public LrtForHla GetInstance(string selectionName, string inputDirectory, string caseName, string hlaFactoryName, double?leakProbabilityOrNull, double pValue) { string patientFileName = string.Format(@"{0}\{1}patient.txt", inputDirectory, caseName); //!!!const string patientFileNameReact = string.Format(@"{0}\{1}react.txt", inputDirectory, caseName); //!!!const string patientFileNameKnown = string.Format(@"{0}\{1}known.txt", inputDirectory, caseName); //!!!const using (DbDataReader datareaderHLA = new StreamDataReader(patientFileName)) using (DbDataReader datareaderReact = new StreamDataReader(patientFileNameReact)) using (DbDataReader datareaderKnown = new StreamDataReader(patientFileNameKnown)) { return(GetInstance(selectionName, datareaderHLA, datareaderReact, datareaderKnown, caseName, hlaFactoryName, leakProbabilityOrNull, pValue)); } }
internal IList <TResult> Run() { var results = new List <TResult>(); var updateResult = _updateTask.Initialize(); while (updateResult.IsNotDone) { if (updateResult.HasResult) { results.Add(updateResult.Result); } Debug.Assert(updateResult.HasMapInput, "MapInput is needed."); var mapinput = updateResult.MapInput; var mapOutputs = new List <TMapOutput>(); foreach (var mapfunc in _mapfunctions) { // We create a copy by doing coding and decoding since the map task might // reuse the fields in next iteration and meanwhile update task might update it. using (MemoryStream mapInputStream = new MemoryStream(), mapOutputStream = new MemoryStream()) { var mapInputWriter = new StreamDataWriter(mapInputStream); _mapInputCodec.Write(mapinput, mapInputWriter); mapInputStream.Position = 0; var mapInputReader = new StreamDataReader(mapInputStream); var output = mapfunc.Map(_mapInputCodec.Read(mapInputReader)); var mapOutputWriter = new StreamDataWriter(mapOutputStream); _mapOutputCodec.Write(output, mapOutputWriter); mapOutputStream.Position = 0; var mapOutputReader = new StreamDataReader(mapOutputStream); output = _mapOutputCodec.Read(mapOutputReader); mapOutputs.Add(output); } } var mapOutput = _reduceTask.Reduce(mapOutputs); updateResult = _updateTask.Update(mapOutput); } if (updateResult.HasResult) { results.Add(updateResult.Result); } return(results); }
public void TestStreamingCodecFunctionCache() { IConfiguration conf = TangFactory.GetTang().NewConfigurationBuilder() .BindImplementation(GenericType <IStreamingCodec <B> > .Class, GenericType <BStreamingCodec> .Class) .Build(); IInjector injector = TangFactory.GetTang().NewInjector(conf); StreamingCodecFunctionCache <A> cache = new StreamingCodecFunctionCache <A>(injector); var readFunc = cache.ReadFunction(typeof(B)); var writeFunc = cache.WriteFunction(typeof(B)); var readAsyncFunc = cache.ReadAsyncFunction(typeof(B)); var writeAsyncFunc = cache.WriteAsyncFunction(typeof(B)); var stream = new MemoryStream(); IDataWriter writer = new StreamDataWriter(stream); IDataReader reader = new StreamDataReader(stream); B val = new B(); val.Value1 = "hello"; val.Value2 = "reef"; writeFunc(val, writer); val.Value1 = "helloasync"; val.Value2 = "reefasync"; CancellationToken token = new CancellationToken(); var asyncResult = writeAsyncFunc.BeginInvoke(val, writer, token, null, null); writeAsyncFunc.EndInvoke(asyncResult); stream.Position = 0; A res = readFunc(reader); B resB1 = res as B; asyncResult = readAsyncFunc.BeginInvoke(reader, token, null, null); res = readAsyncFunc.EndInvoke(asyncResult); B resB2 = res as B; Assert.Equal("hello", resB1.Value1); Assert.Equal("reef", resB1.Value2); Assert.Equal("helloasync", resB2.Value1); Assert.Equal("reefasync", resB2.Value2); }
public void TestConfigurationBroadcastSpec() { FlatTopology <int> topology = new FlatTopology <int>("Operator", "Operator", "task1", "driverid", new BroadcastOperatorSpec("Sender", GetDefaultCodecConfig(), GetDefaultDataConverterConfig())); topology.AddTask("task1"); var conf = topology.GetTaskConfiguration("task1"); IStreamingCodec <int> codec = TangFactory.GetTang().NewInjector(conf).GetInstance <IStreamingCodec <int> >(); var stream = new MemoryStream(); IDataWriter writer = new StreamDataWriter(stream); codec.Write(3, writer); stream.Position = 0; IDataReader reader = new StreamDataReader(stream); int res = codec.Read(reader); Assert.Equal(3, res); }
public async Task TestCommonStreamingCodecs() { IInjector injector = TangFactory.GetTang().NewInjector(); IStreamingCodec <int> intCodec = injector.GetInstance <IntStreamingCodec>(); IStreamingCodec <double> doubleCodec = injector.GetInstance <DoubleStreamingCodec>(); IStreamingCodec <float> floatCodec = injector.GetInstance <FloatStreamingCodec>(); IStreamingCodec <int[]> intArrCodec = injector.GetInstance <IntArrayStreamingCodec>(); IStreamingCodec <double[]> doubleArrCodec = injector.GetInstance <DoubleArrayStreamingCodec>(); IStreamingCodec <float[]> floatArrCodec = injector.GetInstance <FloatArrayStreamingCodec>(); IStreamingCodec <string> stringCodec = injector.GetInstance <StringStreamingCodec>(); CancellationToken token = new CancellationToken(); int obj = 5; int[] intArr = { 1, 2 }; double[] doubleArr = { 1, 2 }; float[] floatArr = { 1, 2 }; string stringObj = "hello"; var stream = new MemoryStream(); IDataWriter writer = new StreamDataWriter(stream); intCodec.Write(obj, writer); await intCodec.WriteAsync(obj + 1, writer, token); doubleCodec.Write(obj + 2, writer); await doubleCodec.WriteAsync(obj + 3, writer, token); floatCodec.Write(obj + 4, writer); await floatCodec.WriteAsync(obj + 5, writer, token); intArrCodec.Write(intArr, writer); await intArrCodec.WriteAsync(intArr, writer, token); doubleArrCodec.Write(doubleArr, writer); await doubleArrCodec.WriteAsync(doubleArr, writer, token); floatArrCodec.Write(floatArr, writer); await floatArrCodec.WriteAsync(floatArr, writer, token); stringCodec.Write(stringObj, writer); await stringCodec.WriteAsync(stringObj, writer, token); stream.Position = 0; IDataReader reader = new StreamDataReader(stream); int res1 = intCodec.Read(reader); int res2 = await intCodec.ReadAsync(reader, token); double res3 = doubleCodec.Read(reader); double res4 = await doubleCodec.ReadAsync(reader, token); float res5 = floatCodec.Read(reader); float res6 = await floatCodec.ReadAsync(reader, token); int[] resArr1 = intArrCodec.Read(reader); int[] resArr2 = await intArrCodec.ReadAsync(reader, token); double[] resArr3 = doubleArrCodec.Read(reader); double[] resArr4 = await doubleArrCodec.ReadAsync(reader, token); float[] resArr5 = floatArrCodec.Read(reader); float[] resArr6 = await floatArrCodec.ReadAsync(reader, token); string resArr7 = stringCodec.Read(reader); string resArr8 = await stringCodec.ReadAsync(reader, token); Assert.Equal(obj, res1); Assert.Equal(obj + 1, res2); Assert.Equal(obj + 2, res3); Assert.Equal(obj + 3, res4); Assert.Equal(obj + 4, res5); Assert.Equal(obj + 5, res6); Assert.Equal(stringObj, resArr7); Assert.Equal(stringObj, resArr8); for (int i = 0; i < intArr.Length; i++) { Assert.Equal(intArr[i], resArr1[i]); Assert.Equal(intArr[i], resArr2[i]); } for (int i = 0; i < doubleArr.Length; i++) { Assert.Equal(doubleArr[i], resArr3[i]); Assert.Equal(doubleArr[i], resArr4[i]); } for (int i = 0; i < floatArr.Length; i++) { Assert.Equal(floatArr[i], resArr5[i]); Assert.Equal(floatArr[i], resArr6[i]); } }