예제 #1
0
        public static async Task <HttpResponseMessage> Run(
            [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
            ILogger log)
        {
            log.LogInformation("Received a json file to transform.");

            string filename = Guid.NewGuid().ToString() + ".csv";

            log.LogInformation("Temp file used to hold data: " + filename);

            var jsonSource = new JsonSource()
            {
                CreateStreamReader = _ => new StreamReader(req.Body)
            };
            var csvFileDest = new CsvDestination(filename);

            jsonSource.LinkTo(csvFileDest);
            await Network.ExecuteAsync(jsonSource);

            log.LogInformation("Successfully stored data in file - now returning the content as response.");

            return(new HttpResponseMessage()
            {
                StatusCode = HttpStatusCode.OK,
                Content = new StreamContent(new FileStream(filename, FileMode.Open))
            });
        }
        protected override void OnNavigatedTo(NavigationEventArgs e) //загрузка данных
        {
            CurrencyContainer.del += Downloaded;
            IDataSource jsonSource = new JsonSource();

            jsonSource.GetCurrencyList(CurrencyContainer.del); //передача в качестве параметра делегата перехода окна
        }
        public void SourceWithDifferentNames()
        {
            //Arrange
            TwoColumnsTableFixture            dest2Columns = new TwoColumnsTableFixture("JsonSource2ColsDynamic");
            RowTransformation <ExpandoObject> trans        = new RowTransformation <ExpandoObject>(
                row =>
            {
                dynamic r = row as ExpandoObject;
                r.Col1    = r.Column1;
                r.Col2    = r.Column2;
                return(r);
            });
            DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>("JsonSource2ColsDynamic", Connection);

            //Act
            JsonSource <ExpandoObject> source = new JsonSource <ExpandoObject>("res/JsonSource/TwoColumnsDifferentNames.json", ResourceType.File);
            var linkTo1 = source.LinkTo(trans);
            var linkTo2 = linkTo1.source.LinkTo(dest);

            using (linkTo1.link)
                using (linkTo2.link)
                {
                    source.Execute();
                    dest.Wait();

                    //Assert
                    dest2Columns.AssertTestData();
                }
        }
 /// <summary>
 /// Initializes a new instance of the <see cref="JsonConfigurationSource"/> class.
 /// </summary>
 /// <param name="filename">The full path to the document.</param>
 public JsonConfigurationSource(string filename)
 {
     path       = new FileInfo(filename ?? throw new ArgumentNullException(nameof(filename)));
     serializer = new JsonSerializer();
     source     = JsonSource.File;
     sync       = new ReaderWriterLockSlim();
 }
예제 #5
0
 void UpdateData()
 {
     DataSource = new JsonDataSource()
     {
         Source = JsonSource.FromUri(new Uri("https://s3.amazonaws.com/utdmakerspace/events.json"))
     };
 }
        public void JsonPathInEpandoObject()
        {
            //Arrange
            TwoColumnsTableFixture            dest2Columns = new TwoColumnsTableFixture("JsonSourceNestedDynamic");
            RowTransformation <ExpandoObject> trans        = new RowTransformation <ExpandoObject>(
                row =>
            {
                dynamic r = row as ExpandoObject;
                r.Col1    = r.Column1;
                return(r);
            });
            DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "JsonSourceNestedDynamic");

            //Act
            JsonSource <ExpandoObject>   source      = new JsonSource <ExpandoObject>("res/JsonSource/NestedData.json", ResourceType.File);
            List <JsonProperty2JsonPath> pathLookups = new List <JsonProperty2JsonPath>()
            {
                new JsonProperty2JsonPath()
                {
                    JsonPropertyName = "Column2",
                    JsonPath         = "Value",
                    NewPropertyName  = "Col2"
                }
            };

            source.JsonSerializer.Converters.Add(new ExpandoJsonPathConverter(pathLookups));

            source.LinkTo(trans).LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
예제 #7
0
        public void JsonSource_FullLoadTest()
        {
            var source = new JsonSource("./JsonTestFile_First.json", InterCommunicator.Empty);

            var fired = source.Rows.Count();

            Assert.AreEqual(3, fired);
        }
예제 #8
0
        public void JsonSource_FullLoadTest()
        {
            var source = new JsonSource("./JsonTestFile_First.json", RuntimeContext.Empty);

            var fired = source.Rows.Count();

            Assert.AreEqual(3, fired);
        }
예제 #9
0
 public GenericListDataPage(string dataSource)
 {
     DataSource = new JsonDataSource()
     {
         Source = JsonSource.FromUri(new Uri(dataSource))
     };
     InitializeComponent();
 }
        protected override void Setup()
        {
            _dbClientMock = new Mock <IDbClient>();
            var dbFactoryMock = new Mock <IDbClientFactory>();

            dbFactoryMock.Setup(x => x.GetJsonClient(It.IsAny <string>())).Returns(_dbClientMock.Object);

            _source = new JsonSource(dbFactoryMock.Object);
        }
예제 #11
0
        public void JsonSource_CancelledLoadTest()
        {
            var tokenSource = new CancellationTokenSource();

            tokenSource.Cancel();
            var source = new JsonSource("./JsonTestFile_First.json", new InterCommunicator(tokenSource.Token));

            var fired = source.Rows.Count();

            Assert.AreEqual(0, fired);
        }
예제 #12
0
        public void JsonFromWebService()
        {
            JsonSource        source = new JsonSource("res/JsonSource/EmptyObject.json", ResourceType.File);
            MemoryDestination dest   = new MemoryDestination();

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            Assert.True(dest.Data.Count == 0);
        }
예제 #13
0
 public ResQuery(string name, string query = null, string desc = null, object parameters = null, bool inSharedZip = false,
                 ResFilType fileType       = ResFilType.Csv, JsonSource jsonSource = default)
 {
     Name        = name;
     Query       = query;
     Desc        = desc;
     Parameters  = parameters;
     InSharedZip = inSharedZip;
     FileType    = fileType;
     JsonSource  = jsonSource;
 }
예제 #14
0
        public void JsonSource_CancelledLoadTest()
        {
            var tokenSource = new CancellationTokenSource();

            tokenSource.Cancel();
            var source = new JsonSource("./JsonTestFile_First.json", new RuntimeContext(tokenSource.Token, new ISchemaColumn[0]));

            var fired = source.Rows.Count();

            Assert.AreEqual(0, fired);
        }
예제 #15
0
        public void ReadEmptyArray()
        {
            JsonSource        source = new JsonSource("res/JsonSource/EmptyArray.json", ResourceType.File);
            MemoryDestination dest   = new MemoryDestination();

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            Assert.True(dest.Data.Count == 0);
        }
예제 #16
0
        public MyConfig() : base(defaultSourceName: nameof(ConfigSourceNames.Combined))
        {
            var configrSource = new ConfigRSource(@"./config.csx",
                                                  autoReloadOnFileChange: true);
            var jsonSource = new JsonSource("./config.json", true);

            this.UseCombinedSource("Combined",
                                   false,
                                   new BaseConfigurationSource[] { configrSource, jsonSource })
            .UseAppSettings()
            .UseEnvironmentVariable("CONFYGURE_OUT_");
        }
예제 #17
0
        /* Data flow
         *
         * JsonSource --> RowTransformation --> Lookup --> Multicast --> DbDestination ("orders" table)
         * (Order data)                        |             |
         *                   CsvSource     <----             --------> TextDestination ("order_data.log")
         *                  ("customer.csv")
         */


        static void Main(string[] args)
        {
            //Preparation
            RecreateTargetTable();

            //Step 1 - creating the components
            var source = new JsonSource <OrderRow>("https://www.etlbox.net/demo/api/orders", ResourceType.Http);

            var rowTransformation = new RowTransformation <OrderRow>();

            rowTransformation.TransformationFunc = row => {
                row.Quantity = int.Parse(row.Description.Split(":").ElementAt(1));
                return(row);
            };

            var lookup = new LookupTransformation <OrderRow, ExpandoObject>();

            lookup.Source = new CsvSource("files/customer.csv");

            lookup.MatchColumns = new[] {
                new MatchColumn()
                {
                    LookupSourcePropertyName = "Id", InputPropertyName = "CustomerId"
                }
            };
            lookup.RetrieveColumns = new[] {
                new RetrieveColumn()
                {
                    LookupSourcePropertyName = "Name", InputPropertyName = "CustomerName"
                }
            };

            var multicast = new Multicast <OrderRow>();

            var dbDest   = new DbDestination <OrderRow>(sqlConnMan, "orders");
            var textDest = new TextDestination <OrderRow>("files/order_data.log");

            textDest.WriteLineFunc = row => {
                return($"{row.OrderNumber}\t{row.CustomerName}\t{row.Quantity}");
            };

            //Step2 - linking components
            source.LinkTo(rowTransformation);
            rowTransformation.LinkTo(lookup);
            lookup.LinkTo(multicast);
            multicast.LinkTo(dbDest);
            multicast.LinkTo(textDest, row => row.CustomerName == "Clark Kent", row => row.CustomerName != "Clark Kent");

            //Step3 - executing the network
            Network.Execute(source);  //Shortcut for Network.ExecuteAsync(source).Wait();
        }
        public void WithNestedArraysInside()
        {
            //Arrange
            MemoryDestination dest = new MemoryDestination();
            RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
                row =>
            {
                dynamic r = row as ExpandoObject;
                return(r);
            });

            //Act
            JsonSource <ExpandoObject>   source      = new JsonSource <ExpandoObject>("res/JsonSource/NestedArray.json", ResourceType.File);
            List <JsonProperty2JsonPath> pathLookups = new List <JsonProperty2JsonPath>()
            {
                new JsonProperty2JsonPath()
                {
                    JsonPropertyName = "Column2",
                    JsonPath         = "$.[*].ArrayCol1",
                    NewPropertyName  = "ArrayCol1"
                },
                new JsonProperty2JsonPath()
                {
                    JsonPropertyName = "Column2",
                    JsonPath         = "$.[*].ArrayCol2",
                    NewPropertyName  = "ArrayCol2"
                }
            };

            source.JsonSerializer.Converters.Add(new ExpandoJsonPathConverter(pathLookups));

            source.LinkTo(trans).LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.Collection <ExpandoObject>(dest.Data,
                                              row => {
                dynamic r = row as ExpandoObject; var lac1 = r.ArrayCol1 as List <object>; var lac2 = r.ArrayCol2;
                Assert.True(r.Column1 == 1 && lac1.Count == 2 && lac2.Count == 2);
            },
                                              row => {
                dynamic r = row as ExpandoObject; var lac1 = r.ArrayCol1 as List <object>; var lac2 = r.ArrayCol2;
                Assert.True(r.Column1 == 2 && lac1.Count == 2 && lac2.Count == 2);
            },
                                              row => {
                dynamic r = row as ExpandoObject; Assert.True(r.Column1 == 3 && r.ArrayCol1 == "E" && r.ArrayCol2 == "TestE");
            }
                                              );
        }
예제 #19
0
        public void JsonFromWebService()
        {
            //Arrange
            MemoryDestination <Todo> dest = new MemoryDestination <Todo>(200);

            //Act
            JsonSource <Todo> source = new JsonSource <Todo>("https://jsonplaceholder.typicode.com/todos");

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.All(dest.Data, item => Assert.True(item.Key > 0));
        }
예제 #20
0
        public void ArrayInObject()
        {
            //Arrange
            TwoColumnsTableFixture      dest2Columns = new TwoColumnsTableFixture("JsonSourceArrayInObject");
            DbDestination <MySimpleRow> dest         = new DbDestination <MySimpleRow>("JsonSourceArrayInObject", SqlConnection);

            //Act
            JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/ArrayInObject.json", ResourceType.File);

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
        public void JsonWithDecimalType()
        {
            //Arrange
            FourColumnsTableFixture       d4c  = new FourColumnsTableFixture("JsonSource4ColsDynamic");
            DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(Connection, "JsonSource4ColsDynamic");

            //Act
            JsonSource <ExpandoObject> source = new JsonSource <ExpandoObject>("res/JsonSource/FourColumns.json", ResourceType.File);

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            d4c.AssertTestData();
        }
        public void JsonPathInJsonPropertyAttribute()
        {
            //Arrange
            TwoColumnsTableFixture      dest2Columns = new TwoColumnsTableFixture("JsonSourceNested");
            DbDestination <MySimpleRow> dest         = new DbDestination <MySimpleRow>(SqlConnection, "JsonSourceNested");

            //Act
            JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/NestedData.json", ResourceType.File);

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
예제 #23
0
        public void SimpleFlowWithStringArray()
        {
            //Arrange
            TwoColumnsTableFixture   dest2Columns = new TwoColumnsTableFixture("JsonSource2ColsNonGen");
            DbDestination <string[]> dest         = new DbDestination <string[]>(Connection, "JsonSource2ColsNonGen");

            //Act
            JsonSource <string[]> source = new JsonSource <string[]>("res/JsonSource/TwoColumnsStringArray.json", ResourceType.File);

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
예제 #24
0
        public void JsonFromFile()
        {
            //Arrange
            TwoColumnsTableFixture      dest2Columns = new TwoColumnsTableFixture("JsonSource2Cols");
            DBDestination <MySimpleRow> dest         = new DBDestination <MySimpleRow>(Connection, "JsonSource2Cols");

            //Act
            JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/TwoColumns.json", ResourceType.File);

            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            dest2Columns.AssertTestData();
        }
예제 #25
0
        public void WithoutErrorLinking()
        {
            //Arrange
            MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();

            //Act
            JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/TwoColumnsErrorLinking.json", ResourceType.File);

            //Assert
            Assert.Throws <Newtonsoft.Json.JsonReaderException>(() =>
            {
                source.LinkTo(dest);
                source.Execute();
                dest.Wait();
            });
        }
        /// <summary>
        /// Initializes a new instance of the <see cref="JsonConfigurationSource"/> class.
        /// </summary>
        /// <param name="src">The source.</param>
        public JsonConfigurationSource(Stream src)
        {
            if (src == null)
            {
                throw new ArgumentNullException(nameof(src));
            }
            if (!src.CanRead)
            {
                throw new IOException("Cannot read content from the provided stream");
            }

            serializer     = new JsonSerializer();
            source         = JsonSource.Stream;
            stream         = src;
            streamPosition = src.Position;
            sync           = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
        }
예제 #27
0
 private static SyncData ReadLastSyncKey()
 {
     try {
         var syncsource = new JsonSource <SyncData>("LastSyncId.json");
         syncsource.DisableLogging = true;
         var memdest = new MemoryDestination <SyncData>();
         memdest.DisableLogging = true;
         syncsource.LinkTo(memdest);
         Network.Execute(syncsource);
         return(memdest.Data.First());
     } catch {
     }
     return(new SyncData()
     {
         SyncId = -1
     });
 }
예제 #28
0
        /// <summary>
        /// Create the TileSource, that provides the data for the IDrawableTileSource
        /// </summary>
        /// <param name="source"></param>
        /// <returns></returns>
        private static ITileSource CreateTileSource(JsonSource source)
        {
            ITileSource tileSource = null;

            if (source.Tiles == null || source.Tiles.Count == 0)
            {
                return(null);
            }

            if (source.Tiles[0].StartsWith("http"))
            {
                tileSource = new HttpTileSource(new GlobalSphericalMercator(
                                                    source.Scheme == "tms" ? YAxis.TMS : YAxis.OSM,
                                                    minZoomLevel: source.ZoomMin ?? 0,
                                                    maxZoomLevel: source.ZoomMax ?? 30
                                                    ),
                                                source.Tiles[0], //"{s}",
                                                source.Tiles,
                                                name: source.Name,
                                                attribution: new Attribution(source.Attribution)
                                                );
            }
            else if (source.Tiles[0].StartsWith("mbtiles://"))
            {
                // We should get the tile source from someone else
                var filename = source.Tiles[0].Substring(10);
                filename = Path.Combine(DirectoryForFiles, filename);
                if (!File.Exists(filename))
                {
                    return(null);
                }
                tileSource = new MbTilesTileSource(new SQLiteConnectionString(filename, false),
                                                   //new GlobalSphericalMercator(
                                                   //    source.Scheme == "tms" ? YAxis.TMS : YAxis.OSM,
                                                   //    minZoomLevel: source.ZoomMin ?? 0,
                                                   //    maxZoomLevel: source.ZoomMax ?? 30)
                                                   null
                                                   );
            }

            return(tileSource);
        }
예제 #29
0
        static void Main(string[] args)
        {
            StartWebServer();
            JsonSource <Order> source = new JsonSource <Order>("https://www.etlbox.net/demo/api/orders", ResourceType.Http);

            source.HttpClient = CreateDefaultHttpClient();

            ColumnRename <Order> rename = new ColumnRename <Order>();

            rename.RenameColumns = new[]
            {
                new RenameColumn()
                {
                    CurrentName = "Id", NewName = "OrderId"
                },
                new RenameColumn()
                {
                    CurrentName = "CustomerId", NewName = "CId"
                },
                new RenameColumn()
                {
                    CurrentName = "Description", RemoveColumn = true
                }
            };
            JsonDestination destination = new JsonDestination();

            destination.ResourceType = ResourceType.Http;
            destination.HttpClient   = CreateDefaultHttpClient();
            destination.HttpRequestMessage.Method = HttpMethod.Post;
            destination.HasNextUri = (streamMetaData, row) => true;
            destination.GetNextUri = (streamMetaData, row) =>
            {
                streamMetaData.HttpRequestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "Some token");
                return($"http://localhost:61456/post/{streamMetaData.ProgressCount}");
            };

            source.LinkTo(rename);
            rename.LinkTo(destination);
            Network.Execute(source);

            WriteServerLog();
        }
예제 #30
0
        public void JsonFromWebService()
        {
            // Arrange
            HttpClient httpClient = MoqJsonResponse(File.ReadAllText("res/JsonSource/Todos.json"));

            //Arrange
            MemoryDestination <Todo> dest = new MemoryDestination <Todo>();

            //Act
            JsonSource <Todo> source = new JsonSource <Todo>("http://test.com/");

            source.HttpClient = httpClient;
            source.LinkTo(dest);
            source.Execute();
            dest.Wait();

            //Assert
            Assert.All(dest.Data, item => Assert.True(item.Key > 0));
            Assert.Equal(5, dest.Data.Count);
        }