public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("Received a json file to transform.");
string filename = Guid.NewGuid().ToString() + ".csv";
log.LogInformation("Temp file used to hold data: " + filename);
var jsonSource = new JsonSource()
{
CreateStreamReader = _ => new StreamReader(req.Body)
};
var csvFileDest = new CsvDestination(filename);
jsonSource.LinkTo(csvFileDest);
await Network.ExecuteAsync(jsonSource);
log.LogInformation("Successfully stored data in file - now returning the content as response.");
return(new HttpResponseMessage()
{
StatusCode = HttpStatusCode.OK,
Content = new StreamContent(new FileStream(filename, FileMode.Open))
});
}
protected override void OnNavigatedTo(NavigationEventArgs e) //загрузка данных
{
CurrencyContainer.del += Downloaded;
IDataSource jsonSource = new JsonSource();
jsonSource.GetCurrencyList(CurrencyContainer.del); //передача в качестве параметра делегата перехода окна
}
public void SourceWithDifferentNames()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("JsonSource2ColsDynamic");
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
row =>
{
dynamic r = row as ExpandoObject;
r.Col1 = r.Column1;
r.Col2 = r.Column2;
return(r);
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>("JsonSource2ColsDynamic", Connection);
//Act
JsonSource <ExpandoObject> source = new JsonSource <ExpandoObject>("res/JsonSource/TwoColumnsDifferentNames.json", ResourceType.File);
var linkTo1 = source.LinkTo(trans);
var linkTo2 = linkTo1.source.LinkTo(dest);
using (linkTo1.link)
using (linkTo2.link)
{
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="JsonConfigurationSource"/> class.
/// </summary>
/// <param name="filename">The full path to the document.</param>
public JsonConfigurationSource(string filename)
{
path = new FileInfo(filename ?? throw new ArgumentNullException(nameof(filename)));
serializer = new JsonSerializer();
source = JsonSource.File;
sync = new ReaderWriterLockSlim();
}
void UpdateData()
{
DataSource = new JsonDataSource()
{
Source = JsonSource.FromUri(new Uri("https://s3.amazonaws.com/utdmakerspace/events.json"))
};
}
public void JsonPathInEpandoObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("JsonSourceNestedDynamic");
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
row =>
{
dynamic r = row as ExpandoObject;
r.Col1 = r.Column1;
return(r);
});
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(SqlConnection, "JsonSourceNestedDynamic");
//Act
JsonSource <ExpandoObject> source = new JsonSource <ExpandoObject>("res/JsonSource/NestedData.json", ResourceType.File);
List <JsonProperty2JsonPath> pathLookups = new List <JsonProperty2JsonPath>()
{
new JsonProperty2JsonPath()
{
JsonPropertyName = "Column2",
JsonPath = "Value",
NewPropertyName = "Col2"
}
};
source.JsonSerializer.Converters.Add(new ExpandoJsonPathConverter(pathLookups));
source.LinkTo(trans).LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void JsonSource_FullLoadTest()
{
var source = new JsonSource("./JsonTestFile_First.json", InterCommunicator.Empty);
var fired = source.Rows.Count();
Assert.AreEqual(3, fired);
}
public void JsonSource_FullLoadTest()
{
var source = new JsonSource("./JsonTestFile_First.json", RuntimeContext.Empty);
var fired = source.Rows.Count();
Assert.AreEqual(3, fired);
}
public GenericListDataPage(string dataSource)
{
DataSource = new JsonDataSource()
{
Source = JsonSource.FromUri(new Uri(dataSource))
};
InitializeComponent();
}
protected override void Setup()
{
_dbClientMock = new Mock <IDbClient>();
var dbFactoryMock = new Mock <IDbClientFactory>();
dbFactoryMock.Setup(x => x.GetJsonClient(It.IsAny <string>())).Returns(_dbClientMock.Object);
_source = new JsonSource(dbFactoryMock.Object);
}
public void JsonSource_CancelledLoadTest()
{
var tokenSource = new CancellationTokenSource();
tokenSource.Cancel();
var source = new JsonSource("./JsonTestFile_First.json", new InterCommunicator(tokenSource.Token));
var fired = source.Rows.Count();
Assert.AreEqual(0, fired);
}
public void JsonFromWebService()
{
JsonSource source = new JsonSource("res/JsonSource/EmptyObject.json", ResourceType.File);
MemoryDestination dest = new MemoryDestination();
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.True(dest.Data.Count == 0);
}
public ResQuery(string name, string query = null, string desc = null, object parameters = null, bool inSharedZip = false,
ResFilType fileType = ResFilType.Csv, JsonSource jsonSource = default)
{
Name = name;
Query = query;
Desc = desc;
Parameters = parameters;
InSharedZip = inSharedZip;
FileType = fileType;
JsonSource = jsonSource;
}
public void JsonSource_CancelledLoadTest()
{
var tokenSource = new CancellationTokenSource();
tokenSource.Cancel();
var source = new JsonSource("./JsonTestFile_First.json", new RuntimeContext(tokenSource.Token, new ISchemaColumn[0]));
var fired = source.Rows.Count();
Assert.AreEqual(0, fired);
}
public void ReadEmptyArray()
{
JsonSource source = new JsonSource("res/JsonSource/EmptyArray.json", ResourceType.File);
MemoryDestination dest = new MemoryDestination();
source.LinkTo(dest);
source.Execute();
dest.Wait();
Assert.True(dest.Data.Count == 0);
}
public MyConfig() : base(defaultSourceName: nameof(ConfigSourceNames.Combined))
{
var configrSource = new ConfigRSource(@"./config.csx",
autoReloadOnFileChange: true);
var jsonSource = new JsonSource("./config.json", true);
this.UseCombinedSource("Combined",
false,
new BaseConfigurationSource[] { configrSource, jsonSource })
.UseAppSettings()
.UseEnvironmentVariable("CONFYGURE_OUT_");
}
/* Data flow
*
* JsonSource --> RowTransformation --> Lookup --> Multicast --> DbDestination ("orders" table)
* (Order data) | |
* CsvSource <---- --------> TextDestination ("order_data.log")
* ("customer.csv")
*/
static void Main(string[] args)
{
//Preparation
RecreateTargetTable();
//Step 1 - creating the components
var source = new JsonSource <OrderRow>("https://www.etlbox.net/demo/api/orders", ResourceType.Http);
var rowTransformation = new RowTransformation <OrderRow>();
rowTransformation.TransformationFunc = row => {
row.Quantity = int.Parse(row.Description.Split(":").ElementAt(1));
return(row);
};
var lookup = new LookupTransformation <OrderRow, ExpandoObject>();
lookup.Source = new CsvSource("files/customer.csv");
lookup.MatchColumns = new[] {
new MatchColumn()
{
LookupSourcePropertyName = "Id", InputPropertyName = "CustomerId"
}
};
lookup.RetrieveColumns = new[] {
new RetrieveColumn()
{
LookupSourcePropertyName = "Name", InputPropertyName = "CustomerName"
}
};
var multicast = new Multicast <OrderRow>();
var dbDest = new DbDestination <OrderRow>(sqlConnMan, "orders");
var textDest = new TextDestination <OrderRow>("files/order_data.log");
textDest.WriteLineFunc = row => {
return($"{row.OrderNumber}\t{row.CustomerName}\t{row.Quantity}");
};
//Step2 - linking components
source.LinkTo(rowTransformation);
rowTransformation.LinkTo(lookup);
lookup.LinkTo(multicast);
multicast.LinkTo(dbDest);
multicast.LinkTo(textDest, row => row.CustomerName == "Clark Kent", row => row.CustomerName != "Clark Kent");
//Step3 - executing the network
Network.Execute(source); //Shortcut for Network.ExecuteAsync(source).Wait();
}
public void WithNestedArraysInside()
{
//Arrange
MemoryDestination dest = new MemoryDestination();
RowTransformation <ExpandoObject> trans = new RowTransformation <ExpandoObject>(
row =>
{
dynamic r = row as ExpandoObject;
return(r);
});
//Act
JsonSource <ExpandoObject> source = new JsonSource <ExpandoObject>("res/JsonSource/NestedArray.json", ResourceType.File);
List <JsonProperty2JsonPath> pathLookups = new List <JsonProperty2JsonPath>()
{
new JsonProperty2JsonPath()
{
JsonPropertyName = "Column2",
JsonPath = "$.[*].ArrayCol1",
NewPropertyName = "ArrayCol1"
},
new JsonProperty2JsonPath()
{
JsonPropertyName = "Column2",
JsonPath = "$.[*].ArrayCol2",
NewPropertyName = "ArrayCol2"
}
};
source.JsonSerializer.Converters.Add(new ExpandoJsonPathConverter(pathLookups));
source.LinkTo(trans).LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.Collection <ExpandoObject>(dest.Data,
row => {
dynamic r = row as ExpandoObject; var lac1 = r.ArrayCol1 as List <object>; var lac2 = r.ArrayCol2;
Assert.True(r.Column1 == 1 && lac1.Count == 2 && lac2.Count == 2);
},
row => {
dynamic r = row as ExpandoObject; var lac1 = r.ArrayCol1 as List <object>; var lac2 = r.ArrayCol2;
Assert.True(r.Column1 == 2 && lac1.Count == 2 && lac2.Count == 2);
},
row => {
dynamic r = row as ExpandoObject; Assert.True(r.Column1 == 3 && r.ArrayCol1 == "E" && r.ArrayCol2 == "TestE");
}
);
}
public void JsonFromWebService()
{
//Arrange
MemoryDestination <Todo> dest = new MemoryDestination <Todo>(200);
//Act
JsonSource <Todo> source = new JsonSource <Todo>("https://jsonplaceholder.typicode.com/todos");
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.All(dest.Data, item => Assert.True(item.Key > 0));
}
public void ArrayInObject()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("JsonSourceArrayInObject");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>("JsonSourceArrayInObject", SqlConnection);
//Act
JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/ArrayInObject.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void JsonWithDecimalType()
{
//Arrange
FourColumnsTableFixture d4c = new FourColumnsTableFixture("JsonSource4ColsDynamic");
DbDestination <ExpandoObject> dest = new DbDestination <ExpandoObject>(Connection, "JsonSource4ColsDynamic");
//Act
JsonSource <ExpandoObject> source = new JsonSource <ExpandoObject>("res/JsonSource/FourColumns.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
d4c.AssertTestData();
}
public void JsonPathInJsonPropertyAttribute()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("JsonSourceNested");
DbDestination <MySimpleRow> dest = new DbDestination <MySimpleRow>(SqlConnection, "JsonSourceNested");
//Act
JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/NestedData.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void SimpleFlowWithStringArray()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("JsonSource2ColsNonGen");
DbDestination <string[]> dest = new DbDestination <string[]>(Connection, "JsonSource2ColsNonGen");
//Act
JsonSource <string[]> source = new JsonSource <string[]>("res/JsonSource/TwoColumnsStringArray.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void JsonFromFile()
{
//Arrange
TwoColumnsTableFixture dest2Columns = new TwoColumnsTableFixture("JsonSource2Cols");
DBDestination <MySimpleRow> dest = new DBDestination <MySimpleRow>(Connection, "JsonSource2Cols");
//Act
JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/TwoColumns.json", ResourceType.File);
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
dest2Columns.AssertTestData();
}
public void WithoutErrorLinking()
{
//Arrange
MemoryDestination <MySimpleRow> dest = new MemoryDestination <MySimpleRow>();
//Act
JsonSource <MySimpleRow> source = new JsonSource <MySimpleRow>("res/JsonSource/TwoColumnsErrorLinking.json", ResourceType.File);
//Assert
Assert.Throws <Newtonsoft.Json.JsonReaderException>(() =>
{
source.LinkTo(dest);
source.Execute();
dest.Wait();
});
}
/// <summary>
/// Initializes a new instance of the <see cref="JsonConfigurationSource"/> class.
/// </summary>
/// <param name="src">The source.</param>
public JsonConfigurationSource(Stream src)
{
if (src == null)
{
throw new ArgumentNullException(nameof(src));
}
if (!src.CanRead)
{
throw new IOException("Cannot read content from the provided stream");
}
serializer = new JsonSerializer();
source = JsonSource.Stream;
stream = src;
streamPosition = src.Position;
sync = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
}
private static SyncData ReadLastSyncKey()
{
try {
var syncsource = new JsonSource <SyncData>("LastSyncId.json");
syncsource.DisableLogging = true;
var memdest = new MemoryDestination <SyncData>();
memdest.DisableLogging = true;
syncsource.LinkTo(memdest);
Network.Execute(syncsource);
return(memdest.Data.First());
} catch {
}
return(new SyncData()
{
SyncId = -1
});
}
/// <summary>
/// Create the TileSource, that provides the data for the IDrawableTileSource
/// </summary>
/// <param name="source"></param>
/// <returns></returns>
private static ITileSource CreateTileSource(JsonSource source)
{
ITileSource tileSource = null;
if (source.Tiles == null || source.Tiles.Count == 0)
{
return(null);
}
if (source.Tiles[0].StartsWith("http"))
{
tileSource = new HttpTileSource(new GlobalSphericalMercator(
source.Scheme == "tms" ? YAxis.TMS : YAxis.OSM,
minZoomLevel: source.ZoomMin ?? 0,
maxZoomLevel: source.ZoomMax ?? 30
),
source.Tiles[0], //"{s}",
source.Tiles,
name: source.Name,
attribution: new Attribution(source.Attribution)
);
}
else if (source.Tiles[0].StartsWith("mbtiles://"))
{
// We should get the tile source from someone else
var filename = source.Tiles[0].Substring(10);
filename = Path.Combine(DirectoryForFiles, filename);
if (!File.Exists(filename))
{
return(null);
}
tileSource = new MbTilesTileSource(new SQLiteConnectionString(filename, false),
//new GlobalSphericalMercator(
// source.Scheme == "tms" ? YAxis.TMS : YAxis.OSM,
// minZoomLevel: source.ZoomMin ?? 0,
// maxZoomLevel: source.ZoomMax ?? 30)
null
);
}
return(tileSource);
}
static void Main(string[] args)
{
StartWebServer();
JsonSource <Order> source = new JsonSource <Order>("https://www.etlbox.net/demo/api/orders", ResourceType.Http);
source.HttpClient = CreateDefaultHttpClient();
ColumnRename <Order> rename = new ColumnRename <Order>();
rename.RenameColumns = new[]
{
new RenameColumn()
{
CurrentName = "Id", NewName = "OrderId"
},
new RenameColumn()
{
CurrentName = "CustomerId", NewName = "CId"
},
new RenameColumn()
{
CurrentName = "Description", RemoveColumn = true
}
};
JsonDestination destination = new JsonDestination();
destination.ResourceType = ResourceType.Http;
destination.HttpClient = CreateDefaultHttpClient();
destination.HttpRequestMessage.Method = HttpMethod.Post;
destination.HasNextUri = (streamMetaData, row) => true;
destination.GetNextUri = (streamMetaData, row) =>
{
streamMetaData.HttpRequestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "Some token");
return($"http://localhost:61456/post/{streamMetaData.ProgressCount}");
};
source.LinkTo(rename);
rename.LinkTo(destination);
Network.Execute(source);
WriteServerLog();
}
public void JsonFromWebService()
{
// Arrange
HttpClient httpClient = MoqJsonResponse(File.ReadAllText("res/JsonSource/Todos.json"));
//Arrange
MemoryDestination <Todo> dest = new MemoryDestination <Todo>();
//Act
JsonSource <Todo> source = new JsonSource <Todo>("http://test.com/");
source.HttpClient = httpClient;
source.LinkTo(dest);
source.Execute();
dest.Wait();
//Assert
Assert.All(dest.Data, item => Assert.True(item.Key > 0));
Assert.Equal(5, dest.Data.Count);
}
