public async Task <PopulationResult> PopulateAsync <TInput>(TInput input)
where TInput : IServiceInput
{
if (MessageMap == null)
{
MessageMap = new MessageProperties("NONE", "Id");
}
PopulationResultCollection rs = await DataCollector.CollectData(this, input);
PopulationResult merged = MergeIntoAggregate(rs);
return(merged);
}
public void TransformationFromMasterListInternalKeytest()
{
FluentOlapConfiguration.TransformationsMasterList
.AddTransformation <int, string>(InternalType.INTEGER, o => $"|{o}|");
var analyzer = new AnalyticalObject <ValueTypeTest>();
analyzer.Property(p => p.x);
var popRs = new PopulationResult(new Dictionary <string, object>()
{
["valuetypetest_x"] = 1
}, analyzer.TypeMap);
Assert.AreEqual(popRs["valuetypetest_x"], "|1|");
}
public void TransformationShadowPropertyTest()
{
var analyzer = new AnalyticalObject <ValueTypeTest>();
var innerAnalyzer = new AnalyticalObject <ValueTypeTestInner>();
innerAnalyzer.Property(x => x.z).HasTransformation(o => o + 2);
analyzer.Property("shadow", innerAnalyzer);
var popRs = new PopulationResult(new Dictionary <string, object>()
{
["valuetypetestinner_z"] = 1
}, analyzer.TypeMap);
Assert.AreEqual(popRs["valuetypetestinner_z"], 3);
}
public async Task Write(DbConnection con, string tableName, PopulationResult populationResult)
{
string columns = string.Join(",", populationResult.Keys);
IList <string> tmpValues = new List <string>();
foreach (object value in populationResult.Values)
{
tmpValues.Add(value != null? $"'{value}'" : "NULL");
}
string values = string.Join(",", tmpValues);
string insert = $"INSERT INTO {tableName} ({columns}) VALUES ({values})";
await con.RunCommandAsync(insert);
}
public void TransformationValueTypeNullTest()
{
var analyzer = new AnalyticalObject <ValueTypeTest>();
analyzer.Property(p => p.x).HasTransformation(o => o + 1);
try
{
var popRs = new PopulationResult(new Dictionary <string, object>()
{
["valuetypetest_x"] = null
}, analyzer.TypeMap);
}
catch (InvalidOperationException e)
{
}
}
public async Task ShadowPropertyInsertionTest()
{
MySqlConnection connection = new MySqlConnection(config.GetConnectionString("TestConnection"));
await connection.RunCommandAsync("DROP TABLE IF EXISTS Post");
await connection.RunCommandAsync("DROP TABLE IF EXISTS AnalyzedModelHashes");
FluentOlapConfiguration.ServiceDefinitions = new ServiceDefinitions
{
["PostsService"] = new HttpService("https://jsonplaceholder.typicode.com/posts/{PostId}"),
["ArbitraryUserService"] = new HttpService("https://jsonplaceholder.typicode.com/users/1"),
};
PostAnalyzer analyzer = new PostAnalyzer();
analyzer.GetFromService("PostsService");
analyzer.Property("arbitaryUser", new AnalyticalObject <User>())
.GetFromService("ArbitraryUserService", new AnalyticalObject <User>());
PopulationResult result = await analyzer.PopulateAsync(new HttpServiceOptions
{
Parameters = new
{
PostId = 1
}
});
DataIngester ingester = new DataIngester(new MariaDbProvider(MariaDbTableEngine.InnoDB, new Dictionary <InternalType, string>()
{
[InternalType.STRING] = "TEXT",
[InternalType.INTEGER] = "BIGINT",
[InternalType.FLOAT] = "FLOAT",
[InternalType.BOOLEAN] = "BOOLEAN",
[InternalType.DATETIME] = "DATETIME"
}));
await ingester.Insert(result, connection);
string username = await connection.RunCommandGetString("select * from Post", "user_username");
Assert.AreEqual(username, "Bret");
await connection.RunCommandAsync("DROP TABLE Post");
await connection.RunCommandAsync("DROP TABLE AnalyzedModelHashes");
}
private PopulationResult MergeIntoAggregate(PopulationResultCollection resultCollection)
{
IDictionary <string, object> merged = new Dictionary <string, object>();
PopulationResult root = resultCollection.Dequeue();
foreach (KeyValuePair <string, object> kv in root)
{
merged.Add(kv);
}
for (int _ = 0; _ < resultCollection.Count; ++_)
{
PopulationResult current = resultCollection.Dequeue();
foreach (KeyValuePair <string, object> kv in current)
{
merged.Add(kv);
}
}
return(new PopulationResult(merged));
}
public async Task BasicPopulationTest()
{
FluentOlapConfiguration.ServiceDefinitions = new ServiceDefinitions
{
["PostsService"] = new HttpService("https://jsonplaceholder.typicode.com/posts/{PostId}")
};
PostAnalyzer analyzer = new PostAnalyzer();
analyzer.ServiceName = "PostsService";
PopulationResult rs = await analyzer.PopulateAsync(new HttpServiceOptions
{
Parameters = new
{
PostId = 1
}
}
);
foreach (string key in rs.Keys)
{
Assert.IsTrue(analyzer.TypeMap.Keys.Count >= rs.Keys.Count());
}
}
public async Task ChangingSchemaTest()
{
MySqlConnection connection = new MySqlConnection(config.GetConnectionString("TestConnection"));
await connection.RunCommandAsync("DROP TABLE IF EXISTS Post");
await connection.RunCommandAsync("DROP TABLE IF EXISTS AnalyzedModelHashes");
FluentOlapConfiguration.ServiceDefinitions = new ServiceDefinitions
{
["PostsService"] = new HttpService("https://jsonplaceholder.typicode.com/posts/{PostId}"),
["UsersService"] = new HttpService("https://jsonplaceholder.typicode.com/users/{userId}"),
};
PostAnalyzer analyzer = new PostAnalyzer();
analyzer.GetFromService("PostsService");
analyzer.Property(p => p.userId).GetFromService("UsersService", new AnalyticalObject <User>());
analyzer.Remove(p => p.Body);
PopulationResult result = await analyzer.PopulateAsync(new HttpServiceOptions
{
Parameters = new
{
PostId = 1
}
});
DataIngester ingester = new DataIngester(new MariaDbProvider(MariaDbTableEngine.InnoDB, new Dictionary <InternalType, string>()
{
[InternalType.STRING] = "TEXT",
[InternalType.INTEGER] = "BIGINT",
[InternalType.FLOAT] = "FLOAT",
[InternalType.BOOLEAN] = "BOOLEAN",
[InternalType.DATETIME] = "DATETIME"
}));
await ingester.Insert(result, connection);
string username = await connection.RunCommandGetString("select * from Post", "user_username");
Assert.AreEqual(username, "Bret");
analyzer.Property(p => p.Body);
PopulationResult result2 = await analyzer.PopulateAsync(new HttpServiceOptions
{
Parameters = new
{
PostId = 1
}
});
await ingester.Insert(result2, connection);
string body = await connection.RunCommandGetString("select * from Post where Id=2", "post_body");
string bodyFromApi =
"quia et suscipit\nsuscipit recusandae consequuntur expedita et cum\nreprehenderit molestiae ut ut quas totam\nnostrum rerum est autem sunt rem eveniet architecto";
Assert.AreEqual(body, bodyFromApi);
await connection.RunCommandAsync("DROP TABLE Post");
await connection.RunCommandAsync("DROP TABLE AnalyzedModelHashes");
}
public static async Task <PopulationResultCollection> CollectData <T>(AnalyticalObject <T> focusedObject,
IServiceInput serviceInput)
{
if (focusedObject.ServiceName == null)
{
throw new Exception("Root analyzer object must have a service defined.");
}
if (!FluentOlapConfiguration.ServiceDefinitions.ContainsKey(focusedObject.ServiceName))
{
throw new Exception($"Service {focusedObject.ServiceName} not defined.");
}
IService <IServiceInput, IServiceOutput> focusedService = FluentOlapConfiguration.ServiceDefinitions[focusedObject.ServiceName];
PopulationResultCollection results = new PopulationResultCollection();
PopulationResult rootResult = await CallService(focusedService, serviceInput);
results.Add(rootResult);
foreach (var expandable in focusedObject.ExpandableChildren)
{
if (expandable.Value.ServiceName == null)
{
throw new Exception("Child analyzer object must have a service defined.");
}
if (!FluentOlapConfiguration.ServiceDefinitions.ContainsKey(expandable.Value.ServiceName))
{
throw new Exception($"Service {expandable.Value.ServiceName} not defined.");
}
IService <IServiceInput, IServiceOutput> expandableService = FluentOlapConfiguration.ServiceDefinitions[expandable.Value.ServiceName];
IServiceInput input = null;
switch (expandableService.Type)
{
case ServiceType.HttpCall:
HttpService expandableHttpService = expandableService as HttpService;
IDictionary <string, string> parameters = new Dictionary <string, string>();
foreach (string innerParam in expandableHttpService.GetRequiredParameters())
{
JToken parsedRoot = JToken.Parse(rootResult.Raw);
parameters.Add(innerParam, parsedRoot[innerParam].Value <string>());
}
input = new HttpServiceOptions
{
PrefixKey = expandable.Value.NodeName,
Parameters = parameters
};
await expandableHttpService.InvokeAsync((HttpServiceOptions)input);
break;
}
results.Add(await CallService(expandableService, input));
}
return(results);
}
private void FindLastTrackedNodesByLabelAndProperties(GraphDatabaseService db, PopulationResult populationResult)
{
using (Transaction ignored = Db.beginTx())
{
Node nodeByUniqueStringProperty = Db.findNode(Label.label(LABEL), UniqueStringProperty, populationResult.MaxPropertyId + "");
Node nodeByStringProperty = Db.findNode(Label.label(LABEL), StringProperty, populationResult.MaxPropertyId + "");
assertNotNull("Should find last inserted node", nodeByStringProperty);
assertEquals("Both nodes should be the same last inserted node", nodeByStringProperty, nodeByUniqueStringProperty);
Node nodeByUniqueLongProperty = Db.findNode(Label.label(LABEL), UniqueLongProperty, populationResult.MaxPropertyId);
Node nodeByLongProperty = Db.findNode(Label.label(LABEL), LongProperty, populationResult.MaxPropertyId);
assertNotNull("Should find last inserted node", nodeByLongProperty);
assertEquals("Both nodes should be the same last inserted node", nodeByLongProperty, nodeByUniqueLongProperty);
}
}
public async Task InsertIntoDb(PopulationResult rs, DbConnection con)
{
await base.Insert(rs, con);
}
/// <summary>
/// Insuring that the existing schemas in the DB are ready to be inserted intp
/// </summary>
/// <param name="rs"></param>
/// <param name="con"></param>
/// <param name="sqlProvider"></param>
/// <returns></returns>
protected async Task EnsureConsistency(PopulationResult rs, DbConnection con)
{
await base.EnsureScheme(rs, con);
}
