public async Task ExportItemsAsync(TextReader itemsDumpReader, LuaModuleFactory moduleFactory)
{
if (itemsDumpReader == null)
{
throw new ArgumentNullException(nameof(itemsDumpReader));
}
if (moduleFactory == null)
{
throw new ArgumentNullException(nameof(moduleFactory));
}
var languages = new List <string>(Languages ?? defaultLanguages);
int items = 0, properties = 0;
var statusReportSw = Stopwatch.StartNew();
foreach (var entity in SerializableEntity.LoadAll(itemsDumpReader))
{
if (entity.Type == EntityType.Item)
{
items++;
}
else if (entity.Type == EntityType.Property)
{
properties++;
}
// Preprocess
entity.Labels = FilterMonolingualTexts(entity.Labels, languages);
entity.Descriptions = FilterMonolingualTexts(entity.Descriptions, languages);
entity.Aliases = FilterMonolingualTexts(entity.Aliases, languages);
// Persist
using (var module = moduleFactory.GetModule(entity.Id))
{
using (var writer = module.Writer)
{
WriteProlog(writer, $"Entity: {entity.Id} ({entity.Labels["en"]})");
using (var luawriter = new JsonLuaWriter(writer)
{
CloseOutput = false
})
{
entity.WriteTo(luawriter);
}
WriteEpilog(writer);
}
await module.SubmitAsync($"Export entity {entity.Id}.");
}
if (statusReportSw.Elapsed > StatusReportInterval)
{
statusReportSw.Restart();
Logger.Information("Exported LUA modules for {Items} items and {Properties} properties.", items, properties);
}
}
Logger.Information("Exported LUA modules for {Items} items and {Properties} properties.", items, properties);
}
public void SerializationTest()
{
var serializer = new JsonSerializer();
var obj = new
{
IntValue = 123,
FloatValue = 456.78,
StringValue = "test",
Value1 = (object)null,
Value2 = new
{
Value21 = 12345.67M
},
Value3 = new Dictionary <string, int>
{
{ "A", 1 },
{ "B", 2 },
{ "C", 3 },
{ "while", 4 }
}
};
using (var sw = new StringWriter())
{
using (var lw = new JsonLuaWriter(sw)
{
CloseOutput = false
})
serializer.Serialize(lw, obj, typeof(object));
Assert.Equal(
"{IntValue=123,FloatValue=456.78,StringValue=\"test\",Value1=nil,Value2={Value21=12345.67},Value3={A=1,B=2,C=3,[\"while\"]=4}}",
sw.ToString());
}
using (var sw = new StringWriter())
{
using (var lw = new JsonLuaWriter(sw)
{
Formatting = Formatting.Indented, CloseOutput = false
})
serializer.Serialize(lw, obj, typeof(object));
Assert.Equal(@"{
IntValue = 123,
FloatValue = 456.78,
StringValue = ""test"",
Value1 = nil,
Value2 = {
Value21 = 12345.67
},
Value3 = {
A = 1,
B = 2,
C = 3,
[""while""] = 4
}
}".Replace("\r\n", sw.NewLine), sw.ToString());
}
}
public async Task ExportModulesAsync()
{
if (SiteConfig == null)
{
throw new ArgumentNullException(nameof(SiteConfig));
}
var clusterDict = new ConcurrentDictionary <string, AotSparqlModuleRoot>();
var queryCounter = 0;
foreach (var(queryName, queryContent) in SiteConfig.Queries)
{
queryCounter++;
Logger.Information("Processing query {Counter}/{Total}: {Name}.",
queryCounter, SiteConfig.Queries.Count, queryName);
try
{
// Execute params query first.
var paramsResult = executor.Execute(queryContent.ParamsQuery);
var clusterVariable = SparqlQuery.ParamsQueryParamPrefix + queryContent.ClusteredBy;
if (!paramsResult.Variables.Contains(clusterVariable))
{
throw new InvalidOperationException(
$"Specified clustering variable {clusterVariable} does not exist in the result set from ParamsQuery.");
}
Logger.Information("Parameter query returned {Count} results.", paramsResult.Count);
if (paramsResult.Count == 0)
{
continue;
}
var paramNames = paramsResult.Variables
.Where(v => v.StartsWith(SparqlQuery.ParamsQueryParamPrefix))
.Select(v => (ParamName: v.Substring(SparqlQuery.ParamsQueryParamPrefix.Length), ResultName: v))
.ToList();
var pivotParamNames = paramNames.Where(p => p.ParamName != queryContent.ClusteredBy).ToList();
var resultVariables = executor.GetResultVariables(queryContent.SourceQuery, paramNames.Select(n => n.ParamName));
int minResultRows = int.MaxValue, maxResultRows = -1;
var paramSetCounter = 0;
// Traverse the params query result set.
foreach (var row in paramsResult)
{
paramSetCounter++;
Logger.Verbose("Processing param set {Counter}/{Total}.", paramSetCounter, paramsResult.Count);
var clusterNode = row[clusterVariable];
var clusterKey = SerializeClusterKey(clusterNode);
var queryParams = paramNames.ToDictionary(p => p.ParamName, p => row.Value(p.ResultName));
var cluster = clusterDict.GetOrAdd(clusterKey,
k => new AotSparqlModuleRoot
{
ResultSets = new SortedDictionary <string, AotSparqlQueryResultSet>()
});
AotSparqlQueryResultSet resultSet;
lock (cluster)
{
if (!cluster.ResultSets.TryGetValue(queryName, out resultSet))
{
resultSet = new AotSparqlQueryResultSet {
Results = new List <AotSparqlQueryResult>()
};
if (pivotParamNames.Count > 0)
{
resultSet.PivotParams = pivotParamNames.Select(p => p.ParamName).ToList();
}
resultSet.Columns = resultVariables;
cluster.ResultSets.Add(queryName, resultSet);
}
}
var queryResult = executor.ExecuteAndSerialize(queryContent.SourceQuery, resultVariables, queryParams);
minResultRows = Math.Min(minResultRows, queryResult.Rows.Count);
maxResultRows = Math.Max(maxResultRows, queryResult.Rows.Count);
lock (cluster)
{
if (pivotParamNames.Count > 0)
{
queryResult.PivotValues = pivotParamNames
.Select(p => executor.SerializeNode(queryParams[p.ParamName]))
.ToList();
}
resultSet.Results.Add(queryResult);
}
}
if (maxResultRows < 0)
{
Debug.Assert(minResultRows == int.MaxValue);
minResultRows = -1;
}
Logger.Information("Executed query {QueryName} with {ParamRows} param sets. Source query results: min: {MinResults}, max: {MaxResults}.",
queryName, paramsResult.Count, minResultRows, maxResultRows);
}
catch (Exception ex)
{
Logger.Error(ex, "Failed to execute query {QueryName}.", queryName);
throw;
}
}
Logger.Information("Normalize clustered modules…");
// Sort result rows to ensure we only update modules when we need to.
foreach (var root in clusterDict.Values)
{
foreach (var resultSet in root.ResultSets.Values)
{
var results = (List <AotSparqlQueryResult>)resultSet.Results;
results.Sort((x, y) => SequenceComparer <object> .Default.Compare(x.PivotValues, y.PivotValues));
foreach (var result in results)
{
var rows = (List <IList <object> >)result.Rows;
rows.Sort(SequenceComparer <object> .Default);
}
}
}
Logger.Information("Writing {Count} clustered modules…", clusterDict.Count);
var statusReportSw = Stopwatch.StartNew();
var writtenCount = 0;
foreach (var(name, root) in clusterDict)
{
using (var module = moduleFactory.GetModule(name))
{
WriteProlog(module.Writer, "Cluster key: " + name);
using (var jwriter = new JsonLuaWriter(module.Writer)
{
CloseOutput = false
})
{
luaModuleJsonSerializer.Serialize(jwriter, root);
}
WriteEpilog(module.Writer);
await module.SubmitAsync("Export clustered SPARQL query result for " + name + ".");
writtenCount++;
if (statusReportSw.Elapsed >= StatusReportInterval)
{
Logger.Information("Written {Count}/{Total} clustered modules.", writtenCount, clusterDict.Count);
}
statusReportSw.Restart();
}
}
Logger.Information("Written {Count}/{Total} clustered modules.", writtenCount, clusterDict.Count);
}
