コード例 #1
0
    public async Task ExportItemsAsync(TextReader itemsDumpReader, LuaModuleFactory moduleFactory)
    {
        if (itemsDumpReader == null)
        {
            throw new ArgumentNullException(nameof(itemsDumpReader));
        }
        if (moduleFactory == null)
        {
            throw new ArgumentNullException(nameof(moduleFactory));
        }
        var languages = new List <string>(Languages ?? defaultLanguages);
        int items = 0, properties = 0;
        var statusReportSw = Stopwatch.StartNew();

        foreach (var entity in SerializableEntity.LoadAll(itemsDumpReader))
        {
            if (entity.Type == EntityType.Item)
            {
                items++;
            }
            else if (entity.Type == EntityType.Property)
            {
                properties++;
            }

            // Preprocess
            entity.Labels       = FilterMonolingualTexts(entity.Labels, languages);
            entity.Descriptions = FilterMonolingualTexts(entity.Descriptions, languages);
            entity.Aliases      = FilterMonolingualTexts(entity.Aliases, languages);

            // Persist
            using (var module = moduleFactory.GetModule(entity.Id))
            {
                using (var writer = module.Writer)
                {
                    WriteProlog(writer, $"Entity: {entity.Id} ({entity.Labels["en"]})");
                    using (var luawriter = new JsonLuaWriter(writer)
                    {
                        CloseOutput = false
                    })
                    {
                        entity.WriteTo(luawriter);
                    }

                    WriteEpilog(writer);
                }

                await module.SubmitAsync($"Export entity {entity.Id}.");
            }

            if (statusReportSw.Elapsed > StatusReportInterval)
            {
                statusReportSw.Restart();
                Logger.Information("Exported LUA modules for {Items} items and {Properties} properties.", items, properties);
            }
        }
        Logger.Information("Exported LUA modules for {Items} items and {Properties} properties.", items, properties);
    }
コード例 #2
0
    public async Task ExportSiteLinksAsync(TextReader itemsDumpReader, LuaModuleFactory moduleFactory, int shardCount)
    {
        if (itemsDumpReader == null)
        {
            throw new ArgumentNullException(nameof(itemsDumpReader));
        }
        if (moduleFactory == null)
        {
            throw new ArgumentNullException(nameof(moduleFactory));
        }
        if (shardCount <= 0)
        {
            throw new ArgumentOutOfRangeException(nameof(shardCount));
        }
        if (ClientSiteName == null)
        {
            throw new ArgumentNullException(nameof(ClientSiteName));
        }

        var shards = Enumerable.Range(0, shardCount).Select(index =>
        {
            var module = moduleFactory.GetModule(index.ToString());
            WriteProlog(module.Writer, $"Shard: {index + 1}/{shardCount}");
            return(module);
        }).ToList();
        var shardLuaWriters = shards.Select(m =>
                                            new LuaTableTextWriter(m.Writer)
        {
            CloseWriter = false, Formatting = Formatting.Prettified
        })
                              .ToList();

        foreach (var writer in shardLuaWriters)
        {
            writer.WriteStartTable();
        }
        try
        {
            foreach (var entity in SerializableEntity.LoadAll(itemsDumpReader))
            {
                var siteLink = entity.SiteLinks.FirstOrDefault(l => l.Site == ClientSiteName);
                if (siteLink == null)
                {
                    continue;
                }
                var shardIndex = Utility.HashString(siteLink.Title) % shardCount;
                var writer     = shardLuaWriters[shardIndex];
                writer.WriteKey(siteLink.Title);
                writer.WriteLiteral(entity.Id);
            }

            Logger.Information("Exporting LUA modules. Shards = {Shards}", shards.Count);
            for (var i = 0; i < shards.Count; i++)
            {
                shardLuaWriters[i].WriteEndTable();
                shardLuaWriters[i].Close();
                WriteEpilog(shards[i].Writer);
                await shards[i].SubmitAsync($"Export SiteLink table. Shard {i + 1}/{shards.Count}.");
            }
        }
        finally
        {
            foreach (var s in shards)
            {
                s.Dispose();
            }
        }
    }
コード例 #3
0
 public AotSparqlModuleExporter(ILogger logger, LuaModuleFactory moduleFactory, AotSparqlExecutor executor)
 {
     this.moduleFactory = moduleFactory;
     this.executor      = executor;
     Logger             = logger ?? throw new ArgumentNullException(nameof(logger));
 }