static void Main(string[] _args)
{
Console.WriteLine("Rozhodnuti-UOHS \n");
Devmasters.Args args = new Devmasters.Args(_args, new[] { "/apiKey" });
logger.Info("Starting with params" + string.Join(" ", args.Arguments));
if (args.MandatoryPresent() == false)
{
Console.WriteLine("/apikey=xxx [/start=] [/new] [/num=]");
return;
}
debug = args.Exists("/debug");
InitDS(args.Exists("/new"), args["/apiKey"]);
httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.Add("Authorization", args["/apiKey"]);
//check last Rozhodnuti
var res = ds.Search("", 1, "DbCreated", true);
int lastId = 0;
if (res.Total > 0)
{
lastId = Convert.ToInt32(res.Results.First().Id);
}
int num = args.GetNumber("/num", 1500).Value;
var start = args.GetNumber("/start", 1500);
if (start.HasValue)
{
lastId = start.Value;
}
if (lastId == 0)
{
num = 90000;
}
ParsePages(datasetId, lastId, num); //stahnuti, parsovani dat z UOHS a vlozeni do Datasetu
}
static void Main(string[] arguments)
{
Console.WriteLine($"Jednání-Rady-ČT - {System.Reflection.Assembly.GetEntryAssembly().GetName().Version}");
Devmasters.Logging.Logger.Root.Info($"Jednání-Rady-ČT - {System.Reflection.Assembly.GetEntryAssembly().GetName().Version}");
Devmasters.Logging.Logger.Root.Debug("Jednání Rady ČT starting with " + string.Join(',', arguments));
var args = new Devmasters.Args(arguments, new string[] { "/mp3path", "/apikey" });
if (args.MandatoryPresent() == false)
{
Help();
}
mp3path = args.Get("/mp3path", null);
if (args.Exists("/utdl"))
{
YTDL = args["/utdl"];
}
else
{
YTDL = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + "\\youtube-dl.exe";
}
startPath = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
apiKey = args["/apikey"];
rewrite = args.Exists("/rewrite");
afterDay = DateTime.Now.Date.AddDays(-1 * args.GetNumber("/daysback", 10000).Value);
if (args.Exists("/ids"))
{
ids = args.GetArray("/ids");
}
skips2t = args.Exists("/skips2t");
int threads = args.GetNumber("/t") ?? 5;
try
{
ds = HlidacStatu.Api.V2.Dataset.Typed.Dataset <Jednani> .OpenDataset(apiKey, DataSetId);
}
catch (ApiException e)
{
ds = HlidacStatu.Api.V2.Dataset.Typed.Dataset <Jednani> .CreateDataset(apiKey, Registration());
}
catch (Exception e)
{
throw;
}
string nextPages = "https://www.ceskatelevize.cz/ivysilani/10000000064-jednani-rady-ceske-televize/dalsi-casti/{0}";
int page = 0;
bool stop = false;
List <Jednani> jednani = new List <Jednani>();
do
{
page++;
using (Devmasters.Net.HttpClient.URLContent net = new Devmasters.Net.HttpClient.URLContent(string.Format(nextPages, page)))
{
Console.WriteLine($"Page {page}");
net.IgnoreHttpErrors = true;
net.Tries = 5;
net.TimeInMsBetweenTries = 2000;
string html = "";
try
{
Devmasters.Logging.Logger.Root.Debug($"downloading {net.Url} ");
html = net.GetContent().Text;
}
catch (Exception e)
{
Devmasters.Logging.Logger.Root.Error($"{net.Url} failed", e);
}
Devmasters.XPath xp = new Devmasters.XPath(html);
var links = xp.GetNodes("//li[contains(@class,'itemBlock')]");
if (links == null || links.Count == 0)
{
break;
}
foreach (var link in links)
{
Jednani j = new Jednani();
j.Odkaz = urlPrefix + Devmasters.XPath.Tools.GetNodeAttributeValue(link, "div/h3/a[@class='itemSetPaging']", "href");
j.Titulek = Devmasters.XPath.Tools.GetNodeText(link, "div/h3/a[@class='itemSetPaging']").Trim();
j.DatumJednani = Devmasters.DT.Util.ToDate(Devmasters.XPath.Tools.GetNodeText(link, "div/p").Trim()) ?? DateTime.MinValue;
j.Id = Devmasters.RegexUtil.GetRegexGroupValue(j.Odkaz, "/ivysilani/10000000064-jednani-rady-ceske-televize/(?<id>\\d{2,})", "id");
if (j.DatumJednani > afterDay &&
(ids == null || ids.Contains(j.Id))
)
{
jednani.Add(j);
}
}
}
} while (stop == false);
//
Devmasters.Logging.Logger.Root.Debug($"Starting {jednani.Count} items ");
Devmasters.Batch.Manager.DoActionForAll <string>(jednani.Select(m => m.Id).Reverse(),
id =>
{
bool exists = ds.ItemExists(id);
if (!string.IsNullOrEmpty(id) &&
(!exists || rewrite)
)
{
Devmasters.Logging.Logger.Root.Debug($"Start parsing {id} ");
var fullJ = ParseJednani(jednani.First(m => m.Id == id));
Devmasters.Logging.Logger.Root.Debug($"Saving {id} ");
ds.AddOrUpdateItem(fullJ, HlidacStatu.Api.V2.Dataset.Typed.ItemInsertMode.rewrite);
}
else if (exists)
{
//check voice2text
var fullJ = ds.GetItemSafe(id);
if (!(fullJ.PrepisAudia?.Count() > 0))
{
Devmasters.Logging.Logger.Root.Debug($"Checking AUDIO text {id} ");
var aud = Audio(fullJ);
if (aud?.Count() > 0)
{
fullJ.PrepisAudia = aud;
ds.AddOrUpdateItem(fullJ, HlidacStatu.Api.V2.Dataset.Typed.ItemInsertMode.rewrite);
}
}
}
return(new Devmasters.Batch.ActionOutputData()
{
Log = id
});
}, true, maxDegreeOfParallelism: threads);
}
static void Main(string[] parameters)
{
var args = new Devmasters.Args(parameters);
logger.Info($"Starting with args {string.Join(' ',parameters)}");
apiKey = args["/apikey"];
force = args.Exists("/force");
var jsonGen = new JSchemaGenerator
{
DefaultRequired = Required.Default
};
var genJsonSchema = jsonGen.Generate(typeof(majitele)).ToString();
HlidacStatu.Api.V2.CoreApi.Model.Registration reg = new HlidacStatu.Api.V2.CoreApi.Model.Registration(
"Skuteční majitelé firem", DatasetNameId,
"https://esm.justice.cz/",
"https://github.com/HlidacStatu/Datasety/tree/master/SkutecniMajitele",
"Evidence skutečných majitelů firem podle zákona č. 37/2021 Sb.",
genJsonSchema, betaversion: true, allowWriteAccess: false,
orderList: new string[, ] {
{ "Podle datumu zápisu", "datum_zapis" },
{ "Podle IČ subjektu", "ico" },
},
defaultOrderBy: "datum_zapis desc",
searchResultTemplate: new HlidacStatu.Api.V2.CoreApi.Model.Template()
{
Body = @"
<!-- scriban {{ date.now }} -->
<table class=""table table-hover"">
<thead>
<tr>
<th>Detail</th>
<th>Subjekt</th>
<th>Skutečný majitel</th>
</tr>
</thead>
<tbody>
{{ for item in model.Result }}
<tr>
<td ><a href=""{{ fn_DatasetItemUrl item.id }}"">{{item.id}}</a></td>
<td >{{fn_RenderCompanyWithLink item.ico}}</td>
<td >
{{ for sk in item.skutecni_majitele }}
{{ if !(fn_IsNullOrEmpty sk.osobaId) }}
{{fn_RenderPersonWithLink2 sk.osobaId}},
{{else }}
{{sk.osoba_jmeno}} {{sk.osoba_prijmeni}},
{{ end }}
{{end }}
</td>
</tr>
{{ end }}
</tbody></table>
"
},
detailTemplate: new HlidacStatu.Api.V2.CoreApi.Model.Template()
{
Body = @"
<!-- scriban {{ date.now }} -->
{{this.item = model}}
<table class=""table table-hover""><tbody>
<tr><td>IČ</td><td ><a href=""{{ fn_DatasetItemUrl item.id }}"">{{item.id}}</a></td></tr>
<tr><td>Subjekt</td><td >{{fn_RenderCompanyWithLink item.ico}}<br/>
{{fn_RenderCompanyInformations item.ico 3}}</td></tr>
<tr><td>Skutečný majitel</td><td >
{{ for sk in item.skutecni_majitele }}
<dl>
<dt>
{{ if !(fn_IsNullOrEmpty sk.osobaId) }}
{{fn_RenderPersonWithLink2 sk.osobaId}}
{{else }}
{{sk.osoba_jmeno}} {{sk.osoba_prijmeni}}
{{end}}
({{sk.udaj_typ_nazev}})
</dt>
<dd>
{{if !(fn_IsNullOrEmpty sk.podil_na_prospechu_hodnota) }}
Podíl na prospěchu ze společnosti: {{sk.podil_na_prospechu_hodnota}}
{{if sk.podil_na_prospechu_typ==""PROCENTA""}}%{{else}}({{sk.podil_na_prospechu_typ}}){{end}}
<br/>
{{end}}
{{if !(fn_IsNullOrEmpty sk.podil_na_hlasovani_hodnota) }}
Podíl na hlasovacích právech: {{sk.podil_na_hlasovani_hodnota}}
{{if sk.podil_na_hlasovani_typ==""PROCENTA""}}%{{else}}({{sk.podil_na_hlasovani_typ}}){{end}}
<br/>
{{end}}
</dd>
</dl>
{{end }}
</td></tr>
</table>
"
}
);
try
{
if (args.Exists("/new"))
{
Configuration configuration = new Configuration();
configuration.AddDefaultHeader("Authorization", apiKey);
HlidacStatu.Api.V2.CoreApi.DatasetyApi datasetyApi = new HlidacStatu.Api.V2.CoreApi.DatasetyApi(configuration);
datasetyApi.ApiV2DatasetyDelete(reg.DatasetId);
}
ds = HlidacStatu.Api.V2.Dataset.Typed.Dataset <majitele> .OpenDataset(apiKey, DatasetNameId);
}
catch (HlidacStatu.Api.V2.CoreApi.Client.ApiException e)
{
ds = HlidacStatu.Api.V2.Dataset.Typed.Dataset <majitele> .CreateDataset(apiKey, reg);
}
catch (Exception e)
{
throw;
}
var wc = new System.Net.WebClient();
var package_list = Newtonsoft.Json.Linq.JObject.Parse(
wc.DownloadString("https://dataor.justice.cz/api/3/action/package_list")
);
var onlyCurrYears = package_list["result"]
.ToArray()
.Select(m => m.Value <string>())
.Where(m => m.EndsWith($"-{DateTime.Now.Year}") && m.Contains("-full-"))
//.Where(m => m == "as-full-ostrava-2021") //DEBUG
;
Devmasters.Batch.Manager.DoActionForAll <string>(onlyCurrYears,
name =>
{
ProcessXML(args, name);
return(new Devmasters.Batch.ActionOutputData());
}, Devmasters.Batch.Manager.DefaultOutputWriter, Devmasters.Batch.Manager.DefaultProgressWriter,
!System.Diagnostics.Debugger.IsAttached,
maxDegreeOfParallelism: 2, prefix: "Get XMLS ");
}
private static void ProcessXML(Devmasters.Args args, string name)
{
logger.Debug($"Starting {name}.xml");
if (System.IO.File.Exists(name + ".xml"))
{
if (args.Exists("/uselocal"))
{
//skip next, use local file
}
else if (force || (DateTime.Now - new System.IO.FileInfo(name + ".xml").LastWriteTime).TotalDays > 4)
{
logger.Debug($"downloading new {name}.xml");
Console.WriteLine($"Downloading new {name}");
DownloadFile(name);
}
}
else
{
logger.Debug($"downloading new {name}.xml");
Console.WriteLine($"Downloading {name}");
DownloadFile(name);
}
if (!System.IO.File.Exists(name + ".xml"))
{
return;
}
rawXML d = null;
Console.WriteLine($"Deserializing {name}");
logger.Debug($"Deserializing {name}.xml");
using (var xmlReader = new System.IO.StreamReader(name + ".xml"))
{
var serializer = new XmlSerializer(typeof(rawXML));
d = (rawXML)serializer.Deserialize(xmlReader);
}
Console.WriteLine($"{d.Subjekt?.Count()} subjects");
Devmasters.Batch.Manager.DoActionForAll <xmlSubjekt>(d.Subjekt //.Where(m=>m.ico== "3493661") //debug
, subj =>
{
majitele item = majitele.GetMajitele(subj);
if (item != null && item?.skutecni_majitele?.Count() > 0)
{
if (!ds.ItemExists(item.ico) || force)
{
item.UpdateOsobaId();
ds.AddOrUpdateItem(item, HlidacStatu.Api.V2.Dataset.Typed.ItemInsertMode.rewrite);
}
else
{
//check change
var old = ds.GetItem(item.ico);
if (old != null)
{
var same = true;
if (old.skutecni_majitele?.Count() != item.skutecni_majitele?.Count())
{
same = false;
}
else if (item.skutecni_majitele?.Count() == old.skutecni_majitele?.Count() && item.skutecni_majitele?.Count() > 0)
{
foreach (var sm in item.skutecni_majitele)
{
same = same && old.skutecni_majitele.Any(m =>
m.osoba_jmeno == sm.osoba_jmeno &&
m.osoba_prijmeni == sm.osoba_prijmeni &&
m.osoba_datum_narozeni == sm.osoba_datum_narozeni &&
m.osoba_titul_pred == sm.osoba_titul_pred &&
m.osoba_titul_za == sm.osoba_titul_za &&
m.adresa_cast_obce == sm.adresa_cast_obce &&
m.adresa_cislo_ev == sm.adresa_cislo_ev &&
m.adresa_cislo_or == sm.adresa_cislo_or &&
m.adresa_cislo_po == sm.adresa_cislo_po &&
m.adresa_obec == sm.adresa_obec &&
m.adresa_okres == sm.adresa_okres &&
m.adresa_psc == sm.adresa_psc &&
m.adresa_stat_nazev == sm.adresa_stat_nazev &&
m.adresa_text == sm.adresa_text &&
m.adresa_ulice == sm.adresa_ulice &&
m.slovni_vyjadreni == sm.slovni_vyjadreni &&
m.podil == sm.podil &&
m.postaveni == sm.postaveni &&
!string.IsNullOrEmpty(m.osobaId)
);
}
}
if (same == false)
{
item.UpdateOsobaId();
ds.AddOrUpdateItem(item, HlidacStatu.Api.V2.Dataset.Typed.ItemInsertMode.rewrite);
}
}
}
}
return(new Devmasters.Batch.ActionOutputData());
}, Devmasters.Batch.Manager.DefaultOutputWriter, Devmasters.Batch.Manager.DefaultProgressWriter,
!System.Diagnostics.Debugger.IsAttached,
maxDegreeOfParallelism: 4, prefix: $"{name} ITEMS ");
}