public void ReadXsd(string source, ref Database.Sqlite database)
{
XmlDocument doc = new XmlDocument();
doc.LoadXml(source);
XmlNodeList listLink = doc.GetElementsByTagName("link:linkbaseRef");
foreach (XmlNode node in listLink)
{
string link = node.Attributes["xlink:href"].InnerText;
if (ListXml == null)
{
ListXml = new List <string>();
}
string[] path = link.Split('/');
//ListXmlに取得済みのタクソノミxmlファイル一覧を追加し 一覧にない場合にxmlをダウンロードして読み込み
if (link.Substring(0, 4) == "http" && !ListXml.Contains(path[path.Length - 1]))
{
ListXml.Add(path[path.Length - 1]);
ImportTaxonomy(link, ref database);
}
}
}
public async Task Download(HttpResponseMessage httpResponseMessage, int id, string field, Database.Sqlite db)
{
string filename = httpResponseMessage.Content.Headers.ContentDisposition.FileName.Replace("\"", "");
System.Net.Http.Headers.MediaTypeHeaderValue contenttype = httpResponseMessage.Content.Headers.ContentType;
using (Stream stream = await httpResponseMessage.Content.ReadAsStreamAsync()) {
using (MemoryStream ms = new MemoryStream()) {
//string filename = $"{docid}_{(int)type}";
stream.CopyTo(ms);
byte[] buffer = ms.ToArray();
stream.Flush();
if (filename == "404.json")
{
filename = httpResponseMessage.RequestMessage.RequestUri.ToString();
}
else
{
int year = 20 * 100 + id / 100000000;
SaveFile(buffer, filename, year);
db.UpdateFilenameOfDisclosure(id, field, filename);
}
#pragma warning disable CS4014
SaveLog(GetLog(httpResponseMessage.StatusCode, RequestType.Archive, contenttype, filename));
#pragma warning restore CS4014
}
}
httpResponseMessage.Dispose();
}
//Responseを返さないのでファイルセーブまで待たないはず
public async Task DownloadAsync(string docid, DocumentType type, int id, Database.Sqlite db, int retry)
{
string url = string.Format("/api/{0}/documents/{1}?type={2}", Version, docid, (int)type);
int i = 0;
do
{
if (i > 0)
{
Debug.Write($"retry notawait Download[{i}] ");
await Task.Delay(2000);
}
try {
debug.ProgramCodeInfo.SetDebugQueue();
using (HttpResponseMessage res = await client.GetAsync(url)) {
debug.ProgramCodeInfo.SetDebugQueue();
string filename = res.Content.Headers.ContentDisposition.FileName.Replace("\"", "");
//string filename = $"{docid}_{(int)type}";
System.Net.Http.Headers.MediaTypeHeaderValue contenttype = res.Content.Headers.ContentType;
#pragma warning disable CS4014
SaveLog(GetLog(res.StatusCode, RequestType.Archive, contenttype, filename, id));
#pragma warning restore CS4014
using (Stream stream = await res.Content.ReadAsStreamAsync()) {
using (MemoryStream ms = new MemoryStream()) {
//SaveLog(GetLog(res.StatusCode, RequestType.Archive, contenttype));
stream.CopyTo(ms);
byte[] buffer = ms.ToArray();
stream.Flush();
if (filename == "404.json")
{
filename = url;
}
else
{
int year = 20 * 100 + id / 100000000;
SaveFile(buffer, filename, year);
db.UpdateFilenameOfDisclosure(id, type.ToString(), filename);
}
return;
}
}
}
} catch (TaskCanceledException ex) {
ArchiveResponse response = new ArchiveResponse();
response.Update(ex);
#pragma warning disable CS4014
SaveLog(GetLog(response));
#pragma warning restore CS4014
debug.ProgramCodeInfo.SetDebugQueue();
} catch (Exception ex) {
ArchiveResponse response = new ArchiveResponse();
response.Update(ex);
#pragma warning disable CS4014
SaveLog(GetLog(response));
#pragma warning restore CS4014
debug.ProgramCodeInfo.SetDebugQueue();
return;
}
debug.ProgramCodeInfo.SetDebugQueue();
i++;
} while (i <= retry);
}
public void Import(Database.Sqlite db)
{
//using (ZipArchive archive = ZipFile.Open(filepath, ZipArchiveMode.Read)) {
// //foreach (ZipArchiveEntry entry in archive.Entries) {
// // FileInfo inf = new FileInfo(entry.FullName);
// // Console.WriteLine(inf.Name);
// // using (ZipArchive entryarchive = entry.Archive) {
// // entryarchive.ExtractToDirectory(@"d:\data\temp");
// // }
// // //if (inf.FullName.Contains("PublicDoc") && inf.Extension == ".xbrl") {
// // // using (Stream stream2 = entry.Open()) {
// // // using (StreamReader reader = new StreamReader(stream2, Encoding.UTF8)) {
// // // return reader.ReadToEnd();
// // // }
// // // }
// // //}
// //}
//}
//Console.WriteLine(filepath);
string extension = Path.GetExtension(filepath).ToLower();
string dir = "temp";
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
//Database.Sqlite db = new Database.Sqlite()
if (extension == ".zip")
{
using (FileStream fs = new FileStream(filepath, FileMode.Open, FileAccess.Read)) {
using (MemoryStream stream = new MemoryStream()) {
fs.CopyTo(stream);
using (ZipArchive archive = new ZipArchive(stream)) {
foreach (ZipArchiveEntry entry in archive.Entries)
{
Console.WriteLine(entry.Name);
//FileInfo inf = new FileInfo(entry.FullName);
//if (inf.FullName.Contains("PublicDoc") && inf.Extension == ".xbrl") {
// using (Stream stream2 = entry.Open()) {
// using (StreamReader reader = new StreamReader(stream2, Encoding.UTF8)) {
// return reader.ReadToEnd();
// }
// }
//}
}
}
}
}
}
else if (extension == ".csv")
{
dir = Directory.GetParent(filepath).FullName;
string[] subdirs = Directory.GetDirectories(dir);
List <string> list = new List <string>();
foreach (string subdir in subdirs)
{
string docid = Path.GetFileName(subdir);
//Console.WriteLine(docid);
list.Add(docid);
}
string filter = $"docid in ('{string.Join("', '", list)}')";
DataTable table = db.ReadDisclosure(filter);
DataView dv = new DataView(table, "", "docid", DataViewRowState.CurrentRows);
if (table.Rows.Count < list.Count)
{
foreach (string docid in list)
{
int index = dv.Find(docid);
if (index < 0)
{
string subdir = Path.Combine(dir, docid);
string[] files = Directory.GetFiles(subdir, "*.xbrl", SearchOption.AllDirectories);
foreach (string file in files)
{
if (file.Contains("PublicDoc"))
{
using (StreamReader reader = new StreamReader(file)) {
string source = reader.ReadToEnd();
Edinet.Xbrl xbrl = new Edinet.Xbrl();
xbrl.Load(source);
string fn = Path.GetFileName(file);
string releasedate = xbrl.GetValue("FilingDateCoverPage");
}
break;
}
}
}
}
}
foreach (string docid in list)
{
string subdir = Path.Combine(dir, docid);
string[] files = Directory.GetFiles(subdir, "*.xbrl", SearchOption.AllDirectories);
foreach (string file in files)
{
if (file.Contains("PublicDoc"))
{
using (StreamReader reader = new StreamReader(file)) {
string source = reader.ReadToEnd();
Edinet.Xbrl xbrl = new Edinet.Xbrl();
xbrl.Load(source);
string fn = Path.GetFileName(file);
string releasedate = xbrl.GetValue("FilingDateCoverPage");
if (releasedate != "" && DateTime.TryParse(releasedate, out DateTime date))
{
int year = date.Year;
}
}
break;
}
}
}
}
//if (File.Exists(filepath)) {
}
public void ImportTaxonomy(string url, ref Database.Sqlite database)
{
string source = null;
//string lastmodified = null;
using (System.Net.WebClient wc = new System.Net.WebClient()) {
using (Stream st = wc.OpenRead(url)) {
using (StreamReader sr = new StreamReader(st, Encoding.UTF8)) {
source = sr.ReadToEnd();
//if (wc.ResponseHeaders["Last-Modified"] != null) {
// lastmodified = wc.ResponseHeaders["Last-Modified"];
//}
}
}
}
string[] path = url.Split('/');
//string fn = path[path.Length - 1];
Console.WriteLine(" reading Taxonomy.. {0}", path[2]);
XmlDocument document = new XmlDocument();
document.LoadXml(source.Replace("<gen:arc", "<link:labelArc")
.Replace("<label:label", "<link:label").Replace("</label:label>", "</link:label>"));
XmlNodeList locs = document.GetElementsByTagName("link:loc");
XmlNodeList labels = document.GetElementsByTagName("link:label");
XmlNodeList arcs = document.GetElementsByTagName("link:labelArc");
if (locs.Count == 0)
{
locs = document.GetElementsByTagName("loc");
}
if (labels.Count == 0)
{
labels = document.GetElementsByTagName("label");
}
if (arcs.Count == 0)
{
arcs = document.GetElementsByTagName("labelArc");
}
if (arcs.Count == 0)
{
arcs = document.GetElementsByTagName("link:definitionArc");
}
if (locs.Count == 0 | labels.Count == 0 | arcs.Count == 0)
{
return;
}
//Dictionary<string, string> dicArc = new Dictionary<string, string>();
Dictionary <string, XmlNode> dicLoc = new Dictionary <string, XmlNode>();
Dictionary <string, string[]> dicLabel = new Dictionary <string, string[]>();
for (int i = 0; i < locs.Count; i++)
{
dicLoc.Add(locs[i].Attributes["xlink:label"].InnerText, locs[i]);
}
for (int i = 0; i < labels.Count; i++)
{
dicLabel.Add(labels[i].Attributes["xlink:label"].InnerText
, new string[] { labels[i].InnerText, labels[i].Attributes["xml:lang"].InnerText });
}
database.SaveTaxonomy(DicTaxonomy, url, arcs, dicLoc, dicLabel);
}
public DialogSearch(Database.Sqlite database)
{
InitializeComponent();
db = database;
buttonOk.Enabled = false;
}
