public void Dispose()
{
receiver?.Dispose();
Log?.Dispose();
WriteProgress?.Dispose();
ParseProgress?.Dispose();
}
private List <Member> FirstParse(Reader reader)
{
int eventIndex = 0;
var members = new List <Member>();
var vals = reader.GetAttributes("name", "/doc/members/member");
foreach (var tuple in vals.Select((v, i) => new { Value = v, Index = i }))
{
var values = reader.GetValues("/doc/members/member[@name=\"{0}\"]/summary".FormatString(tuple.Value));
var value = string.Join("\n", values.Select(s => s));
var ret = reader.GetValue("/doc/members/member[@name=\"{0}\"]/returns".FormatString(tuple.Value), false);
value = RemoveFirstLastBreakLine(value);
var member = ConvertMemberNameToMember(tuple.Value);
member.Value = value;
member.ReturnComment = ret ?? string.Empty;
var xparams = reader.GetAttributes("name", "/doc/members/member[@name=\"{0}\"]/param".FormatString(tuple.Value));
foreach (var param in xparams)
{
var path = "/doc/members/member[@name=\"{0}\"]/param[@name=\"{1}\"]".FormatString(tuple.Value, param);
var value2 = reader.GetValue(path);
value2 = RemoveFirstLastBreakLine(value2);
member.ParameterNames.Add(param, value2);
}
members.Add(member);
ParseProgress?.Invoke(this, new XmlDocumentParseProgressEventArgs(
ParseType.First, vals.Count, ++eventIndex, tuple.Value));
}
return(members);
}
async IAsyncEnumerable <ParsedImage> IImageParsingStrategy.ParseAsync(ParseRequest request, IProgress <ParseProgress> progress)
{
if (request is null)
{
ThrowHelper.ArgumentNull(nameof(request));
}
if (request.EstimatedCount < 1)
{
ThrowHelper.ArgumentOutOfRange(nameof(request.EstimatedCount), request.EstimatedCount, "Value must be 1 or greater!");
}
var imagesPerCategory = (double)request.EstimatedCount / request.Categories.Count();
var capacity = (int)Math.Ceiling(imagesPerCategory / _pageSize) * request.Categories.Sum(x => x.Keywords.Count());
var throttler = new SemaphoreSlim(MaxThreads);
var currentCount = 0;
foreach (var category in request.Categories)
{
var keywordsCount = category.Keywords.Count();
var imagesPerKeyword = (int)Math.Ceiling(imagesPerCategory / keywordsCount);
var allTasks = new List <Task>();
var parsedImages = new List <ParsedImage>(imagesPerKeyword);
var total = (int)Math.Ceiling((double)imagesPerKeyword / _pageSize);
var pages = Enumerable.Range(_startFrom, total);
var disposables = new List <IDisposable>();
foreach (var keyword in category.Keywords)
{
await throttler.WaitAsync();
allTasks.Add(
Task.Run(async() =>
{
try
{
var uri = new Uri(_url).AddParameter("query", keyword);
foreach (var page in pages)
{
var pageUri = uri.AddParameter("per_page", _pageSize)
.AddParameter("page", page);
var take = imagesPerKeyword - (page - 1) * _pageSize;
var response = await _httpClient.GetAsync <Response>(pageUri);
disposables.Add(response.Disposable);
var results = response.Result.Results.Take(take);
foreach (var result in results)
{
var download = await _httpClient.GetAsync(result.Links.Download);
var stream = await download.Content.ReadAsStreamAsync();
var image = Image.FromStream(stream);
var parsedImage = new ParsedImage
{
Category = category.Name,
Image = image,
Keyword = keyword
};
parsedImages.Add(parsedImage);
disposables.Add(download);
disposables.Add(stream);
}
}
}
finally
{
throttler.Release();
}
}));
}
await Task.WhenAll(allTasks);
foreach (var parsedImage in parsedImages)
{
var data = new ParseProgress
{
CurrentCount = ++currentCount,
EstimatedCount = request.EstimatedCount
};
progress?.Report(data);
yield return(parsedImage);
}
foreach (var disposable in disposables)
{
disposable.Dispose();
}
}
}
/// <summary>
/// Start draco process
/// </summary>
public void Run()
{
Log.OnNext("Start Ptolemy.Draco");
Log.OnNext($"InputFile: {request.InputFile}");
Log.OnNext($"TargetDatabaseFile: {request.OutputFile}");
string[] document;
try {
Log.OnNext("Reading InputFile");
using var sr = new StreamReader(request.InputFile);
document = sr.ReadToEnd()
.Split("\n", StringSplitOptions.RemoveEmptyEntries)
.Skip(1)
.ToArray();
}
catch (FileNotFoundException) {
throw new DracoException($"file not found: {request.InputFile}");
}
try {
// input file's format
// signalA signalB
// 0. value value ...
// ...
// Get signal list
var keys = document[0].Split(' ', StringSplitOptions.RemoveEmptyEntries);
var writer = receiver.Buffer(request.BufferSize).Subscribe(
r => {
using var repo = new SqliteRepository(request.OutputFile);
repo.BulkUpsert(r);
WriteProgress.OnNext(Unit.Default);
Log.OnNext($"Write {r.Count} records");
}, () => WriteProgress.OnCompleted());
token.Register(writer.Dispose);
foreach (var line in document.Skip(1).SelectMany(line => {
// Parse: time value value ....
var rt = ResultEntity.Parse(request.Seed, request.Sweep, line, keys);
ParseProgress.OnNext(Unit.Default);
Log.OnNext($"Parsed: {line}");
return(rt);
}))
{
token.ThrowIfCancellationRequested();
receiver.OnNext(line);
}
Log.OnNext($"Finished Parse");
receiver.OnCompleted();
ParseProgress.OnCompleted();
}
catch (IndexOutOfRangeException) {
throw new DracoException($"invalid file format: {request.InputFile}");
}
catch (FormatException) {
throw new DracoException($"データの数値に不正な値があります {request.InputFile}");
}
catch (OperationCanceledException) {
throw new DracoException("Canceled by user");
}
catch (Exception e) {
throw new DracoException($"Unknown error has occured\n\t-->{e}");
}
}
private Element SecondParse(List <Member> members)
{
var root = new Element()
{
Type = ElementType.Root,
Name = "Root",
Members = null
};
int eventIndex = 0;
Element preElem = root;
//Element classElem = root;
var elemMap = new Dictionary <string, Element>();
var classElemMap = new Dictionary <string, Element>();
foreach (var tuple in members.Select((v, i) => new { Value = v, Index = i }))
{
var member = tuple.Value;
var nameSpace = member.Namespace;
while (true)
{
var firstName = nameSpace.GetFirstName();
if (firstName == null)
{
break;
}
var elem = new Element()
{
Type = ElementType.Namespace,
Namespace = member.Namespace,
Name = firstName
};
if (!preElem.HasElement(elem.Name))
{
preElem.Namespaces.Add(elem);
preElem = elem;
NamespaceCount++;
}
else
{
preElem = preElem.Namespaces[preElem.Namespaces.Count - 1];
}
nameSpace = nameSpace.GetNamespaceWithoutFirst();
}
if (member.Type == MethodType.Class)
{
var name = member.Name;
var classElem = new Element()
{
Id = member.Id,
Type = ElementType.Class,
Namespace = member.Namespace,
Name = name,
Value = member.Value,
Namespaces = new List <Element>()
};
classElemMap.Put("{0}.{1}".FormatString(classElem.Namespace.ToString(), classElem.Name), classElem);
preElem.Namespaces.Add(classElem);
ClassCount++;
}
else
{
var classElem = classElemMap.Get(member.Namespace.ToString(), new Element());
if ("{0}.{1}".FormatString(classElem.Namespace.ToString(), classElem.Name).Equals(member.Namespace.ToString()))
{
classElem.Members.Add(member);
}
}
preElem = root;
ParseProgress?.Invoke(this, new XmlDocumentParseProgressEventArgs(
ParseType.Second, members.Count, ++eventIndex, tuple.Value.Id));
}
return(root);
}
