public void Dispose() { receiver?.Dispose(); Log?.Dispose(); WriteProgress?.Dispose(); ParseProgress?.Dispose(); }
private List <Member> FirstParse(Reader reader) { int eventIndex = 0; var members = new List <Member>(); var vals = reader.GetAttributes("name", "/doc/members/member"); foreach (var tuple in vals.Select((v, i) => new { Value = v, Index = i })) { var values = reader.GetValues("/doc/members/member[@name=\"{0}\"]/summary".FormatString(tuple.Value)); var value = string.Join("\n", values.Select(s => s)); var ret = reader.GetValue("/doc/members/member[@name=\"{0}\"]/returns".FormatString(tuple.Value), false); value = RemoveFirstLastBreakLine(value); var member = ConvertMemberNameToMember(tuple.Value); member.Value = value; member.ReturnComment = ret ?? string.Empty; var xparams = reader.GetAttributes("name", "/doc/members/member[@name=\"{0}\"]/param".FormatString(tuple.Value)); foreach (var param in xparams) { var path = "/doc/members/member[@name=\"{0}\"]/param[@name=\"{1}\"]".FormatString(tuple.Value, param); var value2 = reader.GetValue(path); value2 = RemoveFirstLastBreakLine(value2); member.ParameterNames.Add(param, value2); } members.Add(member); ParseProgress?.Invoke(this, new XmlDocumentParseProgressEventArgs( ParseType.First, vals.Count, ++eventIndex, tuple.Value)); } return(members); }
async IAsyncEnumerable <ParsedImage> IImageParsingStrategy.ParseAsync(ParseRequest request, IProgress <ParseProgress> progress) { if (request is null) { ThrowHelper.ArgumentNull(nameof(request)); } if (request.EstimatedCount < 1) { ThrowHelper.ArgumentOutOfRange(nameof(request.EstimatedCount), request.EstimatedCount, "Value must be 1 or greater!"); } var imagesPerCategory = (double)request.EstimatedCount / request.Categories.Count(); var capacity = (int)Math.Ceiling(imagesPerCategory / _pageSize) * request.Categories.Sum(x => x.Keywords.Count()); var throttler = new SemaphoreSlim(MaxThreads); var currentCount = 0; foreach (var category in request.Categories) { var keywordsCount = category.Keywords.Count(); var imagesPerKeyword = (int)Math.Ceiling(imagesPerCategory / keywordsCount); var allTasks = new List <Task>(); var parsedImages = new List <ParsedImage>(imagesPerKeyword); var total = (int)Math.Ceiling((double)imagesPerKeyword / _pageSize); var pages = Enumerable.Range(_startFrom, total); var disposables = new List <IDisposable>(); foreach (var keyword in category.Keywords) { await throttler.WaitAsync(); allTasks.Add( Task.Run(async() => { try { var uri = new Uri(_url).AddParameter("query", keyword); foreach (var page in pages) { var pageUri = uri.AddParameter("per_page", _pageSize) .AddParameter("page", page); var take = imagesPerKeyword - (page - 1) * _pageSize; var response = await _httpClient.GetAsync <Response>(pageUri); disposables.Add(response.Disposable); var results = response.Result.Results.Take(take); foreach (var result in results) { var download = await _httpClient.GetAsync(result.Links.Download); var stream = await download.Content.ReadAsStreamAsync(); var image = Image.FromStream(stream); var parsedImage = new ParsedImage { Category = category.Name, Image = image, Keyword = keyword }; parsedImages.Add(parsedImage); disposables.Add(download); disposables.Add(stream); } } } finally { throttler.Release(); } })); } await Task.WhenAll(allTasks); foreach (var parsedImage in parsedImages) { var data = new ParseProgress { CurrentCount = ++currentCount, EstimatedCount = request.EstimatedCount }; progress?.Report(data); yield return(parsedImage); } foreach (var disposable in disposables) { disposable.Dispose(); } } }
/// <summary> /// Start draco process /// </summary> public void Run() { Log.OnNext("Start Ptolemy.Draco"); Log.OnNext($"InputFile: {request.InputFile}"); Log.OnNext($"TargetDatabaseFile: {request.OutputFile}"); string[] document; try { Log.OnNext("Reading InputFile"); using var sr = new StreamReader(request.InputFile); document = sr.ReadToEnd() .Split("\n", StringSplitOptions.RemoveEmptyEntries) .Skip(1) .ToArray(); } catch (FileNotFoundException) { throw new DracoException($"file not found: {request.InputFile}"); } try { // input file's format // signalA signalB // 0. value value ... // ... // Get signal list var keys = document[0].Split(' ', StringSplitOptions.RemoveEmptyEntries); var writer = receiver.Buffer(request.BufferSize).Subscribe( r => { using var repo = new SqliteRepository(request.OutputFile); repo.BulkUpsert(r); WriteProgress.OnNext(Unit.Default); Log.OnNext($"Write {r.Count} records"); }, () => WriteProgress.OnCompleted()); token.Register(writer.Dispose); foreach (var line in document.Skip(1).SelectMany(line => { // Parse: time value value .... var rt = ResultEntity.Parse(request.Seed, request.Sweep, line, keys); ParseProgress.OnNext(Unit.Default); Log.OnNext($"Parsed: {line}"); return(rt); })) { token.ThrowIfCancellationRequested(); receiver.OnNext(line); } Log.OnNext($"Finished Parse"); receiver.OnCompleted(); ParseProgress.OnCompleted(); } catch (IndexOutOfRangeException) { throw new DracoException($"invalid file format: {request.InputFile}"); } catch (FormatException) { throw new DracoException($"データの数値に不正な値があります {request.InputFile}"); } catch (OperationCanceledException) { throw new DracoException("Canceled by user"); } catch (Exception e) { throw new DracoException($"Unknown error has occured\n\t-->{e}"); } }
private Element SecondParse(List <Member> members) { var root = new Element() { Type = ElementType.Root, Name = "Root", Members = null }; int eventIndex = 0; Element preElem = root; //Element classElem = root; var elemMap = new Dictionary <string, Element>(); var classElemMap = new Dictionary <string, Element>(); foreach (var tuple in members.Select((v, i) => new { Value = v, Index = i })) { var member = tuple.Value; var nameSpace = member.Namespace; while (true) { var firstName = nameSpace.GetFirstName(); if (firstName == null) { break; } var elem = new Element() { Type = ElementType.Namespace, Namespace = member.Namespace, Name = firstName }; if (!preElem.HasElement(elem.Name)) { preElem.Namespaces.Add(elem); preElem = elem; NamespaceCount++; } else { preElem = preElem.Namespaces[preElem.Namespaces.Count - 1]; } nameSpace = nameSpace.GetNamespaceWithoutFirst(); } if (member.Type == MethodType.Class) { var name = member.Name; var classElem = new Element() { Id = member.Id, Type = ElementType.Class, Namespace = member.Namespace, Name = name, Value = member.Value, Namespaces = new List <Element>() }; classElemMap.Put("{0}.{1}".FormatString(classElem.Namespace.ToString(), classElem.Name), classElem); preElem.Namespaces.Add(classElem); ClassCount++; } else { var classElem = classElemMap.Get(member.Namespace.ToString(), new Element()); if ("{0}.{1}".FormatString(classElem.Namespace.ToString(), classElem.Name).Equals(member.Namespace.ToString())) { classElem.Members.Add(member); } } preElem = root; ParseProgress?.Invoke(this, new XmlDocumentParseProgressEventArgs( ParseType.Second, members.Count, ++eventIndex, tuple.Value.Id)); } return(root); }