public async Task <ImportStats> UpdateSongPlays(SetlistShow show, IEnumerable <SetlistSong> songs) { var stats = new ImportStats(); await db.WithConnection(async con => { stats.Created += await con.ExecuteAsync(@" INSERT INTO setlist_songs_plays ( played_setlist_song_id, played_setlist_show_id ) VALUES ( @songId, @showId ) ON CONFLICT ON CONSTRAINT setlist_songs_plays_song_id_show_id_key DO NOTHING ", songs.Select(song => new { showId = show.id, songId = song.id })); stats.Removed += await con.ExecuteAsync(@" DELETE FROM setlist_songs_plays WHERE played_setlist_show_id = @showId AND NOT(played_setlist_song_id = ANY(@songIds)) ", new { showId = show.id, songIds = songs.Select(s => s.id).ToList() }); }); return(stats); }
public static void Run(string data, Organization organization, Person runningPerson) { if (!data.StartsWith("\r\n<html>\r\n<head>\r\n <style>\r\n .tal")) { runningPerson.SendPhoneMessage("The file you uploaded does not appear to be a Payson export file. No processing done. The data has been discarded."); throw new ArgumentException("This does not appear to be a Payson file"); } if (organization.Identity != 1) { runningPerson.SendPhoneMessage("Payson import is currently only supported for PPSE. No processing done. The data has been discarded."); throw new Exception("Payson is only supported for PPSE at the moment."); } ImportResult result = ImportPayson(data); ImportStats stats = ProcessImportedData(result, organization, runningPerson); try { runningPerson.SendPhoneMessage("The Payson file was processed. See mail for more details."); } catch (Exception) { // Ignore error on SMS transmit } string mailBody = string.Empty; mailBody += String.Format("Rows processed: {0,9:N0}\r\n", stats.ProcessedTransactionCount); mailBody += String.Format("Tx imported: {0,9:N0}\r\n", stats.ImportedTransactionCount); mailBody += String.Format("Tx modified: {0,9:N0}\r\n", stats.ModifiedTransactionCount - stats.ImportedTransactionCount); runningPerson.SendNotice("Payson file imported", mailBody, organization.Identity); }
public static void Run(string data, Organization organization, Person runningPerson) { if (!data.StartsWith("Date\t Time\t Time Zone\t Name\t Type")) { runningPerson.SendPhoneMessage( "The file you uploaded does not appear to be a PayPal tab-delimited file of all activity. No processing done. The data has been discarded."); throw new ArgumentException("This does not appear to be a PayPal file"); } ImportResult result = ImportPaypal(data); ImportStats stats = ProcessImportedData(result, organization, runningPerson); try { runningPerson.SendPhoneMessage("The PayPal file was processed. See mail for more details."); } catch (Exception) { // Ignore error on SMS transmit } string mailBody = string.Empty; mailBody += String.Format("Rows processed: {0,9:N0}\r\n", stats.ProcessedTransactionCount); mailBody += String.Format("Tx imported: {0,9:N0}\r\n", stats.ImportedTransactionCount); mailBody += String.Format("Tx modified: {0,9:N0}\r\n", stats.ModifiedTransactionCount - stats.ImportedTransactionCount); runningPerson.SendNotice("PayPal file imported", mailBody, organization.Identity); }
/// <summary> /// Imports custom fields /// </summary> /// <returns></returns> private async Task <ImportStats> _importCustomFields() { _logger.LogDebug($"Custom fields import. Start."); using var client = RedmineXmlImporter.CreateWebClient(); var uri = new Uri($"{_redmineUrl}custom_fields.json?key={_redmineApiKey}"); var json = await client.DownloadStringTaskAsync(uri); var result = new ImportStats(); var opts = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; var list = JsonSerializer.Deserialize <Models.RedmineJson.CustomFieldResponse>(json, opts); using var scope = _services.CreateScope(); using var _db = scope.ServiceProvider.GetRequiredService <DB>(); var types = _db.CustomField.ToList(); result.total = list.FieldList.Count; foreach (var item in list.FieldList) { var dbItem = types.FirstOrDefault(d => d.Id == item.Id); if (dbItem == null) { dbItem = new Redmine.Models.CustomField { Id = item.Id, Created = DateTimeOffset.Now }; await _db.AddAsync(dbItem); result.added++; } dbItem.Name = item.Name; dbItem.Type = item.Type; dbItem.Format = item.Format; dbItem.IsRequired = item.IsRequired; dbItem.IsMultiple = item.IsMultiple; if (_db.Entry(dbItem).State != EntityState.Unchanged) { dbItem.Updated = DateTimeOffset.Now; result.updated++; } } await _db.SaveChangesAsync(); _logger.LogInformation($"Custom fields import. Finish. {list.FieldList.Count} processed."); return(result); }
public ImportCoreDataStats() { Users = new ImportStats(); Statuses = new ImportStats(); Priorities = new ImportStats(); TimeEntries = new ImportStats(); CustomFields = new ImportStats(); IssueTrackers = new ImportStats(); Projects = new ImportStats(); Versions = new ImportStats(); }
public void Add(ImportStats anotherCounter) { if (anotherCounter == null) { return; } total += anotherCounter.total; added += anotherCounter.added; updated += anotherCounter.updated; deleted += anotherCounter.deleted; }
private void UpdateQueryList() { ImportStats objImportStats = (ImportStats)Session["ImportStats"]; if (objImportStats.TotalLeads > 0 && Session["Campaign"] != null) { XmlDocument xDocCampaign = new XmlDocument(); Campaign objCampaign = (Campaign)Session["Campaign"]; xDocCampaign.LoadXml(Serialize.SerializeObject(objCampaign, "Campaign")); CampaignService objCampService = new CampaignService(); objCampService.UpdateCampaignQueriesStats(xDocCampaign); } }
/// <summary> /// Executes any sql to process data adding output parameters to fill result /// </summary> /// <param name="sql">SQL to execute with placeholders for @total, @added, @updated and @deleted output parameters</param> /// <param name="queryParams">Query parameters to add to query call, except output parameters</param> /// <returns></returns> private async Task <ImportStats> _executeSql(string sql, params object[] queryParams) { using var scope = _services.CreateScope(); using var db = scope.ServiceProvider.GetRequiredService <DB>(); var parameters = new List <object>(queryParams); var total = new SqlParameter("@total", System.Data.SqlDbType.Int); total.Direction = System.Data.ParameterDirection.Output; parameters.Add(total); var added = new SqlParameter("@added", System.Data.SqlDbType.Int); added.Direction = System.Data.ParameterDirection.Output; parameters.Add(added); var updated = new SqlParameter("@updated", System.Data.SqlDbType.Int); updated.Direction = System.Data.ParameterDirection.Output; parameters.Add(updated); var deleted = new SqlParameter("@deleted", System.Data.SqlDbType.Int); deleted.Direction = System.Data.ParameterDirection.Output; parameters.Add(deleted); await db.Database.ExecuteSqlRawAsync(sql, parameters); var result = new ImportStats(); if (total.Value != DBNull.Value) { result.total = (int)total.Value; } if (added.Value != DBNull.Value) { result.added = (int)added.Value; } if (updated.Value != DBNull.Value) { result.updated = (int)updated.Value; } if (deleted.Value != DBNull.Value) { result.deleted = (int)deleted.Value; } return(result); }
protected void Page_Load(object sender, EventArgs e) { if (!IsPostBack) { try { UpdateQueryList(); } catch { } } if (Session["ImportStats"] != null) { ImportStats objImportStats = (ImportStats)Session["ImportStats"]; lblTotalLeads.Text = objImportStats.TotalLeads.ToString(); lblImported.Text = objImportStats.LeadsImported.ToString(); lblBlank.Text = objImportStats.LeadsBlankPhoneNumber.ToString(); lblSpChar.Text = objImportStats.LeadsSPCharPhoneNumber.ToString(); lblBadData.Text = objImportStats.LeadsBadData.ToString(); lblDuplicate.Text = objImportStats.LeadsDuplicate.ToString(); lblBadLength.Text = objImportStats.LeadsInvalidNumberLength.ToString(); lblUpdated.Text = objImportStats.LeadsUpdated.ToString(); try { if (hdnExportToClient.Value != "") { ExportToClientMachine(hdnExportToClient.Value, "text/csv"); hdnExportToClient.Value = ""; } else { if (objImportStats.LeadsDuplicate > 0 && Session["DupFilePath"] != null && Session["DupFilePath"].ToString() != "") { if (System.IO.File.Exists(Session["DupFilePath"].ToString())) { hdnExportToClient.Value = Session["DupFilePath"].ToString(); } Session["DupFilePath"] = null; } } } catch (Exception ex) { throw ex; } } }
private async Task __old(PerformContext context) { var artists = (await _artistService.All()).ToList(); context.WriteLine($"--> Updating all {artists.Count} artists"); var progress = context.WriteProgressBar(); var stats = new ImportStats(); await artists.AsyncForEachWithProgress(progress, async artist => { context.WriteLine($"--> Importing {artist.name}"); var artistStats = await _importerService.Import(artist, null, context); context.WriteLine($"--> Imported {artist.name}! " + artistStats); stats += artistStats; }); context.WriteLine("--> Imported all artists! " + stats); }
public static ImportStats ImportArchiveToDatabase(string dbPath, bool cleanDatabase, params string[] arFiles) { var existing = File.Exists(dbPath); var stats = new ImportStats(); using (var db = new SqliteConnection(GetConnectionString(dbPath))) { if (!existing) { db.Execute(_sqlSchema); } db.Execute("BEGIN"); if (cleanDatabase) { db.Execute(@" DELETE FROM fingerprints; DELETE FROM instructions; DELETE FROM masks; DELETE FROM imports; DELETE FROM variants; DELETE FROM relocations;" ); } foreach (var ar in arFiles) { ImportArFile(db, ar, stats); } db.Execute("COMMIT"); } return(stats); }
protected static ImportStats ProcessImportedData(ImportResult import, Organization organization, Person importingPerson) { FinancialAccount payPalAccount = organization.FinancialAccounts.AssetsPaypal; FinancialAccount bankFees = organization.FinancialAccounts.CostsBankFees; FinancialAccount donations = organization.FinancialAccounts.IncomeDonations; int autoDepositLimit = 1000; // TODO: Get from organization parameters int autoWithdrawalLimit = 0; ImportStats result = new ImportStats(); foreach (ImportedRow row in import.Rows) { // Each row is at least a stub, probably more. // If too old, ignore. if (row.DateTime < new DateTime(2008, 12, 4)) { continue; } string importKey = row.SuppliedTransactionId; // If importKey is empty, construct a hash from the data fields. if (string.IsNullOrEmpty(importKey)) { string hashKey = row.HashBase + row.Comment + (row.AmountCentsNet / 100.0).ToString(CultureInfo.InvariantCulture) + row.CurrentBalance.ToString(CultureInfo.InvariantCulture) + row.DateTime.ToString("yyyy-MM-dd-hh-mm-ss"); importKey = SHA1.Hash(hashKey).Replace(" ", ""); } if (importKey.Length > 30) { importKey = importKey.Substring(0, 30); } Int64 amountCents = row.AmountCentsNet; if (amountCents == 0) { amountCents = row.AmountCentsGross; } Dictionary <int, long> nominalTransaction = new Dictionary <int, long>(); FinancialTransaction transaction = null; try { transaction = FinancialTransaction.FromImportKey(organization, importKey); } catch (Exception) { // if we get here, that means the transaction did not yet exist transaction = FinancialTransaction.ImportWithStub(organization.Identity, row.DateTime, payPalAccount.Identity, amountCents, row.Comment, importKey, "" /* new SHA256 field */, importingPerson.Identity); result.ImportedTransactionCount++; if (transaction == null) { // No transaction was created. This is an error condition as it should have been created if it didn't // exist, and the "exist" case is handled in the FromImportKey attempt above. Abort with error. // Throw new exception? continue; } } result.ProcessedTransactionCount++; nominalTransaction[payPalAccount.Identity] = amountCents; // The transaction was created. Examine if the autobook criteria are true. if (amountCents < 0) { if ((-amountCents) < autoWithdrawalLimit * 100) { // Book against autoWithdrawal account. nominalTransaction[bankFees.Identity] = -amountCents; } } else if (amountCents > 0) { if (row.FeeCents < 0) { // This is always an autodeposit, if there is a fee (which is never > 0.0) nominalTransaction[bankFees.Identity] = -row.FeeCents; nominalTransaction[donations.Identity] = -row.AmountCentsGross; } else if (amountCents < autoDepositLimit * 100) { // Book against autoDeposit account. nominalTransaction[donations.Identity] = -amountCents; } } if (transaction.Rows.AmountCentsTotal != 0) // If transaction is unbalanced, balance it { if (transaction.RecalculateTransaction(nominalTransaction, importingPerson)) { result.ModifiedTransactionCount++; } } } return(result); }
public ImportIssuesStats() { List = new ImportStats(); Details = new ImportStats(); }
protected static ImportStats ProcessImportedData(ImportResult import, Organization organization, Person importingPerson) { FinancialAccount payPalAccount = organization.FinancialAccounts.AssetsPaypal; FinancialAccount bankFees = organization.FinancialAccounts.CostsBankFees; FinancialAccount donations = organization.FinancialAccounts.IncomeDonations; int autoDepositLimit = 1000; // TODO: Get from organization parameters int autoWithdrawalLimit = 0; ImportStats result = new ImportStats(); foreach (ImportedRow row in import.Rows) { // Each row is at least a stub, probably more. // If too old, ignore. if (row.DateTime < new DateTime(2008, 12, 4)) { continue; } string importKey = row.SuppliedTransactionId; // If importKey is empty, construct a hash from the data fields. if (string.IsNullOrEmpty(importKey)) { string hashKey = row.HashBase + row.Comment + (row.AmountCentsNet / 100.0).ToString(CultureInfo.InvariantCulture) + row.CurrentBalance.ToString(CultureInfo.InvariantCulture) + row.DateTime.ToString("yyyy-MM-dd-hh-mm-ss"); importKey = SHA1.Hash(hashKey).Replace(" ", ""); } if (importKey.Length > 30) { importKey = importKey.Substring(0, 30); } Int64 amountCents = row.AmountCentsNet; if (amountCents == 0) { amountCents = row.AmountCentsGross; } Dictionary<int, long> nominalTransaction = new Dictionary<int, long>(); FinancialTransaction transaction = null; try { transaction = FinancialTransaction.FromImportKey(organization, importKey); } catch (Exception) { // if we get here, that means the transaction did not yet exist transaction = FinancialTransaction.ImportWithStub(organization.Identity, row.DateTime, payPalAccount.Identity, amountCents, row.Comment, importKey, importingPerson.Identity); result.ImportedTransactionCount++; if (transaction == null) { // No transaction was created. This is an error condition as it should have been created if it didn't // exist, and the "exist" case is handled in the FromImportKey attempt above. Abort with error. // Throw new exception? continue; } } result.ProcessedTransactionCount++; nominalTransaction[payPalAccount.Identity] = amountCents; // The transaction was created. Examine if the autobook criteria are true. if (amountCents < 0) { if ((-amountCents) < autoWithdrawalLimit * 100) { // Book against autoWithdrawal account. nominalTransaction[bankFees.Identity] = -amountCents; } } else if (amountCents > 0) { if (row.FeeCents < 0) { // This is always an autodeposit, if there is a fee (which is never > 0.0) nominalTransaction[bankFees.Identity] = -row.FeeCents; nominalTransaction[donations.Identity] = -row.AmountCentsGross; } else if (amountCents < autoDepositLimit * 100) { // Book against autoDeposit account. nominalTransaction[donations.Identity] = -amountCents; } } if (transaction.Rows.AmountCentsTotal != 0) // If transaction is unbalanced, balance it { if (transaction.RecalculateTransaction(nominalTransaction, importingPerson)) { result.ModifiedTransactionCount++; } } } return result; }
private static void ImportArFile(DbConnection db, string arPath, ImportStats stats = null) { Console.Error.WriteLine($"{arPath}..."); // Dummy value in case none provided stats = stats ?? new ImportStats(); var ar = new Archive(arPath); var constructed = ar.Files .Select(a => new { elf = a.ToElf(), a }) .Where(e => e.elf.Machine == ELFSharp.ELF.Machine.MIPS && e.elf.Endianess == ELFSharp.ELF.Endianess.BigEndian) .Select(e => new { e.elf, e.a, syms = ((ISymbolTable)e.elf.GetSection(".symtab")).Entries.ToArray() }) .Select(e => new { e.a, e.syms, e.elf, text = e.elf.Sections.FirstOrDefault(s => s.Name == ".text")?.GetContents(), fns = e.syms .Select(s => s as SymbolEntry <uint>) .Where(s => s != null && s.Binding == SymbolBinding.Global && s.Size != 0 && s.Type == ELFSharp.ELF.Sections.SymbolType.Function) .Select(s => new { s.Name, s.Size, Location = s.Value }) }) .Select(e => new { e.a, e.syms, e.fns, e.text, e.elf, relocs = e.elf.Sections .Where(s => s.Type == ELFSharp.ELF.Sections.SectionType.Relocation && s.Name.Contains(".text")) .Select(s => new { s, data = s.GetContents().ToWordGroups(2) .Select(g => new { Type = (RelocationType)(g[1] & 0xFF), Address = g[0], T = e.syms[(int)g[1] >> 8] }) //.Where(g => { // if (string.IsNullOrWhiteSpace(g.T?.Name)) // Console.WriteLine("Hit!"); // return true; //}) .Where(g => e.fns.Any(f => g.Address >= f.Location && g.Address < f.Location + f.Size)) .Select(g => { var sym = g.T as SymbolEntry <uint>; var name = sym.Name; if (string.IsNullOrWhiteSpace(name)) { name = sym.PointedSection?.Name; //var preceding = e.syms // .Select(y => y as SymbolEntry<uint>) // .Where(y => y.Binding == SymbolBinding.Global) // .Where(y => y.Type == ELFSharp.ELF.Sections.SymbolType.Function) // .FirstOrDefault(y => y.Value < g.Address); //var offset = sym.Value - sym.PointedSection?.LoadAddress; //var insn = e.text // .Skip((int)g.Address) // .Take(4) // .ToInstructions() // .First(); //switch (g.Type) //{ // case RelocationType.R_MIPS_26: // offset = insn.Target; // break; // case RelocationType.R_MIPS_LO16: // offset = insn.Immediate; // break; //} //name = $"{e.a.Filename}({preceding?.Name}+{offset})"; } return(new { g.Type, g.Address, Name = name }); }) .ToList(), insns = s.Name.Contains(".text") ? e.elf.Sections.First(z => z.Name == s.Name.Replace(".rel", "")).GetContents().ToInstructions() : null }) .FirstOrDefault() ?.data ?.ToDictionary(d => d.Address, d => new { d.Name, d.Type }) }) .Where(e => e.relocs?.Values?.All( r => r.Type == RelocationType.R_MIPS_26 || r.Type == RelocationType.R_MIPS_HI16 || r.Type == RelocationType.R_MIPS_LO16 ) != false) .Select(e => new { e.a, e.elf, fns = e.fns .Select(f => new { f.Name, f.Size, f.Location, Masks = Enumerable.Range(0, (int)f.Size / 4) .Select(i => { var ptr = (uint)(i * 4) + f.Location; if (!(e.relocs?.ContainsKey(ptr)).GetValueOrDefault()) { return(0xFFFFFFFFU); } switch (e.relocs[ptr].Type) { case RelocationType.R_MIPS_26: return(~((1U << 26) - 1)); case RelocationType.R_MIPS_HI16: case RelocationType.R_MIPS_LO16: return(0xFFFF0000U); default: throw new NotImplementedException(); } }) .ToArray() }) .Select(f => new { f.Name, f.Size, f.Location, f.Masks, Instructions = e.text.GetSegment((int)f.Location, (int)f.Size) .ToInstructions() .Zip(f.Masks, (i, m) => new Instruction(i & m)) .ToArray(), Relocs = e.relocs ?.Where(r => r.Key >= f.Location && r.Key < f.Location + f.Size) ?.Select(r => new { Address = r.Key - f.Location, r.Value.Type, r.Value.Name }) ?.ToArray() }) .Select(f => new { f.Name, f.Size, f.Location, f.Masks, f.Instructions, f.Relocs, Hash = CreateMd5Hash( f.Instructions .RemoveTrailingNops() .ToBytes() .ToArray() ) }) .ToArray() }) .Where(e => e.fns.Length > 0) .ToList(); var info = new FileInfo(arPath); int importId = 0; try { importId = db.Query <int>(@" INSERT INTO imports ( import_file_md5, import_file, import_mtime ) VALUES ( @md5, @name, @mtime ); SELECT last_insert_rowid()", new { md5 = CreateMd5Hash(File.ReadAllBytes(arPath)), name = info.Name, mtime = info.LastWriteTimeUtc.ToString("o") } ).Single(); } catch { stats.ArchivesExisting++; return; } var import = db.Query <sqlImport>("SELECT * FROM imports WHERE import_id = @importId", new { importId }) .Single(); foreach (var f in constructed.SelectMany(c => c.fns.Select(f => new { c, fn = f }))) { var needInsnsAndMasks = false; reload: var existing = db.Query <sqlFingerprint>("SELECT * FROM fingerprints WHERE fingerprint_md5 = @md5", new { md5 = f.fn.Hash }) .SingleOrDefault(); if (existing == null) { db.Execute(@" INSERT INTO fingerprints ( fingerprint_md5, fingerprint_length ) VALUES ( @md5, @length );", new { md5 = f.fn.Hash, length = f.fn.Size } ); needInsnsAndMasks = true; goto reload; } if (needInsnsAndMasks) { // Also insert instructions and masks only once var list = f.fn.Masks.Select(m => new { word = m, t = "mask" }) .Concat(f.fn.Instructions.Select(i => new { word = i.Word, t = "instruction" })); foreach (var l in list) { db.Execute( $"INSERT INTO {l.t}s ( fingerprint_id, {l.t}_word ) VALUES ( @fpid, @word )", new { fpid = existing.fingerprint_id, word = (UInt64)l.word } ); } } // Check that variant does not already exist var countExisting = db.Query <int>( "SELECT COUNT(*) FROM variants WHERE fingerprint_id = @fpid AND variant_name = @name", new { fpid = existing.fingerprint_id, f.fn.Name } ).Single(); if (countExisting > 0) { stats.CountExisting++; continue; } else { stats.CountImported++; } var variant = new sqlVariant { fingerprint_id = existing.fingerprint_id, import_id = import.import_id, variant_name = f.fn.Name, variant_source = f.c.a.Filename }; variant.variant_id = db.Query <int>(@" INSERT INTO variants ( fingerprint_id, import_id, variant_name, variant_source ) VALUES ( @fingerprint_id, @import_id, @variant_name, @variant_source ); SELECT last_insert_rowid()", new { variant.fingerprint_id, variant.import_id, variant.variant_name, variant.variant_source } ).Single(); if (f.fn.Relocs == null) { continue; } foreach (var r in f.fn.Relocs) { db.Execute(@" INSERT INTO relocations ( relocation_address, relocation_sym, relocation_value, relocation_type, variant_id ) VALUES ( @Address, @Name, @value, @Type, @variant_id )", new { r.Address, r.Name, value = 0, r.Type, variant.variant_id } ); } } foreach (var c in constructed) { c.elf.Dispose(); } }
/// <summary> /// Imports issues priorities /// </summary> /// <returns></returns> private async Task <ImportStats> _importIssuePriorities() { _logger.LogDebug($"Priorities import. Start."); using var client = RedmineXmlImporter.CreateWebClient(); var uri = new Uri($"{_redmineUrl}enumerations/issue_priorities.json?key={_redmineApiKey}"); var json = await client.DownloadStringTaskAsync(uri); var result = new ImportStats(); var opts = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; var list = JsonSerializer.Deserialize <Models.RedmineJson.IssuePriorityResponse>(json, opts); // priorities are sorted from lowest to highest list.PriorityList.Reverse(); using var scope = _services.CreateScope(); using var _db = scope.ServiceProvider.GetRequiredService <DB>(); var prios = _db.IssuePriority.ToList(); _logger.LogDebug($"priorities"); result.total = list.PriorityList.Count; foreach (var item in list.PriorityList) { var dbItem = prios.FirstOrDefault(d => d.Id == item.Id); if (dbItem == null) { dbItem = new Redmine.Models.IssuePriority { Id = item.Id, Sort = list.PriorityList.IndexOf(item) + 1, // 1 .. 5 Created = DateTimeOffset.Now }; dbItem.Name = item.Name; var code = $"prio{dbItem.Sort}"; if (!prios.Any(d => d.Code == code)) { dbItem.Code = code; } await _db.AddAsync(dbItem); result.added++; } if (_db.Entry(dbItem).State != EntityState.Unchanged) { dbItem.Updated = DateTimeOffset.Now; result.updated++; } } await _db.SaveChangesAsync(); _logger.LogInformation($"Priorities import. Finish. {list.PriorityList.Count} processed."); return(result); }
/// <summary> /// Starts received-processor job for passed list of uris /// </summary> /// <param name="uriList">Uri list to fetch and process</param> /// <param name="processor">Processor function for fetched xml</param> /// <returns>Child elements in received xml</returns> private async Task <ImportStats> _doJob(List <string> uriList, Func <XDocument, Task <ImportStats> > processor) { var result = new ImportStats(); var receiveBlock = new TransformBlock <string, XDocument>( async(uri) => { XDocument xDoc = null; try { xDoc = await RedmineXmlImporter.ReceiveXmlAsync(uri, 0, 0); } catch (Exception e) { var we = e as WebException; if (we == null) { throw; } var resp = we.Response as HttpWebResponse; if (resp?.StatusCode != HttpStatusCode.Forbidden) { throw; } // we've got 403 responsse. It's a project with disabled something. } return(xDoc); }, _opts ); var processBlock = new ActionBlock <XDocument>( async xDoc => { if (xDoc == null) { return; } result.Add(await processor(xDoc)); } ); receiveBlock.LinkTo(processBlock, new DataflowLinkOptions { PropagateCompletion = true }); foreach (var uri in uriList) { receiveBlock.Post(uri); } receiveBlock.Complete(); await receiveBlock.Completion; _logger.LogDebug($"Received all. {processBlock.InputCount} pages waiting for process"); await processBlock.Completion; return(result); }