public async Task <ImportStats> UpdateSongPlays(SetlistShow show, IEnumerable <SetlistSong> songs)
{
var stats = new ImportStats();
await db.WithConnection(async con => {
stats.Created += await con.ExecuteAsync(@"
INSERT
INTO
setlist_songs_plays
(
played_setlist_song_id,
played_setlist_show_id
)
VALUES
(
@songId,
@showId
)
ON CONFLICT
ON CONSTRAINT setlist_songs_plays_song_id_show_id_key
DO NOTHING
", songs.Select(song => new { showId = show.id, songId = song.id }));
stats.Removed += await con.ExecuteAsync(@"
DELETE
FROM
setlist_songs_plays
WHERE
played_setlist_show_id = @showId
AND NOT(played_setlist_song_id = ANY(@songIds))
", new { showId = show.id, songIds = songs.Select(s => s.id).ToList() });
});
return(stats);
}
public static void Run(string data, Organization organization, Person runningPerson)
{
if (!data.StartsWith("\r\n<html>\r\n<head>\r\n <style>\r\n .tal"))
{
runningPerson.SendPhoneMessage("The file you uploaded does not appear to be a Payson export file. No processing done. The data has been discarded.");
throw new ArgumentException("This does not appear to be a Payson file");
}
if (organization.Identity != 1)
{
runningPerson.SendPhoneMessage("Payson import is currently only supported for PPSE. No processing done. The data has been discarded.");
throw new Exception("Payson is only supported for PPSE at the moment.");
}
ImportResult result = ImportPayson(data);
ImportStats stats = ProcessImportedData(result, organization, runningPerson);
try
{
runningPerson.SendPhoneMessage("The Payson file was processed. See mail for more details.");
}
catch (Exception)
{
// Ignore error on SMS transmit
}
string mailBody = string.Empty;
mailBody += String.Format("Rows processed: {0,9:N0}\r\n", stats.ProcessedTransactionCount);
mailBody += String.Format("Tx imported: {0,9:N0}\r\n", stats.ImportedTransactionCount);
mailBody += String.Format("Tx modified: {0,9:N0}\r\n", stats.ModifiedTransactionCount - stats.ImportedTransactionCount);
runningPerson.SendNotice("Payson file imported", mailBody, organization.Identity);
}
public static void Run(string data, Organization organization, Person runningPerson)
{
if (!data.StartsWith("Date\t Time\t Time Zone\t Name\t Type"))
{
runningPerson.SendPhoneMessage(
"The file you uploaded does not appear to be a PayPal tab-delimited file of all activity. No processing done. The data has been discarded.");
throw new ArgumentException("This does not appear to be a PayPal file");
}
ImportResult result = ImportPaypal(data);
ImportStats stats = ProcessImportedData(result, organization, runningPerson);
try
{
runningPerson.SendPhoneMessage("The PayPal file was processed. See mail for more details.");
}
catch (Exception)
{
// Ignore error on SMS transmit
}
string mailBody = string.Empty;
mailBody += String.Format("Rows processed: {0,9:N0}\r\n", stats.ProcessedTransactionCount);
mailBody += String.Format("Tx imported: {0,9:N0}\r\n", stats.ImportedTransactionCount);
mailBody += String.Format("Tx modified: {0,9:N0}\r\n",
stats.ModifiedTransactionCount - stats.ImportedTransactionCount);
runningPerson.SendNotice("PayPal file imported", mailBody, organization.Identity);
}
/// <summary>
/// Imports custom fields
/// </summary>
/// <returns></returns>
private async Task <ImportStats> _importCustomFields()
{
_logger.LogDebug($"Custom fields import. Start.");
using var client = RedmineXmlImporter.CreateWebClient();
var uri = new Uri($"{_redmineUrl}custom_fields.json?key={_redmineApiKey}");
var json = await client.DownloadStringTaskAsync(uri);
var result = new ImportStats();
var opts = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
};
var list = JsonSerializer.Deserialize <Models.RedmineJson.CustomFieldResponse>(json, opts);
using var scope = _services.CreateScope();
using var _db = scope.ServiceProvider.GetRequiredService <DB>();
var types = _db.CustomField.ToList();
result.total = list.FieldList.Count;
foreach (var item in list.FieldList)
{
var dbItem = types.FirstOrDefault(d => d.Id == item.Id);
if (dbItem == null)
{
dbItem = new Redmine.Models.CustomField
{
Id = item.Id,
Created = DateTimeOffset.Now
};
await _db.AddAsync(dbItem);
result.added++;
}
dbItem.Name = item.Name;
dbItem.Type = item.Type;
dbItem.Format = item.Format;
dbItem.IsRequired = item.IsRequired;
dbItem.IsMultiple = item.IsMultiple;
if (_db.Entry(dbItem).State != EntityState.Unchanged)
{
dbItem.Updated = DateTimeOffset.Now;
result.updated++;
}
}
await _db.SaveChangesAsync();
_logger.LogInformation($"Custom fields import. Finish. {list.FieldList.Count} processed.");
return(result);
}
public ImportCoreDataStats()
{
Users = new ImportStats();
Statuses = new ImportStats();
Priorities = new ImportStats();
TimeEntries = new ImportStats();
CustomFields = new ImportStats();
IssueTrackers = new ImportStats();
Projects = new ImportStats();
Versions = new ImportStats();
}
public void Add(ImportStats anotherCounter)
{
if (anotherCounter == null)
{
return;
}
total += anotherCounter.total;
added += anotherCounter.added;
updated += anotherCounter.updated;
deleted += anotherCounter.deleted;
}
private void UpdateQueryList()
{
ImportStats objImportStats = (ImportStats)Session["ImportStats"];
if (objImportStats.TotalLeads > 0 && Session["Campaign"] != null)
{
XmlDocument xDocCampaign = new XmlDocument();
Campaign objCampaign = (Campaign)Session["Campaign"];
xDocCampaign.LoadXml(Serialize.SerializeObject(objCampaign, "Campaign"));
CampaignService objCampService = new CampaignService();
objCampService.UpdateCampaignQueriesStats(xDocCampaign);
}
}
/// <summary>
/// Executes any sql to process data adding output parameters to fill result
/// </summary>
/// <param name="sql">SQL to execute with placeholders for @total, @added, @updated and @deleted output parameters</param>
/// <param name="queryParams">Query parameters to add to query call, except output parameters</param>
/// <returns></returns>
private async Task <ImportStats> _executeSql(string sql, params object[] queryParams)
{
using var scope = _services.CreateScope();
using var db = scope.ServiceProvider.GetRequiredService <DB>();
var parameters = new List <object>(queryParams);
var total = new SqlParameter("@total", System.Data.SqlDbType.Int);
total.Direction = System.Data.ParameterDirection.Output;
parameters.Add(total);
var added = new SqlParameter("@added", System.Data.SqlDbType.Int);
added.Direction = System.Data.ParameterDirection.Output;
parameters.Add(added);
var updated = new SqlParameter("@updated", System.Data.SqlDbType.Int);
updated.Direction = System.Data.ParameterDirection.Output;
parameters.Add(updated);
var deleted = new SqlParameter("@deleted", System.Data.SqlDbType.Int);
deleted.Direction = System.Data.ParameterDirection.Output;
parameters.Add(deleted);
await db.Database.ExecuteSqlRawAsync(sql, parameters);
var result = new ImportStats();
if (total.Value != DBNull.Value)
{
result.total = (int)total.Value;
}
if (added.Value != DBNull.Value)
{
result.added = (int)added.Value;
}
if (updated.Value != DBNull.Value)
{
result.updated = (int)updated.Value;
}
if (deleted.Value != DBNull.Value)
{
result.deleted = (int)deleted.Value;
}
return(result);
}
protected void Page_Load(object sender, EventArgs e)
{
if (!IsPostBack)
{
try
{
UpdateQueryList();
}
catch { }
}
if (Session["ImportStats"] != null)
{
ImportStats objImportStats = (ImportStats)Session["ImportStats"];
lblTotalLeads.Text = objImportStats.TotalLeads.ToString();
lblImported.Text = objImportStats.LeadsImported.ToString();
lblBlank.Text = objImportStats.LeadsBlankPhoneNumber.ToString();
lblSpChar.Text = objImportStats.LeadsSPCharPhoneNumber.ToString();
lblBadData.Text = objImportStats.LeadsBadData.ToString();
lblDuplicate.Text = objImportStats.LeadsDuplicate.ToString();
lblBadLength.Text = objImportStats.LeadsInvalidNumberLength.ToString();
lblUpdated.Text = objImportStats.LeadsUpdated.ToString();
try
{
if (hdnExportToClient.Value != "")
{
ExportToClientMachine(hdnExportToClient.Value, "text/csv");
hdnExportToClient.Value = "";
}
else
{
if (objImportStats.LeadsDuplicate > 0 && Session["DupFilePath"] != null && Session["DupFilePath"].ToString() != "")
{
if (System.IO.File.Exists(Session["DupFilePath"].ToString()))
{
hdnExportToClient.Value = Session["DupFilePath"].ToString();
}
Session["DupFilePath"] = null;
}
}
}
catch (Exception ex)
{
throw ex;
}
}
}
private async Task __old(PerformContext context)
{
var artists = (await _artistService.All()).ToList();
context.WriteLine($"--> Updating all {artists.Count} artists");
var progress = context.WriteProgressBar();
var stats = new ImportStats();
await artists.AsyncForEachWithProgress(progress, async artist =>
{
context.WriteLine($"--> Importing {artist.name}");
var artistStats = await _importerService.Import(artist, null, context);
context.WriteLine($"--> Imported {artist.name}! " + artistStats);
stats += artistStats;
});
context.WriteLine("--> Imported all artists! " + stats);
}
public static ImportStats ImportArchiveToDatabase(string dbPath, bool cleanDatabase, params string[] arFiles)
{
var existing = File.Exists(dbPath);
var stats = new ImportStats();
using (var db = new SqliteConnection(GetConnectionString(dbPath)))
{
if (!existing)
{
db.Execute(_sqlSchema);
}
db.Execute("BEGIN");
if (cleanDatabase)
{
db.Execute(@"
DELETE FROM fingerprints;
DELETE FROM instructions;
DELETE FROM masks;
DELETE FROM imports;
DELETE FROM variants;
DELETE FROM relocations;"
);
}
foreach (var ar in arFiles)
{
ImportArFile(db, ar, stats);
}
db.Execute("COMMIT");
}
return(stats);
}
protected static ImportStats ProcessImportedData(ImportResult import, Organization organization,
Person importingPerson)
{
FinancialAccount payPalAccount = organization.FinancialAccounts.AssetsPaypal;
FinancialAccount bankFees = organization.FinancialAccounts.CostsBankFees;
FinancialAccount donations = organization.FinancialAccounts.IncomeDonations;
int autoDepositLimit = 1000; // TODO: Get from organization parameters
int autoWithdrawalLimit = 0;
ImportStats result = new ImportStats();
foreach (ImportedRow row in import.Rows)
{
// Each row is at least a stub, probably more.
// If too old, ignore.
if (row.DateTime < new DateTime(2008, 12, 4))
{
continue;
}
string importKey = row.SuppliedTransactionId;
// If importKey is empty, construct a hash from the data fields.
if (string.IsNullOrEmpty(importKey))
{
string hashKey = row.HashBase + row.Comment +
(row.AmountCentsNet / 100.0).ToString(CultureInfo.InvariantCulture) +
row.CurrentBalance.ToString(CultureInfo.InvariantCulture) +
row.DateTime.ToString("yyyy-MM-dd-hh-mm-ss");
importKey = SHA1.Hash(hashKey).Replace(" ", "");
}
if (importKey.Length > 30)
{
importKey = importKey.Substring(0, 30);
}
Int64 amountCents = row.AmountCentsNet;
if (amountCents == 0)
{
amountCents = row.AmountCentsGross;
}
Dictionary <int, long> nominalTransaction = new Dictionary <int, long>();
FinancialTransaction transaction = null;
try
{
transaction = FinancialTransaction.FromImportKey(organization, importKey);
}
catch (Exception)
{
// if we get here, that means the transaction did not yet exist
transaction = FinancialTransaction.ImportWithStub(organization.Identity, row.DateTime,
payPalAccount.Identity, amountCents,
row.Comment, importKey, "" /* new SHA256 field */,
importingPerson.Identity);
result.ImportedTransactionCount++;
if (transaction == null)
{
// No transaction was created. This is an error condition as it should have been created if it didn't
// exist, and the "exist" case is handled in the FromImportKey attempt above. Abort with error.
// Throw new exception?
continue;
}
}
result.ProcessedTransactionCount++;
nominalTransaction[payPalAccount.Identity] = amountCents;
// The transaction was created. Examine if the autobook criteria are true.
if (amountCents < 0)
{
if ((-amountCents) < autoWithdrawalLimit * 100)
{
// Book against autoWithdrawal account.
nominalTransaction[bankFees.Identity] = -amountCents;
}
}
else if (amountCents > 0)
{
if (row.FeeCents < 0)
{
// This is always an autodeposit, if there is a fee (which is never > 0.0)
nominalTransaction[bankFees.Identity] = -row.FeeCents;
nominalTransaction[donations.Identity] = -row.AmountCentsGross;
}
else if (amountCents < autoDepositLimit * 100)
{
// Book against autoDeposit account.
nominalTransaction[donations.Identity] = -amountCents;
}
}
if (transaction.Rows.AmountCentsTotal != 0) // If transaction is unbalanced, balance it
{
if (transaction.RecalculateTransaction(nominalTransaction, importingPerson))
{
result.ModifiedTransactionCount++;
}
}
}
return(result);
}
public ImportIssuesStats()
{
List = new ImportStats();
Details = new ImportStats();
}
protected static ImportStats ProcessImportedData(ImportResult import, Organization organization, Person importingPerson)
{
FinancialAccount payPalAccount = organization.FinancialAccounts.AssetsPaypal;
FinancialAccount bankFees = organization.FinancialAccounts.CostsBankFees;
FinancialAccount donations = organization.FinancialAccounts.IncomeDonations;
int autoDepositLimit = 1000; // TODO: Get from organization parameters
int autoWithdrawalLimit = 0;
ImportStats result = new ImportStats();
foreach (ImportedRow row in import.Rows)
{
// Each row is at least a stub, probably more.
// If too old, ignore.
if (row.DateTime < new DateTime(2008, 12, 4))
{
continue;
}
string importKey = row.SuppliedTransactionId;
// If importKey is empty, construct a hash from the data fields.
if (string.IsNullOrEmpty(importKey))
{
string hashKey = row.HashBase + row.Comment + (row.AmountCentsNet / 100.0).ToString(CultureInfo.InvariantCulture) + row.CurrentBalance.ToString(CultureInfo.InvariantCulture) +
row.DateTime.ToString("yyyy-MM-dd-hh-mm-ss");
importKey = SHA1.Hash(hashKey).Replace(" ", "");
}
if (importKey.Length > 30)
{
importKey = importKey.Substring(0, 30);
}
Int64 amountCents = row.AmountCentsNet;
if (amountCents == 0)
{
amountCents = row.AmountCentsGross;
}
Dictionary<int, long> nominalTransaction = new Dictionary<int, long>();
FinancialTransaction transaction = null;
try
{
transaction = FinancialTransaction.FromImportKey(organization, importKey);
}
catch (Exception)
{
// if we get here, that means the transaction did not yet exist
transaction = FinancialTransaction.ImportWithStub(organization.Identity, row.DateTime,
payPalAccount.Identity, amountCents,
row.Comment, importKey,
importingPerson.Identity);
result.ImportedTransactionCount++;
if (transaction == null)
{
// No transaction was created. This is an error condition as it should have been created if it didn't
// exist, and the "exist" case is handled in the FromImportKey attempt above. Abort with error.
// Throw new exception?
continue;
}
}
result.ProcessedTransactionCount++;
nominalTransaction[payPalAccount.Identity] = amountCents;
// The transaction was created. Examine if the autobook criteria are true.
if (amountCents < 0)
{
if ((-amountCents) < autoWithdrawalLimit * 100)
{
// Book against autoWithdrawal account.
nominalTransaction[bankFees.Identity] = -amountCents;
}
}
else if (amountCents > 0)
{
if (row.FeeCents < 0)
{
// This is always an autodeposit, if there is a fee (which is never > 0.0)
nominalTransaction[bankFees.Identity] = -row.FeeCents;
nominalTransaction[donations.Identity] = -row.AmountCentsGross;
}
else if (amountCents < autoDepositLimit * 100)
{
// Book against autoDeposit account.
nominalTransaction[donations.Identity] = -amountCents;
}
}
if (transaction.Rows.AmountCentsTotal != 0) // If transaction is unbalanced, balance it
{
if (transaction.RecalculateTransaction(nominalTransaction, importingPerson))
{
result.ModifiedTransactionCount++;
}
}
}
return result;
}
private static void ImportArFile(DbConnection db, string arPath, ImportStats stats = null)
{
Console.Error.WriteLine($"{arPath}...");
// Dummy value in case none provided
stats = stats ?? new ImportStats();
var ar = new Archive(arPath);
var constructed = ar.Files
.Select(a => new { elf = a.ToElf(), a })
.Where(e => e.elf.Machine == ELFSharp.ELF.Machine.MIPS && e.elf.Endianess == ELFSharp.ELF.Endianess.BigEndian)
.Select(e => new { e.elf, e.a, syms = ((ISymbolTable)e.elf.GetSection(".symtab")).Entries.ToArray() })
.Select(e => new {
e.a,
e.syms,
e.elf,
text = e.elf.Sections.FirstOrDefault(s => s.Name == ".text")?.GetContents(),
fns = e.syms
.Select(s => s as SymbolEntry <uint>)
.Where(s => s != null && s.Binding == SymbolBinding.Global && s.Size != 0 && s.Type == ELFSharp.ELF.Sections.SymbolType.Function)
.Select(s => new { s.Name, s.Size, Location = s.Value })
})
.Select(e => new {
e.a,
e.syms,
e.fns,
e.text,
e.elf,
relocs = e.elf.Sections
.Where(s => s.Type == ELFSharp.ELF.Sections.SectionType.Relocation && s.Name.Contains(".text"))
.Select(s => new {
s,
data = s.GetContents().ToWordGroups(2)
.Select(g => new {
Type = (RelocationType)(g[1] & 0xFF),
Address = g[0],
T = e.syms[(int)g[1] >> 8]
})
//.Where(g => {
// if (string.IsNullOrWhiteSpace(g.T?.Name))
// Console.WriteLine("Hit!");
// return true;
//})
.Where(g => e.fns.Any(f => g.Address >= f.Location && g.Address < f.Location + f.Size))
.Select(g => {
var sym = g.T as SymbolEntry <uint>;
var name = sym.Name;
if (string.IsNullOrWhiteSpace(name))
{
name = sym.PointedSection?.Name;
//var preceding = e.syms
// .Select(y => y as SymbolEntry<uint>)
// .Where(y => y.Binding == SymbolBinding.Global)
// .Where(y => y.Type == ELFSharp.ELF.Sections.SymbolType.Function)
// .FirstOrDefault(y => y.Value < g.Address);
//var offset = sym.Value - sym.PointedSection?.LoadAddress;
//var insn = e.text
// .Skip((int)g.Address)
// .Take(4)
// .ToInstructions()
// .First();
//switch (g.Type)
//{
// case RelocationType.R_MIPS_26:
// offset = insn.Target;
// break;
// case RelocationType.R_MIPS_LO16:
// offset = insn.Immediate;
// break;
//}
//name = $"{e.a.Filename}({preceding?.Name}+{offset})";
}
return(new {
g.Type,
g.Address,
Name = name
});
})
.ToList(),
insns = s.Name.Contains(".text")
? e.elf.Sections.First(z => z.Name == s.Name.Replace(".rel", "")).GetContents().ToInstructions()
: null
})
.FirstOrDefault()
?.data
?.ToDictionary(d => d.Address, d => new { d.Name, d.Type })
})
.Where(e => e.relocs?.Values?.All(
r => r.Type == RelocationType.R_MIPS_26 ||
r.Type == RelocationType.R_MIPS_HI16 ||
r.Type == RelocationType.R_MIPS_LO16
) != false)
.Select(e => new {
e.a,
e.elf,
fns = e.fns
.Select(f => new {
f.Name,
f.Size,
f.Location,
Masks = Enumerable.Range(0, (int)f.Size / 4)
.Select(i => {
var ptr = (uint)(i * 4) + f.Location;
if (!(e.relocs?.ContainsKey(ptr)).GetValueOrDefault())
{
return(0xFFFFFFFFU);
}
switch (e.relocs[ptr].Type)
{
case RelocationType.R_MIPS_26:
return(~((1U << 26) - 1));
case RelocationType.R_MIPS_HI16:
case RelocationType.R_MIPS_LO16:
return(0xFFFF0000U);
default:
throw new NotImplementedException();
}
})
.ToArray()
})
.Select(f => new {
f.Name,
f.Size,
f.Location,
f.Masks,
Instructions = e.text.GetSegment((int)f.Location, (int)f.Size)
.ToInstructions()
.Zip(f.Masks, (i, m) => new Instruction(i & m))
.ToArray(),
Relocs = e.relocs
?.Where(r => r.Key >= f.Location && r.Key < f.Location + f.Size)
?.Select(r => new { Address = r.Key - f.Location, r.Value.Type, r.Value.Name })
?.ToArray()
})
.Select(f => new {
f.Name, f.Size, f.Location, f.Masks, f.Instructions, f.Relocs,
Hash = CreateMd5Hash(
f.Instructions
.RemoveTrailingNops()
.ToBytes()
.ToArray()
)
})
.ToArray()
})
.Where(e => e.fns.Length > 0)
.ToList();
var info = new FileInfo(arPath);
int importId = 0;
try
{
importId = db.Query <int>(@"
INSERT INTO imports ( import_file_md5, import_file, import_mtime ) VALUES (
@md5,
@name,
@mtime
);
SELECT last_insert_rowid()",
new
{
md5 = CreateMd5Hash(File.ReadAllBytes(arPath)),
name = info.Name,
mtime = info.LastWriteTimeUtc.ToString("o")
}
).Single();
}
catch
{
stats.ArchivesExisting++;
return;
}
var import = db.Query <sqlImport>("SELECT * FROM imports WHERE import_id = @importId", new { importId })
.Single();
foreach (var f in constructed.SelectMany(c => c.fns.Select(f => new { c, fn = f })))
{
var needInsnsAndMasks = false;
reload:
var existing =
db.Query <sqlFingerprint>("SELECT * FROM fingerprints WHERE fingerprint_md5 = @md5", new { md5 = f.fn.Hash })
.SingleOrDefault();
if (existing == null)
{
db.Execute(@"
INSERT INTO fingerprints ( fingerprint_md5, fingerprint_length ) VALUES (
@md5, @length
);",
new { md5 = f.fn.Hash, length = f.fn.Size }
);
needInsnsAndMasks = true;
goto reload;
}
if (needInsnsAndMasks)
{
// Also insert instructions and masks only once
var list = f.fn.Masks.Select(m => new { word = m, t = "mask" })
.Concat(f.fn.Instructions.Select(i => new { word = i.Word, t = "instruction" }));
foreach (var l in list)
{
db.Execute(
$"INSERT INTO {l.t}s ( fingerprint_id, {l.t}_word ) VALUES ( @fpid, @word )",
new { fpid = existing.fingerprint_id, word = (UInt64)l.word }
);
}
}
// Check that variant does not already exist
var countExisting = db.Query <int>(
"SELECT COUNT(*) FROM variants WHERE fingerprint_id = @fpid AND variant_name = @name",
new { fpid = existing.fingerprint_id, f.fn.Name }
).Single();
if (countExisting > 0)
{
stats.CountExisting++;
continue;
}
else
{
stats.CountImported++;
}
var variant = new sqlVariant
{
fingerprint_id = existing.fingerprint_id,
import_id = import.import_id,
variant_name = f.fn.Name,
variant_source = f.c.a.Filename
};
variant.variant_id = db.Query <int>(@"
INSERT INTO variants ( fingerprint_id, import_id, variant_name, variant_source ) VALUES (
@fingerprint_id,
@import_id,
@variant_name,
@variant_source
);
SELECT last_insert_rowid()",
new { variant.fingerprint_id, variant.import_id, variant.variant_name, variant.variant_source }
).Single();
if (f.fn.Relocs == null)
{
continue;
}
foreach (var r in f.fn.Relocs)
{
db.Execute(@"
INSERT INTO relocations (
relocation_address,
relocation_sym,
relocation_value,
relocation_type,
variant_id
) VALUES (
@Address,
@Name,
@value,
@Type,
@variant_id
)",
new { r.Address, r.Name, value = 0, r.Type, variant.variant_id }
);
}
}
foreach (var c in constructed)
{
c.elf.Dispose();
}
}
/// <summary>
/// Imports issues priorities
/// </summary>
/// <returns></returns>
private async Task <ImportStats> _importIssuePriorities()
{
_logger.LogDebug($"Priorities import. Start.");
using var client = RedmineXmlImporter.CreateWebClient();
var uri = new Uri($"{_redmineUrl}enumerations/issue_priorities.json?key={_redmineApiKey}");
var json = await client.DownloadStringTaskAsync(uri);
var result = new ImportStats();
var opts = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = true
};
var list = JsonSerializer.Deserialize <Models.RedmineJson.IssuePriorityResponse>(json, opts);
// priorities are sorted from lowest to highest
list.PriorityList.Reverse();
using var scope = _services.CreateScope();
using var _db = scope.ServiceProvider.GetRequiredService <DB>();
var prios = _db.IssuePriority.ToList();
_logger.LogDebug($"priorities");
result.total = list.PriorityList.Count;
foreach (var item in list.PriorityList)
{
var dbItem = prios.FirstOrDefault(d => d.Id == item.Id);
if (dbItem == null)
{
dbItem = new Redmine.Models.IssuePriority
{
Id = item.Id,
Sort = list.PriorityList.IndexOf(item) + 1, // 1 .. 5
Created = DateTimeOffset.Now
};
dbItem.Name = item.Name;
var code = $"prio{dbItem.Sort}";
if (!prios.Any(d => d.Code == code))
{
dbItem.Code = code;
}
await _db.AddAsync(dbItem);
result.added++;
}
if (_db.Entry(dbItem).State != EntityState.Unchanged)
{
dbItem.Updated = DateTimeOffset.Now;
result.updated++;
}
}
await _db.SaveChangesAsync();
_logger.LogInformation($"Priorities import. Finish. {list.PriorityList.Count} processed.");
return(result);
}
/// <summary>
/// Starts received-processor job for passed list of uris
/// </summary>
/// <param name="uriList">Uri list to fetch and process</param>
/// <param name="processor">Processor function for fetched xml</param>
/// <returns>Child elements in received xml</returns>
private async Task <ImportStats> _doJob(List <string> uriList, Func <XDocument, Task <ImportStats> > processor)
{
var result = new ImportStats();
var receiveBlock = new TransformBlock <string, XDocument>(
async(uri) =>
{
XDocument xDoc = null;
try
{
xDoc = await RedmineXmlImporter.ReceiveXmlAsync(uri, 0, 0);
}
catch (Exception e)
{
var we = e as WebException;
if (we == null)
{
throw;
}
var resp = we.Response as HttpWebResponse;
if (resp?.StatusCode != HttpStatusCode.Forbidden)
{
throw;
}
// we've got 403 responsse. It's a project with disabled something.
}
return(xDoc);
},
_opts
);
var processBlock = new ActionBlock <XDocument>(
async xDoc =>
{
if (xDoc == null)
{
return;
}
result.Add(await processor(xDoc));
}
);
receiveBlock.LinkTo(processBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (var uri in uriList)
{
receiveBlock.Post(uri);
}
receiveBlock.Complete();
await receiveBlock.Completion;
_logger.LogDebug($"Received all. {processBlock.InputCount} pages waiting for process");
await processBlock.Completion;
return(result);
}