/// <summary>
/// Analyze the records to determine which should be included in the feed
/// based on dependencies.
/// </summary>
public async Task Analyze()
{
// load some small tables into memory for performance
var cache = new DataLineCache();
await cache.Load(Repo.Lines(), new[] { nameof(CsvOrg), nameof(CsvCourse), nameof(CsvClass) });
var orgIds = new List<string>();
// include Orgs that have been selected for sync
foreach (var org in cache.GetMap<CsvOrg>().Values.Where(IsUnappliedChange))
{
orgIds.Add(org.SourcedId);
IncludeReadyTouch(org);
}
await Repo.Committer.Invoke();
// courses are manually marked for sync, so choose only those
foreach (var course in cache.GetMap<CsvCourse>().Values.Where(l => l.IncludeInSync).Where(IsUnappliedChange))
IncludeReadyTouch(course);
await Repo.Committer.Invoke();
// now walk the classes and include those which map to an included course and are part of the selected orgs.
var classMap = cache.GetMap<CsvClass>();
var courseIds = cache.GetMap<CsvCourse>()
.Values
.Where(l => l.IncludeInSync)
.Select(l => l.SourcedId)
.ToHashSet();
foreach (var _class in classMap.Values.Where(IsUnappliedChangeWithoutIncludedInSync))
{
CsvClass csvClass = JsonConvert.DeserializeObject<CsvClass>(_class.RawData);
if (courseIds.Contains(csvClass.courseSourcedId) && orgIds.Contains(csvClass.schoolSourcedId))
IncludeReadyTouch(_class);
await Repo.Committer.InvokeIfChunk();
}
await Repo.Committer.InvokeIfAny();
// process enrollments in the database associated with the District based on the conditions below (in chunks of 200)
await Repo.Lines<CsvEnrollment>().ForEachInChunksAsync(chunkSize: 200,
action: async (enrollment) =>
{
CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject<CsvEnrollment>(enrollment.RawData);
// figure out if we need to process this enrollment
if (!classMap.ContainsKey(csvEnrollment.classSourcedId) || // look up class associated with enrollment
!classMap[csvEnrollment.classSourcedId].IncludeInSync || // only include enrollment if the class is included
!IsUnappliedChangeWithoutIncludedInSync(enrollment)) // only include if unapplied change in enrollment
return;
var user = await Repo.Lines<CsvUser>().SingleOrDefaultAsync(l => l.SourcedId == csvEnrollment.userSourcedId);
if (user == null) // should never happen
{
enrollment.Error = $"Missing user for {csvEnrollment.userSourcedId}";
Logger.Here().LogError($"Missing user for enrollment for line {enrollment.DataSyncLineId}");
return;
}
// mark enrollment for sync
IncludeReadyTouch(enrollment);
// mark user for sync
//DataSyncLine user = userMap[csvEnrollment.userSourcedId];
if (IsUnappliedChangeWithoutIncludedInSync(user))
IncludeReadyTouch(user);
},
onChunkComplete: async () => await Repo.Committer.Invoke());
// now process any user changes we may have missed
await Repo.Lines<CsvUser>().Where(u => u.IncludeInSync
&& u.LoadStatus != LoadStatus.NoChange
&& u.SyncStatus != SyncStatus.ReadyToApply)
.ForEachInChunksForShrinkingList(chunkSize: 200,
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
action: async (user) => IncludeReadyTouch(user),
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
onChunkComplete: async () => await Repo.Committer.Invoke());
await Repo.Committer.Invoke();
}
//Metoda care citeste un csv si returneaza datele citite intr-un DataTable
private DataTable readCsv(string path)
{
//Creez un tabel de tip DataTable in care o sa introduc valorile din CSV
DataTable tabel = new DataTable("Csv");
tabel.Columns.Add("LogTime", typeof(DateTime));
tabel.Columns.Add("Action", typeof(string));
tabel.Columns.Add("FolderPath", typeof(string));
tabel.Columns.Add("FileName", typeof(string));
tabel.Columns.Add("Username", typeof(string));
tabel.Columns.Add("IpAdress", typeof(string));
tabel.Columns.Add("XferSize", typeof(int));
tabel.Columns.Add("Duration", typeof(double));
tabel.Columns.Add("AgentBrand", typeof(string));
tabel.Columns.Add("AgentVersion", typeof(string));
tabel.Columns.Add("Error", typeof(int));
Log.Info(String.Format("Incepe parsarea fisierului: {0}", path));
try
{
// Pentru citirea datelor din CSV folosesc StreamReader
StreamReader reader = new StreamReader(path);
//Citesc prima linie si nu fac nimic cu ea pentru ca e header-ul CSV-ului
reader.ReadLine();
//Un while in care citesc fiecare linie din CSV pana cand se termina stream-ul
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var element = line.Split(',');
DateTime logTime;
DateTime?logTime2 = null;
//Parsez data din CSV
bool succes = DateTime.TryParseExact(element[0].Trim('"'), "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.None, out logTime);
if (succes == true)
{
logTime2 = logTime;
}
//Instantiez un obiect de tip CsvClass, care are ca proprietati elementele citite pe randul respectiv
CsvClass CsvClass = new CsvClass(logTime2, element[1].Trim('"'), element[2].Trim('"'), element[3].Trim('"'), element[4].Trim('"'),
element[5].Trim('"'), Convert.ToInt32(element[6].Trim('"')), Convert.ToDouble(element[7].Trim('"')),
element[8].Trim('"'), element[9].Trim('"'), Convert.ToInt32(element[10].Trim('"')));
//Adaug pe un rand datele citite
DataRow rand = tabel.NewRow();
if (CsvClass.LogTime == null)
{
rand["LogTime"] = DBNull.Value;
}
else
{
rand["LogTime"] = CsvClass.LogTime;
}
rand["Action"] = CsvClass.Action;
rand["FolderPath"] = CsvClass.FolderPath;
rand["FileName"] = CsvClass.Filename;
rand["Username"] = CsvClass.Username;
rand["IpAdress"] = CsvClass.IpAdress;
rand["XferSize"] = CsvClass.XferSize;
rand["Duration"] = CsvClass.Duration;
rand["AgentBrand"] = CsvClass.AgentBrand;
rand["AgentVersion"] = CsvClass.AgentVersion;
rand["Error"] = CsvClass.Error;
tabel.Rows.Add(rand);
}
reader.Close();
}
catch (Exception e)
{
Log.Error("Au fost probleme la parsarea fisierului fisierului:", e);
}
finally
{
//Dupa ce am citit csv-ul, il mut intr-o alta locatie(marcare)
moveFile(path);
}
return(tabel);
}
//Metoda care citeste un csv si returneaza o lista de obiecte
private List <CsvClass> readCsv(string path)
{
//Creez o lista in care o sa adaug obiectele citite in CSV
List <CsvClass> lista = new List <CsvClass>();
Log.Info(String.Format("Incepe parsarea fisierului: {0}", path));
try
{
// Pentru citirea datelor din CSV folosesc StreamReader
StreamReader reader = new StreamReader(path);
//Citesc prima linie si nu fac nimic cu ea pentru ca e header-ul CSV-ului
reader.ReadLine();
//Un while in care citesc fiecare linie din CSV pana cand se termina stream-ul
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
var element = line.Split(',');
int element6;
int? element6Null = null;
double element7;
double? element7Null = null;
int element10;
int? element10Null = null;
DateTime logTime;
DateTime?logTime2 = null;
//Parsez data din CSV, daca parsarea nu reuseste datetime-ul o sa fie null
bool succes = DateTime.TryParseExact(element[0].Trim('"'), "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture, DateTimeStyles.None, out logTime);
if (succes == true)
{
logTime2 = logTime;
}
//Parsez un element de tip Int,daca parsarea nu reuseste int-ul o sa fie null
bool tryParseElement6 = Int32.TryParse(element[6].Trim('"'), out element6);
if (tryParseElement6 == true)
{
element6Null = element6;
}
//Parsez un element de tip Double,daca parsarea nu reuseste double-ul o sa fie null
bool tryParseElement7 = Double.TryParse(element[7].Trim('"'), out element7);
if (tryParseElement7 == true)
{
element7Null = element7;
}
//Parsez un element de tip Int,daca parsarea nu reuseste int-ul o sa fie null
bool tryParseElement10 = Int32.TryParse(element[10].Trim('"'), out element10);
if (tryParseElement10 == true)
{
element10Null = element10;
}
//Instantiez un obiect de tip CsvClass, care are ca proprietati elementele citite pe randul respectiv
CsvClass CsvClass = new CsvClass(logTime2, element[1].Trim('"'), element[2].Trim('"'), element[3].Trim('"'), element[4].Trim('"'),
element[5].Trim('"'), element6Null, element7Null,
element[8].Trim('"'), element[9].Trim('"'), element10Null);
//Adaug obiectul citit in lista
lista.Add(CsvClass);
}
//Dupa ce stream-ul s-a terminat inchid reader-ul
reader.Close();
}
catch (Exception e)
{
Log.Error("Au fost probleme la parsarea fisierului fisierului:", e);
}
finally
{
//Dupa ce am citit csv-ul, il mut intr-o alta locatie(marcare)
moveFile(path);
}
//Returnez lista de obiecte
return(lista);
}
