public void Run() { using (var dbContext = SidekaDbContext.CreateForTools()) { Console.WriteLine("========= Updating Data ========="); Console.WriteLine("Fetching Desa"); var desas = dbContext.SidekaDesa.OrderBy(d => d.BlogId).ToList(); var contentTypes = new string[] { "penduduk", "pemetaan" }; foreach (var desa in desas) { foreach (var contentType in contentTypes) { Console.WriteLine("Processing Desa {0}-{1} {2}", desa.BlogId, desa.Desa, contentType); var contentQuery = dbContext.SidekaContent .Where(sc => sc.DesaId == desa.BlogId) .Where(sc => sc.ApiVersion == "2.0") .Where(sc => sc.Type == contentType) .OrderByDescending(sc => sc.Id); try { var sidekaContent = contentQuery.AsNoTracking().FirstOrDefault(); if (sidekaContent == null) { continue; } Console.WriteLine("Content Change ID {0}", sidekaContent.ChangeId); var jObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content); var viewModel = new SidekaContentViewModel(jObject); bool hasChanges = false; foreach (var tab in viewModel.Data.Keys.ToArray()) { var data = viewModel.Data[tab]; var columns = viewModel.Columns[tab]; var idGetter = GetIdGetter(columns); var ids = new HashSet <string>(); var duplicates = new List <int>(); for (var i = 0; i < data.Length; i++) { var id = idGetter(data[i]); if (ids.Contains(id)) { duplicates.Add(i); } ids.Add(id); } if (duplicates.Count > 0) { duplicates.Reverse(); Console.WriteLine("Tab {0} has duplicates: {1}", tab, string.Join(", ", duplicates)); var list = data.ToList(); foreach (var i in duplicates) { list.RemoveAt(i); } data = viewModel.Data[tab] = list.ToArray(); hasChanges = true; } } if (hasChanges) { var newContent = JsonConvert.SerializeObject(viewModel); var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Data)); var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Diffs)); var updatedContent = new SidekaContent { Id = sidekaContent.Id, ContentSize = contentSize, Content = newContent, DiffSize = diffSize }; dbContext.Attach(updatedContent); dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true; dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true; dbContext.Entry(updatedContent).Property(c => c.Content).IsModified = true; dbContext.SaveChanges(); Console.WriteLine("Content updated"); } } catch (Exception e) { Console.WriteLine(e); Console.WriteLine(e.StackTrace); } } } } }
public void Run() { using (var dbContext = SidekaDbContext.CreateForTools()) { Console.WriteLine("========= Updating Data ========="); Console.WriteLine("Fetching Desa"); var desas = dbContext.SidekaDesa.Where(d => d.BlogId > 6).OrderBy(d => d.BlogId).ToList(); foreach (var desa in desas) { Console.WriteLine("Processing Desa {0}-{1}", desa.BlogId, desa.Desa); var contentQuery = dbContext.SidekaContent .Where(sc => sc.DesaId == desa.BlogId) .Where(sc => sc.ApiVersion == "2.0"); Console.WriteLine("Fetching Contents For Desa {0}-{1}", desa.BlogId, desa.Desa); var numberOfContents = contentQuery.Count(); var counter = 0; var skip = 0; while (counter < numberOfContents) { Console.WriteLine("Fetching contents skip #{0} take 10", skip); var sidekaContents = contentQuery.AsNoTracking().OrderByDescending(sc => sc.ChangeId).Skip(skip).Take(10).ToList(); foreach (var sidekaContent in sidekaContents) { counter += 1; Console.WriteLine("Processing Content #{0}", counter); var sidekaContentJObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content); if (sidekaContent.ApiVersion == "1.0") { sidekaContentJObject["columns"] = JArray.FromObject(new string[] { "nik", "nama_penduduk", "tempat_lahir", "tanggal_lahir", "jenis_kelamin", "pendidikan", "agama", "status_kawin", "pekerjaan", "pekerjaan_ped", "kewarganegaraan", "kompetensi", "no_telepon", "email", "no_kitas", "no_paspor", "golongan_darah", "status_penduduk", "status_tinggal", "kontrasepsi", "difabilitas", "no_kk", "nama_ayah", "nama_ibu", "hubungan_keluarga", "nama_dusun", "rw", "rt", "alamat_jalan" }); } try { Console.WriteLine("Calculating Sizes...."); var content = new SidekaContentViewModel(sidekaContentJObject); var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Data)); var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Diffs)); Console.WriteLine("Desa {0}-Change Id {1}, Content Size {2}, Diff Size {3}", desa.Desa, sidekaContent.ChangeId, contentSize, diffSize); Console.WriteLine("Saving Size..."); var updatedContent = new SidekaContent { Id = sidekaContent.Id, ContentSize = contentSize, DiffSize = diffSize }; dbContext.Attach(updatedContent); dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true; dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true; //dbContext.Update(updatedContent); dbContext.SaveChanges(); Console.WriteLine("Sizes Have Been Saved"); } catch (Exception ex) { Console.WriteLine("Error When Calculating Size Desa {0}-{1}: {2}", desa.BlogId, desa.Desa, ex.Message); } } skip += 10; } } } }
private Dictionary <string, object> GetDiffsNewerThanClient(int desaId, string contentType, string contentSubtype, int clientChangeId, JObject clientColumns) { var result = new Dictionary <string, object>(); foreach (var key in clientColumns.Properties().Select(c => c.Name)) { result.Add(key, new List <object>()); } var newerQuery = dbContext.SidekaContent .Where(sc => sc.DesaId == desaId) .Where(sc => sc.Type == contentType) .Where(sc => sc.ChangeId > clientChangeId); if (!string.IsNullOrWhiteSpace(contentSubtype)) { newerQuery = newerQuery .Where(sc => sc.Subtype == contentSubtype) .OrderBy(sc => sc.ChangeId); } var contents = newerQuery.Select(sc => sc.Content).ToList(); foreach (var contentString in contents) { var contentJObject = JsonConvert.DeserializeObject <JObject>(contentString); try { var content = new SidekaContentViewModel(contentJObject); if (content.Diffs == null) { continue; } foreach (var diff in content.Diffs) { if (clientColumns[diff.Key] == null) { continue; } var diffTabColumns = JsonConvert.DeserializeObject <JToken>(JsonConvert.SerializeObject(content.Columns[diff.Key])); var clientTabColumns = clientColumns[diff.Key]; foreach (var diffContent in content.Diffs[diff.Key]) { if (JToken.DeepEquals(diffTabColumns, clientTabColumns)) { ((List <object>)result[diff.Key]).Add(diffContent); } else { var transformedDiff = new SidekaDiff { Added = TransformData(diffTabColumns, clientTabColumns, diffContent.Added), Modified = TransformData(diffTabColumns, clientTabColumns, diffContent.Modified), Deleted = TransformData(diffTabColumns, clientTabColumns, diffContent.Deleted) }; transformedDiff.Total = transformedDiff.Added.Length + transformedDiff.Modified.Length + transformedDiff.Deleted.Length; ((List <object>)result[diff.Key]).Add(transformedDiff); } } } } catch (Exception ex) { } } return(result); }
public async Task <IActionResult> PostContentV2([FromBody] JObject contentJObject, int desaId, string contentType, string contentSubtype = null) { Stopwatch sw = new Stopwatch(); sw.Start(); string lockName = desaId + "_" + contentType + "_" + contentSubtype; object writeLock = writeLocks.GetOrAdd(lockName, new object()); lock (writeLock){ var auth = GetAuth(desaId); if (auth == null) { return(StatusCode((int)HttpStatusCode.Forbidden, new Dictionary <string, string>() { { "message", "Invalid or no token" } })); } var permission = contentType; if (new string[] { "perencanaan", "penganggaran", "spp", "penerimaan" }.Contains(contentType)) { permission = "keuangan"; } var roles = (List <string>)auth["roles"]; if (!roles.Contains("administrator") && !roles.Contains(permission)) { return(StatusCode((int)HttpStatusCode.Forbidden, new Dictionary <string, string>() { { "message", "Your account doesn't have the permission" } })); } var content = new SidekaContentViewModel(contentJObject); // Validate foreach (var column in content.Columns) { if (content.Diffs != null && content.Diffs.ContainsKey(column.Key)) { var index = 0; foreach (var diff in content.Diffs[column.Key]) { var location = string.Format("Diff {0} ({1}) tab {2}", index, "added", column.Key); var invalid = Validate(column.Value, diff.Added, location); if (invalid != null) { return(invalid); } location = string.Format("Diff {0} ({1}) tab {2}", index, "modified", column.Key); invalid = Validate(column.Value, diff.Modified, location); if (invalid != null) { return(invalid); } location = string.Format("Diff {0} ({1}) tab {2}", index, "deleted", column.Key); invalid = Validate(column.Value, diff.Deleted, location, false); if (invalid != null) { return(invalid); } } } if (content.Data != null && content.Data.ContainsKey(column.Key)) { var location = string.Format("Data tab {0}", column.Key); var invalid = Validate(column.Value, content.Data[column.Key], location); if (invalid != null) { return(invalid); } } } var clientChangeId = 0; var changeId = QueryStringHelper.GetQueryString <int>(Request.Query, "changeId", 0); if (changeId > 0) { clientChangeId = changeId; } // Find max change id var maxChangeIdQuery = dbContext.SidekaContent .Where(sc => sc.DesaId == desaId) .Where(sc => sc.Type == contentType); if (!string.IsNullOrWhiteSpace(contentSubtype)) { maxChangeIdQuery = maxChangeIdQuery.Where(sc => sc.Subtype == contentSubtype); } var maxChangeId = maxChangeIdQuery.Select(sc => sc.ChangeId).DefaultIfEmpty(0).Max(); // TODO: This is risky!! Consider changing change_id column to serial or autoincrement var newContent = new SidekaContentViewModel(); // Initialize new content to be saved foreach (var column in content.Columns) { newContent.Data[column.Key] = new List <object>().ToArray(); newContent.Columns[column.Key] = column.Value; if (content.Diffs != null && content.Diffs.ContainsKey(column.Key)) { newContent.Diffs[column.Key] = content.Diffs[column.Key]; } else { newContent.Diffs[column.Key] = new List <SidekaDiff>().ToArray(); } } var latestContentQuery = dbContext.SidekaContent .Where(sc => sc.DesaId == desaId) .Where(sc => sc.Type == contentType); if (!string.IsNullOrWhiteSpace(contentSubtype)) { latestContentQuery = latestContentQuery.Where(sc => sc.Subtype == contentSubtype); } var latestContentString = latestContentQuery .OrderByDescending(sc => sc.ChangeId) .Select(sc => sc.Content) .FirstOrDefault(); JObject latestContentJObject = null; if (string.IsNullOrWhiteSpace(latestContentString)) { latestContentJObject = new JObject { { "data", new JObject() }, { "columns", contentJObject["columns"] } }; } else { latestContentJObject = JsonConvert.DeserializeObject <JObject>(latestContentString); } var diffs = GetDiffsNewerThanClient(desaId, contentType, contentSubtype, clientChangeId, (JObject)contentJObject["columns"]); if (latestContentJObject["data"] is JArray && contentType == "penduduk") { newContent.Data["penduduk"] = MergeDiffs(newContent.Columns["penduduk"], newContent.Diffs["penduduk"], new List <object>().ToArray()); } else { var latestContent = new SidekaContentViewModel(latestContentJObject); foreach (var column in content.Columns) { // Initialize so the latest content have the same tab with the posted content if (!latestContent.Columns.ContainsKey(column.Key)) { latestContent.Columns[column.Key] = column.Value; } if (!latestContent.Data.ContainsKey(column.Key)) { latestContent.Data[column.Key] = new List <object>().ToArray(); } if (content.Data != null && content.Data[column.Key] != null && new string[] { "perencanaan", "penganggaran", "penerimaan", "spp" }.Contains(contentType)) { var invalid = ValidateDuplicatesData(column.Key, column.Value, content.Data[column.Key]); if (invalid != null) { return(invalid); } // Special case for client who posted data instead of diffs newContent.Data[column.Key] = content.Data[column.Key]; // Add new diffs to show that the content is rewritten var sidekaDiff = new SidekaDiff { Added = new List <object>().ToArray(), Modified = new List <object>().ToArray(), Deleted = new List <object>().ToArray(), Total = 0, Rewritten = true }; newContent.Diffs[column.Key].Append(sidekaDiff); } else if (newContent.Diffs[column.Key].Length > 0) { // There's diffs in the posted content for this tab, apply them to latest data var latestColumns = latestContent.Columns[column.Key]; var transformedLatestData = TransformData( latestContentJObject["columns"][column.Key], contentJObject["columns"][column.Key], latestContent.Data[column.Key]); var invalid = ValidateDuplicatesDiffs(column.Key, column.Value, content.Diffs[column.Key], transformedLatestData); if (invalid != null) { return(invalid); } var mergedData = MergeDiffs(column.Value, content.Diffs[column.Key], transformedLatestData); newContent.Data[column.Key] = mergedData; newContent.Columns[column.Key] = column.Value; } else { // There's no diffs in the posted content for this tab, use the old data newContent.Data[column.Key] = latestContent.Data[column.Key]; } } } var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(newContent.Data)); var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(newContent.Diffs)); int newChangeId = GetNextChangeId(desaId, contentType, contentSubtype); var sidekaContent = new SidekaContent { DesaId = desaId, Type = contentType, Subtype = contentSubtype, Content = JsonConvert.SerializeObject(newContent), DateCreated = DateTime.Now, CreatedBy = (int)auth["user_id"], ChangeId = newChangeId, ApiVersion = configuration.GetValue <string>("ApiVersion"), ContentSize = contentSize, DiffSize = diffSize }; dbContext.Add(sidekaContent); dbContext.SaveChanges(); var result = new Dictionary <string, object>() { { "success", true }, { "changeId", newChangeId }, { "change_id", newChangeId }, { "diffs", diffs }, { "columns", content.Columns }, }; sw.Stop(); Logs((int)auth["user_id"], desaId, "", "save_content", contentType, contentSubtype, sw.Elapsed.Milliseconds); return(Ok(result)); } }
public async Task <IActionResult> UpdateSizes(int desaId, string contentType, string contentSubtype = null) { var clientChangeId = 0; var changeId = QueryStringHelper.GetQueryString <int>(Request.Query, "changeId", 0); if (changeId > 0) { clientChangeId = changeId; } var contentQuery = dbContext.SidekaContent .Where(sc => sc.DesaId == desaId) .Where(sc => sc.Type == contentType) .Where(sc => sc.Subtype == contentSubtype) .Where(sc => sc.ChangeId >= clientChangeId); var sidekaContents = await contentQuery.OrderByDescending(sc => sc.ChangeId).ToListAsync(); var sizes = new List <Dictionary <string, int> >(); var result = new Dictionary <string, object>() { { "success", true }, { "content", sizes } }; foreach (var sidekaContent in sidekaContents) { var sidekaContentJObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content); var sizeItem = new Dictionary <string, int>(); try { var content = new SidekaContentViewModel(sidekaContentJObject); var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Data)); var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Diffs)); sizeItem.Add("contentSize", contentSize); if (content.Diffs == null) { sizeItem.Add("diffSize", 0); } else { sizeItem.Add("diffSize", diffSize); } sidekaContent.ContentSize = contentSize; sidekaContent.DiffSize = diffSize; dbContext.Update(sidekaContent); await dbContext.SaveChangesAsync(); sizes.Add(sizeItem); } catch (Exception ex) { } } return(Ok(result)); }
public void Run() { using (var dbContext = SidekaDbContext.CreateForTools()) { Console.WriteLine("========= Updating Data ========="); Console.WriteLine("Fetching Desa"); var desas = dbContext.SidekaDesa.OrderBy(d => d.BlogId).ToList(); var contentTypes = new string[] { "penduduk" }; foreach (var desa in desas) { foreach (var contentType in contentTypes) { Console.WriteLine("Processing Desa {0}-{1} {2}", desa.BlogId, desa.Desa, contentType); var contentQuery = dbContext.SidekaContent .Where(sc => sc.DesaId == desa.BlogId) .Where(sc => sc.ApiVersion == "2.0") .Where(sc => sc.Type == contentType) .OrderByDescending(sc => sc.Id); try { var sidekaContent = contentQuery.AsNoTracking().FirstOrDefault(); if (sidekaContent == null) { continue; } Console.WriteLine("Content Change ID {0}", sidekaContent.ChangeId); var jObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content); var viewModel = new SidekaContentViewModel(jObject); bool hasChanges = false; if (!viewModel.Columns.ContainsKey("penduduk") || !viewModel.Data.ContainsKey("penduduk") || viewModel.Columns["penduduk"].Columns.Length != 34 || viewModel.Data["penduduk"].Length == 0) { continue; } bool all30Columns = true; var data = viewModel.Data["penduduk"]; foreach (var row in data) { if (((object[])row).Length != 30) { all30Columns = false; break; } } if (all30Columns) { Console.WriteLine("All 30 columns"); } if (hasChanges) { var newContent = JsonConvert.SerializeObject(viewModel); var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Data)); var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Diffs)); var updatedContent = new SidekaContent { Id = sidekaContent.Id, ContentSize = contentSize, Content = newContent, DiffSize = diffSize }; dbContext.Attach(updatedContent); dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true; dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true; dbContext.Entry(updatedContent).Property(c => c.Content).IsModified = true; dbContext.SaveChanges(); Console.WriteLine("Content updated"); } } catch (Exception e) { Console.WriteLine(e); Console.WriteLine(e.StackTrace); } } } } }