public void run(string regionCode, string subtype)
{
var desa = dbContext.SidekaDesa.FirstOrDefault(s => s.Kode == regionCode);
var pbdt = dbContext.Pbdt2015.FirstOrDefault(s => s.RegionCode == regionCode);
var content = new SidekaContent()
{
DesaId = desa.BlogId,
ApiVersion = configuration.GetValue <string>("ApiVersion"),
ChangeId = 1,
Type = "Kemiskinan",
Content = pbdt.Content,
DiffSize = 0,
Subtype = subtype,
ContentSize = ASCIIEncoding.Unicode.GetByteCount(pbdt.Content)
};
dbContext.Set <SidekaContent>().Add(content);
dbContext.SaveChanges();
}
public void Run()
{
using (var dbContext = SidekaDbContext.CreateForTools())
{
Console.WriteLine("========= Updating Data =========");
Console.WriteLine("Fetching Desa");
var desas = dbContext.SidekaDesa.OrderBy(d => d.BlogId).ToList();
var contentTypes = new string[] { "penduduk", "pemetaan" };
foreach (var desa in desas)
{
foreach (var contentType in contentTypes)
{
Console.WriteLine("Processing Desa {0}-{1} {2}", desa.BlogId, desa.Desa, contentType);
var contentQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desa.BlogId)
.Where(sc => sc.ApiVersion == "2.0")
.Where(sc => sc.Type == contentType)
.OrderByDescending(sc => sc.Id);
try
{
var sidekaContent = contentQuery.AsNoTracking().FirstOrDefault();
if (sidekaContent == null)
{
continue;
}
Console.WriteLine("Content Change ID {0}", sidekaContent.ChangeId);
var jObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content);
var viewModel = new SidekaContentViewModel(jObject);
bool hasChanges = false;
foreach (var tab in viewModel.Data.Keys.ToArray())
{
var data = viewModel.Data[tab];
var columns = viewModel.Columns[tab];
var idGetter = GetIdGetter(columns);
var ids = new HashSet <string>();
var duplicates = new List <int>();
for (var i = 0; i < data.Length; i++)
{
var id = idGetter(data[i]);
if (ids.Contains(id))
{
duplicates.Add(i);
}
ids.Add(id);
}
if (duplicates.Count > 0)
{
duplicates.Reverse();
Console.WriteLine("Tab {0} has duplicates: {1}", tab, string.Join(", ", duplicates));
var list = data.ToList();
foreach (var i in duplicates)
{
list.RemoveAt(i);
}
data = viewModel.Data[tab] = list.ToArray();
hasChanges = true;
}
}
if (hasChanges)
{
var newContent = JsonConvert.SerializeObject(viewModel);
var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Data));
var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Diffs));
var updatedContent = new SidekaContent
{
Id = sidekaContent.Id,
ContentSize = contentSize,
Content = newContent,
DiffSize = diffSize
};
dbContext.Attach(updatedContent);
dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.Content).IsModified = true;
dbContext.SaveChanges();
Console.WriteLine("Content updated");
}
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine(e.StackTrace);
}
}
}
}
}
public void Run()
{
using (var dbContext = SidekaDbContext.CreateForTools())
{
Console.WriteLine("========= Updating Data =========");
Console.WriteLine("Fetching Desa");
var desas = dbContext.SidekaDesa.Where(d => d.BlogId > 6).OrderBy(d => d.BlogId).ToList();
foreach (var desa in desas)
{
Console.WriteLine("Processing Desa {0}-{1}", desa.BlogId, desa.Desa);
var contentQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desa.BlogId)
.Where(sc => sc.ApiVersion == "2.0");
Console.WriteLine("Fetching Contents For Desa {0}-{1}", desa.BlogId, desa.Desa);
var numberOfContents = contentQuery.Count();
var counter = 0;
var skip = 0;
while (counter < numberOfContents)
{
Console.WriteLine("Fetching contents skip #{0} take 10", skip);
var sidekaContents = contentQuery.AsNoTracking().OrderByDescending(sc => sc.ChangeId).Skip(skip).Take(10).ToList();
foreach (var sidekaContent in sidekaContents)
{
counter += 1;
Console.WriteLine("Processing Content #{0}", counter);
var sidekaContentJObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content);
if (sidekaContent.ApiVersion == "1.0")
{
sidekaContentJObject["columns"] = JArray.FromObject(new string[] { "nik", "nama_penduduk", "tempat_lahir", "tanggal_lahir", "jenis_kelamin", "pendidikan", "agama", "status_kawin", "pekerjaan", "pekerjaan_ped", "kewarganegaraan", "kompetensi", "no_telepon", "email", "no_kitas", "no_paspor", "golongan_darah", "status_penduduk", "status_tinggal", "kontrasepsi", "difabilitas", "no_kk", "nama_ayah", "nama_ibu", "hubungan_keluarga", "nama_dusun", "rw", "rt", "alamat_jalan" });
}
try
{
Console.WriteLine("Calculating Sizes....");
var content = new SidekaContentViewModel(sidekaContentJObject);
var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Data));
var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Diffs));
Console.WriteLine("Desa {0}-Change Id {1}, Content Size {2}, Diff Size {3}", desa.Desa, sidekaContent.ChangeId, contentSize, diffSize);
Console.WriteLine("Saving Size...");
var updatedContent = new SidekaContent
{
Id = sidekaContent.Id,
ContentSize = contentSize,
DiffSize = diffSize
};
dbContext.Attach(updatedContent);
dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true;
//dbContext.Update(updatedContent);
dbContext.SaveChanges();
Console.WriteLine("Sizes Have Been Saved");
}
catch (Exception ex)
{
Console.WriteLine("Error When Calculating Size Desa {0}-{1}: {2}", desa.BlogId, desa.Desa, ex.Message);
}
}
skip += 10;
}
}
}
}
public async Task <IActionResult> PostContentV2([FromBody] JObject contentJObject, int desaId, string contentType, string contentSubtype = null)
{
Stopwatch sw = new Stopwatch();
sw.Start();
string lockName = desaId + "_" + contentType + "_" + contentSubtype;
object writeLock = writeLocks.GetOrAdd(lockName, new object());
lock (writeLock){
var auth = GetAuth(desaId);
if (auth == null)
{
return(StatusCode((int)HttpStatusCode.Forbidden, new Dictionary <string, string>()
{
{ "message", "Invalid or no token" }
}));
}
var permission = contentType;
if (new string[] { "perencanaan", "penganggaran", "spp", "penerimaan" }.Contains(contentType))
{
permission = "keuangan";
}
var roles = (List <string>)auth["roles"];
if (!roles.Contains("administrator") && !roles.Contains(permission))
{
return(StatusCode((int)HttpStatusCode.Forbidden, new Dictionary <string, string>()
{
{ "message", "Your account doesn't have the permission" }
}));
}
var content = new SidekaContentViewModel(contentJObject);
// Validate
foreach (var column in content.Columns)
{
if (content.Diffs != null && content.Diffs.ContainsKey(column.Key))
{
var index = 0;
foreach (var diff in content.Diffs[column.Key])
{
var location = string.Format("Diff {0} ({1}) tab {2}", index, "added", column.Key);
var invalid = Validate(column.Value, diff.Added, location);
if (invalid != null)
{
return(invalid);
}
location = string.Format("Diff {0} ({1}) tab {2}", index, "modified", column.Key);
invalid = Validate(column.Value, diff.Modified, location);
if (invalid != null)
{
return(invalid);
}
location = string.Format("Diff {0} ({1}) tab {2}", index, "deleted", column.Key);
invalid = Validate(column.Value, diff.Deleted, location, false);
if (invalid != null)
{
return(invalid);
}
}
}
if (content.Data != null && content.Data.ContainsKey(column.Key))
{
var location = string.Format("Data tab {0}", column.Key);
var invalid = Validate(column.Value, content.Data[column.Key], location);
if (invalid != null)
{
return(invalid);
}
}
}
var clientChangeId = 0;
var changeId = QueryStringHelper.GetQueryString <int>(Request.Query, "changeId", 0);
if (changeId > 0)
{
clientChangeId = changeId;
}
// Find max change id
var maxChangeIdQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desaId)
.Where(sc => sc.Type == contentType);
if (!string.IsNullOrWhiteSpace(contentSubtype))
{
maxChangeIdQuery = maxChangeIdQuery.Where(sc => sc.Subtype == contentSubtype);
}
var maxChangeId = maxChangeIdQuery.Select(sc => sc.ChangeId).DefaultIfEmpty(0).Max();
// TODO: This is risky!! Consider changing change_id column to serial or autoincrement
var newContent = new SidekaContentViewModel();
// Initialize new content to be saved
foreach (var column in content.Columns)
{
newContent.Data[column.Key] = new List <object>().ToArray();
newContent.Columns[column.Key] = column.Value;
if (content.Diffs != null && content.Diffs.ContainsKey(column.Key))
{
newContent.Diffs[column.Key] = content.Diffs[column.Key];
}
else
{
newContent.Diffs[column.Key] = new List <SidekaDiff>().ToArray();
}
}
var latestContentQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desaId)
.Where(sc => sc.Type == contentType);
if (!string.IsNullOrWhiteSpace(contentSubtype))
{
latestContentQuery = latestContentQuery.Where(sc => sc.Subtype == contentSubtype);
}
var latestContentString = latestContentQuery
.OrderByDescending(sc => sc.ChangeId)
.Select(sc => sc.Content)
.FirstOrDefault();
JObject latestContentJObject = null;
if (string.IsNullOrWhiteSpace(latestContentString))
{
latestContentJObject = new JObject
{
{ "data", new JObject() },
{ "columns", contentJObject["columns"] }
};
}
else
{
latestContentJObject = JsonConvert.DeserializeObject <JObject>(latestContentString);
}
var diffs = GetDiffsNewerThanClient(desaId, contentType, contentSubtype, clientChangeId, (JObject)contentJObject["columns"]);
if (latestContentJObject["data"] is JArray && contentType == "penduduk")
{
newContent.Data["penduduk"] = MergeDiffs(newContent.Columns["penduduk"], newContent.Diffs["penduduk"], new List <object>().ToArray());
}
else
{
var latestContent = new SidekaContentViewModel(latestContentJObject);
foreach (var column in content.Columns)
{
// Initialize so the latest content have the same tab with the posted content
if (!latestContent.Columns.ContainsKey(column.Key))
{
latestContent.Columns[column.Key] = column.Value;
}
if (!latestContent.Data.ContainsKey(column.Key))
{
latestContent.Data[column.Key] = new List <object>().ToArray();
}
if (content.Data != null && content.Data[column.Key] != null &&
new string[] { "perencanaan", "penganggaran", "penerimaan", "spp" }.Contains(contentType))
{
var invalid = ValidateDuplicatesData(column.Key, column.Value, content.Data[column.Key]);
if (invalid != null)
{
return(invalid);
}
// Special case for client who posted data instead of diffs
newContent.Data[column.Key] = content.Data[column.Key];
// Add new diffs to show that the content is rewritten
var sidekaDiff = new SidekaDiff
{
Added = new List <object>().ToArray(),
Modified = new List <object>().ToArray(),
Deleted = new List <object>().ToArray(),
Total = 0,
Rewritten = true
};
newContent.Diffs[column.Key].Append(sidekaDiff);
}
else if (newContent.Diffs[column.Key].Length > 0)
{
// There's diffs in the posted content for this tab, apply them to latest data
var latestColumns = latestContent.Columns[column.Key];
var transformedLatestData = TransformData(
latestContentJObject["columns"][column.Key],
contentJObject["columns"][column.Key],
latestContent.Data[column.Key]);
var invalid = ValidateDuplicatesDiffs(column.Key, column.Value, content.Diffs[column.Key], transformedLatestData);
if (invalid != null)
{
return(invalid);
}
var mergedData = MergeDiffs(column.Value, content.Diffs[column.Key], transformedLatestData);
newContent.Data[column.Key] = mergedData;
newContent.Columns[column.Key] = column.Value;
}
else
{
// There's no diffs in the posted content for this tab, use the old data
newContent.Data[column.Key] = latestContent.Data[column.Key];
}
}
}
var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(newContent.Data));
var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(newContent.Diffs));
int newChangeId = GetNextChangeId(desaId, contentType, contentSubtype);
var sidekaContent = new SidekaContent
{
DesaId = desaId,
Type = contentType,
Subtype = contentSubtype,
Content = JsonConvert.SerializeObject(newContent),
DateCreated = DateTime.Now,
CreatedBy = (int)auth["user_id"],
ChangeId = newChangeId,
ApiVersion = configuration.GetValue <string>("ApiVersion"),
ContentSize = contentSize,
DiffSize = diffSize
};
dbContext.Add(sidekaContent);
dbContext.SaveChanges();
var result = new Dictionary <string, object>()
{
{ "success", true },
{ "changeId", newChangeId },
{ "change_id", newChangeId },
{ "diffs", diffs },
{ "columns", content.Columns },
};
sw.Stop();
Logs((int)auth["user_id"], desaId, "", "save_content", contentType, contentSubtype, sw.Elapsed.Milliseconds);
return(Ok(result));
}
}
public async Task <IActionResult> PostContent([FromBody] Dictionary <string, object> data, int desaId,
string contentType, string contentSubtype = null)
{
Stopwatch sw = new Stopwatch();
sw.Start();
var auth = GetAuth(desaId);
if (auth == null)
{
return(StatusCode((int)HttpStatusCode.Forbidden, new Dictionary <string, object>()
{
{ "success", false }
}));
}
if (contentType == "subtypes")
{
return(StatusCode((int)HttpStatusCode.InternalServerError, new Dictionary <string, object>()
{
{ "success", false }
}));
}
var apiVersion = configuration.GetValue <string>("ApiVersion");
var totalData = await dbContext.SidekaContent
.Where(sc => sc.DesaId == desaId)
.Where(sc => sc.Type == contentType)
.Where(sc => sc.ApiVersion == apiVersion)
.CountAsync();
if (totalData > 0)
{
return(StatusCode((int)HttpStatusCode.InternalServerError,
new Dictionary <string, object>()
{
{ "error", "Sideka desktop needs to be updated" }
}));
}
var maxChangeIdQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desaId)
.Where(sc => sc.Type == contentType);
if (!string.IsNullOrWhiteSpace(contentSubtype))
{
maxChangeIdQuery = maxChangeIdQuery.Where(sc => sc.Subtype == contentSubtype);
}
var maxChangeId = await maxChangeIdQuery.Select(sc => sc.ChangeId).DefaultIfEmpty(0).MaxAsync();
var newChangeId = maxChangeId + 1;
var timestamp = (long)data.GetValueOrDefault("timestamp", 0);
var serverTimestamp = DateTimeOffset.Now.ToUnixTimeSeconds();
if (timestamp > serverTimestamp || timestamp <= 0)
{
timestamp = serverTimestamp;
}
var sidekaContent = new SidekaContent
{
DesaId = desaId,
Type = contentType,
Subtype = contentSubtype,
Content = JsonConvert.SerializeObject(data),
Timestamp = timestamp,
DateCreated = DateTime.Now,
CreatedBy = (int)auth["user_id"],
ChangeId = newChangeId,
ApiVersion = "1.0"
};
dbContext.Add(sidekaContent);
await dbContext.SaveChangesAsync();
sw.Stop();
await Logs((int)auth["user_id"], desaId, "", "save_content", contentType, contentSubtype, sw.Elapsed.Milliseconds);
return(Ok(new Dictionary <string, object>()
{
{ "success", true }
}));
}
public void Run()
{
using (var dbContext = SidekaDbContext.CreateForTools())
{
Console.WriteLine("========= Updating Data =========");
Console.WriteLine("Fetching Desa");
var desas = dbContext.SidekaDesa.OrderBy(d => d.BlogId).ToList();
var contentTypes = new string[] { "penduduk" };
foreach (var desa in desas)
{
foreach (var contentType in contentTypes)
{
Console.WriteLine("Processing Desa {0}-{1} {2}", desa.BlogId, desa.Desa, contentType);
var contentQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desa.BlogId)
.Where(sc => sc.ApiVersion == "2.0")
.Where(sc => sc.Type == contentType)
.OrderByDescending(sc => sc.Id);
try
{
var sidekaContent = contentQuery.AsNoTracking().FirstOrDefault();
if (sidekaContent == null)
{
continue;
}
Console.WriteLine("Content Change ID {0}", sidekaContent.ChangeId);
var jObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content);
var viewModel = new SidekaContentViewModel(jObject);
bool hasChanges = false;
if (!viewModel.Columns.ContainsKey("penduduk") ||
!viewModel.Data.ContainsKey("penduduk") ||
viewModel.Columns["penduduk"].Columns.Length != 34 ||
viewModel.Data["penduduk"].Length == 0)
{
continue;
}
bool all30Columns = true;
var data = viewModel.Data["penduduk"];
foreach (var row in data)
{
if (((object[])row).Length != 30)
{
all30Columns = false;
break;
}
}
if (all30Columns)
{
Console.WriteLine("All 30 columns");
}
if (hasChanges)
{
var newContent = JsonConvert.SerializeObject(viewModel);
var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Data));
var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Diffs));
var updatedContent = new SidekaContent
{
Id = sidekaContent.Id,
ContentSize = contentSize,
Content = newContent,
DiffSize = diffSize
};
dbContext.Attach(updatedContent);
dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.Content).IsModified = true;
dbContext.SaveChanges();
Console.WriteLine("Content updated");
}
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine(e.StackTrace);
}
}
}
}
}
