public PbdtToSdContent()
{
configuration = SidekaDbContext.CreateConfigurationForTools();
dbContext = SidekaDbContext.CreateForTools(configuration);
}
public void Run()
{
using (var dbContext = SidekaDbContext.CreateForTools())
{
Console.WriteLine("========= Updating Data =========");
Console.WriteLine("Fetching Desa");
var desas = dbContext.SidekaDesa.OrderBy(d => d.BlogId).ToList();
var contentTypes = new string[] { "penduduk", "pemetaan" };
foreach (var desa in desas)
{
foreach (var contentType in contentTypes)
{
Console.WriteLine("Processing Desa {0}-{1} {2}", desa.BlogId, desa.Desa, contentType);
var contentQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desa.BlogId)
.Where(sc => sc.ApiVersion == "2.0")
.Where(sc => sc.Type == contentType)
.OrderByDescending(sc => sc.Id);
try
{
var sidekaContent = contentQuery.AsNoTracking().FirstOrDefault();
if (sidekaContent == null)
{
continue;
}
Console.WriteLine("Content Change ID {0}", sidekaContent.ChangeId);
var jObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content);
var viewModel = new SidekaContentViewModel(jObject);
bool hasChanges = false;
foreach (var tab in viewModel.Data.Keys.ToArray())
{
var data = viewModel.Data[tab];
var columns = viewModel.Columns[tab];
var idGetter = GetIdGetter(columns);
var ids = new HashSet <string>();
var duplicates = new List <int>();
for (var i = 0; i < data.Length; i++)
{
var id = idGetter(data[i]);
if (ids.Contains(id))
{
duplicates.Add(i);
}
ids.Add(id);
}
if (duplicates.Count > 0)
{
duplicates.Reverse();
Console.WriteLine("Tab {0} has duplicates: {1}", tab, string.Join(", ", duplicates));
var list = data.ToList();
foreach (var i in duplicates)
{
list.RemoveAt(i);
}
data = viewModel.Data[tab] = list.ToArray();
hasChanges = true;
}
}
if (hasChanges)
{
var newContent = JsonConvert.SerializeObject(viewModel);
var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Data));
var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Diffs));
var updatedContent = new SidekaContent
{
Id = sidekaContent.Id,
ContentSize = contentSize,
Content = newContent,
DiffSize = diffSize
};
dbContext.Attach(updatedContent);
dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.Content).IsModified = true;
dbContext.SaveChanges();
Console.WriteLine("Content updated");
}
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine(e.StackTrace);
}
}
}
}
}
public PbdtXlsxToSql()
{
dbContext = SidekaDbContext.CreateForTools();
}
public void Run()
{
using (var dbContext = SidekaDbContext.CreateForTools())
{
Console.WriteLine("========= Updating Data =========");
Console.WriteLine("Fetching Desa");
var desas = dbContext.SidekaDesa.Where(d => d.BlogId > 6).OrderBy(d => d.BlogId).ToList();
foreach (var desa in desas)
{
Console.WriteLine("Processing Desa {0}-{1}", desa.BlogId, desa.Desa);
var contentQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desa.BlogId)
.Where(sc => sc.ApiVersion == "2.0");
Console.WriteLine("Fetching Contents For Desa {0}-{1}", desa.BlogId, desa.Desa);
var numberOfContents = contentQuery.Count();
var counter = 0;
var skip = 0;
while (counter < numberOfContents)
{
Console.WriteLine("Fetching contents skip #{0} take 10", skip);
var sidekaContents = contentQuery.AsNoTracking().OrderByDescending(sc => sc.ChangeId).Skip(skip).Take(10).ToList();
foreach (var sidekaContent in sidekaContents)
{
counter += 1;
Console.WriteLine("Processing Content #{0}", counter);
var sidekaContentJObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content);
if (sidekaContent.ApiVersion == "1.0")
{
sidekaContentJObject["columns"] = JArray.FromObject(new string[] { "nik", "nama_penduduk", "tempat_lahir", "tanggal_lahir", "jenis_kelamin", "pendidikan", "agama", "status_kawin", "pekerjaan", "pekerjaan_ped", "kewarganegaraan", "kompetensi", "no_telepon", "email", "no_kitas", "no_paspor", "golongan_darah", "status_penduduk", "status_tinggal", "kontrasepsi", "difabilitas", "no_kk", "nama_ayah", "nama_ibu", "hubungan_keluarga", "nama_dusun", "rw", "rt", "alamat_jalan" });
}
try
{
Console.WriteLine("Calculating Sizes....");
var content = new SidekaContentViewModel(sidekaContentJObject);
var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Data));
var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(content.Diffs));
Console.WriteLine("Desa {0}-Change Id {1}, Content Size {2}, Diff Size {3}", desa.Desa, sidekaContent.ChangeId, contentSize, diffSize);
Console.WriteLine("Saving Size...");
var updatedContent = new SidekaContent
{
Id = sidekaContent.Id,
ContentSize = contentSize,
DiffSize = diffSize
};
dbContext.Attach(updatedContent);
dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true;
//dbContext.Update(updatedContent);
dbContext.SaveChanges();
Console.WriteLine("Sizes Have Been Saved");
}
catch (Exception ex)
{
Console.WriteLine("Error When Calculating Size Desa {0}-{1}: {2}", desa.BlogId, desa.Desa, ex.Message);
}
}
skip += 10;
}
}
}
}
public SidekaApiController(SidekaDbContext dbContext, IConfiguration configuration)
{
this.dbContext = dbContext;
this.configuration = configuration;
}
public void Run()
{
using (var dbContext = SidekaDbContext.CreateForTools())
{
Console.WriteLine("========= Updating Data =========");
Console.WriteLine("Fetching Desa");
var desas = dbContext.SidekaDesa.OrderBy(d => d.BlogId).ToList();
var contentTypes = new string[] { "penduduk" };
foreach (var desa in desas)
{
foreach (var contentType in contentTypes)
{
Console.WriteLine("Processing Desa {0}-{1} {2}", desa.BlogId, desa.Desa, contentType);
var contentQuery = dbContext.SidekaContent
.Where(sc => sc.DesaId == desa.BlogId)
.Where(sc => sc.ApiVersion == "2.0")
.Where(sc => sc.Type == contentType)
.OrderByDescending(sc => sc.Id);
try
{
var sidekaContent = contentQuery.AsNoTracking().FirstOrDefault();
if (sidekaContent == null)
{
continue;
}
Console.WriteLine("Content Change ID {0}", sidekaContent.ChangeId);
var jObject = JsonConvert.DeserializeObject <JObject>(sidekaContent.Content);
var viewModel = new SidekaContentViewModel(jObject);
bool hasChanges = false;
if (!viewModel.Columns.ContainsKey("penduduk") ||
!viewModel.Data.ContainsKey("penduduk") ||
viewModel.Columns["penduduk"].Columns.Length != 34 ||
viewModel.Data["penduduk"].Length == 0)
{
continue;
}
bool all30Columns = true;
var data = viewModel.Data["penduduk"];
foreach (var row in data)
{
if (((object[])row).Length != 30)
{
all30Columns = false;
break;
}
}
if (all30Columns)
{
Console.WriteLine("All 30 columns");
}
if (hasChanges)
{
var newContent = JsonConvert.SerializeObject(viewModel);
var contentSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Data));
var diffSize = ASCIIEncoding.Unicode.GetByteCount(JsonConvert.SerializeObject(viewModel.Diffs));
var updatedContent = new SidekaContent
{
Id = sidekaContent.Id,
ContentSize = contentSize,
Content = newContent,
DiffSize = diffSize
};
dbContext.Attach(updatedContent);
dbContext.Entry(updatedContent).Property(c => c.ContentSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.DiffSize).IsModified = true;
dbContext.Entry(updatedContent).Property(c => c.Content).IsModified = true;
dbContext.SaveChanges();
Console.WriteLine("Content updated");
}
}
catch (Exception e)
{
Console.WriteLine(e);
Console.WriteLine(e.StackTrace);
}
}
}
}
}
