internal void RefreshLocalEntities(Action <int> setProgress) { var cloudTables = getAllKindNames(); //we get the keys and editdates foreach (var table in cloudTables) { var localCloudData = new CloudLocalStore <CloudEntity>(table.toKind()); var entities = localCloudData.GetUnsyncedLocalEntities( cloudTable: table.toKind(), localTable: getLocalTableName(table).toKind() ); if (entities.Count == 0) { continue; } var entityConverter = new EntityConverter(); var localStore = new CloudLocalStore <LocalEntity>(getLocalTableName(table).toKind()); var flatStore = new FieldValueStore(getTableFieldValueName(table)) { batchSize = 50 }; foreach (var entity in entities) { //we decrypt var localEntity = entityConverter.toLocalEntity(entity); var saved = localStore.Update(localEntity); if (saved == null) { //means we couldn't save, so we do what? //throw exception?? log(string.Format("Couldn't save record for ", table, entity.Id)); continue; } //and deidentify var deid = entityConverter.toDeidEntity(localEntity); var ged = DbSaveableEntity.fromJson <GeneralEntityDataset>( new KindItem(deid.DataBlob) ); //todo: modify so we sync from CloudLocalStore separately than when downloading //and save to localTables flatStore.Save(ged, localEntity, saved.recordId); } //call finalise flatStore.finalise(); } //check if we have these in the local tables //fetch and process records missing //or do a sql comparison }
//select __key__,kindmetadata From appusers //select __key__, kindmetadata From `vmmc_regandproc` where dateadded //select __key__, dateadded From `appusers` where dateadded >= datetime('2016-08-01T00:00:01Z') //select * from appusers where dateadded > datetime('2016-08-16T00:00:00.1Z') //select __key__, dateadded From `appusers` where dateadded >=datetime('2016-08-16T00:00:00.01Z') limit 50 offset 0 order by dateadded asc //https://console.cloud.google.com/datastore/entities/query/gql?project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20* //Key(`__Stat_Total__`, 'total_entity_usage') //https://console.cloud.google.com/datastore/entities/edit?key=0%2F%7C19%2Fpp_client_devicerem%7C37%2Fname:5643a2ae27144cdaa1fe0d0e43b3864b&project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20* //https://console.cloud.google.com/datastore/entities/query/gql?project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20*%20from%20appusers //https://console.cloud.google.com/datastore/entities/edit?key=0%2F%7C8%2Fappusers%7C37%2Fname:524d91cffb024c7085148004cc47854c&project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20*%20from%20appusers //https://console.cloud.google.com/datastore/entities/query/gql?project=jhpzmb-vmmc-odk&ns=&kind=_AE_Backup_Information&gql=select%20*%20from%20appusers%20order%20by%20dateadded%20asc%20limit%20500%20offset%200%20where%20dateadded%20%3E%3D%20datetime(%272016-08-16T00:00:00.01Z%27) private async Task <Key> Save(DbSaveableEntity saveableEntity) { var assets = AppInstance.Instance.ApiAssets; var projectId = assets[Constants.ASSET_PROJECT_ID]; var entity = new Entity() { Key = new Key() { PartitionId = new PartitionId() { NamespaceId = "", ProjectId = projectId }, Path = new List <PathElement>() { new PathElement() { Kind = saveableEntity.kindName.Value, Name = saveableEntity.Id.Value } } }, Properties = new Dictionary <string, Value>() { { "id", new Value() { StringValue = saveableEntity.Id.Value } }, { "entityid", new Value() { StringValue = saveableEntity.EntityId.Value } }, { "dateadded", new Value() { TimestampValue = DateTime.Now } }, { "datablob", new Value() { ExcludeFromIndexes = true, StringValue = saveableEntity .getJson() .Encrypt() .Value } }, { "kindmetadata", new Value() { StringValue = saveableEntity.Entity.KindMetaData ?? string.Empty } } } }; var datastore = GetDatastoreService(GetDefaultCredential(assets), assets); return(await SaveToCloud(datastore, projectId, entity)); }
private int AddToOutQueue(DbSaveableEntity saveableEntity) { var asString = saveableEntity.getJson(); var outEntity = new OutEntity() { Id = saveableEntity.Id.Value, DataBlob = asString }; var res = 0; //new OutDb().DB.InsertOrReplace(outEntity); return(res); }
public static string deidentifyBlob(this string dataBlob, string formName) { var toReturn = string.Empty; //we deserialise the blob to its base form var deserialised = DbSaveableEntity.fromJson <GeneralEntityDataset>( new KindItem(dataBlob) ); var fieldValues = deserialised.FieldValues; //we get the field dictionary for this entity //we read each field, and deidentify if needed based on field attributes //and copy to out object //copy out object to the out entity return(dataBlob); }
public async Task <int> doServerSync(Action <string, ToastLength> makeToast) { makeToast = (string a, ToastLength b) => { }; //var recs = GetRecordsToSync(); //if (recs.Count == 0) //{ // makeToast("No unsynced records", ToastLength.Short); // return 0; //} var hasConnection = await checkConnection(); if (!hasConnection) { isRunning = 0; makeToast?.Invoke("No connection detected", ToastLength.Long); return(0); } makeToast("Connected tested", ToastLength.Short); var recs = new List <OutEntity>(); //await GetCloudEntities1("appusers"); return(recs.Count); var cloudDb = new OutDb().DB; var recIndex = recs.Count - 1; while (recIndex >= 0 && hasConnection) { var outEntity = recs[recIndex]; var ppdataset = DbSaveableEntity.fromJson <GeneralEntityDataset>( new KindItem(outEntity.DataBlob)); var saveable = new DbSaveableEntity(ppdataset) { kindName = new KindName(ppdataset.FormName) }; var saved = false; for (int i = 0; i < 4; i++) { try { await Save(saveable); //we remove this key from the database saved = true; } catch (Google.GoogleApiException gex) { //todo: mark this record as bad to prevent it blocking for life //cloudDb.InsertOrReplace(new OutEntityUnsynced().load(outEntity)); //cloudDb.Delete<OutEntity>(saveable.Id.Value); //break; } catch (System.Net.WebException wex) { //perhaps lost connection //we alllow it to spin for now } catch (Exception ex) { //unknown exception } finally { } if (saved) { try { //we delete the record from downloads db //var deleted = cloudDb.Delete<OutEntity>(saveable.Id.Value); } catch { } break; } else { //lets add a 2 second delay in case it failed the first time await Task.Delay(TimeSpan.FromMilliseconds(2000)); } } recIndex--; hasConnection = await checkConnection(); } return(0); }