internal void RefreshLocalEntities(Action <int> setProgress)
{
var cloudTables = getAllKindNames();
//we get the keys and editdates
foreach (var table in cloudTables)
{
var localCloudData = new CloudLocalStore <CloudEntity>(table.toKind());
var entities = localCloudData.GetUnsyncedLocalEntities(
cloudTable: table.toKind(),
localTable: getLocalTableName(table).toKind()
);
if (entities.Count == 0)
{
continue;
}
var entityConverter = new EntityConverter();
var localStore = new CloudLocalStore <LocalEntity>(getLocalTableName(table).toKind());
var flatStore = new FieldValueStore(getTableFieldValueName(table))
{
batchSize = 50
};
foreach (var entity in entities)
{
//we decrypt
var localEntity = entityConverter.toLocalEntity(entity);
var saved = localStore.Update(localEntity);
if (saved == null)
{
//means we couldn't save, so we do what?
//throw exception??
log(string.Format("Couldn't save record for ", table, entity.Id));
continue;
}
//and deidentify
var deid = entityConverter.toDeidEntity(localEntity);
var ged = DbSaveableEntity.fromJson <GeneralEntityDataset>(
new KindItem(deid.DataBlob)
);
//todo: modify so we sync from CloudLocalStore separately than when downloading
//and save to localTables
flatStore.Save(ged, localEntity, saved.recordId);
}
//call finalise
flatStore.finalise();
}
//check if we have these in the local tables
//fetch and process records missing
//or do a sql comparison
}
//select __key__,kindmetadata From appusers
//select __key__, kindmetadata From `vmmc_regandproc` where dateadded
//select __key__, dateadded From `appusers` where dateadded >= datetime('2016-08-01T00:00:01Z')
//select * from appusers where dateadded > datetime('2016-08-16T00:00:00.1Z')
//select __key__, dateadded From `appusers` where dateadded >=datetime('2016-08-16T00:00:00.01Z') limit 50 offset 0 order by dateadded asc
//https://console.cloud.google.com/datastore/entities/query/gql?project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20*
//Key(`__Stat_Total__`, 'total_entity_usage')
//https://console.cloud.google.com/datastore/entities/edit?key=0%2F%7C19%2Fpp_client_devicerem%7C37%2Fname:5643a2ae27144cdaa1fe0d0e43b3864b&project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20*
//https://console.cloud.google.com/datastore/entities/query/gql?project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20*%20from%20appusers
//https://console.cloud.google.com/datastore/entities/edit?key=0%2F%7C8%2Fappusers%7C37%2Fname:524d91cffb024c7085148004cc47854c&project=jhpzmb-vmmc-odk&ns=&kind=_GAE_MR_TaskPayload&gql=select%20*%20from%20appusers
//https://console.cloud.google.com/datastore/entities/query/gql?project=jhpzmb-vmmc-odk&ns=&kind=_AE_Backup_Information&gql=select%20*%20from%20appusers%20order%20by%20dateadded%20asc%20limit%20500%20offset%200%20where%20dateadded%20%3E%3D%20datetime(%272016-08-16T00:00:00.01Z%27)
private async Task <Key> Save(DbSaveableEntity saveableEntity)
{
var assets = AppInstance.Instance.ApiAssets;
var projectId = assets[Constants.ASSET_PROJECT_ID];
var entity = new Entity()
{
Key = new Key()
{
PartitionId = new PartitionId()
{
NamespaceId = "", ProjectId = projectId
},
Path = new List <PathElement>()
{
new PathElement()
{
Kind = saveableEntity.kindName.Value,
Name = saveableEntity.Id.Value
}
}
},
Properties = new Dictionary <string, Value>()
{
{ "id", new Value()
{
StringValue = saveableEntity.Id.Value
} },
{ "entityid", new Value()
{
StringValue = saveableEntity.EntityId.Value
} },
{ "dateadded", new Value()
{
TimestampValue = DateTime.Now
} },
{ "datablob", new Value()
{
ExcludeFromIndexes = true,
StringValue = saveableEntity
.getJson()
.Encrypt()
.Value
} },
{ "kindmetadata", new Value()
{
StringValue = saveableEntity.Entity.KindMetaData ?? string.Empty
} }
}
};
var datastore = GetDatastoreService(GetDefaultCredential(assets), assets);
return(await SaveToCloud(datastore, projectId, entity));
}
private int AddToOutQueue(DbSaveableEntity saveableEntity)
{
var asString = saveableEntity.getJson();
var outEntity = new OutEntity()
{
Id = saveableEntity.Id.Value,
DataBlob = asString
};
var res = 0;
//new OutDb().DB.InsertOrReplace(outEntity);
return(res);
}
public static string deidentifyBlob(this string dataBlob, string formName)
{
var toReturn = string.Empty;
//we deserialise the blob to its base form
var deserialised = DbSaveableEntity.fromJson <GeneralEntityDataset>(
new KindItem(dataBlob)
);
var fieldValues = deserialised.FieldValues;
//we get the field dictionary for this entity
//we read each field, and deidentify if needed based on field attributes
//and copy to out object
//copy out object to the out entity
return(dataBlob);
}
public async Task <int> doServerSync(Action <string, ToastLength> makeToast)
{
makeToast = (string a, ToastLength b) => { };
//var recs = GetRecordsToSync();
//if (recs.Count == 0)
//{
// makeToast("No unsynced records", ToastLength.Short);
// return 0;
//}
var hasConnection = await checkConnection();
if (!hasConnection)
{
isRunning = 0;
makeToast?.Invoke("No connection detected", ToastLength.Long);
return(0);
}
makeToast("Connected tested", ToastLength.Short);
var recs = new List <OutEntity>();
//await GetCloudEntities1("appusers");
return(recs.Count);
var cloudDb = new OutDb().DB;
var recIndex = recs.Count - 1;
while (recIndex >= 0 && hasConnection)
{
var outEntity = recs[recIndex];
var ppdataset = DbSaveableEntity.fromJson <GeneralEntityDataset>(
new KindItem(outEntity.DataBlob));
var saveable = new DbSaveableEntity(ppdataset)
{
kindName = new KindName(ppdataset.FormName)
};
var saved = false;
for (int i = 0; i < 4; i++)
{
try
{
await Save(saveable);
//we remove this key from the database
saved = true;
}
catch (Google.GoogleApiException gex)
{
//todo: mark this record as bad to prevent it blocking for life
//cloudDb.InsertOrReplace(new OutEntityUnsynced().load(outEntity));
//cloudDb.Delete<OutEntity>(saveable.Id.Value);
//break;
}
catch (System.Net.WebException wex)
{
//perhaps lost connection
//we alllow it to spin for now
}
catch (Exception ex)
{
//unknown exception
}
finally { }
if (saved)
{
try
{
//we delete the record from downloads db
//var deleted = cloudDb.Delete<OutEntity>(saveable.Id.Value);
}
catch
{
}
break;
}
else
{
//lets add a 2 second delay in case it failed the first time
await Task.Delay(TimeSpan.FromMilliseconds(2000));
}
}
recIndex--;
hasConnection = await checkConnection();
}
return(0);
}
