private void CreateObjectsOnServer(List <SpeckleObject> bucketObjects, string baseUrl, ref int numErrors)
{
// Separate objects into sizeable payloads
var payloads = CreatePayloads(bucketObjects);
if (bucketObjects.Count(o => o.Type == "Placeholder") == bucketObjects.Count)
{
numErrors = 0;
return;
}
var payloadTasks = payloads.Select(p => apiClient.ObjectCreateAsync(p, 30000)).ToArray();
// Send objects which are in payload and add to local DB with updated IDs
//foreach (List<SpeckleObject> payload in payloads)
for (var j = 0; j < payloads.Count(); j++)
{
ResponseObject res = null;
try
{
res = payloadTasks[j].Result;
}
catch (Exception ex)
{
numErrors++;
var speckleExceptionContext = ExtractSpeckleExceptionContext(ex);
var errContext = speckleExceptionContext.Concat(new[] { "StreamId=" + StreamID,
"Error in updating the server with a payload of " + payloads[j].Count() + " objects" });
GSA.GsaApp.gsaMessenger.Message(MessageIntent.TechnicalLog, MessageLevel.Error, ex, errContext.ToArray());
}
if (res != null && res.Resources.Count() > 0)
{
for (int i = 0; i < payloads[j].Count(); i++)
{
payloads[j][i]._id = res.Resources[i]._id;
}
}
Task.Run(() =>
{
foreach (SpeckleObject obj in payloads[j].Where(o => o.Hash != null && o._id != null))
{
HelperFunctions.tryCatchWithEvents(() => LocalContext.AddSentObject(obj, baseUrl), "", "Error in updating local db");
}
});
}
int successfulPayloads = payloads.Count() - numErrors;
GSA.GsaApp.gsaMessenger.Message(MessageIntent.Display, MessageLevel.Information,
"Successfully sent " + successfulPayloads + "/" + payloads.Count() + " payloads to the server");
GSA.GsaApp.gsaMessenger.CacheMessage(MessageIntent.TechnicalLog, MessageLevel.Information, "Sent payloads to server",
"NumSuccessful=" + successfulPayloads, "NumErrored=" + numErrors);
}
private void DataSender_Elapsed(object sender, ElapsedEventArgs e)
{
if (MetadataSender.Enabled)
{
// start the timer again, as we need to make sure we're updating
DataSender.Start();
return;
}
this.Message = String.Format("Converting {0} \n objects", BucketObjects.Count);
var convertedObjects = Converter.Serialise(BucketObjects).Select(obj =>
{
if (ObjectCache.ContainsKey(obj.Hash))
{
return new SpecklePlaceholder()
{
Hash = obj.Hash, _id = ObjectCache[obj.Hash]._id
}
}
;
return(obj);
}).ToList();
this.Message = String.Format("Creating payloads");
long totalBucketSize = 0;
long currentBucketSize = 0;
List <List <SpeckleObject> > objectUpdatePayloads = new List <List <SpeckleObject> >();
List <SpeckleObject> currentBucketObjects = new List <SpeckleObject>();
List <SpeckleObject> allObjects = new List <SpeckleObject>();
foreach (SpeckleObject convertedObject in convertedObjects)
{
long size = Converter.getBytes(convertedObject).Length;
currentBucketSize += size;
totalBucketSize += size;
currentBucketObjects.Add(convertedObject);
if (currentBucketSize > 5e5) // restrict max to ~500kb; should it be user config? anyway these functions should go into core. at one point.
{
Debug.WriteLine("Reached payload limit. Making a new one, current #: " + objectUpdatePayloads.Count);
objectUpdatePayloads.Add(currentBucketObjects);
currentBucketObjects = new List <SpeckleObject>();
currentBucketSize = 0;
}
}
// add the last bucket
if (currentBucketObjects.Count > 0)
{
objectUpdatePayloads.Add(currentBucketObjects);
}
Debug.WriteLine("Finished, payload object update count is: " + objectUpdatePayloads.Count + " total bucket size is (kb) " + totalBucketSize / 1000);
if (objectUpdatePayloads.Count > 100)
{
this.AddRuntimeMessage(GH_RuntimeMessageLevel.Error, "This is a humongous update, in the range of ~50mb. For now, create more streams instead of just one massive one! Updates will be faster and snappier, and you can combine them back together at the other end easier.");
return;
}
int k = 0;
List <ResponseObject> responses = new List <ResponseObject>();
foreach (var payload in objectUpdatePayloads)
{
this.Message = String.Format("Sending payload\n{0} / {1}", k++, objectUpdatePayloads.Count);
responses.Add(mySender.ObjectCreateAsync(payload).GetAwaiter().GetResult());
}
this.Message = "Updating stream...";
// create placeholders for stream update payload
List <SpeckleObject> placeholders = new List <SpeckleObject>();
foreach (var myResponse in responses)
{
foreach (var obj in myResponse.Resources)
{
placeholders.Add(new SpecklePlaceholder()
{
_id = obj._id
});
}
}
SpeckleStream updateStream = new SpeckleStream()
{
Layers = BucketLayers,
Name = BucketName,
Objects = placeholders
};
// set some base properties (will be overwritten)
var baseProps = new Dictionary <string, object>();
baseProps["units"] = Rhino.RhinoDoc.ActiveDoc.ModelUnitSystem.ToString();
baseProps["tolerance"] = Rhino.RhinoDoc.ActiveDoc.ModelAbsoluteTolerance;
baseProps["angleTolerance"] = Rhino.RhinoDoc.ActiveDoc.ModelAngleToleranceRadians;
updateStream.BaseProperties = baseProps;
var response = mySender.StreamUpdateAsync(mySender.StreamId, updateStream);
mySender.BroadcastMessage(new { eventType = "update-global" });
// put the objects in the cache
int l = 0;
foreach (var obj in placeholders)
{
ObjectCache[convertedObjects[l].Hash] = placeholders[l];
l++;
}
Log += response.Result.Message;
AddRuntimeMessage(GH_RuntimeMessageLevel.Remark, "Data sent at " + DateTime.Now);
Message = "Data sent\n@" + DateTime.Now.ToString("hh:mm:ss");
}
// NOTE: This is actually triggered when clicking "Push!"
// TODO: Orchestration
// Create buckets, send sequentially, notify ui re upload progress
// NOTE: Problems with local context and cache: we seem to not sucesffuly pass through it
// perhaps we're not storing the right sent object (localcontext.addsentobject)
public override void PushSender(string args)
{
var client = JsonConvert.DeserializeObject <dynamic>(args);
//if it's a category or property filter we need to refresh the list of objects
//if it's a selection filter just use the objects that were stored previously
ISelectionFilter filter = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(client.filter), GetFilterType(client.filter.Type.ToString()));
IEnumerable <SpeckleObject> objects = new List <SpeckleObject>();
objects = GetSelectionFilterObjects(filter, client._id.ToString(), client.streamId.ToString());
var apiClient = new SpeckleApiClient((string)client.account.RestApi)
{
AuthToken = (string)client.account.Token
};
var convertedObjects = new List <SpeckleObject>();
var placeholders = new List <SpeckleObject>();
var units = CurrentDoc.Document.GetUnits().GetFormatOptions(UnitType.UT_Length).DisplayUnits.ToString().ToLowerInvariant().Replace("dut_", "");
InjectScaleInKits(GetScale(units)); // this is used for feet to sane units conversion.
int i = 0;
long currentBucketSize = 0;
var errorMsg = "";
var failedToConvert = 0;
var errors = new List <SpeckleError>();
foreach (var obj in objects)
{
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = true,
isLoadingIndeterminate = false,
loadingProgress = 1f * i++ / objects.Count() * 100,
loadingBlurb = string.Format("Converting and uploading objects: {0} / {1}", i, objects.Count())
}));
var id = 0;
Element revitElement = null;
try
{
revitElement = CurrentDoc.Document.GetElement((string)obj.Properties["revitUniqueId"]);
id = revitElement.Id.IntegerValue;
}
catch (Exception e)
{
errors.Add(new SpeckleError {
Message = "Could not retrieve element", Details = e.Message
});
continue;
}
try
{
var conversionResult = SpeckleCore.Converter.Serialise(new List <object>()
{
revitElement
});
var byteCount = Converter.getBytes(conversionResult).Length;
currentBucketSize += byteCount;
if (byteCount > 2e6)
{
errors.Add(new SpeckleError {
Message = "Element is too big to be sent", Details = $"Element {id} is bigger than 2MB, it will be skipped"
});
continue;
}
convertedObjects.AddRange(conversionResult);
if (currentBucketSize > 5e5 || i >= objects.Count()) // aim for roughly 500kb uncompressed
{
LocalContext.PruneExistingObjects(convertedObjects, apiClient.BaseUrl);
try
{
var chunkResponse = apiClient.ObjectCreateAsync(convertedObjects).Result.Resources;
int m = 0;
foreach (var objConverted in convertedObjects)
{
objConverted._id = chunkResponse[m++]._id;
placeholders.Add(new SpecklePlaceholder()
{
_id = objConverted._id
});
if (objConverted.Type != "Placeholder")
{
LocalContext.AddSentObject(objConverted, apiClient.BaseUrl);
}
}
}
catch (Exception e)
{
errors.Add(new SpeckleError {
Message = $"Failed to send {convertedObjects.Count} objects", Details = e.Message
});
}
currentBucketSize = 0;
convertedObjects = new List <SpeckleObject>(); // reset the chunkness
}
}
catch (Exception e)
{
failedToConvert++;
errors.Add(new SpeckleError {
Message = $"Failed to convert {revitElement.Name}", Details = $"Element id: {id}"
});
//NotifyUi("update-client", JsonConvert.SerializeObject(new
//{
// _id = (string)client._id,
// errors = "Failed to convert " + failedConvert + " objects."
//}));
}
}
if (errors.Any())
{
if (failedToConvert > 0)
{
errorMsg += string.Format("Failed to convert {0} objects ",
failedToConvert,
failedToConvert == 1 ? "" : "s");
}
else
{
errorMsg += string.Format("There {0} {1} error{2} ",
errors.Count() == 1 ? "is" : "are",
errors.Count(),
errors.Count() == 1 ? "" : "s");
}
errorMsg += "<nobr>" + Globals.GetRandomSadFace() + "</nobr>";
}
var myStream = new SpeckleStream()
{
Objects = placeholders
};
var ug = UnitUtils.GetUnitGroup(UnitType.UT_Length);
var baseProps = new Dictionary <string, object>();
baseProps["units"] = units;
baseProps["unitsDictionary"] = GetAndClearUnitDictionary();
myStream.BaseProperties = baseProps;
//myStream.BaseProperties = JsonConvert.SerializeObject(baseProps);
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = true,
isLoadingIndeterminate = true,
loadingBlurb = "Updating stream."
}));
var response = apiClient.StreamUpdateAsync((string)client.streamId, myStream).Result;
var plural = objects.Count() == 1 ? "" : "s";
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = false,
loadingBlurb = "",
message = $"Done sending {objects.Count()} object{plural}.",
errorMsg,
errors
}));
SpeckleTelemetry.RecordStreamUpdated("Revit");
}
/// <summary>
/// Sends the update to the server.
/// </summary>
private void SendUpdate()
{
if (MetadataSender.Enabled)
{
// start the timer again, as we need to make sure we're updating
DataSender.Start();
return;
}
if (IsSendingUpdate)
{
return;
}
IsSendingUpdate = true;
Message = String.Format("Converting {0} \n objects", BucketObjects.Count);
var convertedObjects = Converter.Serialise(BucketObjects).ToList();
Message = String.Format("Creating payloads");
LocalContext.PruneExistingObjects(convertedObjects, Client.BaseUrl);
List <SpeckleObject> persistedObjects = new List <SpeckleObject>();
if (convertedObjects.Count(obj => obj.Type == "Placeholder") != convertedObjects.Count)
{
// create the update payloads
int count = 0;
var objectUpdatePayloads = new List <List <SpeckleObject> >();
long totalBucketSize = 0;
long currentBucketSize = 0;
var currentBucketObjects = new List <SpeckleObject>();
var allObjects = new List <SpeckleObject>();
foreach (SpeckleObject convertedObject in convertedObjects)
{
if (count++ % 100 == 0)
{
Message = "Converted " + count + " objects out of " + convertedObjects.Count() + ".";
}
// size checking & bulk object creation payloads creation
long size = Converter.getBytes(convertedObject).Length;
currentBucketSize += size;
totalBucketSize += size;
currentBucketObjects.Add(convertedObject);
// Object is too big?
if (size > 2e6)
{
AddRuntimeMessage(GH_RuntimeMessageLevel.Warning, "This stream contains a super big object. These will fail. Sorry for the bad error message - we're working on improving this.");
currentBucketObjects.Remove(convertedObject);
}
if (currentBucketSize > 5e5) // restrict max to ~500kb; should it be user config? anyway these functions should go into core. at one point.
{
Debug.WriteLine("Reached payload limit. Making a new one, current #: " + objectUpdatePayloads.Count);
objectUpdatePayloads.Add(currentBucketObjects);
currentBucketObjects = new List <SpeckleObject>();
currentBucketSize = 0;
}
}
// add in the last bucket
if (currentBucketObjects.Count > 0)
{
objectUpdatePayloads.Add(currentBucketObjects);
}
Debug.WriteLine("Finished, payload object update count is: " + objectUpdatePayloads.Count + " total bucket size is (kb) " + totalBucketSize / 1000);
// create bulk object creation tasks
int k = 0;
List <ResponseObject> responses = new List <ResponseObject>();
foreach (var payload in objectUpdatePayloads)
{
Message = String.Format("{0}/{1}", k++, objectUpdatePayloads.Count);
try
{
var objResponse = Client.ObjectCreateAsync(payload).Result;
responses.Add(objResponse);
persistedObjects.AddRange(objResponse.Resources);
int m = 0;
foreach (var oL in payload)
{
oL._id = objResponse.Resources[m++]._id;
}
// push sent objects in the cache non-blocking
Task.Run(() =>
{
foreach (var oL in payload)
{
if (oL.Type != "Placeholder")
{
LocalContext.AddSentObject(oL, Client.BaseUrl);
}
}
});
}
catch (Exception err)
{
AddRuntimeMessage(GH_RuntimeMessageLevel.Error, err.Message);
return;
}
}
}
else
{
persistedObjects = convertedObjects;
}
// create placeholders for stream update payload
List <SpeckleObject> placeholders = new List <SpeckleObject>();
//foreach ( var myResponse in responses )
foreach (var obj in persistedObjects)
{
placeholders.Add(new SpecklePlaceholder()
{
_id = obj._id
});
}
SpeckleStream updateStream = new SpeckleStream()
{
Layers = BucketLayers,
Name = BucketName,
Objects = placeholders
};
// set some base properties (will be overwritten)
var baseProps = new Dictionary <string, object>();
baseProps["units"] = Rhino.RhinoDoc.ActiveDoc.ModelUnitSystem.ToString();
baseProps["tolerance"] = Rhino.RhinoDoc.ActiveDoc.ModelAbsoluteTolerance;
baseProps["angleTolerance"] = Rhino.RhinoDoc.ActiveDoc.ModelAngleToleranceRadians;
updateStream.BaseProperties = baseProps;
var response = Client.StreamUpdateAsync(Client.StreamId, updateStream).Result;
Client.BroadcastMessage("stream", Client.StreamId, new { eventType = "update-global" });
Log += response.Message;
AddRuntimeMessage(GH_RuntimeMessageLevel.Remark, "Data sent at " + DateTime.Now);
Message = "Data sent\n@" + DateTime.Now.ToString("hh:mm:ss");
IsSendingUpdate = false;
State = "Ok";
}
static async Task TestObjects(SpeckleApiClient myClient)
{
var myPoint = new SpecklePoint()
{
Value = new List <double>()
{
1, 2, 3
}
};
var mySecondPoint = new SpecklePoint()
{
Value = new List <double>()
{
23, 33, 12
}
};
var myCircle = new SpeckleCircle()
{
Radius = 21
};
var myPlane = new SpecklePlane()
{
Origin = new SpecklePoint()
{
Value = new List <double>()
{
12, 12, 12
}
}
};
var myArc = new SpeckleArc()
{
Radius = 2, AngleRadians = 2.1, EndAngle = 1, StartAngle = 0
};
myCircle.Properties = new Dictionary <string, object>();
myCircle.Properties.Add("a property", "Hello!");
myCircle.Properties.Add("point", myPoint);
List <SpeckleObject> myList = new List <SpeckleObject>();
Console.WriteLine();
try
{
Console.WriteLine("Creating some objects.");
var Response = await myClient.ObjectCreateAsync(new List <SpeckleObject>() { myPoint, myCircle, myArc, myPlane });
Console.WriteLine("OK: Saved " + Response.Resources.Count + " objects");
myList = Response.Resources;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Console.WriteLine();
try
{
Console.WriteLine("Updating an object");
var Response = await myClient.ObjectUpdateAsync(myList[1]._id, new SpeckleCircle()
{
Radius = 42
});
Console.WriteLine("OK: Saved " + Response.Message);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Console.WriteLine();
try
{
Console.WriteLine("Updating an object's properties");
var Response = await myClient.ObjectUpdatePropertiesAsync(myList[1]._id, new { hello = "World", max = 3.14 });
Console.WriteLine("OK: Saved " + Response.Message);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Console.WriteLine();
try
{
Console.WriteLine("Getting an object");
var Response = await myClient.ObjectGetAsync(myList[1]._id);
Console.WriteLine("OK: Got " + Response.Resource.ToJson());
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Console.WriteLine();
try
{
Console.WriteLine("Getting objects in bulk");
var Response = await myClient.ObjectGetBulkAsync(new string[] { myList[1]._id, myList[0]._id, myList[2]._id }, "fields=properties,radius");
Console.WriteLine("OK: Got " + Response.Resources.Count);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Console.WriteLine();
try
{
Console.WriteLine("Deleting an object");
var Response = await myClient.ObjectDeleteAsync(myList[0]._id);
Console.WriteLine("OK: Got " + Response.Message);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
// TODO: Orchestration
// Create buckets, send sequentially, notify ui re upload progress
// NOTE: Problems with local context and cache: we seem to not sucesffuly pass through it
// perhaps we're not storing the right sent object (localcontext.addsentobject)
public override void UpdateSender(string args)
{
var client = JsonConvert.DeserializeObject <dynamic>(args);
var apiClient = new SpeckleApiClient((string)client.account.RestApi)
{
AuthToken = (string)client.account.Token
};
var convertedObjects = new List <SpeckleObject>();
var placeholders = new List <SpeckleObject>();
var units = CurrentDoc.Document.GetUnits().GetFormatOptions(UnitType.UT_Length).DisplayUnits.ToString().ToLower().Replace("dut_", "");
InjectScaleInKits(GetScale(units)); // this is used for feet to sane units conversion.
int i = 0;
long currentBucketSize = 0;
var errors = "";
var failedSend = 0;
var failedConvert = 0;
foreach (var obj in client.objects)
{
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = true,
isLoadingIndeterminate = false,
loadingProgress = 1f * i++ / client.objects.Count * 100,
loadingBlurb = string.Format("Converting and uploading objects: {0} / {1}", i, client.objects.Count)
}));
try
{
var revitElement = CurrentDoc.Document.GetElement((string)obj.properties["revitUniqueId"]);
var conversionResult = SpeckleCore.Converter.Serialise(new List <object>()
{
revitElement
});
var byteCount = Converter.getBytes(conversionResult).Length;
currentBucketSize += byteCount;
if (byteCount > 2e6)
{
var problemId = revitElement.Id;
}
convertedObjects.AddRange(conversionResult);
if (currentBucketSize > 5e5 || i >= client.objects.Count) // aim for roughly 500kb uncompressed
{
LocalContext.PruneExistingObjects(convertedObjects, apiClient.BaseUrl);
try
{
var chunkResponse = apiClient.ObjectCreateAsync(convertedObjects).Result.Resources;
int m = 0;
foreach (var objConverted in convertedObjects)
{
objConverted._id = chunkResponse[m++]._id;
placeholders.Add(new SpecklePlaceholder()
{
_id = objConverted._id
});
if (objConverted.Type != "Placeholder")
{
LocalContext.AddSentObject(objConverted, apiClient.BaseUrl);
}
}
}
catch (Exception e)
{
failedSend += convertedObjects.Count;
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
errors = "Failed to send " + failedSend + " objects."
}));
}
currentBucketSize = 0;
convertedObjects = new List <SpeckleObject>(); // reset the chunkness
}
}
catch (Exception e)
{
failedConvert++;
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
errors = "Failed to convert " + failedConvert + " objects."
}));
}
}
if (failedConvert > 0)
{
errors += String.Format("Failed to convert a total of {0} objects. ", failedConvert);
}
if (failedSend > 0)
{
errors += String.Format("Failed to send a total of {0} objects. ", failedSend);
}
var myStream = new SpeckleStream()
{
Objects = placeholders
};
var ug = UnitUtils.GetUnitGroup(UnitType.UT_Length);
var baseProps = new Dictionary <string, object>();
baseProps["units"] = units;
baseProps["unitsDictionary"] = GetAndClearUnitDictionary();
myStream.BaseProperties = baseProps;
//myStream.BaseProperties = JsonConvert.SerializeObject(baseProps);
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = true,
isLoadingIndeterminate = true,
loadingBlurb = "Updating stream."
}));
var response = apiClient.StreamUpdateAsync((string)client.streamId, myStream).Result;
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = false,
loadingBlurb = "Done sending.",
errors
}));
}
// Send objects to Speckle server. Triggered on "Push!".
// Create buckets, send sequentially, notify ui re upload progress
public override void PushSender(string args)
{
var client = JsonConvert.DeserializeObject <dynamic>(args);
//if it's a category or property filter we need to refresh the list of objects
//if it's a selection filter just use the objects that were stored previously
ISelectionFilter filter = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(client.filter), GetFilterType(client.filter.Type.ToString()));
IEnumerable <SpeckleObject> objects = new List <SpeckleObject>();
objects = GetSelectionFilterObjects(filter, client._id.ToString(), client.streamId.ToString());
var apiClient = new SpeckleApiClient((string)client.account.RestApi)
{
AuthToken = (string)client.account.Token
};
var task = Task.Run(async() => { await apiClient.IntializeUser(); });
task.Wait();
apiClient.ClientType = "AutoCAD";
var convertedObjects = new List <SpeckleObject>();
var placeholders = new List <SpeckleObject>();
var units = AutocadDataService.GetLengthUnit();
//InjectScaleInKits(GetScale(units)); // this is used for feet to sane units conversion.
int i = 0;
long currentBucketSize = 0;
var errorMsg = "";
var failedToConvert = 0;
var errors = new List <SpeckleError>();
foreach (var obj in objects)
{
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = true,
isLoadingIndeterminate = false,
loadingProgress = 1f * i++ / objects.Count() * 100,
loadingBlurb = string.Format("Converting and uploading objects: {0} / {1}", i, objects.Count())
}));
long handle = 0;
SpeckleObject speckleObject = null;
try
{
handle = (long)obj.Properties["autocadhandle"];
speckleObject = AutocadDataService.GetObject(handle);
if (speckleObject == null)
{
errors.Add(new SpeckleError {
Message = "Could not retrieve element", Details = string.Empty
});
continue;
}
}
catch (Exception e)
{
errors.Add(new SpeckleError {
Message = "Could not retrieve element", Details = e.Message
});
continue;
}
try
{
var conversionResult = new List <SpeckleObject> {
speckleObject
};
var byteCount = Converter.getBytes(conversionResult).Length;
currentBucketSize += byteCount;
if (byteCount > 2e6)
{
errors.Add(new SpeckleError {
Message = "Element is too big to be sent", Details = $"Element {handle} is bigger than 2MB, it will be skipped"
});
continue;
}
convertedObjects.AddRange(conversionResult);
if (currentBucketSize > 5e5 || i >= objects.Count()) // aim for roughly 500kb uncompressed
{
LocalContext.PruneExistingObjects(convertedObjects, apiClient.BaseUrl);
try
{
var chunkResponse = apiClient.ObjectCreateAsync(convertedObjects).Result.Resources;
int m = 0;
foreach (var objConverted in convertedObjects)
{
objConverted._id = chunkResponse[m++]._id;
placeholders.Add(new SpecklePlaceholder()
{
_id = objConverted._id
});
if (objConverted.Type != "Placeholder")
{
LocalContext.AddSentObject(objConverted, apiClient.BaseUrl);
}
}
}
catch (Exception e)
{
errors.Add(new SpeckleError {
Message = $"Failed to send {convertedObjects.Count} objects", Details = e.Message
});
}
currentBucketSize = 0;
convertedObjects = new List <SpeckleObject>(); // reset the chunkness
}
}
catch (Exception e)
{
failedToConvert++;
errors.Add(new SpeckleError {
Message = $"Failed to convert element", Details = $"Element handle: {handle}"
});
//NotifyUi("update-client", JsonConvert.SerializeObject(new
//{
// _id = (string)client._id,
// errors = "Failed to convert " + failedConvert + " objects."
//}));
}
}
if (errors.Any())
{
if (failedToConvert > 0)
{
errorMsg += string.Format("Failed to convert {0} objects ",
failedToConvert,
failedToConvert == 1 ? "" : "s");
}
else
{
errorMsg += string.Format("There {0} {1} error{2} ",
errors.Count() == 1 ? "is" : "are",
errors.Count(),
errors.Count() == 1 ? "" : "s");
}
}
var myStream = new SpeckleStream()
{
Objects = placeholders
};
var baseProps = new Dictionary <string, object>();
baseProps["units"] = units;
//baseProps["unitsDictionary"] = GetAndClearUnitDictionary();
myStream.BaseProperties = baseProps;
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = true,
isLoadingIndeterminate = true,
loadingBlurb = "Updating stream."
}));
apiClient.Stream = myStream;
var response = apiClient.StreamUpdateAsync((string)client.streamId, myStream).Result;
var plural = objects.Count() == 1 ? "" : "s";
NotifyUi("update-client", JsonConvert.SerializeObject(new
{
_id = (string)client._id,
loading = false,
loadingBlurb = "",
message = $"Done sending {objects.Count()} object{plural}.",
errorMsg,
errors
}));
}
/// <summary>
/// Orchestrates the saving of many objects.
/// </summary>
/// <param name="account"></param>
/// <param name="objects"></param>
/// <returns></returns>
static IEnumerable <SpecklePlaceholder> SaveManyObjects(Account account, IEnumerable <SpeckleObject> objects)
{
Console.WriteLine(String.Format("Saving {0} objects.", objects.Count()));
Console.WriteLine();
Console.WriteLine();
// Step 1: Payload creation.
// The approach below keeps request sizes around 500kb each.
// NOTE: Will change in Speckle 2.0
var objectUpdatePayloads = new List <List <SpeckleObject> >();
long totalBucketSize = 0, currentBucketSize = 0;
var currentBucketObjects = new List <SpeckleObject>();
var allObjects = new List <SpeckleObject>();
foreach (var obj in objects)
{
long size = Converter.getBytes(obj).Length;
currentBucketSize += size;
totalBucketSize += size;
currentBucketObjects.Add(obj);
if (currentBucketSize > 5e5) // restrict max to ~500kb;
{
objectUpdatePayloads.Add(currentBucketObjects);
currentBucketObjects = new List <SpeckleObject>();
currentBucketSize = 0;
}
}
// add in the last bucket
if (currentBucketObjects.Count > 0)
{
objectUpdatePayloads.Add(currentBucketObjects);
}
Console.WriteLine();
Console.WriteLine(String.Format("Done making payloads ({0} total).", objectUpdatePayloads.Count));
// Step 2: Actually persist the objects, and return placeholders with their ObjectIds.
// This might take a while.
var client = new SpeckleApiClient(account.RestApi, false, "console_app");
client.AuthToken = account.Token;
var i = 0;
var totalObjs = 0;
foreach (var payload in objectUpdatePayloads)
{
totalObjs += payload.Count;
Console.Write(String.Format("Sending payload #{0} ({1} objects, total saved {2}) ...", i++, payload.Count, totalObjs));
var result = client.ObjectCreateAsync(payload).Result.Resources;
foreach (var placehholder in result)
{
yield return(placehholder as SpecklePlaceholder);
}
Console.Write(" done.");
Console.CursorLeft = 0;
}
}