private void configureHistory(Dictionary <string, object> config = null)
{
ResponseStreamClone response = null;
Task <ResponseStreamClone> respStreamClTask = null;
object enableHistoryObj = null;
if (config != null)
{
config.TryGetValue("EnableHistory", out enableHistoryObj);
}
bool?enableHistory = enableHistoryObj as bool?;
if (enableHistory != null && !(bool)enableHistory)
{
return;
}
// The following line creates a new stream (with a different StreamId), where the current stream content is copied, before it gets modified.
// The streamId of the cloned "backup" is saved among the main Stream "children" field,
// accessible through SpeckleServerAddress/api/v1/streams/streamId
respStreamClTask = SpeckleClient.StreamCloneAsync(SpeckleStreamId);
try
{
response = respStreamClTask?.Result;
}
catch (Exception e) { }
if (response == null)
{
BH.Engine.Reflection.Compute.RecordWarning($"Could not set the EnableHistory option. Task status: {respStreamClTask.Status.ToString()}");
}
}
private void SetupHistory()
{
ResponseStreamClone cloneResult = null;
Task <ResponseStreamClone> respStreamClTask = null;
// The following line creates a new stream (with a different StreamId), where the current stream content is copied, before it gets modified.
// The streamId of the cloned "backup" is saved among the main Stream "children" field,
// accessible through SpeckleServerAddress/api/v1/streams/streamId
respStreamClTask = SpeckleClient.StreamCloneAsync(SpeckleClient.Stream.StreamId);
try
{
cloneResult = respStreamClTask?.Result;
SpeckleClient.Stream.Children = cloneResult.Parent.Children;
}
catch (Exception e)
{
BH.Engine.Reflection.Compute.RecordWarning($"Failed configuring Speckle History. Error: {e.InnerException}");
}
if (cloneResult == null)
{
BH.Engine.Reflection.Compute.RecordWarning($"Failed configuring Speckle History. Task status: {respStreamClTask.Status.ToString()}");
}
}
public override IEnumerable <object> Pull(IRequest query, Dictionary <string, object> config = null)
{
var response = SpeckleClient.StreamGetObjectsAsync(SpeckleStreamId, "").Result;
List <IBHoMObject> bHoMObjects = new List <IBHoMObject>();
List <IObject> iObjects = new List <IObject>();
List <object> reminder = new List <object>();
bool assignSpeckleIdToBHoMObjects = true;
if (query == null)
{
if (!BH.Engine.Speckle.Convert.ToBHoM(response, out bHoMObjects, out iObjects, out reminder, assignSpeckleIdToBHoMObjects))
{
BH.Engine.Reflection.Compute.RecordError("Failed to deserialize and cast the Server response into BHoM objects.");
}
return(bHoMObjects.Concat(iObjects).Concat(reminder));
}
else
{
/// -------------------
/// Base Pull rewritten
/// -------------------
// Make sure this is a FilterQuery
FilterRequest filter = query as FilterRequest;
if (filter == null)
{
Engine.Reflection.Compute.RecordWarning("Please specify a FilterQuery");
return(new List <object>());
}
List <string> speckleIds = QueryToSpeckleIds(filter);
// Read the IBHoMObjects
BH.Engine.Speckle.Convert.ToBHoM(response, out bHoMObjects, out iObjects, out reminder, assignSpeckleIdToBHoMObjects, speckleIds);
// Filter by tag if any
bHoMObjects = filter.Tag == "" ? bHoMObjects : bHoMObjects.Where(x => x.Tags.Contains(filter.Tag)).ToList();
// Return stuff
if (typeof(IBHoMObject).IsAssignableFrom(filter.Type))
{
return(bHoMObjects);
}
else if (typeof(IObject).IsAssignableFrom(filter.Type))
{
return(iObjects);
}
else
{
return(bHoMObjects.Concat(iObjects).Concat(reminder));
}
}
return(new List <object>());
}
// Note: setAssignedId is currently not exposed as an option
// -- waiting for Adapter refactoring LVL 04 to expose a new CRUDconfig input for the Push
// CRUDconfig will become available to all CRUD methods
protected bool CreateAnyObject(List <IObject> objects, bool setAssignedId = true)
{
/// Create the objects
List <SpeckleObject> objs_serialized = SpeckleCore.Converter.Serialise(objects);
SpeckleLayer.ObjectCount += objects.Count();
SpeckleStream.Objects.AddRange(objs_serialized);
/// Assign any other property to the objects before sending them
var objList = objects.ToList();
int i = 0;
foreach (var o in SpeckleStream.Objects)
{
IBHoMObject bhomObj = objList[i] as IBHoMObject;
if (bhomObj != null)
{
SpeckleStream.Objects[i].Name = string.IsNullOrEmpty(bhomObj.Name) ? bhomObj.GetType().ToString() : bhomObj.Name;
//SpeckleStream.Objects[i].Type = string.IsNullOrEmpty(bhomObj.Name) ? bhomObj.GetType().ToString() : bhomObj.Name;
}
i++;
}
/// Send the objects
var updateResponse = SpeckleClient.StreamUpdateAsync(SpeckleStreamId, SpeckleStream).Result;
SpeckleClient.BroadcastMessage("stream", SpeckleStreamId, new { eventType = "update-global" });
/// Read the IBHoMobjects as exported in speckle
/// so we can assign the Speckle-generated id into the BHoMobjects
if (setAssignedId)
{
ResponseObject response = SpeckleClient.StreamGetObjectsAsync(SpeckleStreamId, "").Result;
List <IBHoMObject> bHoMObjects_inSpeckle = new List <IBHoMObject>();
IEnumerable <IBHoMObject> iBhomObjsInSpeckle = BH.Engine.Speckle.Convert.ToBHoM(response, true);
VennDiagram <IBHoMObject> correspondenceDiagram = Engine.Data.Create.VennDiagram(objects.Where(o => o as IBHoMObject != null).Cast <IBHoMObject>(), iBhomObjsInSpeckle, new IBHoMGUIDComparer());
if (correspondenceDiagram.Intersection.Count != objects.Count())
{
var gna = 0;
//Engine.Reflection.Compute.RecordError("Push failed.\nNumber of objects created in Speckle do not correspond to the number of objects pushed.");
//return false;
}
correspondenceDiagram.Intersection.ForEach(o => o.Item1.CustomData[AdapterId] = o.Item2.CustomData[AdapterId]);
}
return(true);
}
public override List <object> Push(IEnumerable <object> objects, string tag = "", PushType pushType = PushType.AdapterDefault, ActionConfig actionConfig = null)
{
// Clone objects for immutability in the UI
List <object> objectsToPush = objects.Select(x => x.DeepClone()).ToList();
// Initialize the SpeckleStream
SpeckleClient.Stream.Objects = new List <SpeckleObject>(); // stream is immutable
// //- Read config
SpecklePushConfig pushConfig = (actionConfig as SpecklePushConfig) ?? new SpecklePushConfig();
// //- Use "Speckle" history: produces a new stream at every push that corresponds to the old version. Enabled by default.
if (pushConfig.EnableHistory)
{
SetupHistory();
}
// Actual creation and add to the stream
for (int i = 0; i < objectsToPush.Count(); i++)
{
SpeckleObject speckleObject = ToSpeckle(objectsToPush[i] as dynamic, pushConfig); // Dynamic dispatch to most appropriate method
// Add objects to the stream
SpeckleClient.Stream.Objects.Add(speckleObject);
}
// // - Send the objects
try
{
// Try the batch upload
BatchUpdateStream(pushConfig);
}
catch (Exception e)
{
try
{
// If the batch upload fails, try the standard SpeckleCore Update as a last resort.
//// - Issue: with `StreamUpdateAsync` Speckle doesn't seem to send anything if the Stream is initially empty.
//// - You need to Push twice if the Stream is initially empty.
var updateResponse = SpeckleClient.StreamUpdateAsync(SpeckleClient.Stream.StreamId, SpeckleClient.Stream).Result;
SpeckleClient.BroadcastMessage("stream", SpeckleClient.Stream.StreamId, new { eventType = "update-global" });
}
catch
{
// If all has failed, return the first error.
BH.Engine.Reflection.Compute.RecordError($"Upload to Speckle failed. Message returned:\n{e.InnerException.Message}");
return(new List <object>());
}
}
return(objectsToPush);
}
public ResponseObject MakeGETRequest(List <string> speckleGuids = null)
{
if (speckleGuids == null)
{
return(SpeckleClient.StreamGetObjectsAsync(SpeckleStreamId, "").Result);
}
else
{
// GET only specific IDs
return(SpeckleClient.ObjectGetBulkAsync(speckleGuids.ToArray(), "omit=displayValue").Result);
}
}
protected bool CreateIObjects(IEnumerable <IObject> objects)
{
IEnumerable <object> newObjects = (IEnumerable <object>)objects; //hacky; assumes T is always a reference type. Should be no problem anyway
List <SpeckleObject> objs_serialized = SpeckleCore.Converter.Serialise(newObjects);
SpeckleLayer.ObjectCount += objects.Count();
SpeckleStream.Objects.AddRange(objs_serialized);
var updateResponse = SpeckleClient.StreamUpdateAsync(SpeckleStreamId, SpeckleStream).Result;
SpeckleClient.BroadcastMessage("stream", SpeckleStreamId, new { eventType = "update-global" });
return(true);
}
public override IEnumerable <object> Pull(IRequest request, PullType pullType = PullType.AdapterDefault, ActionConfig actionConfig = null)
{
// If the request exists, make sure it's a SpeckleRequest
SpeckleRequest speckleRequest = request as SpeckleRequest;
if (request != null && request.GetType() != typeof(FilterRequest) && speckleRequest == null)
{
Engine.Reflection.Compute.RecordError($"SpeckleAdapter supports only {typeof(SpeckleRequest).Name}.");
return(new List <object>());
}
ResponseObject response = null;
string speckleQuery = "";
if (speckleRequest != null)
{
speckleQuery = Speckle.Convert.ToSpeckleQuery(speckleRequest);
}
// Download the objects.
response = SpeckleClient.StreamGetObjectsAsync(SpeckleClient.Stream.StreamId, speckleQuery).Result;
// Conversion configuration.
bool storeSpeckleId = true;
// Extract the configuations from the ActionConfig.
// In this case, only SpecklePullConfig contains an option.
SpecklePullConfig config = actionConfig as SpecklePullConfig;
if (config != null)
{
storeSpeckleId = config.StoreSpeckleId;
}
List <object> converted = Speckle.Convert.FromSpeckle(response.Resources, storeSpeckleId);
return(converted);
}
public void BatchUpdateStream(SpecklePushConfig pushConfig)
{
List <SpeckleObject> convertedObjects = SpeckleClient.Stream.Objects;
SpeckleCore.SpeckleInitializer.Initialize();
SpeckleCore.LocalContext.Init();
LocalContext.PruneExistingObjects(convertedObjects, SpeckleClient.BaseUrl);
List <SpeckleObject> persistedObjects = new List <SpeckleObject>();
OrderedDictionary JobQueue = new OrderedDictionary();
if (convertedObjects.Count(obj => obj.Type == "Placeholder") != convertedObjects.Count)
{
// create the update payloads
int count = 0;
var objectUpdatePayloads = new List <List <SpeckleObject> >();
long totalBucketSize = 0;
long currentBucketSize = 0;
var currentBucketObjects = new List <SpeckleObject>();
var allObjects = new List <SpeckleObject>();
foreach (SpeckleObject convertedObject in convertedObjects)
{
if (count++ % 100 == 0)
{
//Message = "Converted " + count + " objects out of " + convertedObjects.Count() + ".";
}
// size checking & bulk object creation payloads creation
long size = Converter.getBytes(convertedObject).Length;
currentBucketSize += size;
totalBucketSize += size;
currentBucketObjects.Add(convertedObject);
// Object is too big?
if (size > 2e6)
{
BH.Engine.Reflection.Compute.RecordWarning("An object is too big for the current Speckle limitations.");
currentBucketObjects.Remove(convertedObject);
}
if (currentBucketSize > 3e5) // restrict max to ~300kb;
{
//BH.Engine.Reflection.Compute.RecordNote("Reached payload limit. Making a new one, current #: " + objectUpdatePayloads.Count);
objectUpdatePayloads.Add(currentBucketObjects);
currentBucketObjects = new List <SpeckleObject>();
currentBucketSize = 0;
}
}
// add in the last bucket
if (currentBucketObjects.Count > 0)
{
objectUpdatePayloads.Add(currentBucketObjects);
}
if (objectUpdatePayloads.Count > 1)
{
BH.Engine.Reflection.Compute.RecordNote($"Payload has been split in { objectUpdatePayloads.Count } batches. Total size is {totalBucketSize / 1024} kB.");
}
// create bulk object creation tasks
List <ResponseObject> responses = new List <ResponseObject>();
foreach (var payload in objectUpdatePayloads)
{
//Message = String.Format("{0}/{1}", k++, objectUpdatePayloads.Count);
try
{
var objResponse = SpeckleClient.ObjectCreateAsync(payload).Result;
responses.Add(objResponse);
persistedObjects.AddRange(objResponse.Resources);
int m = 0;
foreach (var oL in payload)
{
oL._id = objResponse.Resources[m++]._id;
}
// push sent objects in the cache non-blocking
Task.Run(() =>
{
foreach (var oL in payload)
{
if (oL.Type != "Placeholder")
{
LocalContext.AddSentObject(oL, SpeckleClient.BaseUrl);
}
}
});
}
catch (Exception err)
{
BH.Engine.Reflection.Compute.RecordWarning(err.Message);
continue;
}
}
}
else
{
persistedObjects = convertedObjects;
}
// create placeholders for stream update payload
List <SpeckleObject> placeholders = new List <SpeckleObject>();
//foreach ( var myResponse in responses )
foreach (var obj in persistedObjects)
{
placeholders.Add(new SpecklePlaceholder()
{
_id = obj._id
});
}
SpeckleClient.Stream.Objects = placeholders;
// set some base properties (will be overwritten)
var baseProps = new Dictionary <string, object>();
baseProps["units"] = "m";
baseProps["tolerance"] = "0.001";
baseProps["angleTolerance"] = "0.01";
SpeckleClient.Stream.BaseProperties = baseProps;
var response = SpeckleClient.StreamUpdateAsync(SpeckleClient.StreamId, SpeckleClient.Stream).Result;
SpeckleClient.BroadcastMessage("stream", SpeckleClient.StreamId, new { eventType = "update-global" });
}
/// This method is actually useless --
/// written assuming IBHoMObjects had static GUID (instead they are re-instantiated at any modification).
private bool DiffingByBHoMGuid(List <IObject> objectsToBePushed, out List <IObject> objectsCreated)
{
objectsCreated = new List <IObject>();
List <IObject> objectsToBeCreated = new List <IObject>();
// Dispatch objects to be pushed
List <IBHoMObject> bhomObjectsToBePushed = null;
List <IObject> iObjectsToBePushed = null;
BH.Engine.Speckle.Query.DispatchBHoMObjects(objectsToBePushed, out bhomObjectsToBePushed, out iObjectsToBePushed);
// Receive and dispatch objects already in speckle
ResponseObject response = SpeckleClient.StreamGetObjectsAsync(SpeckleStreamId, "").Result;
List <IBHoMObject> bhomObjectsInSpeckle = null;
List <IObject> iObjectsInSpeckle = null;
List <object> reminderObjectsInSpeckle = null;
BH.Engine.Speckle.Convert.ToBHoM(response, out bhomObjectsInSpeckle, out iObjectsInSpeckle, out reminderObjectsInSpeckle, false);
// If speckle doesn't contain anything, push everything
if (response.Resources.Count == 0)
{
objectsToBeCreated = bhomObjectsToBePushed.Concat(iObjectsToBePushed).ToList();
if (CreateAnyObject(objectsToBeCreated))
{
objectsCreated = objectsToBeCreated;
}
return(true);
}
// Diffing for IBHoMObjects
VennDiagram <IBHoMObject> guidDiagram = Engine.Data.Create.VennDiagram(objectsToBePushed.Where(o => o as IBHoMObject != null).Cast <IBHoMObject>(), bhomObjectsInSpeckle, new IBHoMGUIDComparer());
List <IBHoMObject> newObjects = guidDiagram.OnlySet1.ToList(); // Not having a counterpart in the Speckle Server
List <IBHoMObject> toBeDeleted = guidDiagram.OnlySet2.ToList(); // Objects in the Speckle server that do not exist anymore locally. Just not push them.
List <IBHoMObject> toBeDiffedByProperty = guidDiagram.Intersection.Select(o => o.Item1).ToList(); // Objects that already exist on the Speckle Server, based on their BHoM GUID.
// // - Insert code here for toBeDiffedByProperty to use custom comparers to diff by property.
// At the moment, not having any custom comparer, the toBeDiffedByProperty are all just re-created (like assuming they've all changed, even if they didn't).
objectsToBeCreated = newObjects.Concat(toBeDiffedByProperty).Concat(objectsToBePushed.Where(o => o as IBHoMObject == null)).ToList();
if (objectsToBeCreated.Count == 0)
{
BH.Engine.Reflection.Compute.RecordNote("Speckle already contains every BHoM currently being pushed. They have not been pushed.");
}
// IObjects always have to be recreated as they have no GUID
List <int> objectsToPushHashes = new List <int>();
objectsToPushHashes = objectsToBePushed.Select(o => o.ToString().GetHashCode()).ToList();
foreach (var o in iObjectsToBePushed) //.Concat(reminderObjectsInSpeckle)
{
if (!objectsToPushHashes.Any(hash => hash == o.ToString().GetHashCode()))
{
objectsToBeCreated.Add(o);
}
}
if (CreateAnyObject(objectsToBeCreated))
{
objectsCreated = objectsToBeCreated;
}
return(true);
}
