public void TryGcDeleteSet(StructStore store, Predicate <Item> gcFilter)
{
foreach (var kvp in Clients)
{
var client = kvp.Key;
var deleteItems = kvp.Value;
var structs = store.Clients[client];
for (int di = deleteItems.Count - 1; di >= 0; di--)
{
var deleteItem = deleteItems[di];
var endDeleteItemClock = deleteItem.Clock + deleteItem.Length;
for (int si = StructStore.FindIndexSS(structs, deleteItem.Clock); si < structs.Count; si++)
{
var str = structs[si];
if (str.Id.Clock >= endDeleteItemClock)
{
break;
}
if (str is Item strItem && strItem.Deleted && !strItem.Keep && gcFilter(strItem))
{
strItem.Gc(store, parentGCd: false);
}
}
}
}
}
public YDoc RestoreDocument(YDoc originDoc, YDocOptions opts = null)
{
if (originDoc.Gc)
{
// We should try to restore a GC-ed document, because some of the restored items might have their content deleted.
throw new Exception("originDoc must not be garbage collected");
}
using var encoder = new UpdateEncoderV2();
originDoc.Transact(tr =>
{
int size = StateVector.Count(kvp => kvp.Value /* clock */ > 0);
encoder.RestWriter.WriteVarUint((uint)size);
// Splitting the structs before writing them to the encoder.
foreach (var kvp in StateVector)
{
int client = kvp.Key;
int clock = kvp.Value;
if (clock == 0)
{
continue;
}
if (clock < originDoc.Store.GetState(client))
{
tr.Doc.Store.GetItemCleanStart(tr, new ID(client, clock));
}
var structs = originDoc.Store.Clients[client];
var lastStructIndex = StructStore.FindIndexSS(structs, clock - 1);
// Write # encoded structs.
encoder.RestWriter.WriteVarUint((uint)(lastStructIndex + 1));
encoder.WriteClient(client);
// First clock written is 0.
encoder.RestWriter.WriteVarUint(0);
for (int i = 0; i <= lastStructIndex; i++)
{
structs[i].Write(encoder, 0);
}
}
DeleteSet.Write(encoder);
});
var newDoc = new YDoc(opts ?? originDoc.CloneOptionsWithNewGuid());
newDoc.ApplyUpdateV2(encoder.ToArray(), transactionOrigin: "snapshot");
return(newDoc);
}
/// <param name="gc">Disable garbage collection.</param>
/// <param name="gcFilter">WIll be called before an Item is garbage collected. Return false to keep the item.</param>
public YDoc(YDocOptions opts = null)
{
_opts = opts ?? new YDocOptions();
_transactionCleanups = new List <Transaction>();
ClientId = GenerateNewClientId();
_share = new Dictionary <string, AbstractType>();
Store = new StructStore();
Subdocs = new HashSet <YDoc>();
ShouldLoad = _opts.AutoLoad;
}
internal void CompareStructStores(StructStore ss1, StructStore ss2)
{
Assert.AreEqual(ss1.Clients.Count, ss2.Clients.Count);
foreach (var kvp in ss1.Clients)
{
var client = kvp.Key;
var structs1 = kvp.Value;
Assert.IsTrue(ss2.Clients.TryGetValue(client, out var structs2));
Assert.AreEqual(structs1.Count, structs2.Count);
for (int i = 0; i < structs1.Count; i++)
{
var s1 = structs1[i];
var s2 = structs2[i];
// Checks for abstract struct.
if (!s1.GetType().IsAssignableFrom(s2.GetType()) ||
!ID.Equals(s1.Id, s2.Id) ||
s1.Deleted != s2.Deleted ||
s1.Length != s2.Length)
{
Assert.Fail("Structs don't match");
}
if (s1 is Item s1Item)
{
if (!(s2 is Item s2Item) ||
!((s1Item.Left == null && s2Item.Left == null) || (s1Item.Left != null && s2Item.Left != null && ID.Equals((s1Item.Left as Item)?.LastId, (s2Item.Left as Item)?.LastId))) ||
!CompareItemIds(s1Item.Right as Item, s2Item.Right as Item) ||
!ID.Equals(s1Item.LeftOrigin, s2Item.LeftOrigin) ||
!ID.Equals(s1Item.RightOrigin, s2Item.RightOrigin) ||
!string.Equals(s1Item.ParentSub, s2Item.ParentSub))
{
Assert.Fail("Items don't match");
}
// Make sure that items are connected correctly.
Assert.IsTrue(s1Item.Left == null || (s1Item.Left as Item)?.Right == s1Item);
Assert.IsTrue(s1Item.Right == null || (s1Item.Right as Item)?.Left == s1Item);
Assert.IsTrue((s2 as Item).Left == null || ((s2 as Item).Left as Item).Right == s2);
Assert.IsTrue((s2 as Item).Right == null || ((s2 as Item).Right as Item).Left == s2);
}
}
}
}
/// <param name="structs">All structs by 'client'.</param>
/// <param name="clock">Write structs starting with 'ID(client,clock)'.</param>
public static void WriteStructs(IUpdateEncoder encoder, IList <AbstractStruct> structs, int client, int clock)
{
// Write first id.
int startNewStructs = StructStore.FindIndexSS(structs, clock);
// Write # encoded structs.
encoder.RestWriter.WriteVarUint((uint)(structs.Count - startNewStructs));
encoder.WriteClient(client);
encoder.RestWriter.WriteVarUint((uint)clock);
// Write first struct with offset.
var firstStruct = structs[startNewStructs];
firstStruct.Write(encoder, clock - firstStruct.Id.Clock);
for (int i = startNewStructs + 1; i < structs.Count; i++)
{
structs[i].Write(encoder, 0);
}
}
private void CreateDeleteSetFromStructStore(StructStore ss)
{
foreach (var kvp in ss.Clients)
{
var client = kvp.Key;
var structs = kvp.Value;
var dsItems = new List <DeleteItem>();
for (int i = 0; i < structs.Count; i++)
{
var str = structs[i];
if (str.Deleted)
{
int clock = str.Id.Clock;
int len = str.Length;
while (i + 1 < structs.Count)
{
var next = structs[i + 1];
if ((next.Id.Clock == clock + len) && next.Deleted)
{
len += next.Length;
i++;
}
else
{
break;
}
}
dsItems.Add(new DeleteItem(clock, len));
}
}
if (dsItems.Count > 0)
{
Clients[client] = dsItems;
}
}
}
public static void WriteClientsStructs(IUpdateEncoder encoder, StructStore store, IDictionary <int, int> _sm)
{
// We filter all valid _sm entries into sm.
var sm = new Dictionary <int, int>();
foreach (var kvp in _sm)
{
var client = kvp.Key;
var clock = kvp.Value;
// Only write if new structs are available.
if (store.GetState(client) > clock)
{
sm[client] = clock;
}
}
foreach (var kvp in store.GetStateVector())
{
var client = kvp.Key;
if (!sm.ContainsKey(client))
{
sm[client] = 0;
}
}
// Write # states that were updated.
encoder.RestWriter.WriteVarUint((uint)sm.Count);
// Write items with higher client ids first.
// This heavily improves the conflict resolution algorithm.
var sortedClients = sm.Keys.ToList();
sortedClients.Sort((a, b) => b - a);
foreach (var client in sortedClients)
{
WriteStructs(encoder, store.Clients[client], client, sm[client]);
}
}
public void TryMergeDeleteSet(StructStore store)
{
// Try to merge deleted / gc'd items.
// Merge from right to left for better efficiency and so we don't miss any merge targets.
foreach (var kvp in Clients)
{
var client = kvp.Key;
var deleteItems = kvp.Value;
var structs = store.Clients[client];
for (int di = deleteItems.Count - 1; di >= 0; di--)
{
var deleteItem = deleteItems[di];
// Start with merging the item next to the last deleted item.
var mostRightIndexToCheck = Math.Min(structs.Count - 1, 1 + StructStore.FindIndexSS(structs, deleteItem.Clock + deleteItem.Length - 1));
for (int si = mostRightIndexToCheck; si > 0 && structs[si].Id.Clock >= deleteItem.Clock; si--)
{
TryToMergeWithLeft(structs, si);
}
}
}
}
void IContentEx.Gc(StructStore store)
{
var item = Type._start;
while (item != null)
{
item.Gc(store, parentGCd: true);
item = item.Right as Item;
}
Type._start = null;
foreach (var kvp in Type._map)
{
var valueItem = kvp.Value;
while (valueItem != null)
{
valueItem.Gc(store, parentGCd: true);
valueItem = valueItem.Left as Item;
}
}
Type._map.Clear();
}
internal abstract int?GetMissing(Transaction transaction, StructStore store);
internal static void CleanupTransactions(IList <Transaction> transactionCleanups, int i)
{
if (i < transactionCleanups.Count)
{
var transaction = transactionCleanups[i];
var doc = transaction.Doc;
var store = doc.Store;
var ds = transaction.DeleteSet;
var mergeStructs = transaction._mergeStructs;
var actions = new List <Action>();
try
{
ds.SortAndMergeDeleteSet();
transaction.AfterState = store.GetStateVector();
doc._transaction = null;
actions.Add(() =>
{
doc.InvokeOnBeforeObserverCalls(transaction);
});
actions.Add(() =>
{
foreach (var kvp in transaction.Changed)
{
var itemType = kvp.Key;
var subs = kvp.Value;
if (itemType._item == null || !itemType._item.Deleted)
{
itemType.CallObserver(transaction, subs);
}
}
});
actions.Add(() =>
{
// Deep observe events.
foreach (var kvp in transaction.ChangedParentTypes)
{
var type = kvp.Key;
var events = kvp.Value;
// We need to think about the possibility that the user transforms the YDoc in the event.
if (type._item == null || !type._item.Deleted)
{
foreach (var evt in events)
{
if (evt.Target._item == null || !evt.Target._item.Deleted)
{
evt.CurrentTarget = type;
}
}
// Sort events by path length so that top-level events are fired first.
var sortedEvents = events.ToList();
sortedEvents.Sort((a, b) => a.Path.Count - b.Path.Count);
Debug.Assert(sortedEvents.Count > 0);
actions.Add(() =>
{
type.CallDeepEventHandlerListeners(sortedEvents, transaction);
});
}
}
});
actions.Add(() =>
{
doc.InvokeOnAfterTransaction(transaction);
});
CallAll(actions);
}
finally
{
// Replace deleted items with ItemDeleted / GC.
// This is where content is actually removed from the Yjs Doc.
if (doc.Gc)
{
ds.TryGcDeleteSet(store, doc.GcFilter);
}
ds.TryMergeDeleteSet(store);
// On all affected store.clients props, try to merge.
foreach (var kvp in transaction.AfterState)
{
var client = kvp.Key;
var clock = kvp.Value;
if (!transaction.BeforeState.TryGetValue(client, out int beforeClock))
{
beforeClock = 0;
}
if (beforeClock != clock)
{
var structs = store.Clients[client];
var firstChangePos = Math.Max(StructStore.FindIndexSS(structs, beforeClock), 1);
for (int j = structs.Count - 1; j >= firstChangePos; j--)
{
DeleteSet.TryToMergeWithLeft(structs, j);
}
}
}
// Try to merge mergeStructs.
// TODO: It makes more sense to transform mergeStructs to a DS, sort it, and merge from right to left
// but at the moment DS does not handle duplicates.
for (int j = 0; j < mergeStructs.Count; j++)
{
var client = mergeStructs[j].Id.Client;
var clock = mergeStructs[j].Id.Clock;
var structs = store.Clients[client];
var replacedStructPos = StructStore.FindIndexSS(structs, clock);
if (replacedStructPos + 1 < structs.Count)
{
DeleteSet.TryToMergeWithLeft(structs, replacedStructPos + 1);
}
if (replacedStructPos > 0)
{
DeleteSet.TryToMergeWithLeft(structs, replacedStructPos);
}
}
if (!transaction.Local)
{
if (!transaction.AfterState.TryGetValue(doc.ClientId, out int afterClock))
{
afterClock = -1;
}
if (!transaction.BeforeState.TryGetValue(doc.ClientId, out int beforeClock))
{
beforeClock = -1;
}
if (afterClock != beforeClock)
{
doc.ClientId = YDoc.GenerateNewClientId();
// Debug.WriteLine($"{nameof(Transaction)}: Changed the client-id because another client seems to be using it.");
}
}
// @todo: Merge all the transactions into one and provide send the data as a single update message.
doc.InvokeOnAfterTransactionCleanup(transaction);
doc.InvokeUpdateV2(transaction);
foreach (var subDoc in transaction.SubdocsAdded)
{
doc.Subdocs.Add(subDoc);
}
foreach (var subDoc in transaction.SubdocsRemoved)
{
doc.Subdocs.Remove(subDoc);
}
doc.InvokeSubdocsChanged(transaction.SubdocsLoaded, transaction.SubdocsAdded, transaction.SubdocsRemoved);
foreach (var subDoc in transaction.SubdocsRemoved)
{
subDoc.Destroy();
}
if (transactionCleanups.Count <= i + 1)
{
doc._transactionCleanups.Clear();
doc.InvokeAfterAllTransactions(transactionCleanups);
}
else
{
CleanupTransactions(transactionCleanups, i + 1);
}
}
}
}
/// <summary>
/// Read the next Item in a Decoder and fill this Item with the read data.
/// <br/>
/// This is called when data is received from a remote peer.
/// </summary>
public static void ReadStructs(IUpdateDecoder decoder, Transaction transaction, StructStore store)
{
var clientStructRefs = ReadClientStructRefs(decoder, transaction.Doc);
store.MergeReadStructsIntoPendingReads(clientStructRefs);
store.ResumeStructIntegration(transaction);
store.CleanupPendingStructs();
store.TryResumePendingDeleteReaders(transaction);
}
public void ReadAndApplyDeleteSet(IDSDecoder decoder, Transaction transaction)
{
var unappliedDs = new DeleteSet();
var numClients = decoder.Reader.ReadVarUint();
for (int i = 0; i < numClients; i++)
{
decoder.ResetDsCurVal();
var client = (int)decoder.Reader.ReadVarUint();
var numberOfDeletes = decoder.Reader.ReadVarUint();
if (!Clients.TryGetValue(client, out var structs))
{
structs = new List <AbstractStruct>();
// NOTE: Clients map is not updated.
}
var state = GetState(client);
for (int deleteIndex = 0; deleteIndex < numberOfDeletes; deleteIndex++)
{
var clock = decoder.ReadDsClock();
var clockEnd = clock + decoder.ReadDsLength();
if (clock < state)
{
if (state < clockEnd)
{
unappliedDs.Add(client, state, clockEnd - state);
}
var index = StructStore.FindIndexSS(structs, clock);
// We can ignore the case of GC and Delete structs, because we are going to skip them.
var str = structs[index];
// Split the first item if necessary.
if (!str.Deleted && str.Id.Clock < clock)
{
var splitItem = (str as Item).SplitItem(transaction, clock - str.Id.Clock);
structs.Insert(index + 1, splitItem);
// Increase, we now want to use the next struct.
index++;
}
while (index < structs.Count)
{
str = structs[index++];
if (str.Id.Clock < clockEnd)
{
if (!str.Deleted)
{
if (clockEnd < str.Id.Clock + str.Length)
{
var splitItem = (str as Item).SplitItem(transaction, clockEnd - str.Id.Clock);
structs.Insert(index, splitItem);
}
str.Delete(transaction);
}
}
else
{
break;
}
}
}
else
{
unappliedDs.Add(client, clock, clockEnd - clock);
}
}
}
if (unappliedDs.Clients.Count > 0)
{
// @TODO: No need for encoding+decoding ds anymore.
using var unappliedDsEncoder = new DSEncoderV2();
unappliedDs.Write(unappliedDsEncoder);
_pendingDeleteReaders.Add(new DSDecoderV2(new MemoryStream(unappliedDsEncoder.ToArray())));
}
}
void IContentEx.Gc(StructStore store)
{
// Do nothing.
}
public DeleteSet(StructStore ss)
: this()
{
CreateDeleteSetFromStructStore(ss);
}
public void TryGc(StructStore store, Predicate <Item> gcFilter)
{
TryGcDeleteSet(store, gcFilter);
TryMergeDeleteSet(store);
}
internal override int?GetMissing(Transaction transaction, StructStore store)
{
return(null);
}