private void LoadAll() { var flags = new Dictionary <string, ItemDescriptor>(); var segments = new Dictionary <string, ItemDescriptor>(); foreach (var path in _paths) { try { var content = _fileReader.ReadAllText(path); var data = _parser.Parse(content); _dataMerger.AddToData(data, flags, segments); } catch (FileNotFoundException) when(_skipMissingPaths) { _logger.Debug("{0}: {1}", path, "File not found"); } catch (Exception e) { LogHelpers.LogException(_logger, "Failed to load " + path, e); return; } } var allData = new FullDataSet <ItemDescriptor>( ImmutableDictionary.Create <DataKind, KeyedItems <ItemDescriptor> >() .SetItem(DataModel.Features, new KeyedItems <ItemDescriptor>(flags)) .SetItem(DataModel.Segments, new KeyedItems <ItemDescriptor>(segments)) ); _dataSourceUpdates.Init(allData); _loadedValidData = true; }
internal static void VerifyDataSetOrder(FullDataSet <ItemDescriptor> resultData, FullDataSet <ItemDescriptor> inputData, List <KeyValuePair <DataKind, List <KeyOrderConstraint> > > expectedOrdering) { // Verify that there are the right number of data kinds in the right order Assert.Equal(expectedOrdering.Select(kv => kv.Key).ToList(), resultData.Data.Select(kv => kv.Key).ToList()); foreach (var kindAndItems in resultData.Data) { var inputItemsForKind = inputData.Data.First(kv => kv.Key == kindAndItems.Key).Value; // Verify that all of the input items are present, regardless of order Assert.Equal(new HashSet <KeyValuePair <string, ItemDescriptor> >(kindAndItems.Value.Items), new HashSet <KeyValuePair <string, ItemDescriptor> >(inputItemsForKind.Items)); // Verify that for any two keys where we care about their relative ordering, they are in the right order // (keys where the relative ordering doesn't matter, i.e. there are no prerequisites involved, are simply // omitted from the constraint list) var constraints = expectedOrdering.Where(kv => kv.Key == kindAndItems.Key).Select(kv => kv.Value).FirstOrDefault(); var resultKeys = kindAndItems.Value.Items.Select(kv => kv.Key).ToList(); foreach (var constraint in constraints ?? new List <KeyOrderConstraint>()) { int indexOfEarlierKey = resultKeys.IndexOf(constraint.EarlierKey); int indexOfLaterKey = resultKeys.IndexOf(constraint.LaterKey); Assert.True(indexOfEarlierKey < indexOfLaterKey, String.Format("In \"{0}\", \"{1}\" should be updated before \"{2}\"; actual key order was {3}", kindAndItems.Key.Name, constraint.EarlierKey, constraint.LaterKey, resultKeys)); } } }
public void DataSetIsPassedToDataStoreInCorrectOrder() { // The underlying functionality here is also covered in DataStoreSorterTest, but we want to verify that the // client object is actually *using* DataStoreSorter. var mockStore = new Mock <IDataStore>(); var store = mockStore.Object; FullDataSet <ItemDescriptor> receivedData = new FullDataSet <ItemDescriptor>(); mockStore.Setup(s => s.Init(It.IsAny <FullDataSet <ItemDescriptor> >())) .Callback((FullDataSet <ItemDescriptor> data) => { receivedData = data; }); mockDataSource.Setup(up => up.Start()).Returns(initTask); var config = Configuration.Builder(sdkKey) .DataStore(TestUtils.SpecificDataStore(store)) .DataSource(TestUtils.DataSourceWithData(DataStoreSorterTest.DependencyOrderingTestData)) .Events(Components.NoEvents) .Logging(Components.Logging(testLogging)) .Build(); using (var client = new LdClient(config)) { Assert.NotNull(receivedData); DataStoreSorterTest.VerifyDataSetOrder(receivedData, DataStoreSorterTest.DependencyOrderingTestData, DataStoreSorterTest.ExpectedOrderingForSortedDataSet); } }
internal static PutData ParsePutData(byte[] json) { var r = JReader.FromUtf8Bytes(json); try { string path = null; FullDataSet <ItemDescriptor> data = new FullDataSet <ItemDescriptor>(); for (var obj = r.Object().WithRequiredProperties(_putRequiredProperties); obj.Next(ref r);) { if (obj.Name == "path") { path = r.String(); } else if (obj.Name == "data") { data = ParseFullDataset(ref r); } } return(new PutData(path, data)); } catch (Exception e) { throw r.TranslateException(e); } }
public UCProductAll(FullDataSet source, bool isNds) { thread = new Thread(new ThreadStart(MainForm.DoSplash)); thread.Start(); Thread.Sleep(300); InitializeComponent(); productBindingSource.DataSource = new DataView(source.Product, "IsNds=" + isNds.ToString() + " and ID<>0", null, DataViewRowState.CurrentRows | DataViewRowState.Added | DataViewRowState.ModifiedCurrent); this.unitBindingSource.DataSource = source.Unit; this.storageConditionBindingSource.DataSource = source.StorageCondition; this.substanceBindingSource.DataSource = source.Substance; this.farmGroupLevel2BindingSource.DataSource = source.FarmGroupLevel2; this.packingBindingSource.DataSource = source.Packing; _isnds = isNds; _changes = new DataView(source.Product, null, null, DataViewRowState.Added | DataViewRowState.ModifiedCurrent | DataViewRowState.Deleted); _changes.ListChanged += new ListChangedEventHandler(_changes_ListChanged); _product = source.Product; }
public void Init(User user, FullDataSet data) { _dataStore.Init(user, data, true); ImmutableDictionary <string, FeatureFlag> oldValues, newValues; lock (_lastValuesLock) { _lastValues.TryGetValue(user.Key, out oldValues); var builder = ImmutableDictionary.CreateBuilder <string, FeatureFlag>(); foreach (var newEntry in data.Items) { var newFlag = newEntry.Value.Item; if (newFlag != null) { builder.Add(newEntry.Key, newFlag); } } newValues = builder.ToImmutable(); _lastValues = _lastValues.SetItem(user.Key, newValues); } UpdateStatus(DataSourceState.Valid, null); if (oldValues != null) { List <FlagValueChangeEvent> events = new List <FlagValueChangeEvent>(); foreach (var newEntry in newValues) { var newFlag = newEntry.Value; if (oldValues.TryGetValue(newEntry.Key, out var oldFlag)) { if (newFlag.Variation != oldFlag.Variation) { events.Add(new FlagValueChangeEvent(newEntry.Key, oldFlag.Value, newFlag.Value, false)); } } else { events.Add(new FlagValueChangeEvent(newEntry.Key, LdValue.Null, newFlag.Value, false)); } } foreach (var oldEntry in oldValues) { if (!newValues.ContainsKey(oldEntry.Key)) { events.Add(new FlagValueChangeEvent(oldEntry.Key, oldEntry.Value.Value, LdValue.Null, true)); } } foreach (var e in events) { _taskExecutor.ScheduleEvent(e, FlagValueChanged); } } }
public TReturn Max <TReturn>(Expression <Func <TObject, TReturn> > predicate, Expression <Func <TObject, bool> > clause) { if (FullDataSet.Any(clause)) { return(FullDataSet.Where(clause).Max(predicate)); } return(default(TReturn)); }
private void VerifyNoFlagValues(ILdClient client, FullDataSet flags) { Assert.True(client.Initialized); foreach (var e in flags.Items) { Assert.Equal(LdValue.Null, client.JsonVariation(e.Key, LdValue.Null)); } }
public void Init(FullDataSet <SerializedItemDescriptor> allData) { _db.DataForPrefix(_prefix).Clear(); foreach (var coll in allData.Data) { _db.DataForPrefix(_prefix)[coll.Key] = new Dictionary <string, SerializedItemDescriptor>(coll.Value.Items); } _db.SetInited(_prefix); }
public void RemoveAll(Expression <Func <TObject, bool> > predicate) { var objects = Filter(predicate); foreach (var obj in objects) { FullDataSet.DeleteObject(obj); } }
public void PrerequisiteFlagsAreUpdatedBeforeFlagsThatUseThemWhenInputDataIsReversed() { var inputDataWithReverseOrder = new FullDataSet <ItemDescriptor>(DependencyOrderingTestData.Data.Reverse().Select(kv => new KeyValuePair <DataKind, KeyedItems <ItemDescriptor> >(kv.Key, new KeyedItems <ItemDescriptor>(kv.Value.Items.Reverse())) )); var sortedData = DataStoreSorter.SortAllCollections(inputDataWithReverseOrder); VerifyDataSetOrder(sortedData, DependencyOrderingTestData, ExpectedOrderingForSortedDataSet); }
// Ensures that a data set is sorted by namespace and then by key internal static FullDataSet <ItemDescriptor> NormalizeDataSet(FullDataSet <ItemDescriptor> data) => new FullDataSet <ItemDescriptor>( data.Data.OrderBy(kindAndItems => kindAndItems.Key.Name) .Select(kindAndItems => new KeyValuePair <DataKind, KeyedItems <ItemDescriptor> >( kindAndItems.Key, new KeyedItems <ItemDescriptor>( kindAndItems.Value.Items.OrderBy(keyAndItem => keyAndItem.Key) ) ) ) );
private static async Task Init(IDisposable store, FullDataSet <SerializedItemDescriptor> allData) { if (store is IPersistentDataStore syncStore) { syncStore.Init(allData); } else { await(store as IPersistentDataStoreAsync).InitAsync(allData); } }
private ImmutableDictionary <DataKind, ImmutableDictionary <string, ItemDescriptor> > FullDataSetToMap( FullDataSet <ItemDescriptor> allData) { var builder = ImmutableDictionary.CreateBuilder <DataKind, ImmutableDictionary <string, ItemDescriptor> >(); foreach (var e in allData.Data) { builder.Add(e.Key, e.Value.Items.ToImmutableDictionary()); } return(builder.ToImmutable()); }
public static FullDataSet <ItemDescriptor> SortAllCollections(FullDataSet <ItemDescriptor> allData) { var dataOut = new SortedDictionary <DataKind, KeyedItems <ItemDescriptor> >( PriorityComparer.Instance); foreach (var entry in allData.Data) { var kind = entry.Key; dataOut.Add(kind, new KeyedItems <ItemDescriptor>(SortCollection(kind, entry.Value.Items))); } return(new FullDataSet <ItemDescriptor>(dataOut)); }
internal void DoInit(FullDataSet <ItemDescriptor> data) { _log.Debug("using initial test data:\n{0}", LogValues.Defer(() => string.Join("\n", data.Data.Select(coll => coll.Key.Name + ":\n" + string.Join("\n", coll.Value.Items.Select(kv => coll.Key.Serialize(kv.Value) )) )) )); _updates.Init(data); }
public void Init(FullDataSet <SerializedItemDescriptor> allData) { InitCalledCount++; MaybeThrowError(); Data.Clear(); foreach (var e in allData.Data) { var kind = e.Key; Data[kind] = e.Value.Items.ToDictionary(kv => kv.Key, kv => StorableItem(kind, kv.Value)); } Inited = true; }
private void UpdateDependencyTrackerFromFullDataSet(FullDataSet <ItemDescriptor> allData) { _dependencyTracker.Clear(); foreach (var e0 in allData.Data) { var kind = e0.Key; foreach (var e1 in e0.Value.Items) { var key = e1.Key; _dependencyTracker.UpdateDependenciesFrom(kind, key, e1.Value); } } }
private Exception InitCore(FullDataSet <SerializedItemDescriptor> allData) { try { _core.Init(allData); ProcessError(null); return(null); } catch (Exception e) { ProcessError(e); return(e); } }
internal static JsonTestValue DataSetAsJson(FullDataSet <ItemDescriptor> data) { var ob0 = LdValue.BuildObject(); foreach (var kv0 in data.Data) { var ob1 = LdValue.BuildObject(); foreach (var kv1 in kv0.Value.Items) { ob1.Add(kv1.Key, LdValue.Parse(kv0.Key.Serialize(kv1.Value))); } ob0.Add(kv0.Key.Name, ob1.Build()); } return(JsonTestValue.JsonOf(ob0.Build().ToJsonString())); }
public void Init(FullDataSet <ItemDescriptor> items) { lock (_cachedDataKinds) { _cachedDataKinds.Clear(); foreach (var kv in items.Data) { _cachedDataKinds.Add(kv.Key); } } var serializedItems = items.Data.ToImmutableDictionary( kindAndItems => kindAndItems.Key, kindAndItems => SerializeAll(kindAndItems.Key, kindAndItems.Value.Items) ); Exception failure = InitCore(new FullDataSet <SerializedItemDescriptor>(serializedItems)); if (_itemCache != null && _allCache != null) { _itemCache.Clear(); _allCache.Clear(); if (failure != null && !_caching.IsInfiniteTtl) { // Normally, if the underlying store failed to do the update, we do not want to update the cache - // the idea being that it's better to stay in a consistent state of having old data than to act // like we have new data but then suddenly fall back to old data when the cache expires. However, // if the cache TTL is infinite, then it makes sense to update the cache always. throw failure; } foreach (var e0 in items.Data) { var kind = e0.Key; _allCache.Set(kind, e0.Value.Items.ToImmutableDictionary()); foreach (var e1 in e0.Value.Items) { _itemCache.Set(new CacheKey(kind, e1.Key), e1.Value); } } } if (failure is null || _caching.IsInfiniteTtl) { _inited = true; } if (failure != null) { throw failure; } }
internal static string MakeJsonData(FullDataSet data) { var w = JWriter.New(); using (var ow = w.Object()) { foreach (var item in data.Items) { if (item.Value.Item != null) { FeatureFlagJsonConverter.WriteJsonValue(item.Value.Item, ow.Name(item.Key)); } } } return(w.GetString()); }
public async Task InitAsync(FullDataSet <SerializedItemDescriptor> allData) { // Start by reading the existing keys; we will later delete any of these that weren't in allData. var unusedOldKeys = await ReadExistingKeys(allData.Data.Select(collection => collection.Key)); var requests = new List <WriteRequest>(); var numItems = 0; // Insert or update every provided item foreach (var collection in allData.Data) { var kind = collection.Key; foreach (var keyAndItem in collection.Value.Items) { var encodedItem = MarshalItem(kind, keyAndItem.Key, keyAndItem.Value); if (!CheckSizeLimit(encodedItem)) { continue; } requests.Add(new WriteRequest(new PutRequest(encodedItem))); var combinedKey = new Tuple <string, string>(NamespaceForKind(kind), keyAndItem.Key); unusedOldKeys.Remove(combinedKey); numItems++; } } // Now delete any previously existing items whose keys were not in the current data foreach (var combinedKey in unusedOldKeys) { if (combinedKey.Item1 != InitedKey) { var keys = MakeKeysMap(combinedKey.Item1, combinedKey.Item2); requests.Add(new WriteRequest(new DeleteRequest(keys))); } } // Now set the special key that we check in initializedInternal() var initedItem = MakeKeysMap(InitedKey, InitedKey); requests.Add(new WriteRequest(new PutRequest(initedItem))); await DynamoDBHelpers.BatchWriteRequestsAsync(_client, _tableName, requests); _log.Info("Initialized data store with {0} items", numItems); }
public async Task InitAsync(FullDataSet <SerializedItemDescriptor> allData) { // Start by reading the existing keys; we will later delete any of these that weren't in allData. var keysResult = await _client.KV.Keys(_prefix); var unusedOldKeys = keysResult.Response == null ? new HashSet <string>() : new HashSet <string>(keysResult.Response); var ops = new List <KVTxnOp>(); var numItems = 0; // Insert or update every provided item foreach (var collection in allData.Data) { var kind = collection.Key; foreach (var keyAndItem in collection.Value.Items) { var key = ItemKey(kind, keyAndItem.Key); var op = new KVTxnOp(key, KVTxnVerb.Set) { Value = Encoding.UTF8.GetBytes(keyAndItem.Value.SerializedItem) }; ops.Add(op); unusedOldKeys.Remove(key); numItems++; } } // Now delete any previously existing items whose keys were not in the current data foreach (var oldKey in unusedOldKeys) { ops.Add(new KVTxnOp(oldKey, KVTxnVerb.Delete)); } // Now set the special key that we check in InitializedInternalAsync() var initedOp = new KVTxnOp(InitedKey, KVTxnVerb.Set) { Value = new byte[0] }; ops.Add(initedOp); await BatchOperationsAsync(ops); _log.Info("Initialized data store with {0} items", numItems); }
public static void DataSetsEqual(FullDataSet <ItemDescriptor> expected, FullDataSet <ItemDescriptor> actual) { var expectedNorm = TestUtils.NormalizeDataSet(expected); var actualNorm = TestUtils.NormalizeDataSet(actual); var ok = false; string expectedDesc = null, actualDesc = null; if (expectedNorm.Data.Count() == actualNorm.Data.Count()) { ok = true; for (var i = 0; ok && i < expectedNorm.Data.Count(); i++) { var ec = expectedNorm.Data.ElementAt(i); var ac = actualNorm.Data.ElementAt(i); if (ac.Key != ec.Key) { ok = false; } else { var kind = ac.Key; for (var j = 0; ok && j < ec.Value.Items.Count(); j++) { var ei = ec.Value.Items.ElementAt(j); var ai = ac.Value.Items.ElementAt(j); if (ai.Key != ei.Key || !ItemsEqual(kind, ei.Value, ai.Value)) { expectedDesc = DescribeDataCollection(kind, ei); actualDesc = DescribeDataCollection(kind, ai); ok = false; } } } } } if (!ok) { if (expectedDesc is null) { expectedDesc = DescribeDataSet(expectedNorm); actualDesc = DescribeDataSet(actualNorm); } Assert.True(false, string.Format("data set mismatch:\nexpected: {0}\nactual: {1}", expectedDesc, actualDesc)); } }
public static string ToJsonString(this FullDataSet <ItemDescriptor> data) { var ob0 = LdValue.BuildObject(); foreach (var kv0 in data.Data) { if (kv0.Key == DataModel.Features || kv0.Key == DataModel.Segments) { var ob1 = LdValue.BuildObject(); foreach (var kv1 in kv0.Value.Items) { ob1.Add(kv1.Key, LdValue.Parse(kv0.Key.Serialize(kv1.Value))); } ob0.Add(kv0.Key == DataModel.Features ? "flags" : "segments", ob1.Build()); } } return(ob0.Build().ToJsonString()); }
public void EvaluationUsesDataStoreIfClientIsNotInitedButStoreIsInited() { var dataStore = new InMemoryDataStore(); dataStore.Init(FullDataSet <ItemDescriptor> .Empty()); var flag = new FeatureFlagBuilder("key").OffWithValue(LdValue.Of(1)).Build(); TestUtils.UpsertFlag(dataStore, flag); var config = BasicConfig() .DataStore(dataStore.AsSingletonFactory()) .DataSource(MockDataSourceThatNeverStarts().AsSingletonFactory()) .Build(); using (var client = new LdClient(config)) { Assert.Equal(1, client.IntVariation("key", User.WithKey("user"), 0)); } }
public void Init(FullDataSet <SerializedItemDescriptor> allData) { IDatabase db = _redis.GetDatabase(); ITransaction txn = db.CreateTransaction(); foreach (var collection in allData.Data) { string key = ItemsKey(collection.Key); txn.KeyDeleteAsync(key); foreach (var item in collection.Value.Items) { txn.HashSetAsync(key, item.Key, item.Value.SerializedItem); // Note, these methods are async because this Redis client treats all actions // in a transaction as async - they are only sent to Redis when we execute the // transaction. We don't need to await them. } } txn.StringSetAsync(_initedKey, ""); txn.Execute(); }
public bool Init(FullDataSet <ItemDescriptor> allData) { ImmutableDictionary <DataKind, ImmutableDictionary <string, ItemDescriptor> > oldData = null; try { if (HasFlagChangeListeners()) { // Query the existing data if any, so that after the update we can send events for // whatever was changed var oldDataBuilder = ImmutableDictionary.CreateBuilder <DataKind, ImmutableDictionary <string, ItemDescriptor> >(); foreach (var kind in DataModel.AllDataKinds) { var items = _store.GetAll(kind); oldDataBuilder.Add(kind, items.Items.ToImmutableDictionary()); } oldData = oldDataBuilder.ToImmutable(); } _store.Init(DataStoreSorter.SortAllCollections(allData)); _lastStoreUpdateFailed = false; } catch (Exception e) { ReportStoreFailure(e); return(false); } // We must always update the dependency graph even if we don't currently have any event listeners, because if // listeners are added later, we don't want to have to reread the whole data store to compute the graph UpdateDependencyTrackerFromFullDataSet(allData); // Now, if we previously queried the old data because someone is listening for flag change events, compare // the versions of all items and generate events for those (and any other items that depend on them) if (oldData != null) { SendChangeEvents(ComputeChangedItemsForFullDataSet(oldData, FullDataSetToMap(allData))); } return(true); }
public void EvaluationUsesDataStoreIfClientIsNotInitedButStoreIsInited() { var dataStore = new InMemoryDataStore(); dataStore.Init(FullDataSet <ItemDescriptor> .Empty()); var flag = new FeatureFlagBuilder("key").OffWithValue(LdValue.Of(1)).Build(); TestUtils.UpsertFlag(dataStore, flag); var config = Configuration.Builder(sdkKey).StartWaitTime(TimeSpan.Zero) .DataStore(TestUtils.SpecificDataStore(dataStore)) .DataSource(TestUtils.SpecificDataSource(dataSource)) .Events(Components.NoEvents) .Logging(Components.Logging(testLogging)) .Build(); using (var client = new LdClient(config)) { Assert.Equal(1, client.IntVariation("key", User.WithKey("user"), 0)); } }
public virtual int Fill(FullDataSet.ProductDataTable dataTable, System.Nullable<bool> isnds) { this.Adapter.SelectCommand = this.CommandCollection[0]; if ((isnds.HasValue == true)) { this.Adapter.SelectCommand.Parameters[1].Value = ((bool)(isnds.Value)); } else { this.Adapter.SelectCommand.Parameters[1].Value = System.DBNull.Value; } if ((this.ClearBeforeFill == true)) { dataTable.Clear(); } int returnValue = this.Adapter.Fill(dataTable); return returnValue; }
public virtual int FillByName(FullDataSet.ProductDataTable dataTable, string name) { this.Adapter.SelectCommand = this.CommandCollection[1]; if ((name == null)) { this.Adapter.SelectCommand.Parameters[1].Value = System.DBNull.Value; } else { this.Adapter.SelectCommand.Parameters[1].Value = ((string)(name)); } if ((this.ClearBeforeFill == true)) { dataTable.Clear(); } int returnValue = this.Adapter.Fill(dataTable); return returnValue; }
public virtual int Update(FullDataSet.SubstanceDataTable dataTable) { return this.Adapter.Update(dataTable); }
public virtual int Update(FullDataSet dataSet) { return this.Adapter.Update(dataSet, "Substance"); }
public virtual int Update(FullDataSet dataSet) { return this.Adapter.Update(dataSet, "Unit"); }
public virtual int Fill(FullDataSet.SubstanceDataTable dataTable) { this.Adapter.SelectCommand = this.CommandCollection[0]; if ((this.ClearBeforeFill == true)) { dataTable.Clear(); } int returnValue = this.Adapter.Fill(dataTable); return returnValue; }
public virtual int Update(FullDataSet dataSet) { return this.Adapter.Update(dataSet, "StorageCondition"); }
public virtual int Update(FullDataSet.UnitDataTable dataTable) { return this.Adapter.Update(dataTable); }
public virtual int Update(FullDataSet.StorageConditionDataTable dataTable) { return this.Adapter.Update(dataTable); }
public virtual int Update(FullDataSet dataSet) { return this.Adapter.Update(dataSet, "Manufacturer"); }
public virtual int Update(FullDataSet.ManufacturerDataTable dataTable) { return this.Adapter.Update(dataTable); }
public virtual int Update(FullDataSet dataSet) { return this.Adapter.Update(dataSet, "FarmGroupLevel2"); }
public virtual int Update(FullDataSet.FarmGroupLevel2DataTable dataTable) { return this.Adapter.Update(dataTable); }