public async Task CountersInitialize_with_EnsureDefaultCounterCreation_should_not_overwrite_existing_counters() { using (var server = GetNewServer(port: 8091)) { using (var counterStore = new CounterStore { Url = $"{server.DocumentStore.Url}:{server.Configuration.Port}", Name = DefaultCounterStorageName }) { counterStore.Initialize(true); await counterStore.IncrementAsync("G", "C"); await counterStore.DecrementAsync("G", "C2"); } using (var counterStore = new CounterStore { Url = $"{server.DocumentStore.Url}:{server.Configuration.Port}", Name = DefaultCounterStorageName }) { counterStore.Initialize(true); var summary = await counterStore.Admin.GetCountersByStorage(DefaultCounterStorageName); Assert.Equal(2, summary.Count); Assert.NotNull(summary.SingleOrDefault(x => x.Total == 1 && x.GroupName == "G" && x.CounterName == "C")); Assert.NotNull(summary.SingleOrDefault(x => x.Total == -1 && x.GroupName == "G" && x.CounterName == "C2")); } } }
private async Task ImportIncrementalData(CounterConnectionStringOptions connectionString, Stream stream) { CountingStream sizeStream; JsonTextReader jsonReader; if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false) { throw new InvalidOperationException("Failed to get reader for the data stream."); } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidDataException("StartObject was expected"); } ICounterStore store = null; try { if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } store = new CounterStore { Url = connectionString.Url, Name = connectionString.CounterStoreId, Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials) }; store.Initialize(true); ShowProgress($"Initialized connection to counter store (name = {store.Name})"); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { if (jsonReader.TokenType != JsonToken.StartObject) { continue; } var counterDelta = RavenJToken.ReadFrom(jsonReader).ToObject <CounterState>(); ShowProgress($"Importing counter {counterDelta.GroupName} - {counterDelta.CounterName}"); if (counterDelta.Sign == ValueSign.Negative) { counterDelta.Value = -counterDelta.Value; } store.Batch.ScheduleChange(counterDelta.GroupName, counterDelta.CounterName, counterDelta.Value); } ShowProgress("Finished import of the current file."); await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false); } finally { store?.Dispose(); } }
public async Task Between(SmugglerBetweenOptions <CounterConnectionStringOptions> betweenOptions) { if (betweenOptions.ReportProgress == null) { betweenOptions.ReportProgress = msg => { } } ; using (var source = new CounterStore { Url = betweenOptions.From.Url, Name = betweenOptions.From.CounterStoreId, Credentials = new OperationCredentials(betweenOptions.From.ApiKey, betweenOptions.From.Credentials) }) using (var target = new CounterStore { Url = betweenOptions.To.Url, Name = betweenOptions.To.CounterStoreId, Credentials = new OperationCredentials(betweenOptions.To.ApiKey, betweenOptions.To.Credentials) }) { source.Initialize(true); ShowProgress($"Initialized connection to source counter store (name = {source.Name})"); target.Initialize(true); ShowProgress($"Initialized connection to target counter store (name = {target.Name})"); var existingCounterGroupsAndNames = await target.Admin.GetCounterStorageNameAndGroups(token : CancellationToken).ConfigureAwait(false); var counterSummaries = await source.Advanced.GetCounters(token : CancellationToken).ConfigureAwait(false); ShowProgress($"Fetched counter data from source (there is data about {counterSummaries.Count} counters)"); foreach (var summary in counterSummaries) { if (existingCounterGroupsAndNames.Any(x => x.Group == summary.GroupName && x.Name == summary.CounterName)) { ShowProgress($"Counter {summary.GroupName} - {summary.CounterName} is already there. Reset is performed"); await target.ResetAsync(summary.GroupName, summary.CounterName, CancellationToken) .WithCancellation(CancellationToken) .ConfigureAwait(false); //since it is a full import, the values are overwritten } ShowProgress($"Importing counter {summary.GroupName} - {summary.CounterName}"); target.Batch.ScheduleChange(summary.GroupName, summary.CounterName, summary.Total); } ShowProgress("Finished import..."); await target.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false); } } }
protected ICounterStore NewRemoteCountersStore(string counterStorageName, bool createDefaultCounter = true, OperationCredentials credentials = null, RavenDbServer ravenServer = null) { ravenServer = ravenServer ?? this.RavenDbServer.Value; var serverUrl = ravenServer.SystemDatabase.ServerUrl; serverCount.AddOrUpdate(serverUrl, id => 1, (id, val) => val++); var counterStore = new CounterStore { Url = GetServerUrl(true, serverUrl), Credentials = credentials ?? new OperationCredentials(null, CredentialCache.DefaultNetworkCredentials), Name = counterStorageName + serverCount[serverUrl] }; counterStore.Initialize(createDefaultCounter); return(counterStore); }
public void Cannot_use_admin_endpoint_with_non_admin_apiKey() { string storeName; using (var store = NewRemoteCountersStore("TestStore")) //this will create the TestStore { storeName = store.Name; } Database.Server.Security.Authentication.EnableOnce(); //GoodApiKey is with admin access to <system> ConfigureApiKey(servers[0].SystemDatabase, "thisIsApiKeyName", "thisIsSecret", storeName, true); //BadApiKey is without admin access to <system> ConfigureApiKey(servers[0].SystemDatabase, "NotThisIsApiKeyName", "thisIsSecret", storeName); using (var store = new CounterStore { Url = servers[0].SystemDatabase.ServerUrl, Name = storeName, Credentials = new OperationCredentials(BadApiKey, null) }) { store.Initialize(); var e = Assert.Throws <ErrorResponseException>(() => AsyncHelpers.RunSync(() => store.Admin.GetCounterStoragesNamesAsync())); Assert.Equal(HttpStatusCode.Forbidden, e.StatusCode); } using (var store = new CounterStore { Url = servers[0].SystemDatabase.ServerUrl, Name = storeName, Credentials = new OperationCredentials(GoodApiKey, null) }) { store.Initialize(); IReadOnlyList <string> storageNames = null; Assert.DoesNotThrow(() => storageNames = AsyncHelpers.RunSync(() => store.Admin.GetCounterStoragesNamesAsync())); Assert.Equal(1, storageNames.Count); } }
public async Task When_Counter_has_apiKey_auth_then_operations_with_wrong_apiKey_should_fail() { String storeName; using (var store = NewRemoteCountersStore("TestStore")) //this will create the TestStore { storeName = store.Name; await store.IncrementAsync("G", "C"); } Database.Server.Security.Authentication.EnableOnce(); ConfigureApiKey(servers[0].SystemDatabase, "thisIsApiKeyName", "thisIsSecret", storeName); ConfigureApiKey(servers[0].SystemDatabase, "NotThisIsApiKeyName", "thisIsSecret", "NonExistingResourceName"); using (var store = new CounterStore { Url = servers[0].SystemDatabase.ServerUrl, Name = storeName, Credentials = new OperationCredentials(BadApiKey, null) }) { store.Initialize(); var e = Assert.Throws <ErrorResponseException>(() => AsyncHelpers.RunSync(() => store.IncrementAsync("G", "C"))); Assert.Equal(HttpStatusCode.Forbidden, e.StatusCode); } using (var store = new CounterStore { Url = servers[0].SystemDatabase.ServerUrl, Name = storeName, Credentials = new OperationCredentials(GoodApiKey, null) }) { store.Initialize(); Assert.DoesNotThrow(() => AsyncHelpers.RunSync(() => store.IncrementAsync("G", "C"))); } }
/// <summary> /// Export counter data to specified destination (a file or a stream) /// </summary> /// <param name="exportOptions">options to specify the source and destination of the export</param> /// <exception cref="UnauthorizedAccessException">The caller does not have the required permission.-or- specified a file that is read-only. </exception> /// <exception cref="DirectoryNotFoundException">The specified path is invalid (for example, it is on an unmapped drive). </exception> /// <exception cref="IOException">An I/O error occurred while creating the file. </exception> /// <exception cref="SmugglerException">Encapsulates exception that happens when actually exporting data. See InnerException for details.</exception> public async Task <CounterOperationState> ExportData(SmugglerExportOptions <CounterConnectionStringOptions> exportOptions) { if (exportOptions.From == null) { throw new ArgumentNullException("exportOptions.From"); } if (String.IsNullOrWhiteSpace(exportOptions.ToFile) && exportOptions.ToStream == null) { throw new ArgumentException("ToFile or ToStream property in options must be non-null"); } var result = new CounterOperationState(); var exportFolder = String.Empty; if (Options.Incremental) { ShowProgress("Starting incremental export.."); exportFolder = CalculateExportFile(exportOptions, exportFolder); } else { ShowProgress("Starting full export..."); } SmugglerException lastException = null; var ownedStream = exportOptions.ToStream == null; var stream = exportOptions.ToStream ?? File.Create(exportOptions.ToFile); if (ownedStream) { ShowProgress("Export to dump file " + exportOptions.ToFile); } try { using (var counterStore = new CounterStore { Url = exportOptions.From.Url, Name = exportOptions.From.CounterStoreId, Credentials = new OperationCredentials(exportOptions.From.ApiKey, exportOptions.From.Credentials) }) using (var gZipStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true)) using (var streamWriter = new StreamWriter(gZipStream)) { counterStore.Initialize(); var jsonWriter = new JsonTextWriter(streamWriter) { Formatting = Formatting.Indented }; jsonWriter.WriteStartObject(); jsonWriter.WritePropertyName(Options.Incremental ? "CountersDeltas" : "CounterSnapshots"); //also for human readability jsonWriter.WriteStartArray(); try { if (Options.Incremental) { await ExportIncrementalData(counterStore, exportFolder, jsonWriter).WithCancellation(CancellationToken).ConfigureAwait(false); } else { await ExportFullData(counterStore, jsonWriter).WithCancellation(CancellationToken).ConfigureAwait(false); } } catch (SmugglerException e) { Debug.Assert(e.Data.Keys.Cast <string>().Contains("LastEtag")); result.LastWrittenEtag = (long)e.Data["LastEtag"]; lastException = e; var operation = Options.Incremental ? "Incremental" : "Full"; ShowProgress($"{operation} Export failed. {e}"); } jsonWriter.WriteEndArray(); jsonWriter.WriteEndObject(); streamWriter.Flush(); } if (lastException != null) { throw lastException; } return(result); } finally { if (ownedStream && stream != null) { stream.Flush(); stream.Dispose(); ShowProgress("Finished export and closed file..."); } else { ShowProgress("Finished export..."); } } }
//assumes that the caller has responsibility to handle data stream disposal ("stream" parameter) private async Task ImportFullData(CounterConnectionStringOptions connectionString, Stream stream) { CountingStream sizeStream; JsonTextReader jsonReader; if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false) { throw new InvalidOperationException("Failed to get reader for the data stream."); } if (jsonReader.TokenType != JsonToken.StartObject) { throw new InvalidDataException("StartObject was expected"); } ICounterStore store = null; try { if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray) { throw new InvalidDataException("StartArray was expected"); } store = new CounterStore { Url = connectionString.Url, Name = connectionString.CounterStoreId, Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials) }; store.Initialize(true); ShowProgress($"Initialized connection to counter store (name = {store.Name})"); var existingCounterGroupsAndNames = await store.Admin.GetCounterStorageNameAndGroups(token : CancellationToken) .WithCancellation(CancellationToken) .ConfigureAwait(false); while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray) { if (jsonReader.TokenType != JsonToken.StartObject) { continue; } var counterInfo = (RavenJObject)RavenJToken.ReadFrom(jsonReader); var delta = Math.Abs(counterInfo.Value <long>("Positive")) - Math.Abs(counterInfo.Value <long>("Negative")); var groupName = counterInfo.Value <string>("Group"); var counterName = counterInfo.Value <string>("Name"); if (existingCounterGroupsAndNames.Any(x => x.Group == groupName && x.Name == counterName)) { ShowProgress($"Counter {groupName} - {counterName} is already there. Reset is performed"); await store.ResetAsync(groupName, counterName, CancellationToken).ConfigureAwait(false); //since it is a full import, the values are overwritten } ShowProgress($"Importing counter {groupName} - {counterName}"); store.Batch.ScheduleChange(groupName, counterName, delta); } ShowProgress("Finished import..."); await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false); } finally { store?.Dispose(); } }