public async Task CountersInitialize_with_EnsureDefaultCounterCreation_should_not_overwrite_existing_counters()
{
using (var server = GetNewServer(port: 8091))
{
using (var counterStore = new CounterStore
{
Url = $"{server.DocumentStore.Url}:{server.Configuration.Port}",
Name = DefaultCounterStorageName
})
{
counterStore.Initialize(true);
await counterStore.IncrementAsync("G", "C");
await counterStore.DecrementAsync("G", "C2");
}
using (var counterStore = new CounterStore
{
Url = $"{server.DocumentStore.Url}:{server.Configuration.Port}",
Name = DefaultCounterStorageName
})
{
counterStore.Initialize(true);
var summary = await counterStore.Admin.GetCountersByStorage(DefaultCounterStorageName);
Assert.Equal(2, summary.Count);
Assert.NotNull(summary.SingleOrDefault(x => x.Total == 1 && x.GroupName == "G" && x.CounterName == "C"));
Assert.NotNull(summary.SingleOrDefault(x => x.Total == -1 && x.GroupName == "G" && x.CounterName == "C2"));
}
}
}
private async Task ImportIncrementalData(CounterConnectionStringOptions connectionString, Stream stream)
{
CountingStream sizeStream;
JsonTextReader jsonReader;
if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false)
{
throw new InvalidOperationException("Failed to get reader for the data stream.");
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
ICounterStore store = null;
try
{
if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
store = new CounterStore
{
Url = connectionString.Url,
Name = connectionString.CounterStoreId,
Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials)
};
store.Initialize(true);
ShowProgress($"Initialized connection to counter store (name = {store.Name})");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
if (jsonReader.TokenType != JsonToken.StartObject)
{
continue;
}
var counterDelta = RavenJToken.ReadFrom(jsonReader).ToObject <CounterState>();
ShowProgress($"Importing counter {counterDelta.GroupName} - {counterDelta.CounterName}");
if (counterDelta.Sign == ValueSign.Negative)
{
counterDelta.Value = -counterDelta.Value;
}
store.Batch.ScheduleChange(counterDelta.GroupName, counterDelta.CounterName, counterDelta.Value);
}
ShowProgress("Finished import of the current file.");
await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
finally
{
store?.Dispose();
}
}
public async Task Between(SmugglerBetweenOptions <CounterConnectionStringOptions> betweenOptions)
{
if (betweenOptions.ReportProgress == null)
{
betweenOptions.ReportProgress = msg => { }
}
;
using (var source = new CounterStore
{
Url = betweenOptions.From.Url,
Name = betweenOptions.From.CounterStoreId,
Credentials = new OperationCredentials(betweenOptions.From.ApiKey, betweenOptions.From.Credentials)
})
using (var target = new CounterStore
{
Url = betweenOptions.To.Url,
Name = betweenOptions.To.CounterStoreId,
Credentials = new OperationCredentials(betweenOptions.To.ApiKey, betweenOptions.To.Credentials)
})
{
source.Initialize(true);
ShowProgress($"Initialized connection to source counter store (name = {source.Name})");
target.Initialize(true);
ShowProgress($"Initialized connection to target counter store (name = {target.Name})");
var existingCounterGroupsAndNames = await target.Admin.GetCounterStorageNameAndGroups(token : CancellationToken).ConfigureAwait(false);
var counterSummaries = await source.Advanced.GetCounters(token : CancellationToken).ConfigureAwait(false);
ShowProgress($"Fetched counter data from source (there is data about {counterSummaries.Count} counters)");
foreach (var summary in counterSummaries)
{
if (existingCounterGroupsAndNames.Any(x => x.Group == summary.GroupName && x.Name == summary.CounterName))
{
ShowProgress($"Counter {summary.GroupName} - {summary.CounterName} is already there. Reset is performed");
await target.ResetAsync(summary.GroupName, summary.CounterName, CancellationToken)
.WithCancellation(CancellationToken)
.ConfigureAwait(false); //since it is a full import, the values are overwritten
}
ShowProgress($"Importing counter {summary.GroupName} - {summary.CounterName}");
target.Batch.ScheduleChange(summary.GroupName, summary.CounterName, summary.Total);
}
ShowProgress("Finished import...");
await target.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
}
}
protected ICounterStore NewRemoteCountersStore(string counterStorageName, bool createDefaultCounter = true, OperationCredentials credentials = null, RavenDbServer ravenServer = null)
{
ravenServer = ravenServer ?? this.RavenDbServer.Value;
var serverUrl = ravenServer.SystemDatabase.ServerUrl;
serverCount.AddOrUpdate(serverUrl, id => 1, (id, val) => val++);
var counterStore = new CounterStore
{
Url = GetServerUrl(true, serverUrl),
Credentials = credentials ?? new OperationCredentials(null, CredentialCache.DefaultNetworkCredentials),
Name = counterStorageName + serverCount[serverUrl]
};
counterStore.Initialize(createDefaultCounter);
return(counterStore);
}
public void Cannot_use_admin_endpoint_with_non_admin_apiKey()
{
string storeName;
using (var store = NewRemoteCountersStore("TestStore")) //this will create the TestStore
{
storeName = store.Name;
}
Database.Server.Security.Authentication.EnableOnce();
//GoodApiKey is with admin access to <system>
ConfigureApiKey(servers[0].SystemDatabase, "thisIsApiKeyName", "thisIsSecret", storeName, true);
//BadApiKey is without admin access to <system>
ConfigureApiKey(servers[0].SystemDatabase, "NotThisIsApiKeyName", "thisIsSecret", storeName);
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(BadApiKey, null)
})
{
store.Initialize();
var e = Assert.Throws <ErrorResponseException>(() => AsyncHelpers.RunSync(() => store.Admin.GetCounterStoragesNamesAsync()));
Assert.Equal(HttpStatusCode.Forbidden, e.StatusCode);
}
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(GoodApiKey, null)
})
{
store.Initialize();
IReadOnlyList <string> storageNames = null;
Assert.DoesNotThrow(() => storageNames = AsyncHelpers.RunSync(() => store.Admin.GetCounterStoragesNamesAsync()));
Assert.Equal(1, storageNames.Count);
}
}
public async Task When_Counter_has_apiKey_auth_then_operations_with_wrong_apiKey_should_fail()
{
String storeName;
using (var store = NewRemoteCountersStore("TestStore")) //this will create the TestStore
{
storeName = store.Name;
await store.IncrementAsync("G", "C");
}
Database.Server.Security.Authentication.EnableOnce();
ConfigureApiKey(servers[0].SystemDatabase, "thisIsApiKeyName", "thisIsSecret", storeName);
ConfigureApiKey(servers[0].SystemDatabase, "NotThisIsApiKeyName", "thisIsSecret", "NonExistingResourceName");
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(BadApiKey, null)
})
{
store.Initialize();
var e = Assert.Throws <ErrorResponseException>(() => AsyncHelpers.RunSync(() => store.IncrementAsync("G", "C")));
Assert.Equal(HttpStatusCode.Forbidden, e.StatusCode);
}
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(GoodApiKey, null)
})
{
store.Initialize();
Assert.DoesNotThrow(() => AsyncHelpers.RunSync(() => store.IncrementAsync("G", "C")));
}
}
/// <summary>
/// Export counter data to specified destination (a file or a stream)
/// </summary>
/// <param name="exportOptions">options to specify the source and destination of the export</param>
/// <exception cref="UnauthorizedAccessException">The caller does not have the required permission.-or- specified a file that is read-only. </exception>
/// <exception cref="DirectoryNotFoundException">The specified path is invalid (for example, it is on an unmapped drive). </exception>
/// <exception cref="IOException">An I/O error occurred while creating the file. </exception>
/// <exception cref="SmugglerException">Encapsulates exception that happens when actually exporting data. See InnerException for details.</exception>
public async Task <CounterOperationState> ExportData(SmugglerExportOptions <CounterConnectionStringOptions> exportOptions)
{
if (exportOptions.From == null)
{
throw new ArgumentNullException("exportOptions.From");
}
if (String.IsNullOrWhiteSpace(exportOptions.ToFile) && exportOptions.ToStream == null)
{
throw new ArgumentException("ToFile or ToStream property in options must be non-null");
}
var result = new CounterOperationState();
var exportFolder = String.Empty;
if (Options.Incremental)
{
ShowProgress("Starting incremental export..");
exportFolder = CalculateExportFile(exportOptions, exportFolder);
}
else
{
ShowProgress("Starting full export...");
}
SmugglerException lastException = null;
var ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(exportOptions.ToFile);
if (ownedStream)
{
ShowProgress("Export to dump file " + exportOptions.ToFile);
}
try
{
using (var counterStore = new CounterStore
{
Url = exportOptions.From.Url,
Name = exportOptions.From.CounterStoreId,
Credentials = new OperationCredentials(exportOptions.From.ApiKey, exportOptions.From.Credentials)
})
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
counterStore.Initialize();
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName(Options.Incremental ? "CountersDeltas" : "CounterSnapshots"); //also for human readability
jsonWriter.WriteStartArray();
try
{
if (Options.Incremental)
{
await ExportIncrementalData(counterStore, exportFolder, jsonWriter).WithCancellation(CancellationToken).ConfigureAwait(false);
}
else
{
await ExportFullData(counterStore, jsonWriter).WithCancellation(CancellationToken).ConfigureAwait(false);
}
}
catch (SmugglerException e)
{
Debug.Assert(e.Data.Keys.Cast <string>().Contains("LastEtag"));
result.LastWrittenEtag = (long)e.Data["LastEtag"];
lastException = e;
var operation = Options.Incremental ? "Incremental" : "Full";
ShowProgress($"{operation} Export failed. {e}");
}
jsonWriter.WriteEndArray();
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
if (lastException != null)
{
throw lastException;
}
return(result);
}
finally
{
if (ownedStream && stream != null)
{
stream.Flush();
stream.Dispose();
ShowProgress("Finished export and closed file...");
}
else
{
ShowProgress("Finished export...");
}
}
}
//assumes that the caller has responsibility to handle data stream disposal ("stream" parameter)
private async Task ImportFullData(CounterConnectionStringOptions connectionString, Stream stream)
{
CountingStream sizeStream;
JsonTextReader jsonReader;
if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false)
{
throw new InvalidOperationException("Failed to get reader for the data stream.");
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
ICounterStore store = null;
try
{
if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
store = new CounterStore
{
Url = connectionString.Url,
Name = connectionString.CounterStoreId,
Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials)
};
store.Initialize(true);
ShowProgress($"Initialized connection to counter store (name = {store.Name})");
var existingCounterGroupsAndNames = await store.Admin.GetCounterStorageNameAndGroups(token : CancellationToken)
.WithCancellation(CancellationToken)
.ConfigureAwait(false);
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
if (jsonReader.TokenType != JsonToken.StartObject)
{
continue;
}
var counterInfo = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var delta = Math.Abs(counterInfo.Value <long>("Positive")) - Math.Abs(counterInfo.Value <long>("Negative"));
var groupName = counterInfo.Value <string>("Group");
var counterName = counterInfo.Value <string>("Name");
if (existingCounterGroupsAndNames.Any(x => x.Group == groupName && x.Name == counterName))
{
ShowProgress($"Counter {groupName} - {counterName} is already there. Reset is performed");
await store.ResetAsync(groupName, counterName, CancellationToken).ConfigureAwait(false); //since it is a full import, the values are overwritten
}
ShowProgress($"Importing counter {groupName} - {counterName}");
store.Batch.ScheduleChange(groupName, counterName, delta);
}
ShowProgress("Finished import...");
await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
finally
{
store?.Dispose();
}
}
