internal CountersBatchOperation(CounterStore parent, string counterStorageName, CountersBatchOptions batchOptions = null)
: base(parent, counterStorageName)
{
if (batchOptions != null && batchOptions.BatchSizeLimit < 1)
{
throw new ArgumentException("batchOptions.BatchSizeLimit cannot be negative", "batchOptions");
}
defaultOptions = batchOptions ?? new CountersBatchOptions(); //defaults do exist
streamingStarted = new AsyncManualResetEvent();
batchOperationTcs = new TaskCompletionSource <bool>();
cts = new CancellationTokenSource();
changesQueue = new BlockingCollection <CounterChange>(defaultOptions.BatchSizeLimit);
singleAuthUrl = string.Format("{0}/cs/{1}/singleAuthToken", ServerUrl, counterStorageName);
OperationId = Guid.NewGuid();
disposed = false;
batchOperationTask = StartBatchOperation();
if (AsyncHelpers.RunSync(() => streamingStarted.WaitAsync(TimeSpan.FromMilliseconds(DefaultOptions.StreamingInitializeTimeout))) == false ||
batchOperationTask.IsFaulted)
{
throw new InvalidOperationException("Failed to start streaming batch.", batchOperationTask.Exception);
}
closeAndReopenStreamingTimer = CreateNewTimer();
}
public async Task CountersInitialize_with_EnsureDefaultCounterCreation_should_not_overwrite_existing_counters()
{
using (var server = GetNewServer(port: 8091))
{
using (var counterStore = new CounterStore
{
Url = $"{server.DocumentStore.Url}:{server.Configuration.Port}",
Name = DefaultCounterStorageName
})
{
counterStore.Initialize(true);
await counterStore.IncrementAsync("G", "C");
await counterStore.DecrementAsync("G", "C2");
}
using (var counterStore = new CounterStore
{
Url = $"{server.DocumentStore.Url}:{server.Configuration.Port}",
Name = DefaultCounterStorageName
})
{
counterStore.Initialize(true);
var summary = await counterStore.Admin.GetCountersByStorage(DefaultCounterStorageName);
Assert.Equal(2, summary.Count);
Assert.NotNull(summary.SingleOrDefault(x => x.Total == 1 && x.GroupName == "G" && x.CounterName == "C"));
Assert.NotNull(summary.SingleOrDefault(x => x.Total == -1 && x.GroupName == "G" && x.CounterName == "C2"));
}
}
}
protected CounterOperationsBase(CounterStore parent, string counterStorageName)
{
credentials = parent.Credentials;
jsonRequestFactory = parent.JsonRequestFactory;
ServerUrl = parent.Url;
CounterStorageUrl = string.Format(CultureInfo.InvariantCulture, "{0}/cs/{1}", ServerUrl, counterStorageName);
countersConvention = parent.CountersConvention;
}
private async Task ImportIncrementalData(CounterConnectionStringOptions connectionString, Stream stream)
{
CountingStream sizeStream;
JsonTextReader jsonReader;
if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false)
{
throw new InvalidOperationException("Failed to get reader for the data stream.");
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
ICounterStore store = null;
try
{
if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
store = new CounterStore
{
Url = connectionString.Url,
Name = connectionString.CounterStoreId,
Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials)
};
store.Initialize(true);
ShowProgress($"Initialized connection to counter store (name = {store.Name})");
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
if (jsonReader.TokenType != JsonToken.StartObject)
{
continue;
}
var counterDelta = RavenJToken.ReadFrom(jsonReader).ToObject <CounterState>();
ShowProgress($"Importing counter {counterDelta.GroupName} - {counterDelta.CounterName}");
if (counterDelta.Sign == ValueSign.Negative)
{
counterDelta.Value = -counterDelta.Value;
}
store.Batch.ScheduleChange(counterDelta.GroupName, counterDelta.CounterName, counterDelta.Value);
}
ShowProgress("Finished import of the current file.");
await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
finally
{
store?.Dispose();
}
}
public void TestMetricsMin()
{
var cs = new CounterStore();
var c = cs.GetCounter("1");
foreach (var i in Enumerable.Range(0, 60))
{
c.RegisterEvent(new DateTime(2012, 01, 01, 3, i, i));
c.RegisterEvent(new DateTime(2012, 01, 01, 3, 59 - i, 59 - i));
}
Assert.AreEqual(2, c.AvgEventsPerMinute);
}
internal CounterReplicationInformer(HttpJsonRequestFactory requestFactory, CounterStore counterStore, CountersConvention countersConvention, int delayTimeInMiliSec = 1000)
{
currentReadStripingBase = 0;
ReplicationDestinations = new List <CounterReplicationDestination>();
this.requestFactory = requestFactory;
this.counterStore = counterStore;
this.countersConvention = countersConvention;
this.delayTimeInMiliSec = delayTimeInMiliSec;
failureCounters = new FailureCounters();
firstTime = true;
lastReplicationUpdate = SystemTime.UtcNow;
MaxIntervalBetweenUpdatesInMilliseconds = TimeSpan.FromMinutes(DefaultIntervalBetweenUpdatesInMinutes).TotalMilliseconds;
}
public void Test2()
{
var cs = new CounterStore();
var c = cs.GetCounter("1");
c.TotalEvents = 88;
c.RegisterEvent();
Assert.AreEqual(89, c.TotalEvents);
Assert.AreEqual(null, c.Comment);
c.Comment = "123";
Assert.AreEqual("123", c.Comment);
}
public async Task Between(SmugglerBetweenOptions <CounterConnectionStringOptions> betweenOptions)
{
if (betweenOptions.ReportProgress == null)
{
betweenOptions.ReportProgress = msg => { }
}
;
using (var source = new CounterStore
{
Url = betweenOptions.From.Url,
Name = betweenOptions.From.CounterStoreId,
Credentials = new OperationCredentials(betweenOptions.From.ApiKey, betweenOptions.From.Credentials)
})
using (var target = new CounterStore
{
Url = betweenOptions.To.Url,
Name = betweenOptions.To.CounterStoreId,
Credentials = new OperationCredentials(betweenOptions.To.ApiKey, betweenOptions.To.Credentials)
})
{
source.Initialize(true);
ShowProgress($"Initialized connection to source counter store (name = {source.Name})");
target.Initialize(true);
ShowProgress($"Initialized connection to target counter store (name = {target.Name})");
var existingCounterGroupsAndNames = await target.Admin.GetCounterStorageNameAndGroups(token : CancellationToken).ConfigureAwait(false);
var counterSummaries = await source.Advanced.GetCounters(token : CancellationToken).ConfigureAwait(false);
ShowProgress($"Fetched counter data from source (there is data about {counterSummaries.Count} counters)");
foreach (var summary in counterSummaries)
{
if (existingCounterGroupsAndNames.Any(x => x.Group == summary.GroupName && x.Name == summary.CounterName))
{
ShowProgress($"Counter {summary.GroupName} - {summary.CounterName} is already there. Reset is performed");
await target.ResetAsync(summary.GroupName, summary.CounterName, CancellationToken)
.WithCancellation(CancellationToken)
.ConfigureAwait(false); //since it is a full import, the values are overwritten
}
ShowProgress($"Importing counter {summary.GroupName} - {summary.CounterName}");
target.Batch.ScheduleChange(summary.GroupName, summary.CounterName, summary.Total);
}
ShowProgress("Finished import...");
await target.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
}
}
public async Task SynchronizationTopologyDiscovererSimpleTest()
{
using (CounterStore storeA = (CounterStore)NewRemoteCountersStore(DefaultCounterStorageName + "A"))
using (var storeB = NewRemoteCountersStore(DefaultCounterStorageName + "B"))
using (var storeC = NewRemoteCountersStore(DefaultCounterStorageName + "C"))
using (var storeD = NewRemoteCountersStore(DefaultCounterStorageName + "D"))
using (var storeE = NewRemoteCountersStore(DefaultCounterStorageName + "E"))
{
await SetupReplicationAsync(storeA, storeB);
await SetupReplicationAsync(storeB, storeC);
await SetupReplicationAsync(storeC, storeD);
await SetupReplicationAsync(storeD, storeE);
await SetupReplicationAsync(storeE, storeA);
await storeA.ChangeAsync("group", "counter", 2);
await WaitForReplicationBetween(storeA, storeE, "group", "counter");
var url = storeA.Url.ForCounter(DefaultCounterStorageName + "A1") + "/admin/replication/topology/view";
var request = storeA
.JsonRequestFactory
.CreateHttpJsonRequest(new CreateHttpJsonRequestParams(null, url, HttpMethods.Post, storeA.Credentials, storeA.CountersConvention));
var json = (RavenJObject)request.ReadResponseJson();
var topology = json.Deserialize <CountersReplicationTopology>(new DocumentConvention());
Assert.NotNull(topology);
Assert.Equal(5, topology.Servers.Count);
Assert.Equal(5, topology.Connections.Count);
topology.Connections.Single(x => x.Destination == storeA.Url.ForCounter(storeA.Name) && x.Source == storeE.Url.ForCounter(storeE.Name));
topology.Connections.Single(x => x.Destination == storeB.Url.ForCounter(storeB.Name) && x.Source == storeA.Url.ForCounter(storeA.Name));
topology.Connections.Single(x => x.Destination == storeC.Url.ForCounter(storeC.Name) && x.Source == storeB.Url.ForCounter(storeB.Name));
topology.Connections.Single(x => x.Destination == storeD.Url.ForCounter(storeD.Name) && x.Source == storeC.Url.ForCounter(storeC.Name));
topology.Connections.Single(x => x.Destination == storeE.Url.ForCounter(storeE.Name) && x.Source == storeD.Url.ForCounter(storeD.Name));
foreach (var connection in topology.Connections)
{
Assert.Equal(ReplicatonNodeState.Online, connection.SourceToDestinationState);
Assert.Equal(ReplicatonNodeState.Online, connection.DestinationToSourceState);
Assert.NotNull(connection.Source);
Assert.NotNull(connection.Destination);
Assert.NotNull(connection.LastEtag);
Assert.NotNull(connection.SendServerId);
Assert.NotNull(connection.StoredServerId);
}
}
}
internal static void ScriptOrderPrevRange()
{
if (HttpContext.Current == null)
{
return;
}
if (HttpContext.Current.Items[ResourceManager.scriptOrderkey] == null)
{
return;
}
CounterStore c = ((CounterStore)HttpContext.Current.Items[ResourceManager.scriptOrderkey]);
c.counter = c.counter - 1000000;
}
public void TestMetricsHourSec()
{
var cs = new CounterStore();
var c = cs.GetCounter("1");
foreach (var i in Enumerable.Range(0, 60))
{
foreach (var h in Enumerable.Range(0, 24))
{
c.RegisterEvent(new DateTime(2012, 01, 01, h, 1, i));
}
}
// Assert.AreEqual(120, c.TotalEvents);
Assert.AreEqual(24, c.AvgEventsPerSecond);
Assert.AreEqual(60, c.AvgEventsPerHour);
}
protected ICounterStore NewRemoteCountersStore(string counterStorageName, bool createDefaultCounter = true, OperationCredentials credentials = null, RavenDbServer ravenServer = null)
{
ravenServer = ravenServer ?? this.RavenDbServer.Value;
var serverUrl = ravenServer.SystemDatabase.ServerUrl;
serverCount.AddOrUpdate(serverUrl, id => 1, (id, val) => val++);
var counterStore = new CounterStore
{
Url = GetServerUrl(true, serverUrl),
Credentials = credentials ?? new OperationCredentials(null, CredentialCache.DefaultNetworkCredentials),
Name = counterStorageName + serverCount[serverUrl]
};
counterStore.Initialize(createDefaultCounter);
return(counterStore);
}
public void Cannot_use_admin_endpoint_with_non_admin_apiKey()
{
string storeName;
using (var store = NewRemoteCountersStore("TestStore")) //this will create the TestStore
{
storeName = store.Name;
}
Database.Server.Security.Authentication.EnableOnce();
//GoodApiKey is with admin access to <system>
ConfigureApiKey(servers[0].SystemDatabase, "thisIsApiKeyName", "thisIsSecret", storeName, true);
//BadApiKey is without admin access to <system>
ConfigureApiKey(servers[0].SystemDatabase, "NotThisIsApiKeyName", "thisIsSecret", storeName);
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(BadApiKey, null)
})
{
store.Initialize();
var e = Assert.Throws <ErrorResponseException>(() => AsyncHelpers.RunSync(() => store.Admin.GetCounterStoragesNamesAsync()));
Assert.Equal(HttpStatusCode.Forbidden, e.StatusCode);
}
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(GoodApiKey, null)
})
{
store.Initialize();
IReadOnlyList <string> storageNames = null;
Assert.DoesNotThrow(() => storageNames = AsyncHelpers.RunSync(() => store.Admin.GetCounterStoragesNamesAsync()));
Assert.Equal(1, storageNames.Count);
}
}
private async System.Threading.Tasks.Task UpdateState()
{
var events = await CounterStore.AllEventsAfter(lastSequence).ConfigureAwait(false);
foreach (var counterEvent in events)
{
switch (counterEvent.Value.EventCase)
{
case EventOneofCase.Added:
await DispatcherHelper.RunAsync(() => Counters.Add(new AutomatedCounter(counterEvent.Value.Id.ToGuid(), counterEvent.Value.Added.Name)));
break;
case EventOneofCase.Removed:
{
var counterId = counterEvent.Value.Id.ToGuid();
var counterToRemove = Counters.SingleOrDefault(x => x.Id == counterId);
if (counterToRemove != null)
{
await DispatcherHelper.RunAsync(() => Counters.Remove(counterToRemove));
}
}
break;
case EventOneofCase.NameChanged:
{
var counterId = counterEvent.Value.Id.ToGuid();
var counterToUpdate = Counters.SingleOrDefault(x => x.Id == counterId);
if (counterToUpdate != null)
{
await DispatcherHelper.RunAsync(() => counterToUpdate.Name = counterEvent.Value.NameChanged.Name);
}
}
break;
}
lastSequence = counterEvent.Key;
}
}
public async Task When_Counter_has_apiKey_auth_then_operations_with_wrong_apiKey_should_fail()
{
String storeName;
using (var store = NewRemoteCountersStore("TestStore")) //this will create the TestStore
{
storeName = store.Name;
await store.IncrementAsync("G", "C");
}
Database.Server.Security.Authentication.EnableOnce();
ConfigureApiKey(servers[0].SystemDatabase, "thisIsApiKeyName", "thisIsSecret", storeName);
ConfigureApiKey(servers[0].SystemDatabase, "NotThisIsApiKeyName", "thisIsSecret", "NonExistingResourceName");
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(BadApiKey, null)
})
{
store.Initialize();
var e = Assert.Throws <ErrorResponseException>(() => AsyncHelpers.RunSync(() => store.IncrementAsync("G", "C")));
Assert.Equal(HttpStatusCode.Forbidden, e.StatusCode);
}
using (var store = new CounterStore
{
Url = servers[0].SystemDatabase.ServerUrl,
Name = storeName,
Credentials = new OperationCredentials(GoodApiKey, null)
})
{
store.Initialize();
Assert.DoesNotThrow(() => AsyncHelpers.RunSync(() => store.IncrementAsync("G", "C")));
}
}
public void TestStore()
{
var cs = new CounterStore();
var c1 = cs.GetCounter("c1");
var c2 = cs.GetCounter("c2");
Assert.AreEqual(0, c1.TotalEvents);
Assert.AreEqual(0, c2.TotalEvents);
c1.RegisterEvent();
c1.RegisterEvent();
c2.RegisterEvent();
Assert.AreEqual(2, c1.TotalEvents);
Assert.AreEqual(1 / 60, c1.AvgEventsPerMinute);
Assert.AreEqual(1 / 24, c1.AvgEventsPerHour);
Assert.AreEqual(1, c2.TotalEvents);
var counterNames = cs.EnumerateCounters().OrderBy(a => a);
Assert.AreEqual(2, counterNames.Count());
Assert.AreEqual("c1", counterNames.First());
Assert.AreEqual("c2", counterNames.Last());
var c11 = cs.GetCounter("c1");
Assert.AreEqual(2, c11.TotalEvents);
cs.DeleteCounter("c1");
counterNames = cs.EnumerateCounters().OrderBy(a => a);
Assert.AreEqual(1, counterNames.Count());
Assert.AreEqual("c2", counterNames.Last());
}
protected override void OnInitialized()
{
base.OnInitialized();
CounterStore.AddStateChangeListener(UpdateView); //attach listener to the store
}
/// <summary>
/// Export counter data to specified destination (a file or a stream)
/// </summary>
/// <param name="exportOptions">options to specify the source and destination of the export</param>
/// <exception cref="UnauthorizedAccessException">The caller does not have the required permission.-or- specified a file that is read-only. </exception>
/// <exception cref="DirectoryNotFoundException">The specified path is invalid (for example, it is on an unmapped drive). </exception>
/// <exception cref="IOException">An I/O error occurred while creating the file. </exception>
/// <exception cref="SmugglerException">Encapsulates exception that happens when actually exporting data. See InnerException for details.</exception>
public async Task <CounterOperationState> ExportData(SmugglerExportOptions <CounterConnectionStringOptions> exportOptions)
{
if (exportOptions.From == null)
{
throw new ArgumentNullException("exportOptions.From");
}
if (String.IsNullOrWhiteSpace(exportOptions.ToFile) && exportOptions.ToStream == null)
{
throw new ArgumentException("ToFile or ToStream property in options must be non-null");
}
var result = new CounterOperationState();
var exportFolder = String.Empty;
if (Options.Incremental)
{
ShowProgress("Starting incremental export..");
exportFolder = CalculateExportFile(exportOptions, exportFolder);
}
else
{
ShowProgress("Starting full export...");
}
SmugglerException lastException = null;
var ownedStream = exportOptions.ToStream == null;
var stream = exportOptions.ToStream ?? File.Create(exportOptions.ToFile);
if (ownedStream)
{
ShowProgress("Export to dump file " + exportOptions.ToFile);
}
try
{
using (var counterStore = new CounterStore
{
Url = exportOptions.From.Url,
Name = exportOptions.From.CounterStoreId,
Credentials = new OperationCredentials(exportOptions.From.ApiKey, exportOptions.From.Credentials)
})
using (var gZipStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true))
using (var streamWriter = new StreamWriter(gZipStream))
{
counterStore.Initialize();
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName(Options.Incremental ? "CountersDeltas" : "CounterSnapshots"); //also for human readability
jsonWriter.WriteStartArray();
try
{
if (Options.Incremental)
{
await ExportIncrementalData(counterStore, exportFolder, jsonWriter).WithCancellation(CancellationToken).ConfigureAwait(false);
}
else
{
await ExportFullData(counterStore, jsonWriter).WithCancellation(CancellationToken).ConfigureAwait(false);
}
}
catch (SmugglerException e)
{
Debug.Assert(e.Data.Keys.Cast <string>().Contains("LastEtag"));
result.LastWrittenEtag = (long)e.Data["LastEtag"];
lastException = e;
var operation = Options.Incremental ? "Incremental" : "Full";
ShowProgress($"{operation} Export failed. {e}");
}
jsonWriter.WriteEndArray();
jsonWriter.WriteEndObject();
streamWriter.Flush();
}
if (lastException != null)
{
throw lastException;
}
return(result);
}
finally
{
if (ownedStream && stream != null)
{
stream.Flush();
stream.Dispose();
ShowProgress("Finished export and closed file...");
}
else
{
ShowProgress("Finished export...");
}
}
}
//assumes that the caller has responsibility to handle data stream disposal ("stream" parameter)
private async Task ImportFullData(CounterConnectionStringOptions connectionString, Stream stream)
{
CountingStream sizeStream;
JsonTextReader jsonReader;
if (SmugglerHelper.TryGetJsonReaderForStream(stream, out jsonReader, out sizeStream) == false)
{
throw new InvalidOperationException("Failed to get reader for the data stream.");
}
if (jsonReader.TokenType != JsonToken.StartObject)
{
throw new InvalidDataException("StartObject was expected");
}
ICounterStore store = null;
try
{
if (jsonReader.Read() == false && jsonReader.TokenType != JsonToken.StartArray)
{
throw new InvalidDataException("StartArray was expected");
}
store = new CounterStore
{
Url = connectionString.Url,
Name = connectionString.CounterStoreId,
Credentials = new OperationCredentials(connectionString.ApiKey, connectionString.Credentials)
};
store.Initialize(true);
ShowProgress($"Initialized connection to counter store (name = {store.Name})");
var existingCounterGroupsAndNames = await store.Admin.GetCounterStorageNameAndGroups(token : CancellationToken)
.WithCancellation(CancellationToken)
.ConfigureAwait(false);
while (jsonReader.Read() && jsonReader.TokenType != JsonToken.EndArray)
{
if (jsonReader.TokenType != JsonToken.StartObject)
{
continue;
}
var counterInfo = (RavenJObject)RavenJToken.ReadFrom(jsonReader);
var delta = Math.Abs(counterInfo.Value <long>("Positive")) - Math.Abs(counterInfo.Value <long>("Negative"));
var groupName = counterInfo.Value <string>("Group");
var counterName = counterInfo.Value <string>("Name");
if (existingCounterGroupsAndNames.Any(x => x.Group == groupName && x.Name == counterName))
{
ShowProgress($"Counter {groupName} - {counterName} is already there. Reset is performed");
await store.ResetAsync(groupName, counterName, CancellationToken).ConfigureAwait(false); //since it is a full import, the values are overwritten
}
ShowProgress($"Importing counter {groupName} - {counterName}");
store.Batch.ScheduleChange(groupName, counterName, delta);
}
ShowProgress("Finished import...");
await store.Batch.FlushAsync().WithCancellation(CancellationToken).ConfigureAwait(false);
}
finally
{
store?.Dispose();
}
}
public void Dispose()
{
CounterStore.RemoveStateChangeListener(UpdateView);
}
