/// <summary>
/// Removes a Docker volume.
/// </summary>
/// <param name="nameOrId">The volume name or ID.</param>
/// <param name="cancellationToken">Optional cancellation token.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
public async Task VolumeRemove(string nameOrId, CancellationToken cancellationToken = default)
{
await SyncContext.ClearAsync;
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(nameOrId), nameof(nameOrId));
await JsonClient.DeleteAsync(GetUri("volumes", nameOrId), cancellationToken : cancellationToken);
}
public async Task DeletetAsync()
{
// Ensure that DELETE returning an explict type works.
using (new MockHttpServer(baseUri,
async context =>
{
var request = context.Request;
var response = context.Response;
if (request.Method != "DELETE")
{
response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
if (request.Path.ToString() != "/info")
{
response.StatusCode = (int)HttpStatusCode.NotFound;
return;
}
var output = new ReplyDoc()
{
Value1 = "Hello World!"
};
response.ContentType = "application/json";
await response.WriteAsync(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
var reply = (await jsonClient.DeleteAsync(baseUri + "info")).As <ReplyDoc>();
Assert.Equal("Hello World!", reply.Value1);
reply = await jsonClient.DeleteAsync <ReplyDoc>(baseUri + "info");
Assert.Equal("Hello World!", reply.Value1);
}
}
}
public async Task DeleteAsync_Headers()
{
// Ensure that DELETE with headers work.
using (new MockHttpServer(baseUri,
async context =>
{
var request = context.Request;
var response = context.Response;
if (request.Method != "DELETE")
{
response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
if (request.Path.ToString() != "/info")
{
response.StatusCode = (int)HttpStatusCode.NotFound;
return;
}
var output = new ReplyDoc()
{
Value1 = request.Headers["arg1"],
Value2 = request.Headers["arg2"]
};
response.ContentType = "application/json";
await response.WriteAsync(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
var headers = new ArgDictionary()
{
{ "arg1", "test1" },
{ "arg2", "test2" }
};
var reply = (await jsonClient.DeleteAsync(baseUri + "info", headers: headers)).As <ReplyDoc>();
Assert.Equal("test1", reply.Value1);
Assert.Equal("test2", reply.Value2);
}
}
}
public async Task DeleteAsync_Error()
{
// Ensure that DELETE returning a hard error works.
using (new MockHttpServer(baseUri,
context =>
{
var response = context.Response;
response.StatusCode = (int)HttpStatusCode.NotFound;
}))
{
using (var jsonClient = new JsonClient())
{
await Assert.ThrowsAsync <HttpException>(async() => await jsonClient.DeleteAsync(baseUri + "info"));
}
}
}
public async Task DeleteAsync_Args()
{
// Ensure that DELETE with query arguments work.
using (new MockHttpServer(baseUri,
context =>
{
var request = context.Request;
var response = context.Response;
if (request.Method != "DELETE")
{
response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
if (request.Path.ToString() != "/info")
{
response.StatusCode = (int)HttpStatusCode.NotFound;
return;
}
var output = new ReplyDoc()
{
Value1 = request.QueryGet("arg1"),
Value2 = request.QueryGet("arg2")
};
response.ContentType = "application/json";
response.Write(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
var reply = (await jsonClient.DeleteAsync(baseUri + "info?arg1=test1&arg2=test2")).As <ReplyDoc>();
Assert.Equal("test1", reply.Value1);
Assert.Equal("test2", reply.Value2);
}
}
}
public async Task DeleteAsync_Dynamic_NotJson()
{
// Ensure that DELETE returning non-JSON returns a NULL dynamic document.
using (new MockHttpServer(baseUri,
async context =>
{
var request = context.Request;
var response = context.Response;
if (request.Method != "DELETE")
{
response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
if (request.Path.ToString() != "/info")
{
response.StatusCode = (int)HttpStatusCode.NotFound;
return;
}
var output = new ReplyDoc()
{
Value1 = "Hello World!"
};
response.ContentType = "application/not-json";
await response.WriteAsync(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
var reply = (await jsonClient.DeleteAsync(baseUri + "info")).AsDynamic();
Assert.Null(reply);
}
}
}
public async Task DeleteAsync_Retry()
{
// Ensure that DELETE will retry after soft errors.
var attemptCount = 0;
using (new MockHttpServer(baseUri,
async context =>
{
var request = context.Request;
var response = context.Response;
if (attemptCount++ == 0)
{
response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
return;
}
var output = new ReplyDoc()
{
Value1 = "Hello World!"
};
response.ContentType = "application/json";
await response.WriteAsync(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
var reply = (await jsonClient.DeleteAsync(baseUri + "info")).AsDynamic();
Assert.Equal(2, attemptCount);
Assert.Equal("Hello World!", (string)reply.Value1);
}
}
}
public async Task DeleteAsync_NoRetryExplicit()
{
// Ensure that DELETE won't retry if [retryPolicy=NoRetryPolicy]
var attemptCount = 0;
using (new MockHttpServer(baseUri,
async context =>
{
var request = context.Request;
var response = context.Response;
if (attemptCount++ == 0)
{
response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
return;
}
var output = new ReplyDoc()
{
Value1 = "Hello World!"
};
response.ContentType = "application/json";
await response.WriteAsync(NeonHelper.JsonSerialize(output));
}))
{
using (var jsonClient = new JsonClient())
{
await Assert.ThrowsAsync <HttpException>(async() => await jsonClient.DeleteAsync(NoRetryPolicy.Instance, baseUri + "info"));
Assert.Equal(1, attemptCount);
}
}
}
/// <summary>
/// Handles purging of old <b>logstash</b> and <b>metricbeat</b> Elasticsearch indexes.
/// </summary>
/// <returns>The tracking <see cref="Task"/>.</returns>
public async Task LogPurgerAsync(TimeSpan logPurgerInterval, int retentionDays)
{
using (var jsonClient = new JsonClient())
{
jsonClient.BaseAddress = KubernetesClientConfiguration.IsInCluster()
? this.ServiceMap[NeonServices.Elasticsearch].Endpoints.Default.Uri
: new Uri($"http://localhost:{this.ServiceMap[NeonServices.Elasticsearch].Endpoints.Default.Port}");
var periodicTask =
new AsyncPeriodicTask(
logPurgerInterval,
onTaskAsync:
async() =>
{
// We're going to list the indexes and look for [logstash]
// and [metricbeat] indexes that encode the index date like:
//
// logstash-2018.06.06
// metricbeat-6.1.1-2018.06.06
//
// The date is simply encodes the day covered by the index.
var utcNow = DateTime.UtcNow;
var deleteBeforeDate = new DateTime(utcNow.Year, utcNow.Month, utcNow.Day) - TimeSpan.FromDays(retentionDays);
var indexList = await jsonClient.GetAsync <JObject>("_aliases");
foreach (var indexProperty in indexList.Properties())
{
var indexName = indexProperty.Name;
// We're only purging [logstash] and [metricbeat] indexes.
if (!indexName.StartsWith("logstash-") && !indexName.StartsWith("metricbeat-"))
{
continue;
}
// Extract the date from the index name.
var pos = indexName.LastIndexOf('-');
if (pos == -1)
{
Log.LogWarn(() => $"LOG-PURGER: Cannot extract date from index named [{indexName}].");
continue;
}
var date = indexName.Substring(pos + 1);
var fields = date.Split('.');
var indexDate = default(DateTime);
try
{
indexDate = new DateTime(int.Parse(fields[0]), int.Parse(fields[1]), int.Parse(fields[2]));
}
catch
{
Log.LogWarn(() => $"LOG-PURGER: Cannot extract date from index named [{indexName}].");
continue;
}
if (indexDate < deleteBeforeDate)
{
Log.LogInfo(() => $"LOG-PURGER: Deleting index [{indexName}].");
await jsonClient.DeleteAsync <JObject>(indexName);
Log.LogInfo(() => $"LOG-PURGER: [{indexName}] was deleted.");
}
}
Log.LogDebug("LOG-PURGER: Scan finished.");
return(await Task.FromResult(false));
},
onExceptionAsync:
async e =>
{
Log.LogError("LOG-PURGER", e);
return(await Task.FromResult(false));
},
onTerminateAsync:
async() =>
{
Log.LogInfo(() => "LOG-PURGER: Terminating");
await Task.CompletedTask;
});
await periodicTask.Run();
}
}
/// <summary>
/// Handles purging of old <b>logstash</b> and <b>metricbeat</b> Elasticsearch indexes.
/// </summary>
/// <returns>The tracking <see cref="Task"/>.</returns>
private static async Task LogPurgerAsync()
{
using (var jsonClient = new JsonClient())
{
var periodicTask =
new AsyncPeriodicTask(
logPurgerInterval,
onTaskAsync:
async() =>
{
if (IsSetupPending)
{
log.LogInfo(() => "LOG-PURGER: Delaying because hive setup is still in progress.");
return(false);
}
var manager = hive.GetReachableManager();
log.LogDebug(() => "LOG-PURGER: Scanning for old Elasticsearch indexes ready for removal.");
// We're going to list the indexes and look for [logstash]
// and [metricbeat] indexes that encode the index date like:
//
// logstash-2018.06.06
// metricbeat-6.1.1-2018.06.06
//
// The date is simply encodes the day covered by the index.
if (!hive.Globals.TryGetInt(HiveGlobals.UserLogRetentionDays, out var retentionDays))
{
retentionDays = 14;
}
var utcNow = DateTime.UtcNow;
var deleteBeforeDate = new DateTime(utcNow.Year, utcNow.Month, utcNow.Day) - TimeSpan.FromDays(retentionDays);
var indexList = await jsonClient.GetAsync <JObject>($"http://{manager.PrivateAddress}:{HiveHostPorts.ProxyPrivateHttpLogEsData}/_aliases");
foreach (var indexProperty in indexList.Properties())
{
var indexName = indexProperty.Name;
// We're only purging [logstash] and [metricbeat] indexes.
if (!indexName.StartsWith("logstash-") && !indexName.StartsWith("metricbeat-"))
{
continue;
}
// Extract the date from the index name.
var pos = indexName.LastIndexOf('-');
if (pos == -1)
{
log.LogWarn(() => $"LOG-PURGER: Cannot extract date from index named [{indexName}].");
continue;
}
var date = indexName.Substring(pos + 1);
var fields = date.Split('.');
var indexDate = default(DateTime);
try
{
indexDate = new DateTime(int.Parse(fields[0]), int.Parse(fields[1]), int.Parse(fields[2]));
}
catch
{
log.LogWarn(() => $"LOG-PURGER: Cannot extract date from index named [{indexName}].");
continue;
}
if (indexDate < deleteBeforeDate)
{
log.LogInfo(() => $"LOG-PURGER: Deleting index [{indexName}].");
await jsonClient.DeleteAsync <JObject>($"http://{manager.PrivateAddress}:{HiveHostPorts.ProxyPrivateHttpLogEsData}/{indexName}");
log.LogInfo(() => $"LOG-PURGER: [{indexName}] was deleted.");
}
}
log.LogDebug("LOG-PURGER: Scan finished.");
return(await Task.FromResult(false));
},
/// <summary>
/// Removes a Docker network.
/// </summary>
/// <param name="nameOrId">The network name or ID.</param>
/// <param name="cancellationToken">Optional cancellation token.</param>
/// <returns>The tracking <see cref="Task"/>.</returns>
public async Task NetworkRemove(string nameOrId, CancellationToken cancellationToken = default)
{
Covenant.Requires <ArgumentNullException>(!string.IsNullOrEmpty(nameOrId));
await JsonClient.DeleteAsync(GetUri("networks", nameOrId), cancellationToken : cancellationToken);
}