private static async Task Run(Service service)
{
await service.LogOnAsync(SdkHelper.Splunk.Username, SdkHelper.Splunk.Password);
string savedSearchName = "example_search";
string savedSearchQuery = "search index=_internal | head 10";
// Delete the saved search if it exists before we start.
SavedSearch savedSearch = await service.SavedSearches.GetOrNullAsync(savedSearchName);
if (savedSearch != null)
{
await savedSearch.RemoveAsync();
}
savedSearch = await service.SavedSearches.CreateAsync(savedSearchName, savedSearchQuery);
Job savedSearchJob = await savedSearch.DispatchAsync();
using (SearchResultStream stream = await savedSearchJob.GetSearchResultsAsync())
{
foreach (SearchResult result in stream)
{
Console.WriteLine(result);
}
}
await savedSearch.RemoveAsync();
}
public async Task JobPreviewDefaultsToAll()
{
using (var service = await SdkHelper.CreateService())
{
JobArgs jobArgs = new JobArgs();
var job = await service.Jobs.CreateAsync("search index=_* | head 101", args : jobArgs);
for (int delay = 1000; delay < 5000; delay += 1000)
{
try
{
await job.TransitionAsync(DispatchState.Done, delay);
break;
}
catch (TaskCanceledException)
{ }
}
using (SearchResultStream stream = await job.GetSearchPreviewAsync())
{
// Is the result preview count greater than the default of 100?
Assert.Equal(101, job.ResultPreviewCount);
}
await job.CancelAsync();
}
}
public void Test()
{
var service = new Mock <Service>();
var job = new Mock <Job>();
service.Setup(s => s.LogOnAsync(
It.IsAny <string>(),
It.IsAny <string>()))
.Returns(Task.FromResult(true));
service.Setup(s => s.LogOffAsync()).Returns(Task.FromResult(true));
service.Setup(s => s.SearchAsync(
It.IsAny <string>(), // search
It.IsAny <int>(), // count
It.IsAny <ExecutionMode>(), // mode
It.IsAny <JobArgs>(), // args
It.IsAny <CustomJobArgs>())) // customArgs
.Returns(Task.FromResult(job.Object));
var message = new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StreamContent(new FileStream("search-results.xml", FileMode.Open)),
};
var resultStream = SearchResultStream.CreateAsync(message);
job.Setup(j => j.GetSearchResultsAsync(It.IsAny <int>())).Returns(resultStream);
Run(service.Object).Wait();
}
async Task CanHandleInFlightErrorsReportedBySplunk()
{
var path = Path.Combine(TestAtomFeed.Directory, "Service.ExportSearchResults-failure.xml");
var message = new HttpResponseMessage(HttpStatusCode.OK);
message.Content = new StreamContent(new FileStream(path, FileMode.Open, FileAccess.Read));
SearchResultStream stream = null;
try
{
stream = await SearchResultStream.CreateAsync(message);
int count = 0;
foreach (var result in stream)
{
++count;
}
Assert.False(true, "Expected RequestException");
}
catch (RequestException e)
{
Assert.Equal(e.Message, "Fatal: JournalSliceDirectory: Cannot seek to 0");
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
public void StreamWithoutItemsDoesntThrowExceptions()
{
var asyncEnumerable = AsyncEnumerableBuilder.FromArray(new string[0]);
using var sut = new SearchResultStream(asyncEnumerable);
using var streamReader = new StreamReader(sut);
var actual = ReadAllLines(streamReader).ToList();
CollectionAssert.IsEmpty(actual);
}
public void StreamReturnsInputItems()
{
var expected = new[] { "Item1", "Item2", "Item3" };
var asyncEnumerable = AsyncEnumerableBuilder.FromArray(expected);
using var sut = new SearchResultStream(asyncEnumerable);
using var streamReader = new StreamReader(sut);
var actual = ReadAllLines(streamReader).ToList();
CollectionAssert.AreEqual(expected, actual);
}
public static void PrintResults(SearchResultStream results, ListBox logger)
{
logger.Items.Clear();
var recordNumber = 0;
foreach (var result in results)
{
logger.Items.Add("\n" + result.GetValue("_time") + " : " + result.GetValue("Value"));
Console.WriteLine(string.Format("{0:D8}: {1}", ++recordNumber, result));
}
}
private async void DisplaySearchResult(string searchStr)
{
SearchExportArgs jobArgs = new SearchExportArgs();
if (this.searchEarliestTime != null)
{
jobArgs.EarliestTime = this.searchEarliestTime;
}
if (this.searchLatestTime != null)
{
jobArgs.LatestTime = this.searchLatestTime;
}
using (SearchResultStream resultStream = await MainPage.SplunkService.ExportSearchResultsAsync(searchStr, jobArgs))
{
titleGrid.Visibility = Visibility.Visible;
this.allResults.Clear();
Task task = this.GetResultTask(resultStream);
bool showFirstPage = false;
do
{
if (this.allResults.Count > 0)
{
if (!showFirstPage && this.ShowResultPage(this.allResults, 0, this.itemsPerPage))
{
showFirstPage = true;
}
if (this.currentShownPageIndex < 0)
{
ShowPagingLink(0);
}
}
await Task.Delay(1000);
} while (!(task.Status == TaskStatus.RanToCompletion || task.Status == TaskStatus.Faulted || task.Status == TaskStatus.Canceled));
if (!showFirstPage)
{
this.ShowResultPage(this.allResults, 0, this.itemsPerPage);
}
if (this.currentShownPageIndex < 0)
{
ShowPagingLink(0);
}
this.PageContentReset();
}
}
/// <summary>
/// Called when [search].
/// </summary>
/// <param name="service">The service.</param>
/// <returns></returns>
static async Task Run(Service service)
{
await service.LogOnAsync(SdkHelper.Splunk.Username, SdkHelper.Splunk.Password);
//// Simple oneshot search
using (SearchResultStream stream = await service.SearchOneShotAsync("search index=_internal | head 5"))
{
foreach (SearchResult result in stream)
{
Console.WriteLine(result);
}
}
}
static async Task Run(Service service)
{
await service.LogOnAsync(SdkHelper.Splunk.Username, SdkHelper.Splunk.Password);
Console.WriteLine("Press return to cancel.");
string searchQuery = "search index=_internal | stats count by method";
Job realtimeJob = await service.Jobs.CreateAsync(searchQuery, args : new JobArgs
{
SearchMode = SearchMode.RealTime,
EarliestTime = "rt-1h",
LatestTime = "rt",
});
var tokenSource = new CancellationTokenSource();
#pragma warning disable 4014
//// Because this call is not awaited, execution of the current
//// method continues before the call is completed. Consider
//// applying the 'await' operator to the result of the call.
Task.Run(async() =>
{
Console.ReadLine();
await realtimeJob.CancelAsync();
tokenSource.Cancel();
});
#pragma warning restore 4014
while (!tokenSource.IsCancellationRequested)
{
using (SearchResultStream stream = await realtimeJob.GetSearchPreviewAsync())
{
Console.WriteLine("fieldnames: " + string.Join(";", stream.FieldNames));
Console.WriteLine("fieldname count: " + stream.FieldNames.Count);
Console.WriteLine("final result: " + stream.IsFinal);
foreach (SearchResult result in stream)
{
Console.WriteLine(result);
}
Console.WriteLine("");
await Task.Delay(2000, tokenSource.Token);
}
}
}
public async Task JobResultsDefaultsToAll()
{
using (var service = await SdkHelper.CreateService())
{
JobArgs jobArgs = new JobArgs();
var job = await service.Jobs.CreateAsync("search index=_* | head 101", args : jobArgs);
using (SearchResultStream stream = await job.GetSearchResultsAsync())
{
// Is the result count greater than the default of 100?
Assert.Equal(101, job.ResultCount);
}
await job.CancelAsync();
}
}
public async Task <IActionResult> GetMany([FromQuery] string dataType, [FromQuery] string whereArguments, [FromQuery] string orderByArguments, [FromQuery] uint?limit = null)
{
// Validate
if (string.IsNullOrEmpty(dataType))
{
return(BadRequest("Data type not specified"));
}
// Authorize
var loggedInUsername = UsernameNormalizer.Normalize(HttpContext.User.Identity.Name);
var resourceDescription = new GetDataResourceDescription(dataType);
var authorizationResult = await authorizationModule.AuthorizeAsync(resourceDescription, loggedInUsername);
if (!authorizationResult.IsAuthorized)
{
return(StatusCode((int)HttpStatusCode.Unauthorized, "Not authorized"));
}
IRdDataStorage rdDataStorage;
try
{
rdDataStorage = await dataRouter.GetSourceSystemAsync(dataType);
}
catch (KeyNotFoundException e)
{
return(BadRequest(e.Message));
}
apiEventLogger.Log(LogLevel.Info, $"User '{authorizationResult.User.UserName}' requested objects of type '{dataType}' matching '{whereArguments?.RemoveLineBreaks()}' ordered by '{orderByArguments?.RemoveLineBreaks()}'");
try
{
var getManyResult = rdDataStorage.GetManyAsync(dataType, whereArguments, orderByArguments, limit);
var stream = new SearchResultStream(getManyResult.Select(x => DataEncoder.DecodeToJson(x.Data)));
return(new FileStreamResult(stream, Conventions.JsonContentType));
}
catch (FormatException formatException)
{
return(BadRequest(formatException.Message));
}
catch (Exception e)
{
return(StatusCode((int)HttpStatusCode.InternalServerError, e.InnermostException().Message));
}
}
private void GetAllResults(SearchResultStream resultStream, ref List <ResultData> allResults)
{
int resultCount = 0;
try
{
foreach (SearchResult result in resultStream)
{
List <string> results = this.ParseResult(result);
allResults.Add(new ResultData(++resultCount, results[0], results[1]));
}
}
catch (Exception ex)
{
//the stream has some broken fields
// Enumeration ended prematurely : System.IO.InvalidDataException: Read <fieldOrder> where </fieldOrder> was expected.
}
}
async Task CanSkipEmptyResults()
{
var baseFileName = Path.Combine(TestAtomFeed.Directory, "DVPL-5873");
var message = new HttpResponseMessage(HttpStatusCode.OK);
message.Content = new StreamContent(new FileStream(baseFileName + ".xml", FileMode.Open, FileAccess.Read));
using (var stream = await SearchResultStream.CreateAsync(message))
{
int count = 0;
foreach (var observedResult in stream)
{
++count;
}
Assert.Equal(count, stream.ReadCount);
}
}
async Task CanHandleBlankAndEmptyValues()
{
var builder = ImmutableSortedSet.CreateBuilder <string>();
builder.Add("");
var blankTaggedTextElement = new TaggedFieldValue("text", builder.ToImmutableSortedSet <string>());
builder.Clear();
builder.Add("tag");
var taggedBlankTextElement = new TaggedFieldValue("", builder.ToImmutableSortedSet <string>());
var path = Path.Combine(TestAtomFeed.Directory, "BlankAndEmptySearchResults.xml");
var message = new HttpResponseMessage(HttpStatusCode.OK);
message.Content = new StreamContent(new FileStream(path, FileMode.Open, FileAccess.Read));
using (var stream = await SearchResultStream.CreateAsync(message))
{
int count = 0;
foreach (dynamic observedResult in stream)
{
Assert.Equal(observedResult._raw, string.Empty);
Assert.Null(observedResult.SegmentedRaw);
Assert.Equal(observedResult.blank_tagged_text_element, blankTaggedTextElement);
Assert.Equal(observedResult.empty_tagged_text_element, blankTaggedTextElement);
Assert.Equal(observedResult.blank_text_element, string.Empty);
Assert.Equal(observedResult.blank_value_element, string.Empty);
Assert.Equal(observedResult.empty_text_element, string.Empty);
Assert.Equal(observedResult.tagged_blank_text_element, taggedBlankTextElement);
Assert.Equal(observedResult.tagged_empty_text_element, taggedBlankTextElement);
++count;
}
}
}
async Task CanEnumerateResults()
{
var baseFileName = Path.Combine(TestAtomFeed.Directory, "TaggedSearchResults");
using (var expectedResults = new StreamReader(baseFileName + ".expected.text", encoding: Encoding.UTF8))
{
var message = new HttpResponseMessage(HttpStatusCode.OK);
message.Content = new StreamContent(new FileStream(baseFileName + ".xml", FileMode.Open, FileAccess.Read));
using (var stream = await SearchResultStream.CreateAsync(message))
{
int count = 0;
foreach (var observedResult in stream)
{
string expectedResult = null;
++count;
try
{
expectedResult = await expectedResults.ReadLineAsync();
continue;
}
catch (Exception e)
{
Assert.False(true, string.Format("Error while reading expected results: {0}", e.Message));
}
Assert.NotNull(expectedResult);
Assert.Equal(expectedResult, observedResult.ToString());
}
Assert.Null(expectedResults.ReadLine());
Assert.Equal(count, stream.ReadCount);
}
}
}
public async Task JobEventsTruncationModeArgument()
{
using (var service = await SdkHelper.CreateService())
{
JobArgs jobArgs = new JobArgs();
await ForEachEnum(typeof(TruncationMode), async enumValue =>
{
var job = await service.Jobs.CreateAsync(Search, args: jobArgs);
var args = new SearchEventArgs
{
TruncationMode = (TruncationMode)Enum.Parse(typeof(TruncationMode), enumValue)
};
using (SearchResultStream stream = await job.GetSearchEventsAsync(args))
{ }
await job.CancelAsync();
});
}
}
private Task GetResultTask(SearchResultStream resultStream)
{
int resultCount = 0;
//start a task to get all results
return(Task.Factory.StartNew(() =>
{
foreach (SearchResult result in resultStream)
{
List <string> results = this.ParseResult(result);
allResults.Add(new ResultData(++resultCount, results[0], results[1]));
if (resultCount > this.totalPage * itemsPerPage)
{
this.totalPage++;
}
if (this.cancelSearchTokenSource.Token.IsCancellationRequested)
{
break;
}
}
}));
}
public static FileStreamResult ToFileStreamResult(this IAsyncEnumerable <string> asyncEnumerable)
{
var stream = new SearchResultStream(asyncEnumerable);
return(new FileStreamResult(stream, Conventions.JsonContentType));
}
private void GetAllResults(SearchResultStream resultStream, ref List<ResultData> allResults)
{
int resultCount = 0;
try
{
foreach (SearchResult result in resultStream)
{
List<string> results = this.ParseResult(result);
allResults.Add(new ResultData(++resultCount, results[0], results[1]));
}
}
catch (Exception ex)
{
//the stream has some broken fields
// Enumeration ended prematurely : System.IO.InvalidDataException: Read <fieldOrder> where </fieldOrder> was expected.
}
}
private Task GetResultTask(SearchResultStream resultStream)
{
int resultCount = 0;
//start a task to get all results
return Task.Factory.StartNew(() =>
{
foreach (SearchResult result in resultStream)
{
List<string> results = this.ParseResult(result);
allResults.Add(new ResultData(++resultCount, results[0], results[1]));
if (resultCount > this.totalPage * itemsPerPage)
{
this.totalPage++;
}
if (this.cancelSearchTokenSource.Token.IsCancellationRequested)
{
break;
}
}
});
}
private async void DisplaySearchPreviewResult(string searchStr)
{
int maxResultCount = 10000;
JobArgs args = new JobArgs();
args.EarliestTime = this.searchEarliestTime;
args.LatestTime = this.searchLatestTime;
args.SearchMode = SearchMode.RealTime;
Job realtimeJob = await MainPage.SplunkService.Jobs.CreateAsync(searchStr, count : maxResultCount, args : args);
Stopwatch watch = new Stopwatch();
watch.Start();
this.allResults.Clear();
int resultCount = 0;
do
{
using (SearchResultStream resultStream = await realtimeJob.GetSearchPreviewAsync())
{
resultCount = resultStream.Count();
}
} while (resultCount == 0 && watch.Elapsed.TotalSeconds <= 10 && !this.cancelSearchTokenSource.Token.IsCancellationRequested);
bool showFirstPage = false;
SearchResultArgs searchArgs = new SearchResultArgs();
searchArgs.Count = maxResultCount;
using (SearchResultStream resultStream = await realtimeJob.GetSearchPreviewAsync(searchArgs))
{
titleGrid.Visibility = Visibility.Visible;
Task task = this.GetResultTask(resultStream);
//start a task to get all results
do
{
if (this.allResults.Count > 0)
{
if (!showFirstPage && this.ShowResultPage(this.allResults, 0, this.itemsPerPage))
{
showFirstPage = true;
}
if (this.currentShownPageIndex < 0)
{
ShowPagingLink(0);
}
}
await Task.Delay(1000);
} while (!(task.Status == TaskStatus.RanToCompletion || task.Status == TaskStatus.Faulted || task.Status == TaskStatus.Canceled));
if (!showFirstPage)
{
this.ShowResultPage(this.allResults, 0, this.itemsPerPage);
}
if (this.currentShownPageIndex < 0)
{
ShowPagingLink(0);
}
this.PageContentReset();
await realtimeJob.CancelAsync();
}
}
private static IActionResult BuildCsvResultAsync(IAsyncEnumerable <BsonDocument> documents)
{
var searchResultStream = new SearchResultStream(ToCsv(documents));
return(new FileStreamResult(searchResultStream, "text/csv"));
}
public static FileStreamResult ToFileStreamResult <T>(this IAsyncEnumerable <T> asyncEnumerable)
{
var stream = new SearchResultStream(asyncEnumerable.Select(x => JsonConvert.SerializeObject(x)));
return(new FileStreamResult(stream, Conventions.JsonContentType));
}
public static FileStreamResult ToFileStreamResult <T>(this IAsyncEnumerable <T> asyncEnumerable, Func <T, string> selector)
{
var stream = new SearchResultStream(asyncEnumerable.Select(selector));
return(new FileStreamResult(stream, Conventions.JsonContentType));
}
public async Task Transmitter1()
{
string indexName = string.Format("delete-me-{0}", Guid.NewGuid());
using (var service = await SdkHelper.CreateService())
{
Index index = await service.Indexes.RecreateAsync(indexName);
Assert.False(index.Disabled);
await Task.Delay(2000);
// Submit event using TransmitterArgs
const string Source = "splunk-sdk-tests";
const string SourceType = "splunk-sdk-test-event";
const string Host = "test-host";
var transmitterArgs = new TransmitterArgs
{
Host = Host,
Source = Source,
SourceType = SourceType,
};
Transmitter transmitter = service.Transmitter;
SearchResult result;
//// TODO: Check contentss
result = await transmitter.SendAsync(
MockContext.GetOrElse(string.Format("1, {0}, {1}, simple event", DateTime.Now, indexName)),
indexName, transmitterArgs);
Assert.NotNull(result);
result = await transmitter.SendAsync(
MockContext.GetOrElse(string.Format("2, {0}, {1}, simple event", DateTime.Now, indexName)),
indexName, transmitterArgs);
Assert.NotNull(result);
using (MemoryStream stream = new MemoryStream())
{
using (StreamWriter writer = new StreamWriter(stream, Encoding.UTF8, 4096, leaveOpen: true))
{
writer.WriteLine(
MockContext.GetOrElse(string.Format("1, {0}, {1}, stream event", DateTime.Now, indexName)));
writer.WriteLine(
MockContext.GetOrElse(string.Format("2, {0}, {1}, stream event", DateTime.Now, indexName)));
}
stream.Seek(0, SeekOrigin.Begin);
await transmitter.SendAsync(stream, indexName, transmitterArgs);
}
await index.PollForUpdatedEventCount(4);
var search = string.Format(
"search index={0} host={1} source={2} sourcetype={3}",
indexName,
Host,
Source,
SourceType);
using (SearchResultStream stream = await service.SearchOneShotAsync(search))
{
Assert.Equal(0, stream.FieldNames.Count);
Assert.False(stream.IsFinal);
Assert.Equal(0, stream.ReadCount);
foreach (SearchResult record in stream)
{
var fieldNames = stream.FieldNames;
Assert.Equal(14, fieldNames.Count);
Assert.Equal(14, record.FieldNames.Count);
Assert.Equal(fieldNames.AsEnumerable(), record.FieldNames.AsEnumerable());
var memberNames = record.GetDynamicMemberNames();
var intersection = fieldNames.Intersect(memberNames);
Assert.Equal(memberNames, intersection);
}
Assert.Equal(14, stream.FieldNames.Count);
Assert.Equal(4, stream.ReadCount);
}
}
}
private static IActionResult BuildJsonResultAsync(IAsyncEnumerable <BsonDocument> documents)
{
var searchResultStream = new SearchResultStream(documents.Select(DataEncoder.DecodeToJson));
return(new FileStreamResult(searchResultStream, Conventions.JsonContentType));
}