Example #1
0
        private static async Task Run(Service service)
        {
            await service.LogOnAsync(SdkHelper.Splunk.Username, SdkHelper.Splunk.Password);

            string savedSearchName  = "example_search";
            string savedSearchQuery = "search index=_internal | head 10";

            // Delete the saved search if it exists before we start.
            SavedSearch savedSearch = await service.SavedSearches.GetOrNullAsync(savedSearchName);

            if (savedSearch != null)
            {
                await savedSearch.RemoveAsync();
            }

            savedSearch = await service.SavedSearches.CreateAsync(savedSearchName, savedSearchQuery);

            Job savedSearchJob = await savedSearch.DispatchAsync();

            using (SearchResultStream stream = await savedSearchJob.GetSearchResultsAsync())
            {
                foreach (SearchResult result in stream)
                {
                    Console.WriteLine(result);
                }
            }

            await savedSearch.RemoveAsync();
        }
        public async Task JobPreviewDefaultsToAll()
        {
            using (var service = await SdkHelper.CreateService())
            {
                JobArgs jobArgs = new JobArgs();

                var job = await service.Jobs.CreateAsync("search index=_* | head 101", args : jobArgs);

                for (int delay = 1000; delay < 5000; delay += 1000)
                {
                    try
                    {
                        await job.TransitionAsync(DispatchState.Done, delay);

                        break;
                    }
                    catch (TaskCanceledException)
                    { }
                }

                using (SearchResultStream stream = await job.GetSearchPreviewAsync())
                {
                    // Is the result preview count greater than the default of 100?
                    Assert.Equal(101, job.ResultPreviewCount);
                }

                await job.CancelAsync();
            }
        }
Example #3
0
        public void Test()
        {
            var service = new Mock <Service>();
            var job     = new Mock <Job>();

            service.Setup(s => s.LogOnAsync(
                              It.IsAny <string>(),
                              It.IsAny <string>()))
            .Returns(Task.FromResult(true));

            service.Setup(s => s.LogOffAsync()).Returns(Task.FromResult(true));

            service.Setup(s => s.SearchAsync(
                              It.IsAny <string>(),         // search
                              It.IsAny <int>(),            // count
                              It.IsAny <ExecutionMode>(),  // mode
                              It.IsAny <JobArgs>(),        // args
                              It.IsAny <CustomJobArgs>())) // customArgs
            .Returns(Task.FromResult(job.Object));

            var message = new HttpResponseMessage(HttpStatusCode.OK)
            {
                Content = new StreamContent(new FileStream("search-results.xml", FileMode.Open)),
            };

            var resultStream = SearchResultStream.CreateAsync(message);

            job.Setup(j => j.GetSearchResultsAsync(It.IsAny <int>())).Returns(resultStream);
            Run(service.Object).Wait();
        }
Example #4
0
        async Task CanHandleInFlightErrorsReportedBySplunk()
        {
            var path    = Path.Combine(TestAtomFeed.Directory, "Service.ExportSearchResults-failure.xml");
            var message = new HttpResponseMessage(HttpStatusCode.OK);

            message.Content = new StreamContent(new FileStream(path, FileMode.Open, FileAccess.Read));
            SearchResultStream stream = null;

            try
            {
                stream = await SearchResultStream.CreateAsync(message);

                int count = 0;

                foreach (var result in stream)
                {
                    ++count;
                }

                Assert.False(true, "Expected RequestException");
            }
            catch (RequestException e)
            {
                Assert.Equal(e.Message, "Fatal: JournalSliceDirectory: Cannot seek to 0");
            }
            finally
            {
                if (stream != null)
                {
                    stream.Dispose();
                }
            }
        }
Example #5
0
        public void StreamWithoutItemsDoesntThrowExceptions()
        {
            var asyncEnumerable = AsyncEnumerableBuilder.FromArray(new string[0]);

            using var sut          = new SearchResultStream(asyncEnumerable);
            using var streamReader = new StreamReader(sut);
            var actual = ReadAllLines(streamReader).ToList();

            CollectionAssert.IsEmpty(actual);
        }
Example #6
0
        public void StreamReturnsInputItems()
        {
            var expected        = new[] { "Item1", "Item2", "Item3" };
            var asyncEnumerable = AsyncEnumerableBuilder.FromArray(expected);

            using var sut          = new SearchResultStream(asyncEnumerable);
            using var streamReader = new StreamReader(sut);
            var actual = ReadAllLines(streamReader).ToList();

            CollectionAssert.AreEqual(expected, actual);
        }
        public static void PrintResults(SearchResultStream results, ListBox logger)
        {
            logger.Items.Clear();
            var recordNumber = 0;

            foreach (var result in results)
            {
                logger.Items.Add("\n" + result.GetValue("_time") + "                  : " + result.GetValue("Value"));
                Console.WriteLine(string.Format("{0:D8}: {1}", ++recordNumber, result));
            }
        }
        private async void DisplaySearchResult(string searchStr)
        {
            SearchExportArgs jobArgs = new SearchExportArgs();

            if (this.searchEarliestTime != null)
            {
                jobArgs.EarliestTime = this.searchEarliestTime;
            }

            if (this.searchLatestTime != null)
            {
                jobArgs.LatestTime = this.searchLatestTime;
            }

            using (SearchResultStream resultStream = await MainPage.SplunkService.ExportSearchResultsAsync(searchStr, jobArgs))
            {
                titleGrid.Visibility = Visibility.Visible;

                this.allResults.Clear();
                Task task = this.GetResultTask(resultStream);

                bool showFirstPage = false;
                do
                {
                    if (this.allResults.Count > 0)
                    {
                        if (!showFirstPage && this.ShowResultPage(this.allResults, 0, this.itemsPerPage))
                        {
                            showFirstPage = true;
                        }

                        if (this.currentShownPageIndex < 0)
                        {
                            ShowPagingLink(0);
                        }
                    }

                    await Task.Delay(1000);
                } while (!(task.Status == TaskStatus.RanToCompletion || task.Status == TaskStatus.Faulted || task.Status == TaskStatus.Canceled));

                if (!showFirstPage)
                {
                    this.ShowResultPage(this.allResults, 0, this.itemsPerPage);
                }

                if (this.currentShownPageIndex < 0)
                {
                    ShowPagingLink(0);
                }

                this.PageContentReset();
            }
        }
Example #9
0
        /// <summary>
        /// Called when [search].
        /// </summary>
        /// <param name="service">The service.</param>
        /// <returns></returns>
        static async Task Run(Service service)
        {
            await service.LogOnAsync(SdkHelper.Splunk.Username, SdkHelper.Splunk.Password);

            //// Simple oneshot search

            using (SearchResultStream stream = await service.SearchOneShotAsync("search index=_internal | head 5"))
            {
                foreach (SearchResult result in stream)
                {
                    Console.WriteLine(result);
                }
            }
        }
Example #10
0
        static async Task Run(Service service)
        {
            await service.LogOnAsync(SdkHelper.Splunk.Username, SdkHelper.Splunk.Password);

            Console.WriteLine("Press return to cancel.");

            string searchQuery = "search index=_internal | stats count by method";

            Job realtimeJob = await service.Jobs.CreateAsync(searchQuery, args : new JobArgs
            {
                SearchMode   = SearchMode.RealTime,
                EarliestTime = "rt-1h",
                LatestTime   = "rt",
            });

            var tokenSource = new CancellationTokenSource();

            #pragma warning disable 4014
            //// Because this call is not awaited, execution of the current
            //// method continues before the call is completed. Consider
            //// applying the 'await' operator to the result of the call.

            Task.Run(async() =>
            {
                Console.ReadLine();

                await realtimeJob.CancelAsync();
                tokenSource.Cancel();
            });

            #pragma warning restore 4014

            while (!tokenSource.IsCancellationRequested)
            {
                using (SearchResultStream stream = await realtimeJob.GetSearchPreviewAsync())
                {
                    Console.WriteLine("fieldnames: " + string.Join(";", stream.FieldNames));
                    Console.WriteLine("fieldname count: " + stream.FieldNames.Count);
                    Console.WriteLine("final result: " + stream.IsFinal);

                    foreach (SearchResult result in stream)
                    {
                        Console.WriteLine(result);
                    }

                    Console.WriteLine("");
                    await Task.Delay(2000, tokenSource.Token);
                }
            }
        }
        public async Task JobResultsDefaultsToAll()
        {
            using (var service = await SdkHelper.CreateService())
            {
                JobArgs jobArgs = new JobArgs();

                var job = await service.Jobs.CreateAsync("search index=_* | head 101", args : jobArgs);

                using (SearchResultStream stream = await job.GetSearchResultsAsync())
                {
                    // Is the result count greater than the default of 100?
                    Assert.Equal(101, job.ResultCount);
                }

                await job.CancelAsync();
            }
        }
Example #12
0
        public async Task <IActionResult> GetMany([FromQuery] string dataType, [FromQuery] string whereArguments, [FromQuery] string orderByArguments, [FromQuery] uint?limit = null)
        {
            // Validate
            if (string.IsNullOrEmpty(dataType))
            {
                return(BadRequest("Data type not specified"));
            }

            // Authorize
            var loggedInUsername    = UsernameNormalizer.Normalize(HttpContext.User.Identity.Name);
            var resourceDescription = new GetDataResourceDescription(dataType);
            var authorizationResult = await authorizationModule.AuthorizeAsync(resourceDescription, loggedInUsername);

            if (!authorizationResult.IsAuthorized)
            {
                return(StatusCode((int)HttpStatusCode.Unauthorized, "Not authorized"));
            }

            IRdDataStorage rdDataStorage;

            try
            {
                rdDataStorage = await dataRouter.GetSourceSystemAsync(dataType);
            }
            catch (KeyNotFoundException e)
            {
                return(BadRequest(e.Message));
            }

            apiEventLogger.Log(LogLevel.Info, $"User '{authorizationResult.User.UserName}' requested objects of type '{dataType}' matching '{whereArguments?.RemoveLineBreaks()}' ordered by '{orderByArguments?.RemoveLineBreaks()}'");
            try
            {
                var getManyResult = rdDataStorage.GetManyAsync(dataType, whereArguments, orderByArguments, limit);
                var stream        = new SearchResultStream(getManyResult.Select(x => DataEncoder.DecodeToJson(x.Data)));
                return(new FileStreamResult(stream, Conventions.JsonContentType));
            }
            catch (FormatException formatException)
            {
                return(BadRequest(formatException.Message));
            }
            catch (Exception e)
            {
                return(StatusCode((int)HttpStatusCode.InternalServerError, e.InnermostException().Message));
            }
        }
        private void GetAllResults(SearchResultStream resultStream, ref List <ResultData> allResults)
        {
            int resultCount = 0;

            try
            {
                foreach (SearchResult result in resultStream)
                {
                    List <string> results = this.ParseResult(result);
                    allResults.Add(new ResultData(++resultCount, results[0], results[1]));
                }
            }
            catch (Exception ex)
            {
                //the stream has some broken fields
                // Enumeration ended prematurely : System.IO.InvalidDataException: Read <fieldOrder> where </fieldOrder> was expected.
            }
        }
Example #14
0
        async Task CanSkipEmptyResults()
        {
            var baseFileName = Path.Combine(TestAtomFeed.Directory, "DVPL-5873");
            var message      = new HttpResponseMessage(HttpStatusCode.OK);

            message.Content = new StreamContent(new FileStream(baseFileName + ".xml", FileMode.Open, FileAccess.Read));

            using (var stream = await SearchResultStream.CreateAsync(message))
            {
                int count = 0;

                foreach (var observedResult in stream)
                {
                    ++count;
                }

                Assert.Equal(count, stream.ReadCount);
            }
        }
Example #15
0
        async Task CanHandleBlankAndEmptyValues()
        {
            var builder = ImmutableSortedSet.CreateBuilder <string>();

            builder.Add("");

            var blankTaggedTextElement = new TaggedFieldValue("text", builder.ToImmutableSortedSet <string>());

            builder.Clear();
            builder.Add("tag");

            var taggedBlankTextElement = new TaggedFieldValue("", builder.ToImmutableSortedSet <string>());

            var path    = Path.Combine(TestAtomFeed.Directory, "BlankAndEmptySearchResults.xml");
            var message = new HttpResponseMessage(HttpStatusCode.OK);

            message.Content = new StreamContent(new FileStream(path, FileMode.Open, FileAccess.Read));

            using (var stream = await SearchResultStream.CreateAsync(message))
            {
                int count = 0;

                foreach (dynamic observedResult in stream)
                {
                    Assert.Equal(observedResult._raw, string.Empty);
                    Assert.Null(observedResult.SegmentedRaw);

                    Assert.Equal(observedResult.blank_tagged_text_element, blankTaggedTextElement);
                    Assert.Equal(observedResult.empty_tagged_text_element, blankTaggedTextElement);

                    Assert.Equal(observedResult.blank_text_element, string.Empty);
                    Assert.Equal(observedResult.blank_value_element, string.Empty);
                    Assert.Equal(observedResult.empty_text_element, string.Empty);

                    Assert.Equal(observedResult.tagged_blank_text_element, taggedBlankTextElement);
                    Assert.Equal(observedResult.tagged_empty_text_element, taggedBlankTextElement);

                    ++count;
                }
            }
        }
Example #16
0
        async Task CanEnumerateResults()
        {
            var baseFileName = Path.Combine(TestAtomFeed.Directory, "TaggedSearchResults");

            using (var expectedResults = new StreamReader(baseFileName + ".expected.text", encoding: Encoding.UTF8))
            {
                var message = new HttpResponseMessage(HttpStatusCode.OK);

                message.Content = new StreamContent(new FileStream(baseFileName + ".xml", FileMode.Open, FileAccess.Read));

                using (var stream = await SearchResultStream.CreateAsync(message))
                {
                    int count = 0;

                    foreach (var observedResult in stream)
                    {
                        string expectedResult = null;
                        ++count;

                        try
                        {
                            expectedResult = await expectedResults.ReadLineAsync();

                            continue;
                        }
                        catch (Exception e)
                        {
                            Assert.False(true, string.Format("Error while reading expected results: {0}", e.Message));
                        }

                        Assert.NotNull(expectedResult);
                        Assert.Equal(expectedResult, observedResult.ToString());
                    }

                    Assert.Null(expectedResults.ReadLine());
                    Assert.Equal(count, stream.ReadCount);
                }
            }
        }
        public async Task JobEventsTruncationModeArgument()
        {
            using (var service = await SdkHelper.CreateService())
            {
                JobArgs jobArgs = new JobArgs();

                await ForEachEnum(typeof(TruncationMode), async enumValue =>
                {
                    var job = await service.Jobs.CreateAsync(Search, args: jobArgs);

                    var args = new SearchEventArgs
                    {
                        TruncationMode = (TruncationMode)Enum.Parse(typeof(TruncationMode), enumValue)
                    };

                    using (SearchResultStream stream = await job.GetSearchEventsAsync(args))
                    { }

                    await job.CancelAsync();
                });
            }
        }
        private Task GetResultTask(SearchResultStream resultStream)
        {
            int resultCount = 0;

            //start a task to get all results
            return(Task.Factory.StartNew(() =>
            {
                foreach (SearchResult result in resultStream)
                {
                    List <string> results = this.ParseResult(result);
                    allResults.Add(new ResultData(++resultCount, results[0], results[1]));

                    if (resultCount > this.totalPage * itemsPerPage)
                    {
                        this.totalPage++;
                    }

                    if (this.cancelSearchTokenSource.Token.IsCancellationRequested)
                    {
                        break;
                    }
                }
            }));
        }
Example #19
0
        public static FileStreamResult ToFileStreamResult(this IAsyncEnumerable <string> asyncEnumerable)
        {
            var stream = new SearchResultStream(asyncEnumerable);

            return(new FileStreamResult(stream, Conventions.JsonContentType));
        }
        private void GetAllResults(SearchResultStream resultStream, ref List<ResultData> allResults)
        {
            int resultCount = 0;

            try
            {
                foreach (SearchResult result in resultStream)
                {
                    List<string> results = this.ParseResult(result);
                    allResults.Add(new ResultData(++resultCount, results[0], results[1]));
                }
            }
            catch (Exception ex)
            {
                //the stream has some broken fields
                // Enumeration ended prematurely : System.IO.InvalidDataException: Read <fieldOrder> where </fieldOrder> was expected.   
            }
        }
        private Task GetResultTask(SearchResultStream resultStream)
        {
            int resultCount = 0;

            //start a task to get all results
            return Task.Factory.StartNew(() =>
            {
                foreach (SearchResult result in resultStream)
                {
                    List<string> results = this.ParseResult(result);
                    allResults.Add(new ResultData(++resultCount, results[0], results[1]));

                    if (resultCount > this.totalPage * itemsPerPage)
                    {
                        this.totalPage++;
                    }

                    if (this.cancelSearchTokenSource.Token.IsCancellationRequested)
                    {
                        break;
                    }
                }
            });
        }
        private async void DisplaySearchPreviewResult(string searchStr)
        {
            int     maxResultCount = 10000;
            JobArgs args           = new JobArgs();

            args.EarliestTime = this.searchEarliestTime;
            args.LatestTime   = this.searchLatestTime;
            args.SearchMode   = SearchMode.RealTime;
            Job realtimeJob = await MainPage.SplunkService.Jobs.CreateAsync(searchStr, count : maxResultCount, args : args);

            Stopwatch watch = new Stopwatch();

            watch.Start();

            this.allResults.Clear();
            int resultCount = 0;

            do
            {
                using (SearchResultStream resultStream = await realtimeJob.GetSearchPreviewAsync())
                {
                    resultCount = resultStream.Count();
                }
            } while (resultCount == 0 && watch.Elapsed.TotalSeconds <= 10 && !this.cancelSearchTokenSource.Token.IsCancellationRequested);

            bool             showFirstPage = false;
            SearchResultArgs searchArgs    = new SearchResultArgs();

            searchArgs.Count = maxResultCount;
            using (SearchResultStream resultStream = await realtimeJob.GetSearchPreviewAsync(searchArgs))
            {
                titleGrid.Visibility = Visibility.Visible;
                Task task = this.GetResultTask(resultStream);

                //start a task to get all results
                do
                {
                    if (this.allResults.Count > 0)
                    {
                        if (!showFirstPage && this.ShowResultPage(this.allResults, 0, this.itemsPerPage))
                        {
                            showFirstPage = true;
                        }

                        if (this.currentShownPageIndex < 0)
                        {
                            ShowPagingLink(0);
                        }
                    }

                    await Task.Delay(1000);
                } while (!(task.Status == TaskStatus.RanToCompletion || task.Status == TaskStatus.Faulted || task.Status == TaskStatus.Canceled));


                if (!showFirstPage)
                {
                    this.ShowResultPage(this.allResults, 0, this.itemsPerPage);
                }
                if (this.currentShownPageIndex < 0)
                {
                    ShowPagingLink(0);
                }

                this.PageContentReset();
                await realtimeJob.CancelAsync();
            }
        }
Example #23
0
        private static IActionResult BuildCsvResultAsync(IAsyncEnumerable <BsonDocument> documents)
        {
            var searchResultStream = new SearchResultStream(ToCsv(documents));

            return(new FileStreamResult(searchResultStream, "text/csv"));
        }
Example #24
0
        public static FileStreamResult ToFileStreamResult <T>(this IAsyncEnumerable <T> asyncEnumerable)
        {
            var stream = new SearchResultStream(asyncEnumerable.Select(x => JsonConvert.SerializeObject(x)));

            return(new FileStreamResult(stream, Conventions.JsonContentType));
        }
Example #25
0
        public static FileStreamResult ToFileStreamResult <T>(this IAsyncEnumerable <T> asyncEnumerable, Func <T, string> selector)
        {
            var stream = new SearchResultStream(asyncEnumerable.Select(selector));

            return(new FileStreamResult(stream, Conventions.JsonContentType));
        }
Example #26
0
        public async Task Transmitter1()
        {
            string indexName = string.Format("delete-me-{0}", Guid.NewGuid());

            using (var service = await SdkHelper.CreateService())
            {
                Index index = await service.Indexes.RecreateAsync(indexName);

                Assert.False(index.Disabled);

                await Task.Delay(2000);

                // Submit event using TransmitterArgs

                const string Source     = "splunk-sdk-tests";
                const string SourceType = "splunk-sdk-test-event";
                const string Host       = "test-host";

                var transmitterArgs = new TransmitterArgs
                {
                    Host       = Host,
                    Source     = Source,
                    SourceType = SourceType,
                };

                Transmitter  transmitter = service.Transmitter;
                SearchResult result;

                //// TODO: Check contentss
                result = await transmitter.SendAsync(
                    MockContext.GetOrElse(string.Format("1, {0}, {1}, simple event", DateTime.Now, indexName)),
                    indexName, transmitterArgs);

                Assert.NotNull(result);

                result = await transmitter.SendAsync(
                    MockContext.GetOrElse(string.Format("2, {0}, {1}, simple event", DateTime.Now, indexName)),
                    indexName, transmitterArgs);

                Assert.NotNull(result);

                using (MemoryStream stream = new MemoryStream())
                {
                    using (StreamWriter writer = new StreamWriter(stream, Encoding.UTF8, 4096, leaveOpen: true))
                    {
                        writer.WriteLine(
                            MockContext.GetOrElse(string.Format("1, {0}, {1}, stream event", DateTime.Now, indexName)));
                        writer.WriteLine(
                            MockContext.GetOrElse(string.Format("2, {0}, {1}, stream event", DateTime.Now, indexName)));
                    }

                    stream.Seek(0, SeekOrigin.Begin);

                    await transmitter.SendAsync(stream, indexName, transmitterArgs);
                }

                await index.PollForUpdatedEventCount(4);

                var search = string.Format(
                    "search index={0} host={1} source={2} sourcetype={3}",
                    indexName,
                    Host,
                    Source,
                    SourceType);

                using (SearchResultStream stream = await service.SearchOneShotAsync(search))
                {
                    Assert.Equal(0, stream.FieldNames.Count);
                    Assert.False(stream.IsFinal);
                    Assert.Equal(0, stream.ReadCount);

                    foreach (SearchResult record in stream)
                    {
                        var fieldNames = stream.FieldNames;

                        Assert.Equal(14, fieldNames.Count);
                        Assert.Equal(14, record.FieldNames.Count);
                        Assert.Equal(fieldNames.AsEnumerable(), record.FieldNames.AsEnumerable());

                        var memberNames  = record.GetDynamicMemberNames();
                        var intersection = fieldNames.Intersect(memberNames);

                        Assert.Equal(memberNames, intersection);
                    }

                    Assert.Equal(14, stream.FieldNames.Count);
                    Assert.Equal(4, stream.ReadCount);
                }
            }
        }
Example #27
0
        private static IActionResult BuildJsonResultAsync(IAsyncEnumerable <BsonDocument> documents)
        {
            var searchResultStream = new SearchResultStream(documents.Select(DataEncoder.DecodeToJson));

            return(new FileStreamResult(searchResultStream, Conventions.JsonContentType));
        }