public void CanRunIndexerAndGetIndexerStatus()
        {
            Run(() =>
            {
                SearchServiceClient searchClient = Data.GetSearchServiceClient();

                Indexer indexer = Data.CreateTestIndexer();

                IndexerDefinitionResponse createResponse = searchClient.Indexers.Create(indexer);
                Assert.Equal(HttpStatusCode.Created, createResponse.StatusCode);

                IndexerGetStatusResponse statusResponse = searchClient.Indexers.GetStatus(indexer.Name);
                Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode);

                IndexerExecutionInfo info = statusResponse.ExecutionInfo;
                Assert.Equal(IndexerStatus.Running, info.Status);

                AzureOperationResponse runResponse = searchClient.Indexers.Run(indexer.Name);
                Assert.Equal(HttpStatusCode.Accepted, runResponse.StatusCode);

                // Set handler that injects mock_status query string, which results in service
                // returning a well-known mock response
                searchClient.AddHandlerToPipeline(new MockStatusDelegatingHandler());

                statusResponse = searchClient.Indexers.GetStatus(indexer.Name);
                Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode);

                info = statusResponse.ExecutionInfo;
                Assert.Equal(IndexerStatus.Running, info.Status);

                Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
                Assert.Equal(3, info.ExecutionHistory.Count);

                IndexerExecutionResult newestResult = info.ExecutionHistory[0];
                IndexerExecutionResult middleResult = info.ExecutionHistory[1];
                IndexerExecutionResult oldestResult = info.ExecutionHistory[2];

                Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
                Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
                AssertStartAndEndTimeValid(newestResult);

                Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
                AssertStartAndEndTimeValid(middleResult);

                Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
                Assert.Equal(124876, oldestResult.ItemCount);
                Assert.Equal(2, oldestResult.FailedItemCount);
                Assert.Equal("100", oldestResult.InitialTrackingState);
                Assert.Equal("200", oldestResult.FinalTrackingState);
                AssertStartAndEndTimeValid(oldestResult);

                Assert.Equal(2, oldestResult.Errors.Count);

                Assert.Equal("1", oldestResult.Errors[0].Key);
                Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);

                Assert.Equal("121713", oldestResult.Errors[1].Key);
                Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
            });
        }
Exemplo n.º 2
0
 private static void AssertStartAndEndTimeValid(IndexerExecutionResult result)
 {
     Assert.True(result.StartTime.HasValue);
     Assert.NotEqual(new DateTimeOffset(), result.StartTime.Value);
     Assert.True(result.EndTime.HasValue);
     Assert.NotEqual(new DateTimeOffset(), result.EndTime.Value);
 }
        private static void CheckIndexerStatus(SearchIndexerClient indexerClient, SearchIndexer indexer)
        {
            try
            {
                string indexerName           = "hotels-sql-idxr";
                SearchIndexerStatus execInfo = indexerClient.GetIndexerStatus(indexerName);

                Console.WriteLine("Indexer has run {0} times.", execInfo.ExecutionHistory.Count);
                Console.WriteLine("Indexer Status: " + execInfo.Status.ToString());

                IndexerExecutionResult result = execInfo.LastResult;

                Console.WriteLine("Latest run");
                Console.WriteLine("Run Status: {0}", result.Status.ToString());
                Console.WriteLine("Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount);

                TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value;
                Console.WriteLine("StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed);

                string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage;
                Console.WriteLine("ErrorMessage: {0}", errorMsg);
                Console.WriteLine(" Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count);
            }
            catch (Exception e)
            {
                // Handle exception
            }
        }
Exemplo n.º 4
0
        public static async Task RunAsync(
            [BlobTrigger("datum/{name}", Connection = "Search")] CloudBlockBlob myBlob,
            [CosmosDB("taskDatabase", "TaskCollection", ConnectionStringSetting = "CosmosDB")] IAsyncCollector <object> todos, string name, ILogger log)
        {
            SearchServiceClient serviceClient = new SearchServiceClient(searchServiceName: "jfk-search-service-qymjpzort5hho", credentials: new SearchCredentials("023DB82430FFA416AD39EEF8A6FDFF2A"));
            ISearchIndexClient  indexClient   = serviceClient.Indexes.GetClient("jfkindex");
            await serviceClient.Indexers.RunAsync("jfkindexer");

            try
            {
                IndexerExecutionInfo execInfo = serviceClient.Indexers.GetStatus("jfkindexer");
                Console.WriteLine("{0} veces ejecutadas.", execInfo.ExecutionHistory.Count);
                Console.WriteLine("Indexer Status: " + execInfo.Status.ToString());

                IndexerExecutionResult result = execInfo.LastResult;

                Console.WriteLine("Latest run");
                Console.WriteLine("  Run Status: {0}", result.Status.ToString());
                Console.WriteLine("  Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount);

                TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value;
                Console.WriteLine("  StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed);

                string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage;
                Console.WriteLine("  ErrorMessage: {0}", errorMsg);
                Console.WriteLine("  Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count);
            }
            catch (Exception e) { Console.WriteLine("Error: ", e); }
            var param = new SearchParameters
            {
                Select = new[] { "*" },
                //IncludeTotalResultCount = true
            };

            System.Threading.Thread.Sleep(19000);
            log.LogInformation($"espera terminada{name}");
            var results = indexClient.Documents.Search <Book>($"{name}", param);

            log.LogInformation($"enviando a cosmos.... {myBlob.Name}");
            await todos.AddAsync(results);
        }
Exemplo n.º 5
0
        private static void SyncProductDataFromAzureSQL()
        {
            // This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
            Console.WriteLine("{0}", "Creating Product Data Source...\n");
            var dataSource =
                DataSource.AzureSql(
                    name: MowidoDataSource,
                    sqlConnectionString: "Server=tcp:mowido.database.windows.net,1433;Initial Catalog=MowidoDB;Persist Security Info=False;User ID=mowido;Password=Billions123;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;",
                    tableOrViewName: "Products",
                    description: "Mowido Dataset");

            try
            {
                _searchClient.DataSources.Create(dataSource);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating data source: {0}", ex.Message);
                return;
            }

            Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");

            var indexer =
                new Indexer()
            {
                Name            = MowidoIndexer,
                Description     = "Mowido start search product data indexer",
                DataSourceName  = dataSource.Name,
                TargetIndexName = MowidoProductIndex
            };

            try
            {
                _searchClient.Indexers.Create(indexer);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
                return;
            }

            bool running = true;

            Console.WriteLine("{0}", "Synchronization running...\n");
            while (running)
            {
                IndexerExecutionInfo status = null;

                try
                {
                    status = _searchClient.Indexers.GetStatus(indexer.Name);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
                    return;
                }

                IndexerExecutionResult lastResult = status.LastResult;
                if (lastResult != null)
                {
                    switch (lastResult.Status)
                    {
                    case IndexerExecutionStatus.InProgress:
                        Console.WriteLine("{0}", "Synchronization running...\n");
                        Thread.Sleep(1000);
                        break;

                    case IndexerExecutionStatus.Success:
                        running = false;
                        Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
                        break;

                    default:
                        running = false;
                        Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
                        break;
                    }
                }
            }
        }
Exemplo n.º 6
0
        private static async Task <bool> RunAsync(string[] args)
        {
            bool result = true;

            int instances = int.Parse(ConfigurationManager.AppSettings["PartitionCount"] ?? "1");

            var instanceFilter = GetNumberRangesFromArguments(args);

            Console.WriteLine("Indexer Partitions: {0}", instances);

            // use numeric partitioning by default
            Func <object, object> getNextPartitionValue = (object value) => (int)value + 1;
            Func <object, string> formatPartitionValue  = (object value) => ((int)value).ToString(ConfigurationManager.AppSettings["PartitionFormatString"] ?? "0");
            object partitionStartValue = 1;
            object partitionEndValue;

            // use a date time partitioner if Partition Dates are used
            if (!string.IsNullOrWhiteSpace(ConfigurationManager.AppSettings["PartitionStartDate"]) && !string.IsNullOrWhiteSpace(ConfigurationManager.AppSettings["PartitionEndDate"]))
            {
                DateTimeOffset partitionStartDate, partitionEndDate;
                if (!DateTimeOffset.TryParse(ConfigurationManager.AppSettings["PartitionStartDate"], out partitionStartDate))
                {
                    partitionStartDate = DateTimeOffset.FromUnixTimeSeconds(long.Parse(ConfigurationManager.AppSettings["PartitionStartDate"]));
                }
                if (!DateTimeOffset.TryParse(ConfigurationManager.AppSettings["PartitionEndDate"], out partitionEndDate))
                {
                    partitionEndDate = DateTimeOffset.FromUnixTimeSeconds(long.Parse(ConfigurationManager.AppSettings["PartitionEndDate"]));
                }

                TimeSpan interval = TimeSpan.FromTicks((long)((partitionEndDate - partitionStartDate).Ticks / (double)instances));
                getNextPartitionValue = (object value) => (DateTimeOffset)value + interval;
                formatPartitionValue  = (object value) => ((DateTimeOffset)value).ToUnixTimeSeconds().ToString();
            }

            for (int i = 1; i <= instances; i++)
            {
                if (instanceFilter.Any() && !instanceFilter.Contains(i))
                {
                    continue;
                }

                partitionEndValue = getNextPartitionValue(partitionStartValue);
                string suffix      = instances > 1 ? "-" + instances.ToString("000") + "-" + i.ToString("000") : string.Empty;
                string iName       = ConfigurationManager.AppSettings["IndexerNamePrefix"] + suffix;
                string dsName      = string.IsNullOrEmpty(ConfigurationManager.AppSettings["DatasourceNamePrefix"]) ? iName : ConfigurationManager.AppSettings["DatasourceNamePrefix"] + suffix;
                string description = string.Format("Partition {0} of {1} from {2} to {3} ({4}-{5})", i, instances, partitionStartValue, partitionEndValue, formatPartitionValue(partitionStartValue), formatPartitionValue(partitionEndValue));

                if (args.Contains("delete", StringComparer.InvariantCultureIgnoreCase) || args.Contains("del", StringComparer.InvariantCultureIgnoreCase))
                {
                    result = await DeleteObject(iName, "indexers") && await DeleteObject(dsName, "datasources");

                    if (!result)
                    {
                        return(result);
                    }
                }

                if (args.Contains("create", StringComparer.InvariantCultureIgnoreCase))
                {
                    Console.WriteLine();
                    Console.WriteLine(description);
                    result = await CreateDataSource(dsName, description, formatPartitionValue(partitionStartValue), formatPartitionValue(partitionEndValue));

                    if (!result)
                    {
                        return(result);
                    }

                    result = await CreateIndexer(iName, dsName, description, disabled : true);

                    if (!result)
                    {
                        return(result);
                    }
                }

                if (args.Contains("disable", StringComparer.InvariantCultureIgnoreCase))
                {
                    result = await CreateIndexer(iName, dsName, description, disabled : true);

                    if (!result)
                    {
                        Console.WriteLine("Could not disable " + iName);
                    }
                }

                if (args.Contains("enable", StringComparer.InvariantCultureIgnoreCase))
                {
                    result = await CreateIndexer(iName, dsName, description, disabled : false);

                    if (!result)
                    {
                        Console.WriteLine("Could not enable " + iName);
                    }
                }

                if (args.Contains("reset", StringComparer.InvariantCultureIgnoreCase))
                {
                    result = await ResetIndexer(iName);

                    Console.WriteLine("Could not reset " + iName);
                }

                if (args.Contains("run", StringComparer.InvariantCultureIgnoreCase))
                {
                    result = await RunIndexer(iName);

                    if (!result)
                    {
                        Console.WriteLine("Could not run " + iName);
                    }
                }

                if (args.Contains("status", StringComparer.InvariantCultureIgnoreCase) || args.Contains("stats", StringComparer.InvariantCultureIgnoreCase) || args.Contains("stat", StringComparer.InvariantCultureIgnoreCase))
                {
                    var stat = await GetIndexerStatus(iName);

                    IndexerExecutionResult lastresult = stat?.LastResult;

                    // combine the results
                    if (lastresult != null)
                    {
                        int take = args.Where(a => a.StartsWith("/take:", StringComparison.InvariantCultureIgnoreCase)).Select(a => (int?)int.Parse(a.Substring(6))).FirstOrDefault() ?? 1;
                        int skip = args.Where(a => a.StartsWith("/skip:", StringComparison.InvariantCultureIgnoreCase)).Select(a => (int?)int.Parse(a.Substring(6))).FirstOrDefault() ?? 0;
                        var hist = stat.ExecutionHistory.Where(h => h.StartTime != stat.LastResult.StartTime);

                        hist       = new[] { lastresult }.Union(hist).Skip(skip).Take(take);
                        lastresult = !hist.Any() ? null : new IndexerExecutionResult(
                            hist.First().Status,
                            string.Join(" | ", hist.Select(h => h.ErrorMessage)),
                            hist.Last().StartTime,
                            hist.First().EndTime,
                            hist.Reverse().SelectMany(h => h.Errors).ToList(),
                            hist.Reverse().SelectMany(h => h.Warnings).ToList(),
                            hist.Sum(h => h.ItemCount),
                            hist.Sum(h => h.FailedItemCount),
                            hist.Last().InitialTrackingState,
                            hist.First().FinalTrackingState
                            );
                    }

                    Console.WriteLine("{0,18} ({7}): {2,10} - Documents Processed: {3,-3}  Failed: {4,-3}  Warn({8}) Err({6}): {5}",
                                      iName,
                                      stat.Status,
                                      lastresult?.Status,
                                      lastresult?.ItemCount,
                                      lastresult?.FailedItemCount,
                                      lastresult?.ErrorMessage,
                                      lastresult?.Errors?.Count,
                                      ((lastresult?.EndTime ?? DateTimeOffset.UtcNow) - lastresult?.StartTime)?.ToString(@"hh\:mm\:ss"),
                                      lastresult?.Warnings?.Count
                                      );
                    if (args.Contains("errors", StringComparer.InvariantCultureIgnoreCase) || args.Contains("error", StringComparer.InvariantCultureIgnoreCase) || args.Contains("err", StringComparer.InvariantCultureIgnoreCase))
                    {
                        if (lastresult?.Errors?.Count > 0)
                        {
                            Console.WriteLine("Errors:");
                            foreach (var err in lastresult.Errors)
                            {
                                Console.WriteLine("      {0} - {1}",
                                                  GetKeyValue(err.Key),
                                                  err.ErrorMessage);
                            }
                        }
                    }
                    if (args.Contains("warnings", StringComparer.InvariantCultureIgnoreCase) || args.Contains("warning", StringComparer.InvariantCultureIgnoreCase) || args.Contains("warn", StringComparer.InvariantCultureIgnoreCase))
                    {
                        if (lastresult?.Warnings?.Count > 0)
                        {
                            Console.WriteLine("Warnings:");
                            foreach (var err in lastresult.Warnings)
                            {
                                Console.WriteLine("      {0} - {1}",
                                                  GetKeyValue(err.Key),
                                                  err.Message);
                            }
                        }
                    }
                }

                partitionStartValue = partitionEndValue;
            }


            if (args.Contains("import", StringComparer.InvariantCultureIgnoreCase))
            {
                await ImportData(instances);
            }

            return(result);
        }
        private static void SyncDataFromAzureSQL()
        {
            // This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
            Console.WriteLine("{0}", "Creating Data Source...\n");
            var dataSource =
                DataSource.AzureSql(
                    name: UsgsDataSource,
                    sqlConnectionString: "Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;",
                    tableOrViewName: "GeoNamesRI",
                    description: "USGS Dataset");

            try
            {
                _searchClient.DataSources.Create(dataSource);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating data source: {0}", ex.Message);
                return;
            }

            Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");

            var indexer =
                new Indexer()
            {
                Name            = UsgsIndexer,
                Description     = "USGS data indexer",
                DataSourceName  = dataSource.Name,
                TargetIndexName = GeoNamesIndex
            };

            try
            {
                _searchClient.Indexers.Create(indexer);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
                return;
            }

            bool running = true;

            Console.WriteLine("{0}", "Synchronization running...\n");
            while (running)
            {
                IndexerExecutionInfo status = null;

                try
                {
                    status = _searchClient.Indexers.GetStatus(indexer.Name);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
                    return;
                }

                IndexerExecutionResult lastResult = status.LastResult;
                if (lastResult != null)
                {
                    switch (lastResult.Status)
                    {
                    case IndexerExecutionStatus.InProgress:
                        Console.WriteLine("{0}", "Synchronization running...\n");
                        Thread.Sleep(1000);
                        break;

                    case IndexerExecutionStatus.Success:
                        running = false;
                        Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
                        break;

                    default:
                        running = false;
                        Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
                        break;
                    }
                }
            }
        }
Exemplo n.º 8
0
        private static void CreateAndSyncIndexer()
        {
            // Create a new indexer and sync it
            try
            {
                var        creds = new DataSourceCredentials("Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30");
                DataSource ds    = new DataSource("usgs-datasource", DataSourceType.AzureSql, creds, new DataContainer("GeoNamesRI"));
                ds.Description = "USGS Dataset";

                Indexer idx = new Indexer();
                idx.Name                              = "usgs-indexer";
                idx.Description                       = "USGS data indexer";
                idx.DataSourceName                    = "usgs-datasource";
                idx.TargetIndexName                   = "geonames";
                idx.Parameters                        = new IndexingParameters();
                idx.Parameters.MaxFailedItems         = 10;
                idx.Parameters.MaxFailedItemsPerBatch = 5;
                idx.Parameters.Base64EncodeKeys       = false;

                //Delete indexer and datasource if it existed
                _searchClient.DataSources.Delete("usgs-datasource");
                _searchClient.Indexers.Delete("usgs-indexer");

                //Create indexer and datasource
                _searchClient.DataSources.Create(ds);
                _searchClient.Indexers.Create(idx);

                //Launch the sync and then monitor progress until complete
                bool running = true;

                Console.WriteLine("{0}", "Synchronization running...\n");
                while (running)
                {
                    IndexerExecutionInfo status = null;
                    try
                    {
                        status = _searchClient.Indexers.GetStatus(idx.Name);
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
                        return;
                    }

                    IndexerExecutionResult lastResult = status.LastResult;
                    if (lastResult != null)
                    {
                        switch (lastResult.Status)
                        {
                        case IndexerExecutionStatus.InProgress:
                            Console.WriteLine("{0}", "Synchronization running...\n");
                            Thread.Sleep(1000);
                            break;

                        case IndexerExecutionStatus.Success:
                            running = false;
                            Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
                            break;

                        default:
                            running = false;
                            Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
                            break;
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating indexer: {0}: \n", ex.Message.ToString());
            }
        }
Exemplo n.º 9
0
        public void CanRunIndexerAndGetIndexerStatus()
        {
            Run(() =>
            {
                // Set handler that injects mock_status query string, which results in service
                // returning a well-known mock response
                SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler());

                Indexer indexer = Data.CreateTestIndexer();

                searchClient.Indexers.Create(indexer);

                IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name);
                Assert.Equal(IndexerStatus.Running, info.Status);

                AzureOperationResponse runResponse =
                    searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result;
                Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode);

                info = searchClient.Indexers.GetStatus(indexer.Name);
                Assert.Equal(IndexerStatus.Running, info.Status);

                Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
                Assert.Equal(3, info.ExecutionHistory.Count);

                Assert.NotNull(info.Limits);
                Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract);
                Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize);
                Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime);

                IndexerExecutionResult newestResult = info.ExecutionHistory[0];
                IndexerExecutionResult middleResult = info.ExecutionHistory[1];
                IndexerExecutionResult oldestResult = info.ExecutionHistory[2];

                Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
                Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
                AssertStartAndEndTimeValid(newestResult);

                Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
                AssertStartAndEndTimeValid(middleResult);

                Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
                Assert.Equal(124876, oldestResult.ItemCount);
                Assert.Equal(2, oldestResult.FailedItemCount);
                Assert.Equal("100", oldestResult.InitialTrackingState);
                Assert.Equal("200", oldestResult.FinalTrackingState);
                AssertStartAndEndTimeValid(oldestResult);

                Assert.Equal(2, oldestResult.Errors.Count);

                Assert.Equal("1", oldestResult.Errors[0].Key);
                Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);

                Assert.Equal("121713", oldestResult.Errors[1].Key);
                Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);

                Assert.Equal(1, oldestResult.Warnings.Count);
                Assert.Equal("2", oldestResult.Warnings[0].Key);
                Assert.Equal("This is the first and last warning", oldestResult.Warnings[0].Message);
            });
        }
        private static void SyncDataFromBlobSorage()
        {
            // This will use the Azure Search Indexer to synchronize data from Blob Storage to Azure Search
            Console.WriteLine("{0}", "Creating Data Source...\n");
            var dataSource =
                DataSource.AzureBlobStorage(
                    name: InterpreterIntelligenceDataSource,
                    storageConnectionString:
                    containerName: "");

            try
            {
                _searchClient.DataSources.Create(dataSource);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating data source: {0}", ex.Message);
                return;
            }

            Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");

            var indexer =
                new Indexer()
            {
                Name            = InterpreterIntelligenceIndexer,
                Description     = "Interpreter Intelligence indexer",
                DataSourceName  = dataSource.Name,
                TargetIndexName = InterpreterIntelligenceIndex
            };

            try
            {
                _searchClient.Indexers.Create(indexer);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
                return;
            }

            bool running = true;

            Console.WriteLine("{0}", "Synchronization running...\n");
            while (running)
            {
                IndexerExecutionInfo status = null;

                try
                {
                    status = _searchClient.Indexers.GetStatus(indexer.Name);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
                    return;
                }

                IndexerExecutionResult lastResult = status.LastResult;
                if (lastResult != null)
                {
                    switch (lastResult.Status)
                    {
                    case IndexerExecutionStatus.InProgress:
                        Console.WriteLine("{0}", "Synchronization running...\n");
                        Thread.Sleep(1000);
                        break;

                    case IndexerExecutionStatus.Success:
                        running = false;
                        Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
                        break;

                    default:
                        running = false;
                        Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
                        break;
                    }
                }
            }
        }
Exemplo n.º 11
0
        public void CanRunIndexerAndGetIndexerStatus()
        {
            Run(() =>
            {
                // Set handler that injects mock_status query string, which results in service
                // returning a well-known mock response
                SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler());

                Indexer indexer = Data.CreateTestIndexer();

                searchClient.Indexers.Create(indexer);

                IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name);
                Assert.Equal(IndexerStatus.Running, info.Status);

                AzureOperationResponse runResponse =
                    searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result;
                Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode);

                info = searchClient.Indexers.GetStatus(indexer.Name);
                Assert.Equal(IndexerStatus.Running, info.Status);

                Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
                Assert.Equal(3, info.ExecutionHistory.Count);

                Assert.NotNull(info.Limits);
                Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract);
                Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize);
                Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime);

                IndexerExecutionResult newestResult = info.ExecutionHistory[0];
                IndexerExecutionResult middleResult = info.ExecutionHistory[1];
                IndexerExecutionResult oldestResult = info.ExecutionHistory[2];

                Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
                Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
                AssertStartAndEndTimeValid(newestResult);

                Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
                AssertStartAndEndTimeValid(middleResult);

                Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
                Assert.Equal(124876, oldestResult.ItemCount);
                Assert.Equal(2, oldestResult.FailedItemCount);
                Assert.Equal("100", oldestResult.InitialTrackingState);
                Assert.Equal("200", oldestResult.FinalTrackingState);
                AssertStartAndEndTimeValid(oldestResult);

                Assert.Equal(2, oldestResult.Errors.Count);

                Assert.Equal("1", oldestResult.Errors[0].Key);
                Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);
                Assert.Equal("DocumentExtraction.AzureBlob.MyDataSource", oldestResult.Errors[0].Name);
                Assert.Equal("The file could not be parsed.", oldestResult.Errors[0].Details);
                Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[0].DocumentationLink);

                Assert.Equal("121713", oldestResult.Errors[1].Key);
                Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
                Assert.Equal("DocumentExtraction.AzureBlob.DataReader", oldestResult.Errors[1].Name);
                Assert.Equal("Blob size cannot exceed 256 MB.", oldestResult.Errors[1].Details);
                Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[1].DocumentationLink);

                Assert.Equal(1, oldestResult.Warnings.Count);
                Assert.Equal("2", oldestResult.Warnings[0].Key);
                Assert.Equal("Document was truncated to 50000 characters.", oldestResult.Warnings[0].Message);
                Assert.Equal("Enrichment.LanguageDetectionSkill.#4", oldestResult.Warnings[0].Name);
                Assert.Equal("Try to split the input into smaller chunks using Split skill.", oldestResult.Warnings[0].Details);
                Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2099692", oldestResult.Warnings[0].DocumentationLink);
            });
        }
Exemplo n.º 12
0
        private static void SyncDataFromAzureSQL()
        {
            // This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
            Console.WriteLine("{0}", "Creating Data Source...\n");
            var dataSource =
                new DataSource()
            {
                Name        = AssetDataSource,
                Description = "AssetManagement DataSet",
                Type        = DataSourceType.AzureSql,
                Credentials = new DataSourceCredentials("Server=fckpvybjkl.database.windows.net,1433;Database=AssetManagement;User ID=brucewright18;Password=Mallard18;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;"),
                Container   = new DataContainer("vAssetPwoCustomerSearch")
            };

            try
            {
                _searchClient.DataSources.Create(dataSource);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating data source: {0}", ex.Message);
                return;
            }

            Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");

            var indexer =
                new Indexer()
            {
                Name            = AssetIndexer,
                Description     = "Asset Management data indexer",
                DataSourceName  = dataSource.Name,
                TargetIndexName = AssetIndex
            };

            try
            {
                _searchClient.Indexers.Create(indexer);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
                return;
            }

            bool running = true;

            Console.WriteLine("{0}", "Synchronization running...\n");
            while (running)
            {
                IndexerExecutionInfo status = null;

                try
                {
                    status = _searchClient.Indexers.GetStatus(indexer.Name);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
                    return;
                }

                IndexerExecutionResult lastResult = status.LastResult;
                if (lastResult != null)
                {
                    switch (lastResult.Status)
                    {
                    case IndexerExecutionStatus.InProgress:
                        Console.WriteLine("{0}", "Synchronization running...\n");
                        Thread.Sleep(1000);
                        break;

                    case IndexerExecutionStatus.Success:
                        running = false;
                        Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
                        break;

                    default:
                        running = false;
                        Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
                        break;
                    }
                }
            }
        }
Exemplo n.º 13
0
        private static void SyncDataFromAzureSQL()
        {
            // This will use the Azure Search Indexer to synchronize data from Azure Blob Storage to Azure Search
            Console.WriteLine("{0}", "Creating Data Source...\n");
            var dataSource =
                DataSource.AzureBlobStorage(
                    name: UsgsDataSource,
                    storageConnectionString: "DefaultEndpointsProtocol=https;AccountName=ronanblobstorage;AccountKey=Z8nPhGaeduop96WgvZ2FP6QKGNjXeN/G8RmyoJtX97Cycusq5WOaIymucItSNnrv31ChYMICh04nmyHbCmySYw==;EndpointSuffix=core.windows.net;",
                    containerName: "testmovies",
                    description: "movies Json");

            try
            {
                _searchClient.DataSources.Create(dataSource);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating data source: {0}", ex.Message);
                return;
            }

            Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");

            var indexer =
                new Indexer()
            {
                Name            = UsgsIndexer,
                Description     = "USGS data indexer",
                DataSourceName  = dataSource.Name,
                TargetIndexName = GeoNamesIndex
            };

            try
            {
                _searchClient.Indexers.Create(indexer);
            }
            catch (Exception ex)
            {
                Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
                return;
            }

            bool running = true;

            Console.WriteLine("{0}", "Synchronization running...\n");
            while (running)
            {
                IndexerExecutionInfo status = null;

                try
                {
                    status = _searchClient.Indexers.GetStatus(indexer.Name);
                }
                catch (Exception ex)
                {
                    Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
                    return;
                }

                IndexerExecutionResult lastResult = status.LastResult;
                if (lastResult != null)
                {
                    switch (lastResult.Status)
                    {
                    case IndexerExecutionStatus.InProgress:
                        Console.WriteLine("{0}", "Synchronization running...\n");
                        Thread.Sleep(1000);
                        break;

                    case IndexerExecutionStatus.Success:
                        running = false;
                        Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
                        break;

                    default:
                        running = false;
                        Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
                        break;
                    }
                }
            }
        }
Exemplo n.º 14
0
 public static SearchIndexerStatus SearchIndexerStatus(IndexerStatus status = default, IndexerExecutionResult lastResult = default, IReadOnlyList <IndexerExecutionResult> executionHistory = default, SearchIndexerLimits limits = default)
 {
     executionHistory ??= new List <IndexerExecutionResult>();
     return(new SearchIndexerStatus(status, lastResult, executionHistory, limits));
 }
        private static void SyncDataFromAzureSql()
        {
            // This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
            Console.WriteLine("Creating Data Source...");
            string geoNamesIndex  = ConfigurationManager.AppSettings["SearchGeoNamesIndex"];
            string usgsIndexer    = ConfigurationManager.AppSettings["SearchUsageindexer"];
            string usgsDataSource = ConfigurationManager.AppSettings["SearchUsageDataSource"];

            var dataSource =
                DataSource.AzureSql(
                    usgsDataSource,
                    ConfigurationManager.AppSettings["SearchSqlSourceConnectionString"],
                    ConfigurationManager.AppSettings["SearchSqlSourceTableOrView"],
                    description: ConfigurationManager.AppSettings["SearchSqlSourceDescription"]);

            try
            {
                _searchClient.DataSources.Create(dataSource);
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Error creating data source: {ex.Message}");
                return;
            }

            Console.WriteLine("Creating Indexer and syncing data...");

            Stopwatch stopwatch = new Stopwatch();

            stopwatch.Start();

            var indexer = new Indexer
            {
                Name            = usgsIndexer,
                Description     = "Alpha data indexer",
                DataSourceName  = dataSource.Name,
                TargetIndexName = geoNamesIndex
            };

            try
            {
                _searchClient.Indexers.Create(indexer);
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Error creating and running indexer: {ex.Message}");
                return;
            }

            Console.WriteLine("{0}", "Synchronization running...");

            _keepSpinning = true;
            Thread spinner = new Thread(Spin);

            spinner.Start();

            while (_keepSpinning)
            {
                IndexerExecutionInfo status;

                try
                {
                    status = _searchClient.Indexers.GetStatus(indexer.Name);
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"Error polling for indexer status: {ex.Message}");
                    return;
                }

                IndexerExecutionResult lastResult = status.LastResult;
                if (lastResult == null)
                {
                    continue;
                }

                switch (lastResult.Status)
                {
                case IndexerExecutionStatus.InProgress:
                    break;

                case IndexerExecutionStatus.Success:
                    _keepSpinning = false;
                    stopwatch.Stop();
                    _keepSpinning = false;
                    Console.WriteLine($"Synchronized {lastResult.ItemCount} rows in {stopwatch.ElapsedMilliseconds}ms");
                    break;

                case IndexerExecutionStatus.TransientFailure:
                    _keepSpinning = false;
                    Console.WriteLine($"Transient Failure: {lastResult.ErrorMessage}");
                    break;

                case IndexerExecutionStatus.Reset:
                    Console.WriteLine("Reset");
                    break;

                default:
                    _keepSpinning = false;
                    Console.WriteLine($"Synchronization failed: {lastResult.ErrorMessage}");
                    break;
                }
            }
        }