public void CanRunIndexerAndGetIndexerStatus()
{
Run(() =>
{
SearchServiceClient searchClient = Data.GetSearchServiceClient();
Indexer indexer = Data.CreateTestIndexer();
IndexerDefinitionResponse createResponse = searchClient.Indexers.Create(indexer);
Assert.Equal(HttpStatusCode.Created, createResponse.StatusCode);
IndexerGetStatusResponse statusResponse = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode);
IndexerExecutionInfo info = statusResponse.ExecutionInfo;
Assert.Equal(IndexerStatus.Running, info.Status);
AzureOperationResponse runResponse = searchClient.Indexers.Run(indexer.Name);
Assert.Equal(HttpStatusCode.Accepted, runResponse.StatusCode);
// Set handler that injects mock_status query string, which results in service
// returning a well-known mock response
searchClient.AddHandlerToPipeline(new MockStatusDelegatingHandler());
statusResponse = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode);
info = statusResponse.ExecutionInfo;
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
Assert.Equal(3, info.ExecutionHistory.Count);
IndexerExecutionResult newestResult = info.ExecutionHistory[0];
IndexerExecutionResult middleResult = info.ExecutionHistory[1];
IndexerExecutionResult oldestResult = info.ExecutionHistory[2];
Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
AssertStartAndEndTimeValid(newestResult);
Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
AssertStartAndEndTimeValid(middleResult);
Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
Assert.Equal(124876, oldestResult.ItemCount);
Assert.Equal(2, oldestResult.FailedItemCount);
Assert.Equal("100", oldestResult.InitialTrackingState);
Assert.Equal("200", oldestResult.FinalTrackingState);
AssertStartAndEndTimeValid(oldestResult);
Assert.Equal(2, oldestResult.Errors.Count);
Assert.Equal("1", oldestResult.Errors[0].Key);
Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);
Assert.Equal("121713", oldestResult.Errors[1].Key);
Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
});
}
static void CheckIndexerStatus(Indexer indexer, SearchServiceClient searchService)
{
try
{
IndexerExecutionInfo execInfo = searchService.Indexers.GetStatus(indexer.Name);
Console.WriteLine("Indexer has run {0} times.", execInfo.ExecutionHistory.Count);
Console.WriteLine("Indexer Status: " + execInfo.Status.ToString());
IndexerExecutionResult result = execInfo.LastResult;
Console.WriteLine("Latest run");
Console.WriteLine(" Run Status: {0}", result.Status.ToString());
Console.WriteLine(" Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount);
TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value;
Console.WriteLine(" StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed);
string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage;
Console.WriteLine(" ErrorMessage: {0}", errorMsg);
Console.WriteLine(" Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count);
}
catch (Exception e)
{
// Handle exception
}
}
private static void CheckIndexerOverallStatus(SearchServiceClient searchClient, Indexer indexer)
{
try
{
// Get the index status
IndexerExecutionInfo indexerExecutionInfo = searchClient.Indexers.GetStatus(indexer.Name);
switch (indexerExecutionInfo.Status)
{
case IndexerStatus.Error:
Console.WriteLine("\nIndexer has error status. Check the Azure Portal to further understand the error.");
break;
case IndexerStatus.Running:
Console.WriteLine("\nIndexer is running");
break;
case IndexerStatus.Unknown:
Console.WriteLine("\nIndexer status is unknown");
break;
default:
Console.WriteLine("\nNo indexer information");
break;
}
}
catch (Exception e)
{
Console.WriteLine("\nFailed to get indexer overall status\n Exception message: {0}\n", e.Message);
}
}
/*
* This function checks the status of the indexer every 5 seconds
* and writes the current status to the console
*/
private static async Task <bool> CheckIndexerStatus()
{
Console.WriteLine("Waiting for indexing to complete...");
IndexerExecutionStatus requestStatus = IndexerExecutionStatus.InProgress;
try
{
await _searchClient.Indexers.GetAsync(IndexerName);
while (requestStatus.Equals(IndexerExecutionStatus.InProgress))
{
Thread.Sleep(5000);
IndexerExecutionInfo info = await _searchClient.Indexers.GetStatusAsync(IndexerName);
requestStatus = info.LastResult.Status;
if (DebugMode)
{
Console.WriteLine("Indexer Status: " + requestStatus.ToString());
}
}
}
catch (Exception ex)
{
if (DebugMode)
{
Console.WriteLine("ERROR RETREIVING INDEXER STATUS: " + ex.Message + "\n\n" + ex.StackTrace);
}
return(false);
}
return(requestStatus.Equals(IndexerExecutionStatus.Success));
}
private static async Task <bool> CheckIndexerStatus()
{
Console.WriteLine("Waiting for indexing to complete...");
IndexerExecutionStatus requestStatus = IndexerExecutionStatus.InProgress;
try
{
await _searchClient.Indexers.GetAsync(IndexerName);
while (requestStatus.Equals(IndexerExecutionStatus.InProgress))
{
Thread.Sleep(3000);
IndexerExecutionInfo info = await _searchClient.Indexers.GetStatusAsync(IndexerName);
requestStatus = info.LastResult.Status;
if (DebugMode)
{
Console.WriteLine("Current indexer status: {0}", requestStatus.ToString());
}
}
}
catch (Exception ex)
{
if (DebugMode)
{
Console.WriteLine("Error retrieving indexer status: {0}", ex.Message);
}
return(false);
}
return(requestStatus.Equals(IndexerExecutionStatus.Success));
}
private async Task CheckIndexerStatus(string[] fileNames, Guid anonymizationId, int referralId)
{
Console.WriteLine("Waiting for indexing to complete...");
IndexerExecutionStatus requestStatus = IndexerExecutionStatus.InProgress;
try
{
await _searchClient.Indexers.GetAsync(_configuration["IndexerName"]);
while (requestStatus.Equals(IndexerExecutionStatus.InProgress))
{
Thread.Sleep(3000);
IndexerExecutionInfo info = await _searchClient.Indexers.GetStatusAsync(_configuration["IndexerName"]);
requestStatus = info.LastResult.Status;
Console.WriteLine("Current indexer status: {0}", requestStatus.ToString());
}
}
catch (Exception ex)
{
Console.WriteLine("Error retrieving indexer status: {0}", ex.Message);
}
ISearchIndexClient indexClient = _searchClient.Indexes.GetClient(_configuration["IndexName"]);
foreach (var f in fileNames)
{
var searchResult = await indexClient.Documents.SearchAsync("*", new SearchParameters { Filter = $"(search.in(fileName,'{f}', '|'))" });
foreach (var result in searchResult.Results)
{
if (result.Document["fileName"].ToString() == f)
{
var uploadBatch = new List <IndexSchema>();
var indexDoc = new IndexSchema
{
id = result.Document["id"].ToString(),
anonymizationId = anonymizationId.ToString(),
referralId = referralId.ToString()
};
uploadBatch.Add(indexDoc);
var batch = IndexBatch.MergeOrUpload(uploadBatch);
await indexClient.Documents.IndexAsync(batch);
uploadBatch.Clear();
}
}
}
}
public void CanResetIndexerAndGetIndexerStatus()
{
Run(() =>
{
SearchServiceClient searchClient = Data.GetSearchServiceClient();
Indexer indexer = Data.CreateTestIndexer();
searchClient.Indexers.Create(indexer);
searchClient.Indexers.Reset(indexer.Name);
IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.Reset, info.LastResult.Status);
});
}
public static async Task RunAsync(
[BlobTrigger("datum/{name}", Connection = "Search")] CloudBlockBlob myBlob,
[CosmosDB("taskDatabase", "TaskCollection", ConnectionStringSetting = "CosmosDB")] IAsyncCollector <object> todos, string name, ILogger log)
{
SearchServiceClient serviceClient = new SearchServiceClient(searchServiceName: "jfk-search-service-qymjpzort5hho", credentials: new SearchCredentials("023DB82430FFA416AD39EEF8A6FDFF2A"));
ISearchIndexClient indexClient = serviceClient.Indexes.GetClient("jfkindex");
await serviceClient.Indexers.RunAsync("jfkindexer");
try
{
IndexerExecutionInfo execInfo = serviceClient.Indexers.GetStatus("jfkindexer");
Console.WriteLine("{0} veces ejecutadas.", execInfo.ExecutionHistory.Count);
Console.WriteLine("Indexer Status: " + execInfo.Status.ToString());
IndexerExecutionResult result = execInfo.LastResult;
Console.WriteLine("Latest run");
Console.WriteLine(" Run Status: {0}", result.Status.ToString());
Console.WriteLine(" Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount);
TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value;
Console.WriteLine(" StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed);
string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage;
Console.WriteLine(" ErrorMessage: {0}", errorMsg);
Console.WriteLine(" Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count);
}
catch (Exception e) { Console.WriteLine("Error: ", e); }
var param = new SearchParameters
{
Select = new[] { "*" },
//IncludeTotalResultCount = true
};
System.Threading.Thread.Sleep(19000);
log.LogInformation($"espera terminada{name}");
var results = indexClient.Documents.Search <Book>($"{name}", param);
log.LogInformation($"enviando a cosmos.... {myBlob.Name}");
await todos.AddAsync(results);
}
private bool IndexBlobStorageStatus(SearchServiceClient client)
{
try
{
IndexerExecutionInfo demoIndexerExecutionInfo = client.Indexers.GetStatus(indexerName);
switch (demoIndexerExecutionInfo.Status)
{
case IndexerStatus.Error:
WriteLine("Indexer has error status");
return(false);
case IndexerStatus.Running:
WriteLine("Indexer is running");
if (demoIndexerExecutionInfo.LastResult.Status == IndexerExecutionStatus.Success)
{
return(true);
}
else
{
System.Threading.Thread.Sleep(5000);
return(IndexBlobStorageStatus(client));
}
case IndexerStatus.Unknown:
WriteLine("Indexer status is unknown");
return(false);
default:
WriteLine("No indexer information. Assumign Completed");
return(true);
}
}
catch (Exception ex)
{
WriteLine("Indexer Failed - " + ex.ToString());
return(false);
}
}
public void CanResetIndexerAndGetIndexerStatus()
{
Run(() =>
{
SearchServiceClient searchClient = Data.GetSearchServiceClient();
Indexer indexer = Data.CreateTestIndexer();
IndexerDefinitionResponse createResponse = searchClient.Indexers.Create(indexer);
Assert.Equal(HttpStatusCode.Created, createResponse.StatusCode);
AzureOperationResponse resetResponse = searchClient.Indexers.Reset(indexer.Name);
Assert.Equal(HttpStatusCode.NoContent, resetResponse.StatusCode);
IndexerGetStatusResponse statusResponse = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode);
IndexerExecutionInfo info = statusResponse.ExecutionInfo;
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.Reset, info.LastResult.Status);
});
}
private static void SyncDataFromBlobSorage()
{
// This will use the Azure Search Indexer to synchronize data from Blob Storage to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
DataSource.AzureBlobStorage(
name: InterpreterIntelligenceDataSource,
storageConnectionString:
containerName: "");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = InterpreterIntelligenceIndexer,
Description = "Interpreter Intelligence indexer",
DataSourceName = dataSource.Name,
TargetIndexName = InterpreterIntelligenceIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static void SyncDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure Blob Storage to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
DataSource.AzureBlobStorage(
name: UsgsDataSource,
storageConnectionString: "DefaultEndpointsProtocol=https;AccountName=ronanblobstorage;AccountKey=Z8nPhGaeduop96WgvZ2FP6QKGNjXeN/G8RmyoJtX97Cycusq5WOaIymucItSNnrv31ChYMICh04nmyHbCmySYw==;EndpointSuffix=core.windows.net;",
containerName: "testmovies",
description: "movies Json");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = UsgsIndexer,
Description = "USGS data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = GeoNamesIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static void SyncProductDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
Console.WriteLine("{0}", "Creating Product Data Source...\n");
var dataSource =
DataSource.AzureSql(
name: MowidoDataSource,
sqlConnectionString: "Server=tcp:mowido.database.windows.net,1433;Initial Catalog=MowidoDB;Persist Security Info=False;User ID=mowido;Password=Billions123;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;",
tableOrViewName: "Products",
description: "Mowido Dataset");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = MowidoIndexer,
Description = "Mowido start search product data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = MowidoProductIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static void SyncDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
new DataSource()
{
Name = AssetDataSource,
Description = "AssetManagement DataSet",
Type = DataSourceType.AzureSql,
Credentials = new DataSourceCredentials("Server=fckpvybjkl.database.windows.net,1433;Database=AssetManagement;User ID=brucewright18;Password=Mallard18;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;"),
Container = new DataContainer("vAssetPwoCustomerSearch")
};
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = AssetIndexer,
Description = "Asset Management data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = AssetIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static void SyncDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
DataSource.AzureSql(
name: UsgsDataSource,
sqlConnectionString: "Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;",
tableOrViewName: "GeoNamesRI",
description: "USGS Dataset");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = UsgsIndexer,
Description = "USGS data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = GeoNamesIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static void CreateAndSyncIndexer()
{
// Create a new indexer and sync it
try
{
var creds = new DataSourceCredentials("Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30");
DataSource ds = new DataSource("usgs-datasource", DataSourceType.AzureSql, creds, new DataContainer("GeoNamesRI"));
ds.Description = "USGS Dataset";
Indexer idx = new Indexer();
idx.Name = "usgs-indexer";
idx.Description = "USGS data indexer";
idx.DataSourceName = "usgs-datasource";
idx.TargetIndexName = "geonames";
idx.Parameters = new IndexingParameters();
idx.Parameters.MaxFailedItems = 10;
idx.Parameters.MaxFailedItemsPerBatch = 5;
idx.Parameters.Base64EncodeKeys = false;
//Delete indexer and datasource if it existed
_searchClient.DataSources.Delete("usgs-datasource");
_searchClient.Indexers.Delete("usgs-indexer");
//Create indexer and datasource
_searchClient.DataSources.Create(ds);
_searchClient.Indexers.Create(idx);
//Launch the sync and then monitor progress until complete
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(idx.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("Error creating indexer: {0}: \n", ex.Message.ToString());
}
}
public void CanRunIndexerAndGetIndexerStatus()
{
Run(() =>
{
// Set handler that injects mock_status query string, which results in service
// returning a well-known mock response
SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler());
Indexer indexer = Data.CreateTestIndexer();
searchClient.Indexers.Create(indexer);
IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
AzureOperationResponse runResponse =
searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result;
Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode);
info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
Assert.Equal(3, info.ExecutionHistory.Count);
Assert.NotNull(info.Limits);
Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract);
Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize);
Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime);
IndexerExecutionResult newestResult = info.ExecutionHistory[0];
IndexerExecutionResult middleResult = info.ExecutionHistory[1];
IndexerExecutionResult oldestResult = info.ExecutionHistory[2];
Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
AssertStartAndEndTimeValid(newestResult);
Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
AssertStartAndEndTimeValid(middleResult);
Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
Assert.Equal(124876, oldestResult.ItemCount);
Assert.Equal(2, oldestResult.FailedItemCount);
Assert.Equal("100", oldestResult.InitialTrackingState);
Assert.Equal("200", oldestResult.FinalTrackingState);
AssertStartAndEndTimeValid(oldestResult);
Assert.Equal(2, oldestResult.Errors.Count);
Assert.Equal("1", oldestResult.Errors[0].Key);
Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);
Assert.Equal("121713", oldestResult.Errors[1].Key);
Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
Assert.Equal(1, oldestResult.Warnings.Count);
Assert.Equal("2", oldestResult.Warnings[0].Key);
Assert.Equal("This is the first and last warning", oldestResult.Warnings[0].Message);
});
}
public async Task CreateIndex(string searchAdminKey, string cosmosConnectionString, SearchConfig searchConfig)
{
Console.WriteLine("{0}", "Retrieving Search Service\n");
var azureSearchService = SearchHelper.GetSearchServiceClient(searchConfig.SearchServiceName, searchAdminKey);
if (azureSearchService == null)
{
throw new WebException("Unable to find Search Service");
}
Console.WriteLine("{0}", "Deleting Index...\n");
SearchHelper.DeleteIndexIfExists(searchConfig.CustomerSearchIndexName);
Console.WriteLine("{0}", "Creating Index Model...\n");
var indexModelForCustomer = IndexModelHelper.CreateIndexModelForCustomer(searchConfig.CustomerSearchIndexName);
if (indexModelForCustomer == null)
{
Console.WriteLine("Unable to create Index Model");
throw new Exception("Unable to create Index Model");
}
Console.WriteLine("{0}", "Creating Index...\n");
SearchHelper.CreateIndex(indexModelForCustomer);
Console.WriteLine("{0}", "Creating Data Source object...\n");
var dataSource = DataSourceHelper.CreateDataSource(searchConfig.CustomerSearchDataSourceQuery, searchConfig.CustomerCollectionId,
searchConfig.CustomerSearchIndexName, searchConfig.CustomerSearchDataSourceName, cosmosConnectionString);
try
{
Console.WriteLine("{0}", "Attempting to Create/Update Data Source...\n");
await azureSearchService.DataSources.CreateOrUpdateWithHttpMessagesAsync(dataSource);
}
catch (CloudException e)
{
Console.WriteLine(e.ToString());
throw;
}
Indexer indexer;
try
{
Console.WriteLine("{0}", "Attempting to Create Indexer...\n");
indexer = IndexerHelper.CreateIndexer(azureSearchService, searchConfig.CustomerSearchIndexerName, indexModelForCustomer, searchConfig.CustomerSearchDataSourceName);
}
catch (CloudException e)
{
Console.WriteLine("{0}{1}", "Unable to Create Indexer...\n", e);
throw;
}
Console.WriteLine("{0}", "Run Indexer...\n");
try
{
azureSearchService.Indexers.Run(indexer.Name);
}
catch (CloudException e)
{
Console.WriteLine("{0} {1}", "Unable to get data for the Indexer...\n", e);
throw;
}
var running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = azureSearchService.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
throw;
}
var lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.Reset:
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0} Status: {1}, Item Count: {2}", "Synchronization running...\n", lastResult.Status, lastResult.ItemCount);
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount.ToString());
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
Console.WriteLine("{0} Status: {1}", "Completed... ", HttpStatusCode.Created);
}
public void CanRunIndexerAndGetIndexerStatus()
{
Run(() =>
{
// Set handler that injects mock_status query string, which results in service
// returning a well-known mock response
SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler());
Indexer indexer = Data.CreateTestIndexer();
searchClient.Indexers.Create(indexer);
IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
AzureOperationResponse runResponse =
searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result;
Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode);
info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
Assert.Equal(3, info.ExecutionHistory.Count);
Assert.NotNull(info.Limits);
Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract);
Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize);
Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime);
IndexerExecutionResult newestResult = info.ExecutionHistory[0];
IndexerExecutionResult middleResult = info.ExecutionHistory[1];
IndexerExecutionResult oldestResult = info.ExecutionHistory[2];
Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
AssertStartAndEndTimeValid(newestResult);
Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
AssertStartAndEndTimeValid(middleResult);
Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
Assert.Equal(124876, oldestResult.ItemCount);
Assert.Equal(2, oldestResult.FailedItemCount);
Assert.Equal("100", oldestResult.InitialTrackingState);
Assert.Equal("200", oldestResult.FinalTrackingState);
AssertStartAndEndTimeValid(oldestResult);
Assert.Equal(2, oldestResult.Errors.Count);
Assert.Equal("1", oldestResult.Errors[0].Key);
Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);
Assert.Equal("DocumentExtraction.AzureBlob.MyDataSource", oldestResult.Errors[0].Name);
Assert.Equal("The file could not be parsed.", oldestResult.Errors[0].Details);
Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[0].DocumentationLink);
Assert.Equal("121713", oldestResult.Errors[1].Key);
Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
Assert.Equal("DocumentExtraction.AzureBlob.DataReader", oldestResult.Errors[1].Name);
Assert.Equal("Blob size cannot exceed 256 MB.", oldestResult.Errors[1].Details);
Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[1].DocumentationLink);
Assert.Equal(1, oldestResult.Warnings.Count);
Assert.Equal("2", oldestResult.Warnings[0].Key);
Assert.Equal("Document was truncated to 50000 characters.", oldestResult.Warnings[0].Message);
Assert.Equal("Enrichment.LanguageDetectionSkill.#4", oldestResult.Warnings[0].Name);
Assert.Equal("Try to split the input into smaller chunks using Split skill.", oldestResult.Warnings[0].Details);
Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2099692", oldestResult.Warnings[0].DocumentationLink);
});
}
