public void CanRunIndexerAndGetIndexerStatus()
{
Run(() =>
{
SearchServiceClient searchClient = Data.GetSearchServiceClient();
Indexer indexer = Data.CreateTestIndexer();
IndexerDefinitionResponse createResponse = searchClient.Indexers.Create(indexer);
Assert.Equal(HttpStatusCode.Created, createResponse.StatusCode);
IndexerGetStatusResponse statusResponse = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode);
IndexerExecutionInfo info = statusResponse.ExecutionInfo;
Assert.Equal(IndexerStatus.Running, info.Status);
AzureOperationResponse runResponse = searchClient.Indexers.Run(indexer.Name);
Assert.Equal(HttpStatusCode.Accepted, runResponse.StatusCode);
// Set handler that injects mock_status query string, which results in service
// returning a well-known mock response
searchClient.AddHandlerToPipeline(new MockStatusDelegatingHandler());
statusResponse = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode);
info = statusResponse.ExecutionInfo;
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
Assert.Equal(3, info.ExecutionHistory.Count);
IndexerExecutionResult newestResult = info.ExecutionHistory[0];
IndexerExecutionResult middleResult = info.ExecutionHistory[1];
IndexerExecutionResult oldestResult = info.ExecutionHistory[2];
Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
AssertStartAndEndTimeValid(newestResult);
Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
AssertStartAndEndTimeValid(middleResult);
Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
Assert.Equal(124876, oldestResult.ItemCount);
Assert.Equal(2, oldestResult.FailedItemCount);
Assert.Equal("100", oldestResult.InitialTrackingState);
Assert.Equal("200", oldestResult.FinalTrackingState);
AssertStartAndEndTimeValid(oldestResult);
Assert.Equal(2, oldestResult.Errors.Count);
Assert.Equal("1", oldestResult.Errors[0].Key);
Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);
Assert.Equal("121713", oldestResult.Errors[1].Key);
Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
});
}
private static void AssertStartAndEndTimeValid(IndexerExecutionResult result)
{
Assert.True(result.StartTime.HasValue);
Assert.NotEqual(new DateTimeOffset(), result.StartTime.Value);
Assert.True(result.EndTime.HasValue);
Assert.NotEqual(new DateTimeOffset(), result.EndTime.Value);
}
private static void CheckIndexerStatus(SearchIndexerClient indexerClient, SearchIndexer indexer)
{
try
{
string indexerName = "hotels-sql-idxr";
SearchIndexerStatus execInfo = indexerClient.GetIndexerStatus(indexerName);
Console.WriteLine("Indexer has run {0} times.", execInfo.ExecutionHistory.Count);
Console.WriteLine("Indexer Status: " + execInfo.Status.ToString());
IndexerExecutionResult result = execInfo.LastResult;
Console.WriteLine("Latest run");
Console.WriteLine("Run Status: {0}", result.Status.ToString());
Console.WriteLine("Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount);
TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value;
Console.WriteLine("StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed);
string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage;
Console.WriteLine("ErrorMessage: {0}", errorMsg);
Console.WriteLine(" Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count);
}
catch (Exception e)
{
// Handle exception
}
}
public static async Task RunAsync(
[BlobTrigger("datum/{name}", Connection = "Search")] CloudBlockBlob myBlob,
[CosmosDB("taskDatabase", "TaskCollection", ConnectionStringSetting = "CosmosDB")] IAsyncCollector <object> todos, string name, ILogger log)
{
SearchServiceClient serviceClient = new SearchServiceClient(searchServiceName: "jfk-search-service-qymjpzort5hho", credentials: new SearchCredentials("023DB82430FFA416AD39EEF8A6FDFF2A"));
ISearchIndexClient indexClient = serviceClient.Indexes.GetClient("jfkindex");
await serviceClient.Indexers.RunAsync("jfkindexer");
try
{
IndexerExecutionInfo execInfo = serviceClient.Indexers.GetStatus("jfkindexer");
Console.WriteLine("{0} veces ejecutadas.", execInfo.ExecutionHistory.Count);
Console.WriteLine("Indexer Status: " + execInfo.Status.ToString());
IndexerExecutionResult result = execInfo.LastResult;
Console.WriteLine("Latest run");
Console.WriteLine(" Run Status: {0}", result.Status.ToString());
Console.WriteLine(" Total Documents: {0}, Failed: {1}", result.ItemCount, result.FailedItemCount);
TimeSpan elapsed = result.EndTime.Value - result.StartTime.Value;
Console.WriteLine(" StartTime: {0:T}, EndTime: {1:T}, Elapsed: {2:t}", result.StartTime.Value, result.EndTime.Value, elapsed);
string errorMsg = (result.ErrorMessage == null) ? "none" : result.ErrorMessage;
Console.WriteLine(" ErrorMessage: {0}", errorMsg);
Console.WriteLine(" Document Errors: {0}, Warnings: {1}\n", result.Errors.Count, result.Warnings.Count);
}
catch (Exception e) { Console.WriteLine("Error: ", e); }
var param = new SearchParameters
{
Select = new[] { "*" },
//IncludeTotalResultCount = true
};
System.Threading.Thread.Sleep(19000);
log.LogInformation($"espera terminada{name}");
var results = indexClient.Documents.Search <Book>($"{name}", param);
log.LogInformation($"enviando a cosmos.... {myBlob.Name}");
await todos.AddAsync(results);
}
private static void SyncProductDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
Console.WriteLine("{0}", "Creating Product Data Source...\n");
var dataSource =
DataSource.AzureSql(
name: MowidoDataSource,
sqlConnectionString: "Server=tcp:mowido.database.windows.net,1433;Initial Catalog=MowidoDB;Persist Security Info=False;User ID=mowido;Password=Billions123;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;",
tableOrViewName: "Products",
description: "Mowido Dataset");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = MowidoIndexer,
Description = "Mowido start search product data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = MowidoProductIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static async Task <bool> RunAsync(string[] args)
{
bool result = true;
int instances = int.Parse(ConfigurationManager.AppSettings["PartitionCount"] ?? "1");
var instanceFilter = GetNumberRangesFromArguments(args);
Console.WriteLine("Indexer Partitions: {0}", instances);
// use numeric partitioning by default
Func <object, object> getNextPartitionValue = (object value) => (int)value + 1;
Func <object, string> formatPartitionValue = (object value) => ((int)value).ToString(ConfigurationManager.AppSettings["PartitionFormatString"] ?? "0");
object partitionStartValue = 1;
object partitionEndValue;
// use a date time partitioner if Partition Dates are used
if (!string.IsNullOrWhiteSpace(ConfigurationManager.AppSettings["PartitionStartDate"]) && !string.IsNullOrWhiteSpace(ConfigurationManager.AppSettings["PartitionEndDate"]))
{
DateTimeOffset partitionStartDate, partitionEndDate;
if (!DateTimeOffset.TryParse(ConfigurationManager.AppSettings["PartitionStartDate"], out partitionStartDate))
{
partitionStartDate = DateTimeOffset.FromUnixTimeSeconds(long.Parse(ConfigurationManager.AppSettings["PartitionStartDate"]));
}
if (!DateTimeOffset.TryParse(ConfigurationManager.AppSettings["PartitionEndDate"], out partitionEndDate))
{
partitionEndDate = DateTimeOffset.FromUnixTimeSeconds(long.Parse(ConfigurationManager.AppSettings["PartitionEndDate"]));
}
TimeSpan interval = TimeSpan.FromTicks((long)((partitionEndDate - partitionStartDate).Ticks / (double)instances));
getNextPartitionValue = (object value) => (DateTimeOffset)value + interval;
formatPartitionValue = (object value) => ((DateTimeOffset)value).ToUnixTimeSeconds().ToString();
}
for (int i = 1; i <= instances; i++)
{
if (instanceFilter.Any() && !instanceFilter.Contains(i))
{
continue;
}
partitionEndValue = getNextPartitionValue(partitionStartValue);
string suffix = instances > 1 ? "-" + instances.ToString("000") + "-" + i.ToString("000") : string.Empty;
string iName = ConfigurationManager.AppSettings["IndexerNamePrefix"] + suffix;
string dsName = string.IsNullOrEmpty(ConfigurationManager.AppSettings["DatasourceNamePrefix"]) ? iName : ConfigurationManager.AppSettings["DatasourceNamePrefix"] + suffix;
string description = string.Format("Partition {0} of {1} from {2} to {3} ({4}-{5})", i, instances, partitionStartValue, partitionEndValue, formatPartitionValue(partitionStartValue), formatPartitionValue(partitionEndValue));
if (args.Contains("delete", StringComparer.InvariantCultureIgnoreCase) || args.Contains("del", StringComparer.InvariantCultureIgnoreCase))
{
result = await DeleteObject(iName, "indexers") && await DeleteObject(dsName, "datasources");
if (!result)
{
return(result);
}
}
if (args.Contains("create", StringComparer.InvariantCultureIgnoreCase))
{
Console.WriteLine();
Console.WriteLine(description);
result = await CreateDataSource(dsName, description, formatPartitionValue(partitionStartValue), formatPartitionValue(partitionEndValue));
if (!result)
{
return(result);
}
result = await CreateIndexer(iName, dsName, description, disabled : true);
if (!result)
{
return(result);
}
}
if (args.Contains("disable", StringComparer.InvariantCultureIgnoreCase))
{
result = await CreateIndexer(iName, dsName, description, disabled : true);
if (!result)
{
Console.WriteLine("Could not disable " + iName);
}
}
if (args.Contains("enable", StringComparer.InvariantCultureIgnoreCase))
{
result = await CreateIndexer(iName, dsName, description, disabled : false);
if (!result)
{
Console.WriteLine("Could not enable " + iName);
}
}
if (args.Contains("reset", StringComparer.InvariantCultureIgnoreCase))
{
result = await ResetIndexer(iName);
Console.WriteLine("Could not reset " + iName);
}
if (args.Contains("run", StringComparer.InvariantCultureIgnoreCase))
{
result = await RunIndexer(iName);
if (!result)
{
Console.WriteLine("Could not run " + iName);
}
}
if (args.Contains("status", StringComparer.InvariantCultureIgnoreCase) || args.Contains("stats", StringComparer.InvariantCultureIgnoreCase) || args.Contains("stat", StringComparer.InvariantCultureIgnoreCase))
{
var stat = await GetIndexerStatus(iName);
IndexerExecutionResult lastresult = stat?.LastResult;
// combine the results
if (lastresult != null)
{
int take = args.Where(a => a.StartsWith("/take:", StringComparison.InvariantCultureIgnoreCase)).Select(a => (int?)int.Parse(a.Substring(6))).FirstOrDefault() ?? 1;
int skip = args.Where(a => a.StartsWith("/skip:", StringComparison.InvariantCultureIgnoreCase)).Select(a => (int?)int.Parse(a.Substring(6))).FirstOrDefault() ?? 0;
var hist = stat.ExecutionHistory.Where(h => h.StartTime != stat.LastResult.StartTime);
hist = new[] { lastresult }.Union(hist).Skip(skip).Take(take);
lastresult = !hist.Any() ? null : new IndexerExecutionResult(
hist.First().Status,
string.Join(" | ", hist.Select(h => h.ErrorMessage)),
hist.Last().StartTime,
hist.First().EndTime,
hist.Reverse().SelectMany(h => h.Errors).ToList(),
hist.Reverse().SelectMany(h => h.Warnings).ToList(),
hist.Sum(h => h.ItemCount),
hist.Sum(h => h.FailedItemCount),
hist.Last().InitialTrackingState,
hist.First().FinalTrackingState
);
}
Console.WriteLine("{0,18} ({7}): {2,10} - Documents Processed: {3,-3} Failed: {4,-3} Warn({8}) Err({6}): {5}",
iName,
stat.Status,
lastresult?.Status,
lastresult?.ItemCount,
lastresult?.FailedItemCount,
lastresult?.ErrorMessage,
lastresult?.Errors?.Count,
((lastresult?.EndTime ?? DateTimeOffset.UtcNow) - lastresult?.StartTime)?.ToString(@"hh\:mm\:ss"),
lastresult?.Warnings?.Count
);
if (args.Contains("errors", StringComparer.InvariantCultureIgnoreCase) || args.Contains("error", StringComparer.InvariantCultureIgnoreCase) || args.Contains("err", StringComparer.InvariantCultureIgnoreCase))
{
if (lastresult?.Errors?.Count > 0)
{
Console.WriteLine("Errors:");
foreach (var err in lastresult.Errors)
{
Console.WriteLine(" {0} - {1}",
GetKeyValue(err.Key),
err.ErrorMessage);
}
}
}
if (args.Contains("warnings", StringComparer.InvariantCultureIgnoreCase) || args.Contains("warning", StringComparer.InvariantCultureIgnoreCase) || args.Contains("warn", StringComparer.InvariantCultureIgnoreCase))
{
if (lastresult?.Warnings?.Count > 0)
{
Console.WriteLine("Warnings:");
foreach (var err in lastresult.Warnings)
{
Console.WriteLine(" {0} - {1}",
GetKeyValue(err.Key),
err.Message);
}
}
}
}
partitionStartValue = partitionEndValue;
}
if (args.Contains("import", StringComparer.InvariantCultureIgnoreCase))
{
await ImportData(instances);
}
return(result);
}
private static void SyncDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
DataSource.AzureSql(
name: UsgsDataSource,
sqlConnectionString: "Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;",
tableOrViewName: "GeoNamesRI",
description: "USGS Dataset");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = UsgsIndexer,
Description = "USGS data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = GeoNamesIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static void CreateAndSyncIndexer()
{
// Create a new indexer and sync it
try
{
var creds = new DataSourceCredentials("Server=tcp:azs-playground.database.windows.net,1433;Database=usgs;User ID=reader;Password=EdrERBt3j6mZDP;Trusted_Connection=False;Encrypt=True;Connection Timeout=30");
DataSource ds = new DataSource("usgs-datasource", DataSourceType.AzureSql, creds, new DataContainer("GeoNamesRI"));
ds.Description = "USGS Dataset";
Indexer idx = new Indexer();
idx.Name = "usgs-indexer";
idx.Description = "USGS data indexer";
idx.DataSourceName = "usgs-datasource";
idx.TargetIndexName = "geonames";
idx.Parameters = new IndexingParameters();
idx.Parameters.MaxFailedItems = 10;
idx.Parameters.MaxFailedItemsPerBatch = 5;
idx.Parameters.Base64EncodeKeys = false;
//Delete indexer and datasource if it existed
_searchClient.DataSources.Delete("usgs-datasource");
_searchClient.Indexers.Delete("usgs-indexer");
//Create indexer and datasource
_searchClient.DataSources.Create(ds);
_searchClient.Indexers.Create(idx);
//Launch the sync and then monitor progress until complete
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(idx.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("Error creating indexer: {0}: \n", ex.Message.ToString());
}
}
public void CanRunIndexerAndGetIndexerStatus()
{
Run(() =>
{
// Set handler that injects mock_status query string, which results in service
// returning a well-known mock response
SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler());
Indexer indexer = Data.CreateTestIndexer();
searchClient.Indexers.Create(indexer);
IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
AzureOperationResponse runResponse =
searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result;
Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode);
info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
Assert.Equal(3, info.ExecutionHistory.Count);
Assert.NotNull(info.Limits);
Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract);
Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize);
Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime);
IndexerExecutionResult newestResult = info.ExecutionHistory[0];
IndexerExecutionResult middleResult = info.ExecutionHistory[1];
IndexerExecutionResult oldestResult = info.ExecutionHistory[2];
Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
AssertStartAndEndTimeValid(newestResult);
Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
AssertStartAndEndTimeValid(middleResult);
Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
Assert.Equal(124876, oldestResult.ItemCount);
Assert.Equal(2, oldestResult.FailedItemCount);
Assert.Equal("100", oldestResult.InitialTrackingState);
Assert.Equal("200", oldestResult.FinalTrackingState);
AssertStartAndEndTimeValid(oldestResult);
Assert.Equal(2, oldestResult.Errors.Count);
Assert.Equal("1", oldestResult.Errors[0].Key);
Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);
Assert.Equal("121713", oldestResult.Errors[1].Key);
Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
Assert.Equal(1, oldestResult.Warnings.Count);
Assert.Equal("2", oldestResult.Warnings[0].Key);
Assert.Equal("This is the first and last warning", oldestResult.Warnings[0].Message);
});
}
private static void SyncDataFromBlobSorage()
{
// This will use the Azure Search Indexer to synchronize data from Blob Storage to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
DataSource.AzureBlobStorage(
name: InterpreterIntelligenceDataSource,
storageConnectionString:
containerName: "");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = InterpreterIntelligenceIndexer,
Description = "Interpreter Intelligence indexer",
DataSourceName = dataSource.Name,
TargetIndexName = InterpreterIntelligenceIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
public void CanRunIndexerAndGetIndexerStatus()
{
Run(() =>
{
// Set handler that injects mock_status query string, which results in service
// returning a well-known mock response
SearchServiceClient searchClient = Data.GetSearchServiceClient(new MockStatusDelegatingHandler());
Indexer indexer = Data.CreateTestIndexer();
searchClient.Indexers.Create(indexer);
IndexerExecutionInfo info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
AzureOperationResponse runResponse =
searchClient.Indexers.RunWithHttpMessagesAsync(indexer.Name).Result;
Assert.Equal(HttpStatusCode.Accepted, runResponse.Response.StatusCode);
info = searchClient.Indexers.GetStatus(indexer.Name);
Assert.Equal(IndexerStatus.Running, info.Status);
Assert.Equal(IndexerExecutionStatus.InProgress, info.LastResult.Status);
Assert.Equal(3, info.ExecutionHistory.Count);
Assert.NotNull(info.Limits);
Assert.Equal(100000, info.Limits.MaxDocumentContentCharactersToExtract);
Assert.Equal(1000, info.Limits.MaxDocumentExtractionSize);
Assert.Equal(TimeSpan.FromDays(1), info.Limits.MaxRunTime);
IndexerExecutionResult newestResult = info.ExecutionHistory[0];
IndexerExecutionResult middleResult = info.ExecutionHistory[1];
IndexerExecutionResult oldestResult = info.ExecutionHistory[2];
Assert.Equal(IndexerExecutionStatus.TransientFailure, newestResult.Status);
Assert.Equal("The indexer could not connect to the data source", newestResult.ErrorMessage);
AssertStartAndEndTimeValid(newestResult);
Assert.Equal(IndexerExecutionStatus.Reset, middleResult.Status);
AssertStartAndEndTimeValid(middleResult);
Assert.Equal(IndexerExecutionStatus.Success, oldestResult.Status);
Assert.Equal(124876, oldestResult.ItemCount);
Assert.Equal(2, oldestResult.FailedItemCount);
Assert.Equal("100", oldestResult.InitialTrackingState);
Assert.Equal("200", oldestResult.FinalTrackingState);
AssertStartAndEndTimeValid(oldestResult);
Assert.Equal(2, oldestResult.Errors.Count);
Assert.Equal("1", oldestResult.Errors[0].Key);
Assert.Equal("Key field contains unsafe characters", oldestResult.Errors[0].ErrorMessage);
Assert.Equal("DocumentExtraction.AzureBlob.MyDataSource", oldestResult.Errors[0].Name);
Assert.Equal("The file could not be parsed.", oldestResult.Errors[0].Details);
Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[0].DocumentationLink);
Assert.Equal("121713", oldestResult.Errors[1].Key);
Assert.Equal("Item is too large", oldestResult.Errors[1].ErrorMessage);
Assert.Equal("DocumentExtraction.AzureBlob.DataReader", oldestResult.Errors[1].Name);
Assert.Equal("Blob size cannot exceed 256 MB.", oldestResult.Errors[1].Details);
Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2049388", oldestResult.Errors[1].DocumentationLink);
Assert.Equal(1, oldestResult.Warnings.Count);
Assert.Equal("2", oldestResult.Warnings[0].Key);
Assert.Equal("Document was truncated to 50000 characters.", oldestResult.Warnings[0].Message);
Assert.Equal("Enrichment.LanguageDetectionSkill.#4", oldestResult.Warnings[0].Name);
Assert.Equal("Try to split the input into smaller chunks using Split skill.", oldestResult.Warnings[0].Details);
Assert.Equal("https://go.microsoft.com/fwlink/?linkid=2099692", oldestResult.Warnings[0].DocumentationLink);
});
}
private static void SyncDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
new DataSource()
{
Name = AssetDataSource,
Description = "AssetManagement DataSet",
Type = DataSourceType.AzureSql,
Credentials = new DataSourceCredentials("Server=fckpvybjkl.database.windows.net,1433;Database=AssetManagement;User ID=brucewright18;Password=Mallard18;Trusted_Connection=False;Encrypt=True;Connection Timeout=30;"),
Container = new DataContainer("vAssetPwoCustomerSearch")
};
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = AssetIndexer,
Description = "Asset Management data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = AssetIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
private static void SyncDataFromAzureSQL()
{
// This will use the Azure Search Indexer to synchronize data from Azure Blob Storage to Azure Search
Console.WriteLine("{0}", "Creating Data Source...\n");
var dataSource =
DataSource.AzureBlobStorage(
name: UsgsDataSource,
storageConnectionString: "DefaultEndpointsProtocol=https;AccountName=ronanblobstorage;AccountKey=Z8nPhGaeduop96WgvZ2FP6QKGNjXeN/G8RmyoJtX97Cycusq5WOaIymucItSNnrv31ChYMICh04nmyHbCmySYw==;EndpointSuffix=core.windows.net;",
containerName: "testmovies",
description: "movies Json");
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine("Error creating data source: {0}", ex.Message);
return;
}
Console.WriteLine("{0}", "Creating Indexer and syncing data...\n");
var indexer =
new Indexer()
{
Name = UsgsIndexer,
Description = "USGS data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = GeoNamesIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine("Error creating and running indexer: {0}", ex.Message);
return;
}
bool running = true;
Console.WriteLine("{0}", "Synchronization running...\n");
while (running)
{
IndexerExecutionInfo status = null;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine("Error polling for indexer status: {0}", ex.Message);
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult != null)
{
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
Console.WriteLine("{0}", "Synchronization running...\n");
Thread.Sleep(1000);
break;
case IndexerExecutionStatus.Success:
running = false;
Console.WriteLine("Synchronized {0} rows...\n", lastResult.ItemCount);
break;
default:
running = false;
Console.WriteLine("Synchronization failed: {0}\n", lastResult.ErrorMessage);
break;
}
}
}
}
public static SearchIndexerStatus SearchIndexerStatus(IndexerStatus status = default, IndexerExecutionResult lastResult = default, IReadOnlyList <IndexerExecutionResult> executionHistory = default, SearchIndexerLimits limits = default)
{
executionHistory ??= new List <IndexerExecutionResult>();
return(new SearchIndexerStatus(status, lastResult, executionHistory, limits));
}
private static void SyncDataFromAzureSql()
{
// This will use the Azure Search Indexer to synchronize data from Azure SQL to Azure Search
Console.WriteLine("Creating Data Source...");
string geoNamesIndex = ConfigurationManager.AppSettings["SearchGeoNamesIndex"];
string usgsIndexer = ConfigurationManager.AppSettings["SearchUsageindexer"];
string usgsDataSource = ConfigurationManager.AppSettings["SearchUsageDataSource"];
var dataSource =
DataSource.AzureSql(
usgsDataSource,
ConfigurationManager.AppSettings["SearchSqlSourceConnectionString"],
ConfigurationManager.AppSettings["SearchSqlSourceTableOrView"],
description: ConfigurationManager.AppSettings["SearchSqlSourceDescription"]);
try
{
_searchClient.DataSources.Create(dataSource);
}
catch (Exception ex)
{
Console.WriteLine($"Error creating data source: {ex.Message}");
return;
}
Console.WriteLine("Creating Indexer and syncing data...");
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
var indexer = new Indexer
{
Name = usgsIndexer,
Description = "Alpha data indexer",
DataSourceName = dataSource.Name,
TargetIndexName = geoNamesIndex
};
try
{
_searchClient.Indexers.Create(indexer);
}
catch (Exception ex)
{
Console.WriteLine($"Error creating and running indexer: {ex.Message}");
return;
}
Console.WriteLine("{0}", "Synchronization running...");
_keepSpinning = true;
Thread spinner = new Thread(Spin);
spinner.Start();
while (_keepSpinning)
{
IndexerExecutionInfo status;
try
{
status = _searchClient.Indexers.GetStatus(indexer.Name);
}
catch (Exception ex)
{
Console.WriteLine($"Error polling for indexer status: {ex.Message}");
return;
}
IndexerExecutionResult lastResult = status.LastResult;
if (lastResult == null)
{
continue;
}
switch (lastResult.Status)
{
case IndexerExecutionStatus.InProgress:
break;
case IndexerExecutionStatus.Success:
_keepSpinning = false;
stopwatch.Stop();
_keepSpinning = false;
Console.WriteLine($"Synchronized {lastResult.ItemCount} rows in {stopwatch.ElapsedMilliseconds}ms");
break;
case IndexerExecutionStatus.TransientFailure:
_keepSpinning = false;
Console.WriteLine($"Transient Failure: {lastResult.ErrorMessage}");
break;
case IndexerExecutionStatus.Reset:
Console.WriteLine("Reset");
break;
default:
_keepSpinning = false;
Console.WriteLine($"Synchronization failed: {lastResult.ErrorMessage}");
break;
}
}
}
