public async Task OneTimeSetUp()
{
var connectionStringEnvironmentVariableName = "CosmosDBPersistence_ConnectionString";
var connectionString = GetEnvironmentVariable(connectionStringEnvironmentVariableName,
fallbackEmulatorConnectionString: "AccountEndpoint = https://localhost:8081/;AccountKey=C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==");
ContainerName = $"{DateTime.UtcNow.Ticks}_{Path.GetFileNameWithoutExtension(Path.GetTempFileName())}";
var builder = new CosmosClientBuilder(connectionString);
builder.AddCustomHandlers(new LoggingHandler(), new TransactionalBatchCounterHandler());
CosmosDbClient = builder.Build();
var installer = new Installer(new CosmosClientProvidedByConfiguration
{
Client = CosmosDbClient
}, new InstallerSettings
{
ContainerName = ContainerName,
DatabaseName = DatabaseName,
Disabled = false,
PartitionKeyPath = PartitionPathKey
});
await installer.Install("");
var database = CosmosDbClient.GetDatabase(DatabaseName);
Container = database.GetContainer(ContainerName);
}
static async Task Main(string[] args)
{
// The Azure Cosmos DB endpoint for running this sample.
string EndpointUri = "";
// The primary key for the Azure Cosmos account.
string PrimaryKey = "";
// The name of the database and container we will create
string databaseId = "MyProductCatalog";
string containerId = "Products";
// The Cosmos client instance
CosmosClient cosmosClient = new CosmosClient(EndpointUri, PrimaryKey);
// The database we will create
Database database = Task.Run(async() => await cosmosClient.CreateDatabaseIfNotExistsAsync(databaseId)).Result;
// The container we will create.
Container container = Task.Run(async() => await database.CreateContainerIfNotExistsAsync(containerId, "/Id")).Result;
CosmosDbClient <Product> mClient = new CosmosDbClient <Product>(cosmosClient, databaseId, containerId);
Product product = new Product
{
Id = "1",
Product_Name = "Football",
Product_Description = "A grey foorball with black spots",
Product_Price = "15.00 USD",
Product_Availability = "Unavailable"
};
await mClient.AddItemAsync(product);
Console.WriteLine($"Added product: {product}");
}
public async Task Initialize()
{
this.client = new CosmosDbClient(
connectionString: this.configuration["CosmosDbEndpoint"],
authKeyOrResourceToken: this.configuration["CosmosDbKey"],
databaseId: this.configuration["DatabaseId"],
containerId: this.configuration["ContainerId"]);
await this.client.Initialize();
}
public CosmosDbTests()
{
_cosmosDb = CreateCosmosDbClient(serviceEndpoint: _serviceEndpoint,
authKey: AuthKey,
databaseId: DatabaseId,
collectionId: CollectionId,
preferredLocations: _preferredLocations,
microserviceName: null);
DeleteAllDatabaseDocuments();
}
/// <summary>
/// Entry point to the application.
/// </summary>
/// <param name="args">
/// None - Uploads the articles to Cosmos
/// seed - Creates the database and collections (not needed)
/// query - Performs a query on processed images
/// </param>
static void Main(string[] args)
{
List <String> arguments = new List <string>(args);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// Load the configuration settings that contain the CosmosDB, Azure Storage, and RSS feed information
/////////////////////////////////////////////////////////////////////////////////////////////////////
Configuration config = Configuration.GetConfiguration();
/////////////////////////////////////////////////////////////////////////////////////////////////////
// Creat the Azure Storage Utility
/////////////////////////////////////////////////////////////////////////////////////////////////////
AzureStorageUtility storageUtility = new AzureStorageUtility(config.StorageConnectionString);
/////////////////////////////////////////////////////////////////////////////////////////////////////
// Create the CosmosDB Client
/////////////////////////////////////////////////////////////////////////////////////////////////////
String returnResult = "Jobs completed";
bool bWaitForUser = true;
using (CosmosDbClient client = new CosmosDbClient(config.CosmosUri, config.CosmosKey))
{
if (arguments.Contains("seed"))
{
bWaitForUser = false;
returnResult = Program.SeedDatabase(config, client);
}
else if (arguments.Contains("query"))
{
returnResult = Program.QueryProcessedRecords(config, client);
}
else
{
returnResult = Program.UploadRssFeeds(config, client, storageUtility);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
// Dispose of the static hash algorithm
/////////////////////////////////////////////////////////////////////////////////////////////////////
if (HashGenerator.HashAlgorithm != null)
{
HashGenerator.HashAlgorithm.Dispose();
}
Console.WriteLine(returnResult);
if (bWaitForUser)
{
Console.WriteLine("Press any key to exit.");
Console.ReadLine();
}
}
private static String SeedDatabase(Configuration config, CosmosDbClient client)
{
/////////////////////////////////////////////////////////////////////////////////////////////////
// Create the database and collections if needed
/////////////////////////////////////////////////////////////////////////////////////////////////
Console.WriteLine("Seed CosmosDB database and collections");
foreach (String coll in config.CosmosCollectionList)
{
client.CreateCollection(config.CosmosDatabase, coll).Wait();
}
return("Finsihed seeding database.");
}
/// <summary>
/// Look into the Processed collection and find all images. Get the original image ID
/// and then query the Ingest table to determine which original article it belonged to.
///
/// This function is an example on how to retrieve records by collection or by using
/// a query string.
/// </summary>
/// <param name="config">Configuration with CosmosDB information</param>
/// <param name="client">CLient to do work</param>
/// <returns>String to present to user.</returns>
public static String QueryProcessedRecords(Configuration config, CosmosDbClient client)
{
String returnValue = "Failed to query CosmosDB";
client.Connect();
List <Newtonsoft.Json.Linq.JObject> objects = client.RetrieveRecords(client.DocClient, config.CosmosDatabase, "Processed").Result;
List <String> imageIds = new List <string>();
foreach (Newtonsoft.Json.Linq.JObject jobj in objects)
{
String at = jobj.Value <string>("artifact_type");
if (String.Compare(at, "image", true) == 0)
{
imageIds.Add(jobj.Value <string>("parent"));
}
}
if (imageIds.Count > 0)
{
returnValue = String.Format("Processed Images paired with parent Articles{0}{0}", Environment.NewLine);
foreach (String imageId in imageIds)
{
returnValue += String.Format("Image ID: {0}{1}Parent ID:{1}", imageId, Environment.NewLine);
String sqlQuery = String.Format(ARTICLE_QUERY_WITH_MEDIA_ID, imageId);
List <Newtonsoft.Json.Linq.JObject> parentArticles = client.PerformQuery(client.DocClient, config.CosmosDatabase, "Ingest", sqlQuery).Result;
foreach (Newtonsoft.Json.Linq.JObject jobj in parentArticles)
{
returnValue += String.Format("\t{0}{1}", jobj.Value <string>("id"), Environment.NewLine);
}
}
}
return(returnValue);
}
private static String UploadRssFeeds(Configuration config, CosmosDbClient client, AzureStorageUtility storageUtility)
{
/////////////////////////////////////////////////////////////////////////////////////////////////
// Looop through each of the RSS feeds, collect the articles, then upload them.
/////////////////////////////////////////////////////////////////////////////////////////////////
foreach (RssFeedInfo feed in config.Feeds)
{
Console.WriteLine("Processing feed : " + feed.RSSFeed);
using (RSSFeedReader rssReader = new RSSFeedReader(feed.RSSFeed))
{
/////////////////////////////////////////////////////////////////////////////////////////
// The the batch of articles.....
/////////////////////////////////////////////////////////////////////////////////////////
int feedItemCount = 0;
List <RSSFeedItem> feedItems = rssReader.ReadFeed();
Console.WriteLine("Feed : " + feed.RSSFeed + " has " + feedItems.Count + " items");
/////////////////////////////////////////////////////////////////////////////////////////
// For each article, upload it's image(s) and content.
/////////////////////////////////////////////////////////////////////////////////////////
foreach (RSSFeedItem item in feedItems)
{
Console.WriteLine("Inserting : " + item.Title);
Article mainArticle = new Article();
List <Article> imageContent = new List <Article>();
// Set up the images
foreach (RSSImageItem image in item.Images)
{
String blobUri = storageUtility.UploadBlob(image.Path, feed.AzureStorageContainer).Result;
if (!String.IsNullOrEmpty(blobUri))
{
Article media = new Article();
media.ArtifactType = "image";
media.AssetHash = image.Hash;
media.UniqueIdentifier = image.Id;
media.SetProperty(ArticleProperties.RetrievalDate, DateTime.Now.ToString("O"));
media.SetProperty(ArticleProperties.OriginalUri, image.Uri);
media.SetProperty(ArticleProperties.InternalUri, blobUri);
imageContent.Add(media);
}
}
// Now set up the article iteself
mainArticle.SetProperty(ArticleProperties.OriginalUri, item.Uri);
mainArticle.SetProperty(ArticleProperties.RetrievalDate, DateTime.Now.ToString("O"));
mainArticle.SetProperty(ArticleProperties.PostDate, item.PublishedDate);
mainArticle.SetProperty(ArticleProperties.Title, Program.CleanInput(item.Title));
mainArticle.SetProperty(ArticleProperties.Body, Program.CleanInput(item.Summary));
List <Dictionary <string, string> > childFiles = new List <Dictionary <string, string> >();
foreach (Article file in imageContent)
{
Dictionary <String, String> obj = new Dictionary <string, string>();
obj.Add(Article.MEDIA_ID, file.UniqueIdentifier);
obj.Add(Article.MEDIA_ORIG_URI, file.GetProperty(ArticleProperties.OriginalUri).ToString());
obj.Add(Article.MEDIA_INTERNAL_URI, file.GetProperty(ArticleProperties.InternalUri).ToString());
childFiles.Add(obj);
}
mainArticle.SetProperty(ArticleProperties.ChildImages, childFiles);
mainArticle.SetProperty(ArticleProperties.ChildVideos, null);
mainArticle.SetProperty(ArticleProperties.Author, null);
mainArticle.SetProperty(ArticleProperties.HeroImage, null);
// Insert the media files first
foreach (Article imageArticle in imageContent)
{
try
{
bool imageResult = client.CreateDocument(config.CosmosDatabase, config.CosmosIngestCollection, imageArticle).Result;
Console.WriteLine("Image Insert: " + imageResult.ToString());
}
catch (Exception ex) { }
}
// Wait briefly....
System.Threading.Thread.Sleep(500);
bool articleResult = client.CreateDocument(config.CosmosDatabase, config.CosmosIngestCollection, mainArticle).Result;
Console.WriteLine("Article Insert: " + articleResult.ToString());
// Only allow one for each feed for now
if (++feedItemCount > 5)
{
break;
}
}
}
}
return("Finished uploading current articles.");
}
public async Task OneTimeTearDown()
{
await Container.DeleteContainerStreamAsync();
CosmosDbClient.Dispose();
}
public ValuesController(CosmosDbClient cosmosClient)
{
_cosmosClient = cosmosClient;
}
