static void WriteCsv(CsvZipPackage csvz, string csv)
{
Console.WriteLine($"Processing {Path.GetFileName(csv)}: ");
var data = CsvDataReader.Create(csv);
var analyzer = new SchemaAnalyzer();
Console.Write($" Analyzing. ");
var sw = Stopwatch.StartNew();
var result = analyzer.Analyze(data);
Console.WriteLine(sw.Elapsed.ToString());
var schema = result.GetSchema();
var csvSchema = new CsvSchema(schema);
var csvOpts = new CsvDataReaderOptions {
Schema = csvSchema
};
Console.Write($" Writing. ");
data = CsvDataReader.Create(csv, csvOpts);
sw = Stopwatch.StartNew();
var entry = csvz.CreateEntry(Path.GetFileName(csv));
entry.WriteData(data);
Console.WriteLine(sw.Elapsed.ToString());
}
private static void ReadCompounds(string filename, FlavorNetworkContext context)
{
var compoundsCsvSchema = new CsvSchema();
compoundsCsvSchema.Properties.Add(new SchemaProperty("Id"));
compoundsCsvSchema.Properties.Add(new SchemaProperty("Name"));
compoundsCsvSchema.Properties.Add(new SchemaProperty("CasNumber"));
var compoundsCsvReader = new CsvReader <Compound>();
compoundsCsvReader.CsvDelimiter = "\t";
compoundsCsvReader.UseSingleLineHeaderInCsv = true;
compoundsCsvReader.Schema = compoundsCsvSchema;
var list = compoundsCsvReader.Read(File.OpenRead(filename));
foreach (var item in list)
{
item.Id = item.Id + 1;
item.CompoundFlavors = new List <CompoundFlavor>();
item.IngredientCompounds = new List <IngredientCompound>();
context.Compounds.Add(item);
//Console.WriteLine($"ID: '{item.Id}', Name: '{item.Name}', CAS: '{item.CasNumber}'");
}
}
private static void ReadIngredientCategory(string filename, FlavorNetworkContext context)
{
var ingredientCategoryCsvSchema = new CsvSchema();
ingredientCategoryCsvSchema.Properties.Add(new SchemaProperty("Id"));
ingredientCategoryCsvSchema.Properties.Add(new SchemaProperty("Name"));
ingredientCategoryCsvSchema.Properties.Add(new SchemaProperty("Category"));
var ingredientCategoryCsvReader = new CsvReader <IngredientAndCategory>();
ingredientCategoryCsvReader.CsvDelimiter = "\t";
ingredientCategoryCsvReader.UseSingleLineHeaderInCsv = true;
ingredientCategoryCsvReader.Schema = ingredientCategoryCsvSchema;
var list = ingredientCategoryCsvReader.Read(File.OpenRead(filename));
int id = 1;
foreach (var item in list.Distinct(new CategoryEqualityComparer()))
{
var category = new IngredientCategory();
category.Id = id;
category.Name = item.Category;
category.Ingredients = new List <Ingredient>();
context.IngredientCategories.Add(category);
id++;
}
foreach (var item in list)
{
item.Id = item.Id + 1;
var ingredient = new Ingredient();
ingredient.Id = item.Id;
ingredient.Name = GetIngredientName(item.Name);
ingredient.IngredientCompounds = new List <IngredientCompound>();
ingredient.IngredientContributions = new List <IngredientContribution>();
ingredient.RecipeIngredients = new List <RecipeIngredient>();
var category = context.IngredientCategories.Local.SingleOrDefault(r => r.Name.Equals(item.Category));
if (category != null)
{
ingredient.IngredientCategory = category;
ingredient.IngredientCategoryId = category.Id;
category.Ingredients.Add(ingredient);
context.Ingredients.Add(ingredient);
}
else
{
Console.WriteLine($"Category '{item.Category}' not found!!!");
}
//Console.WriteLine($"Id: '{item.Id}', N: '{item.Name}', C: '{item.Category}'");
}
}
public CsvDataReader(TextReader reader, CsvLayout csvLayout, CsvBehaviour csvBehaviour, CultureInfo cultureInfo)
{
_parser = new CsvParser(reader, csvLayout, csvBehaviour);
_header = _parser.Header;
_line = null;
_converter = new Converter(cultureInfo ?? CultureInfo.InvariantCulture);
_enumerator = _parser.GetEnumerator();
_schema = csvLayout.Schema;
}
private static void ReadIngredientCompounds(string filename, FlavorNetworkContext context)
{
var compoundsCsvSchema = new CsvSchema();
compoundsCsvSchema.Properties.Add(new SchemaProperty("Ingredient"));
compoundsCsvSchema.Properties.Add(new SchemaProperty("Compound"));
var compoundsCsvReader = new CsvReader <IngredientAndCompound>();
compoundsCsvReader.CsvDelimiter = "\t";
compoundsCsvReader.UseSingleLineHeaderInCsv = true;
compoundsCsvReader.Schema = compoundsCsvSchema;
var list = compoundsCsvReader.Read(File.OpenRead(filename));
foreach (var item in list)
{
item.Ingredient = item.Ingredient + 1;
item.Compound = item.Compound + 1;
var ingredient = context.Ingredients.Local.SingleOrDefault(i => i.Id == item.Ingredient);
var compound = context.Compounds.Local.SingleOrDefault(c => c.Id == item.Compound);
if (ingredient != null)
{
if (compound != null)
{
var ingredientCompound = new IngredientCompound();
ingredientCompound.Ingredient = ingredient;
ingredientCompound.IngredientId = ingredient.Id;
ingredientCompound.Compound = compound;
ingredientCompound.CompoundId = compound.Id;
ingredient.IngredientCompounds.Add(ingredientCompound);
compound.IngredientCompounds.Add(ingredientCompound);
context.IngredientCompounds.Add(ingredientCompound);
}
else
{
Console.WriteLine($"Compound with id '{item.Compound}' not found!!!");
}
}
else
{
Console.WriteLine($"Ingredient with id '{item.Ingredient}' not found!!!");
}
}
}
private readonly string _endpoint;// = "http://acornbox.webui";
void IGenerateSchema.GenerateSchema(Guid id)
{
System.Threading.Thread.Sleep(60 * 1000);
string resourceUrl = _endpoint + $"/api/FileEntry/{id}";
FileEntryDto fileEntry;
try
{
fileEntry = resourceUrl.GetJsonAsync <FileEntryDto>()
.GetAwaiter()
.GetResult();
}
catch (FlurlHttpException ex)
{
throw;
}
using FileStream fs = new FileStream(fileEntry.Path, FileMode.Open, FileAccess.Read, FileShare.Read);
using StreamReader sr = new StreamReader(fs, Encoding.Default, true);
using CsvReader csvReader = new CsvReader(sr);
CsvSchemaBuilder csvSchemaBuilder = new CsvSchemaBuilder(csvReader.HeaderRecord);
csvSchemaBuilder.Refine(csvReader);
CsvSchema csvSchema = csvSchemaBuilder.Build();
string csvSchemaStr = Newtonsoft.Json.JsonConvert.SerializeObject(csvSchema, Newtonsoft.Json.Formatting.Indented);
fileEntry.Schema = csvSchemaStr;
fileEntry.GenerateSchemaWorker = Environment.MachineName;
try
{
resourceUrl.PutJsonAsync(fileEntry).GetAwaiter().GetResult();
}
catch (FlurlHttpException ex)
{
throw;
}
}
/// <summary>
/// Read a string as CSV, using specific behaviour, layout and conversion options
/// </summary>
/// <param name="input">The CSV input</param>
/// <param name="quote">The quote character. Default '"'</param>
/// <param name="delimiter">Field delimiter. Default ','</param>
/// <param name="escape">Quote escape character (for quotes inside fields). Default '\'</param>
/// <param name="comment">Comment marker. Default '#'</param>
/// <param name="hasHeaders">Is the first line a header line (default false)?</param>
/// <param name="trimmingOptions">How should fields be trimmed?</param>
/// <param name="missingFieldAction">What should happen when a field is missing from a line?</param>
/// <param name="skipEmptyLines">Should empty lines be skipped?</param>
/// <param name="quotesInsideQuotedFieldAction">What should happen when a quote is found inside a quoted field?</param>
/// <param name="schema">The CSV schema.</param>
/// <param name="cultureInfo">Culture info to be used for parsing culture-sensitive data (such as date/time and decimal numbers)</param>
/// <returns>a DataReader instance to read the contents of the CSV file</returns>
public static IDataReader FromString(
string input,
char quote = '"',
char delimiter = ',',
char escape = '"',
char comment = '#',
bool hasHeaders = false,
ValueTrimmingOptions trimmingOptions = ValueTrimmingOptions.None,
MissingFieldAction missingFieldAction = MissingFieldAction.ParseError,
bool skipEmptyLines = true,
QuotesInsideQuotedFieldAction quotesInsideQuotedFieldAction = QuotesInsideQuotedFieldAction.Ignore,
CsvSchema schema = null,
CultureInfo cultureInfo = null)
{
var reader = new StringReader(input);
var layout = new CsvLayout(quote, delimiter, escape, comment, hasHeaders, schema);
var behaviour = new CsvBehaviour(trimmingOptions, missingFieldAction, skipEmptyLines, quotesInsideQuotedFieldAction);
return(FromReader(reader, layout, behaviour, cultureInfo));
}
/// <summary>
/// Read a file as CSV, using specific behaviour, layout and conversion options. Make sure to dispose the DataReader.
/// </summary>
/// <param name="path">The full or relative path name</param>
/// <param name="encoding">The encoding of the file.</param>
/// <param name="quote">The quote character. Default '"'</param>
/// <param name="delimiter">Field delimiter. Default ','</param>
/// <param name="escape">Quote escape character (for quotes inside fields). Default '\'</param>
/// <param name="comment">Comment marker. Default '#'</param>
/// <param name="hasHeaders">Is the first line a header line (default false)?</param>
/// <param name="trimmingOptions">How should fields be trimmed?</param>
/// <param name="missingFieldAction">What should happen when a field is missing from a line?</param>
/// <param name="skipEmptyLines">Should empty lines be skipped?</param>
/// <param name="quotesInsideQuotedFieldAction">What should happen when a quote is found inside a quoted field?</param>
/// <param name="schema">The CSV schema.</param>
/// <param name="cultureInfo">Culture info to be used for parsing culture-sensitive data (such as date/time and decimal numbers)</param>
/// <returns>a DataReader instance to read the contents of the CSV file</returns>
public static IDataReader FromFile(
string path,
Encoding encoding = null,
char quote = '"',
char delimiter = ',',
char escape = '"',
char comment = '#',
bool hasHeaders = false,
ValueTrimmingOptions trimmingOptions = ValueTrimmingOptions.UnquotedOnly,
MissingFieldAction missingFieldAction = MissingFieldAction.ParseError,
bool skipEmptyLines = true,
QuotesInsideQuotedFieldAction quotesInsideQuotedFieldAction = QuotesInsideQuotedFieldAction.Ignore,
CsvSchema schema = null,
CultureInfo cultureInfo = null)
{
// caller should dispose IDataReader, which will indirectly also close the stream
var layout = new CsvLayout(quote, delimiter, escape, comment, hasHeaders, schema);
var behaviour = new CsvBehaviour(trimmingOptions, missingFieldAction, skipEmptyLines, quotesInsideQuotedFieldAction);
var stream = File.OpenRead(path);
var reader = new StreamReader(stream, encoding ?? Encoding.UTF8, encoding == null);
return(FromReader(reader, layout, behaviour, cultureInfo));
}
public void Init()
{
instance = new CsvSchema();
}
private static void ReadCuisineRegions(string filename, FlavorNetworkContext context)
{
var countryRegionCsvSchema = new CsvSchema();
countryRegionCsvSchema.Properties.Add(new SchemaProperty("Cuisine"));
countryRegionCsvSchema.Properties.Add(new SchemaProperty("Region"));
var countryRegionCsvReader = new CsvReader <CuisineRegion>();
countryRegionCsvReader.CsvDelimiter = "\t";
countryRegionCsvReader.UseSingleLineHeaderInCsv = false;
countryRegionCsvReader.Schema = countryRegionCsvSchema;
var list = countryRegionCsvReader.Read(File.OpenRead(filename))
.Select(w => NormalizeCuisine(w))
.Distinct(new CuisineEqualityComparer())
.OrderBy(w => w.Cuisine);
/*foreach (var item in list)
* {
* Console.WriteLine($"C: '{item.Cuisine}', R: '{item.Region}'");
* }*/
int id = 1;
foreach (var item in list.Distinct(new RegionEqualityComparer()))
{
var region = new Region();
region.Id = id;
region.Name = item.Region;
region.Cuisines = new List <Cuisine>();
context.Regions.Add(region);
id++;
}
id = 1;
foreach (var item in list)
{
var cuisine = new Cuisine();
cuisine.Id = id;
cuisine.Name = item.Cuisine;
cuisine.IngredientContributions = new List <IngredientContribution>();
cuisine.Recipes = new List <Recipe>();
var region = context.Regions.Local.SingleOrDefault(r => r.Name.Equals(item.Region));
if (region != null)
{
cuisine.Region = region;
cuisine.RegionId = region.Id;
region.Cuisines.Add(cuisine);
context.Cuisines.Add(cuisine);
id++;
}
else
{
Console.WriteLine($"Region '{item.Region}' not found!!!");
}
}
}
