private Guid PopulateIdentification(string identityField, RecordMap recordMap)
{
var id = Guid.NewGuid();
recordMap.Add(identityField, id.ToTableRecordValueString());
return(id);
}
private string CreateInsertQuery(string tableName, RecordMap recordMap)
{
var keys = String.Join(JoinSeparator, recordMap.Keys);
var values = String.Join(JoinSeparator, recordMap.Values);
return(String.Format(InsertQueryTemplate, tableName, keys, values));
}
private async Task <Guid> GenerateRecordAsync(RecordMap recordMap)
{
foreach (var field in table.Fields)
{
var value = field.GetNewValue();
if (value != null)
{
recordMap.Add(field.Name, value);
}
await log.WriteLineAsync($"{table.Name}: field '{field.Name}' generated value = {value}");
}
await log.WriteLineAsync();
return(await DatabaseContext.Instance.CreateRecordAsync(table.Name, table.IdentityField, recordMap));
}
public static FileInfo ToCsv(this List <Record> records, string path)
{
Directory.CreateDirectory(Path.GetDirectoryName(path));
using (FileStream output = new FileStream(path, FileMode.Create))
using (StreamWriter writer = new StreamWriter(output, Encoding.UTF8))
using (var csv = new CsvWriter(writer))
{
var map = new RecordMap()
.Map(r => r.Selector, "Territorial Authority")
.Map(r => r.Parent, "Topic");
map
.Map(r => r.Measures)
.Name("Measure")
.ConvertUsing(r => r.MeasureFormatted());
map
.Map(r => r.Categories)
.Name("Category")
.ConvertUsing(r => r.CategoryFormatted());
map
.Map(r => r.Value, "Value")
.Map(r => r.ValueUnit, "ValueUnit")
.Map(r => r.ValueLabel, "ValueLabel")
.Map(r => r.NullReason, "NullReason")
.Map(r => r.Date, "Date")
.Map(r => r.DateLabel, "DateLabel");
map
.Map(r => r.Uri)
.Name("Source")
.ConvertUsing(r => r.UriFormatted());
csv.Configuration.RegisterClassMap(map);
csv.WriteRecords(records);
}
FileInfo file = new FileInfo(path);
Console.WriteLine($"Wrote '{file.FullName}'");
return(file);
}
private async Task <Guid> ProcessTableRecord()
{
var recordMap = new RecordMap();
if (table.Dependencies.Count > 0)
{
foreach (var dependency in table.Dependencies)
{
var dependencyValue = await new DependencyContentGenerator(dependency, log).StartAsync();
if (dependencyValue != null)
{
recordMap.Add(dependency.TargetIdentityFieldName, dependencyValue);
}
}
}
return(await GenerateRecordAsync(recordMap));
}
public async Task <Guid> CreateRecordAsync(string tableName, string identityField, RecordMap recordMap)
{
var id = PopulateIdentification(identityField, recordMap);
var query = CreateInsertQuery(tableName, recordMap);
await Database.ExecuteSqlCommandAsync(query);
return(id);
}
public static string GetHtml(this RecordMap recordMap, Guid pageId = default, int maxBlocks = 0, bool throwIfBlockMissing = true, bool stopBeforeFirstSubHeader = false, bool throwIfCantDecodeTextData = true)
{
var sb = new StringBuilder();
TransformOptions transformOptions = null !;
transformOptions = new TransformOptions
{
//Exclude code, page, bookmark
#if !DEBUG
AcceptedBlockTypes = new List <string> {
"text", "header", "sub_header", "sub_sub_header", "bulleted_list", "image", "quote", "column_list", "column", "callout"
},
#endif
ThrowIfBlockMissing = throwIfBlockMissing,
ThrowIfCantDecodeTextData = throwIfCantDecodeTextData,
MaxBlocks = maxBlocks,
TransformHeader = (data, block) =>
{
sb.Append("<h1 class=\"notion-header-block\">").AppendText(data).AppendLine("</h1>");
return(true);
},
TransformSubHeader = (data, block) =>
{
if (stopBeforeFirstSubHeader)
{
return(false);
}
sb.Append("<h2 class=\"notion-sub_header-block\">").AppendText(data).AppendLine("</h2>");
return(true);
},
TransformSubSubHeader = (data, block) =>
{
sb.Append("<h3 class=\"notion-sub_sub_header-block\">").AppendText(data).AppendLine("</h3>");
return(true);
},
TransformText = (data, block) =>
{
//if (data != null)
sb.Append("<div class=\"notion-text-block\">").AppendText(data).AppendLine("</div>");
return(true);
},
TransformBulletedList = (data, block) =>
{
sb.Append("<ul class=\"notion-bulleted_list-block\"><li>").AppendText(data).AppendLine("</li>");
return(true, () => sb.AppendLine("</ul>"));
},
TransformQuote = (data, block) =>
{
sb.Append("<div class=\"notion-quote-block\">").AppendText(data).AppendLine("</div>");
return(true);
},
TransformImage = (data, block) =>
{
if (data != null)
{
sb.Append("<div class=\"notion-image-block\">").AppendImage(data, block.Id).AppendLine("</div>");
}
return(true);
},
TransformColumnList = (data, block) =>
{
//Start of notion-column_list-block
sb.Append("<div class=\"notion-column_list-block\"><div style=\"display: flex\">");
var totalColumns = data.Columns.Count;
return(true,
StartColumn : (columnIndex, column) =>
{
sb.Append(FormattableString.Invariant($"<div class=\"notion-column\" style=\"width: calc((100% - {46*(totalColumns-1)}px) * {column.Ratio});\">"));
return () => sb.Append("</div>");
},
TransformColumnSeparator : columnIndex => sb.Append("<div class=\"notion-column-separator\"><div class=\"notion-column-separator-line\"></div></div>"),
EndColumnList : () => sb.Append("</div></div>"));
},
TransformCallout = (data, block) =>
{
sb.AppendLine(@$ "<div class=" "notion-callout-block notion-block-color-{data.Format.BlockColor}" ">");
if (Uri.TryCreate(data.Format.PageIcon, UriKind.Absolute, out var iconUrl))
{
sb.AppendLine(@$ "<img class=" "notion-icon" " src=" "{iconUrl}" " />");
}
else
{
sb.AppendLine(@$ "<img class=" "notion-emoji" " src=" "{GetTwitterEmojiUrl(data.Format.PageIcon)}" " />");
}
transformOptions.TransformText(data.Text, block);
sb.AppendLine("</div>");
return(true);
},
#if DEBUG
TransformOther = block =>
{
return(true);
}
#endif
};
recordMap.Transform(transformOptions, pageId);
return(sb.ToString());
}
public static string GetHtmlAbstract(this RecordMap recordMap, Guid pageId = default) => recordMap.GetHtml(pageId, throwIfBlockMissing: false, stopBeforeFirstSubHeader: true, throwIfCantDecodeTextData: false);
public override void OnActionExecuting(ActionExecutingContext filterContext)
{
HttpContextBase httpContextBase = filterContext.HttpContext;
HttpRequestBase request = httpContextBase.Request;
HttpResponseBase response = httpContextBase.Response;
string operKey = request.UserHostAddress + request.Path.ToLower();
OperRecord operRecord = null;
lock (RecordMap)
{
if (RecordMap.ContainsKey(operKey))
{
operRecord = RecordMap[operKey];
long totalMilliseconds = Convert.ToInt64((DateTime.Now - operRecord.LastOperDate).TotalMilliseconds);
int count = Convert.ToInt32((totalMilliseconds - totalMilliseconds % ClearMillisecond) / ClearMillisecond);
operRecord.Count = operRecord.Count > count ? operRecord.Count - count : 0;
}
else
{
RecordMap.Add(operKey, operRecord = new OperRecord
{
Count = 1,
OperKey = operKey,
LastOperDate = DateTime.Now
});
}
}
if (operRecord.Count >= CountLimit)
{
response.ContentEncoding = Encoding.UTF8;
response.ContentType = "application/json;charset=UTF-8";
response.StatusCode = 200;
response.Write(JsonConvert.SerializeObject(new Result
{
code = -1,
msg = "操作过于频繁、请稍后重试！",
}));
filterContext.Result = new EmptyResult();
}
else
{
lock (RecordMap)
{
operRecord.Count++;
operRecord.LastOperDate = DateTime.Now;
}
base.OnActionExecuting(filterContext);
}
if ((DateTime.Now - LastClearDate).TotalSeconds > 60)
{
lock (RecordMap)
{
long sumMillisecond = ClearMillisecond * CountLimit;
foreach (string tkey in new HashSet <string>(RecordMap.Keys))
{
if (tkey.EndsWith(request.Path))
{
operRecord = RecordMap[tkey];
long totalMilliseconds = Convert.ToInt64((DateTime.Now - operRecord.LastOperDate).TotalMilliseconds);
if (totalMilliseconds >= sumMillisecond)
{
RecordMap.Remove(tkey);
}
}
}
}
LastClearDate = DateTime.Now;
}
}
public override void OnActionExecuting(ActionExecutingContext filterContext)
{
HttpContextBase httpContextBase = filterContext.HttpContext;
HttpRequestBase request = httpContextBase.Request;
string key = request.UserHostAddress + request.Path.ToLower();
bool temp = true;
lock (RecordMap)
{
if (RecordMap.ContainsKey(key))
{
temp = (DateTime.Now - RecordMap[key]).TotalMilliseconds > IntervalMillisecond;
}
else
{
RecordMap.Add(key, DateTime.Now);
}
}
if (temp)
{
lock (RecordMap)
{
if (RecordMap.ContainsKey(key))
{
RecordMap[key] = DateTime.Now;
}
else
{
RecordMap.Add(key, DateTime.Now);
}
}
base.OnActionExecuting(filterContext);
}
else
{
HttpResponseBase response = httpContextBase.Response;
response.ContentEncoding = Encoding.UTF8;
response.ContentType = "application/json;charset=UTF-8";
response.StatusCode = 200;
response.Write(JsonConvert.SerializeObject(new Result
{
code = -1,
msg = "操作过于频繁、请稍后重试！",
}));
filterContext.Result = new EmptyResult();
}
if ((DateTime.Now - LastClearDate).TotalSeconds > 60)
{
lock (RecordMap)
{
foreach (string tkey in new HashSet <string>(RecordMap.Keys))
{
if (tkey.EndsWith(request.Path) && RecordMap.ContainsKey(tkey) && (DateTime.Now - RecordMap[tkey]).TotalMilliseconds > IntervalMillisecond)
{
RecordMap.Remove(tkey);
}
}
}
LastClearDate = DateTime.Now;
}
}
public static List <Record> ToRecords(this Dataset dataset, string csvFile, string term)
{
using (StreamReader reader = new StreamReader(csvFile, Encoding.UTF8))
using (CsvReader csv = new CsvReader(reader))
{
Console.WriteLine($"Processing '{Path.GetFileName(csvFile)}'");
// Danger, Will Robinson!
//
// CsvHelper stores property names in couple of Dictionaries that use a string key.
// These dictionaries are case sensitive, because C# supports properties the same name as long as the cases are different (e.g. "discriminator" and "Discriminator" are different properties)
//
// But, we want a case _insensitive_ match for when a new csv file differs in column header case only (e.g. "Territorial Authority" and "Territorial authority") so we don't have to fix our Dataset configuration
// As per CsvHelper's documentation, we're expected to use the PrepareHeaderForMatch property to call ToLower() on all column headers:
// - https://joshclose.github.io/CsvHelper/getting-started/
// - https://github.com/JoshClose/CsvHelper/issues/1183
//
// That violates Microsoft's sting comparison guidelines, however: https://docs.microsoft.com/en-us/dotnet/csharp/how-to/compare-strings
//
// I'd prefer to replace the Dictionaries with ones that use a case insensitive match:
//
// - csv.Context.NamedIndexes = new Dictionary<string, List<int>>(StringComparer.OrdinalIgnoreCase);
// - csv.Context.NamedIndexCache = new Dictionary<string, (string, int)>(StringComparer.OrdinalIgnoreCase);
//
// ...which works right now, but reaching into CsvHelper's internals feels kind of fragile.
//
// So, we'll grumble and go ahead with ToLower()
csv.Configuration.PrepareHeaderForMatch = (header, index) => header.ToLowerInvariant();
// If we have more or fewer headers or properties than we expect, just keep going
csv.Configuration.HeaderValidated = null;
csv.Configuration.MissingFieldFound = null;
csv.Configuration.ReadingExceptionOccurred = exception =>
{
if (exception is TypeConverterException)
{
// Some text in the Value column, instead of a number
// In most datasets, Value is null and we're supplied a null reason to indicate why we don't have a value
// However, some datasets include the null reason in the value column and cause type conversion errors
// We'll just skip these rows
return(false);
}
return(true);
};
var map = new RecordMap()
.Map(r => r.Selector, dataset.Selector ?? "Territorial Authority")
.Map(r => r.Date, dataset.Date);
map.Map(r => r.Measures)
.ConvertUsing(rr => dataset
.Measure
.Select((measure, index) => new { measure, index })
.ToDictionary(
mi => mi.measure.Name,
mi => new ColumnValue
{
Index = mi.index,
Column = mi.measure.Name,
Separator = mi.measure.Separator ?? " — ",
Value = rr.GetField <string>(mi.measure.Name)
})
);
map.Map(r => r.Categories)
.ConvertUsing(rr => dataset
.Category
.Select((category, index) => new { category, index })
.ToDictionary(ci =>
ci.category.Name,
ci => new ColumnValue
{
Index = ci.index,
Column = ci.category.Name,
Separator = ci.category.Separator ?? " — ",
Value = rr.GetField <string>(ci.category.Name)
})
);
map
.Map(r => r.Value, dataset.Value ?? "Value")
.Map(r => r.ValueUnit, dataset.ValueUnit ?? "Value Unit")
.Map(r => r.ValueLabel, dataset.ValueLabel ?? "Value Label")
.Map(r => r.NullReason, dataset.NullReason ?? "Null Reason");
csv.Configuration.RegisterClassMap(map);
return(dataset.ToRecords(csv, term));
}
}
public void InitializeRecordMap(RecordMap recordMap, System.Data.Common.DbTransaction transaction)
{
throw new Exception("The method or operation is not implemented.");
}
public TradeStats(Rylobot bot)
{
Bot = bot;
Records = new RecordMap();
Predictors = new Dictionary <string, Predictor>();
}