private Guid PopulateIdentification(string identityField, RecordMap recordMap) { var id = Guid.NewGuid(); recordMap.Add(identityField, id.ToTableRecordValueString()); return(id); }
private string CreateInsertQuery(string tableName, RecordMap recordMap) { var keys = String.Join(JoinSeparator, recordMap.Keys); var values = String.Join(JoinSeparator, recordMap.Values); return(String.Format(InsertQueryTemplate, tableName, keys, values)); }
private async Task <Guid> GenerateRecordAsync(RecordMap recordMap) { foreach (var field in table.Fields) { var value = field.GetNewValue(); if (value != null) { recordMap.Add(field.Name, value); } await log.WriteLineAsync($"{table.Name}: field '{field.Name}' generated value = {value}"); } await log.WriteLineAsync(); return(await DatabaseContext.Instance.CreateRecordAsync(table.Name, table.IdentityField, recordMap)); }
public static FileInfo ToCsv(this List <Record> records, string path) { Directory.CreateDirectory(Path.GetDirectoryName(path)); using (FileStream output = new FileStream(path, FileMode.Create)) using (StreamWriter writer = new StreamWriter(output, Encoding.UTF8)) using (var csv = new CsvWriter(writer)) { var map = new RecordMap() .Map(r => r.Selector, "Territorial Authority") .Map(r => r.Parent, "Topic"); map .Map(r => r.Measures) .Name("Measure") .ConvertUsing(r => r.MeasureFormatted()); map .Map(r => r.Categories) .Name("Category") .ConvertUsing(r => r.CategoryFormatted()); map .Map(r => r.Value, "Value") .Map(r => r.ValueUnit, "ValueUnit") .Map(r => r.ValueLabel, "ValueLabel") .Map(r => r.NullReason, "NullReason") .Map(r => r.Date, "Date") .Map(r => r.DateLabel, "DateLabel"); map .Map(r => r.Uri) .Name("Source") .ConvertUsing(r => r.UriFormatted()); csv.Configuration.RegisterClassMap(map); csv.WriteRecords(records); } FileInfo file = new FileInfo(path); Console.WriteLine($"Wrote '{file.FullName}'"); return(file); }
private async Task <Guid> ProcessTableRecord() { var recordMap = new RecordMap(); if (table.Dependencies.Count > 0) { foreach (var dependency in table.Dependencies) { var dependencyValue = await new DependencyContentGenerator(dependency, log).StartAsync(); if (dependencyValue != null) { recordMap.Add(dependency.TargetIdentityFieldName, dependencyValue); } } } return(await GenerateRecordAsync(recordMap)); }
public async Task <Guid> CreateRecordAsync(string tableName, string identityField, RecordMap recordMap) { var id = PopulateIdentification(identityField, recordMap); var query = CreateInsertQuery(tableName, recordMap); await Database.ExecuteSqlCommandAsync(query); return(id); }
public static string GetHtml(this RecordMap recordMap, Guid pageId = default, int maxBlocks = 0, bool throwIfBlockMissing = true, bool stopBeforeFirstSubHeader = false, bool throwIfCantDecodeTextData = true) { var sb = new StringBuilder(); TransformOptions transformOptions = null !; transformOptions = new TransformOptions { //Exclude code, page, bookmark #if !DEBUG AcceptedBlockTypes = new List <string> { "text", "header", "sub_header", "sub_sub_header", "bulleted_list", "image", "quote", "column_list", "column", "callout" }, #endif ThrowIfBlockMissing = throwIfBlockMissing, ThrowIfCantDecodeTextData = throwIfCantDecodeTextData, MaxBlocks = maxBlocks, TransformHeader = (data, block) => { sb.Append("<h1 class=\"notion-header-block\">").AppendText(data).AppendLine("</h1>"); return(true); }, TransformSubHeader = (data, block) => { if (stopBeforeFirstSubHeader) { return(false); } sb.Append("<h2 class=\"notion-sub_header-block\">").AppendText(data).AppendLine("</h2>"); return(true); }, TransformSubSubHeader = (data, block) => { sb.Append("<h3 class=\"notion-sub_sub_header-block\">").AppendText(data).AppendLine("</h3>"); return(true); }, TransformText = (data, block) => { //if (data != null) sb.Append("<div class=\"notion-text-block\">").AppendText(data).AppendLine("</div>"); return(true); }, TransformBulletedList = (data, block) => { sb.Append("<ul class=\"notion-bulleted_list-block\"><li>").AppendText(data).AppendLine("</li>"); return(true, () => sb.AppendLine("</ul>")); }, TransformQuote = (data, block) => { sb.Append("<div class=\"notion-quote-block\">").AppendText(data).AppendLine("</div>"); return(true); }, TransformImage = (data, block) => { if (data != null) { sb.Append("<div class=\"notion-image-block\">").AppendImage(data, block.Id).AppendLine("</div>"); } return(true); }, TransformColumnList = (data, block) => { //Start of notion-column_list-block sb.Append("<div class=\"notion-column_list-block\"><div style=\"display: flex\">"); var totalColumns = data.Columns.Count; return(true, StartColumn : (columnIndex, column) => { sb.Append(FormattableString.Invariant($"<div class=\"notion-column\" style=\"width: calc((100% - {46*(totalColumns-1)}px) * {column.Ratio});\">")); return () => sb.Append("</div>"); }, TransformColumnSeparator : columnIndex => sb.Append("<div class=\"notion-column-separator\"><div class=\"notion-column-separator-line\"></div></div>"), EndColumnList : () => sb.Append("</div></div>")); }, TransformCallout = (data, block) => { sb.AppendLine(@$ "<div class=" "notion-callout-block notion-block-color-{data.Format.BlockColor}" ">"); if (Uri.TryCreate(data.Format.PageIcon, UriKind.Absolute, out var iconUrl)) { sb.AppendLine(@$ "<img class=" "notion-icon" " src=" "{iconUrl}" " />"); } else { sb.AppendLine(@$ "<img class=" "notion-emoji" " src=" "{GetTwitterEmojiUrl(data.Format.PageIcon)}" " />"); } transformOptions.TransformText(data.Text, block); sb.AppendLine("</div>"); return(true); }, #if DEBUG TransformOther = block => { return(true); } #endif }; recordMap.Transform(transformOptions, pageId); return(sb.ToString()); }
public static string GetHtmlAbstract(this RecordMap recordMap, Guid pageId = default) => recordMap.GetHtml(pageId, throwIfBlockMissing: false, stopBeforeFirstSubHeader: true, throwIfCantDecodeTextData: false);
public override void OnActionExecuting(ActionExecutingContext filterContext) { HttpContextBase httpContextBase = filterContext.HttpContext; HttpRequestBase request = httpContextBase.Request; HttpResponseBase response = httpContextBase.Response; string operKey = request.UserHostAddress + request.Path.ToLower(); OperRecord operRecord = null; lock (RecordMap) { if (RecordMap.ContainsKey(operKey)) { operRecord = RecordMap[operKey]; long totalMilliseconds = Convert.ToInt64((DateTime.Now - operRecord.LastOperDate).TotalMilliseconds); int count = Convert.ToInt32((totalMilliseconds - totalMilliseconds % ClearMillisecond) / ClearMillisecond); operRecord.Count = operRecord.Count > count ? operRecord.Count - count : 0; } else { RecordMap.Add(operKey, operRecord = new OperRecord { Count = 1, OperKey = operKey, LastOperDate = DateTime.Now }); } } if (operRecord.Count >= CountLimit) { response.ContentEncoding = Encoding.UTF8; response.ContentType = "application/json;charset=UTF-8"; response.StatusCode = 200; response.Write(JsonConvert.SerializeObject(new Result { code = -1, msg = "操作过于频繁、请稍后重试!", })); filterContext.Result = new EmptyResult(); } else { lock (RecordMap) { operRecord.Count++; operRecord.LastOperDate = DateTime.Now; } base.OnActionExecuting(filterContext); } if ((DateTime.Now - LastClearDate).TotalSeconds > 60) { lock (RecordMap) { long sumMillisecond = ClearMillisecond * CountLimit; foreach (string tkey in new HashSet <string>(RecordMap.Keys)) { if (tkey.EndsWith(request.Path)) { operRecord = RecordMap[tkey]; long totalMilliseconds = Convert.ToInt64((DateTime.Now - operRecord.LastOperDate).TotalMilliseconds); if (totalMilliseconds >= sumMillisecond) { RecordMap.Remove(tkey); } } } } LastClearDate = DateTime.Now; } }
public override void OnActionExecuting(ActionExecutingContext filterContext) { HttpContextBase httpContextBase = filterContext.HttpContext; HttpRequestBase request = httpContextBase.Request; string key = request.UserHostAddress + request.Path.ToLower(); bool temp = true; lock (RecordMap) { if (RecordMap.ContainsKey(key)) { temp = (DateTime.Now - RecordMap[key]).TotalMilliseconds > IntervalMillisecond; } else { RecordMap.Add(key, DateTime.Now); } } if (temp) { lock (RecordMap) { if (RecordMap.ContainsKey(key)) { RecordMap[key] = DateTime.Now; } else { RecordMap.Add(key, DateTime.Now); } } base.OnActionExecuting(filterContext); } else { HttpResponseBase response = httpContextBase.Response; response.ContentEncoding = Encoding.UTF8; response.ContentType = "application/json;charset=UTF-8"; response.StatusCode = 200; response.Write(JsonConvert.SerializeObject(new Result { code = -1, msg = "操作过于频繁、请稍后重试!", })); filterContext.Result = new EmptyResult(); } if ((DateTime.Now - LastClearDate).TotalSeconds > 60) { lock (RecordMap) { foreach (string tkey in new HashSet <string>(RecordMap.Keys)) { if (tkey.EndsWith(request.Path) && RecordMap.ContainsKey(tkey) && (DateTime.Now - RecordMap[tkey]).TotalMilliseconds > IntervalMillisecond) { RecordMap.Remove(tkey); } } } LastClearDate = DateTime.Now; } }
public static List <Record> ToRecords(this Dataset dataset, string csvFile, string term) { using (StreamReader reader = new StreamReader(csvFile, Encoding.UTF8)) using (CsvReader csv = new CsvReader(reader)) { Console.WriteLine($"Processing '{Path.GetFileName(csvFile)}'"); // Danger, Will Robinson! // // CsvHelper stores property names in couple of Dictionaries that use a string key. // These dictionaries are case sensitive, because C# supports properties the same name as long as the cases are different (e.g. "discriminator" and "Discriminator" are different properties) // // But, we want a case _insensitive_ match for when a new csv file differs in column header case only (e.g. "Territorial Authority" and "Territorial authority") so we don't have to fix our Dataset configuration // As per CsvHelper's documentation, we're expected to use the PrepareHeaderForMatch property to call ToLower() on all column headers: // - https://joshclose.github.io/CsvHelper/getting-started/ // - https://github.com/JoshClose/CsvHelper/issues/1183 // // That violates Microsoft's sting comparison guidelines, however: https://docs.microsoft.com/en-us/dotnet/csharp/how-to/compare-strings // // I'd prefer to replace the Dictionaries with ones that use a case insensitive match: // // - csv.Context.NamedIndexes = new Dictionary<string, List<int>>(StringComparer.OrdinalIgnoreCase); // - csv.Context.NamedIndexCache = new Dictionary<string, (string, int)>(StringComparer.OrdinalIgnoreCase); // // ...which works right now, but reaching into CsvHelper's internals feels kind of fragile. // // So, we'll grumble and go ahead with ToLower() csv.Configuration.PrepareHeaderForMatch = (header, index) => header.ToLowerInvariant(); // If we have more or fewer headers or properties than we expect, just keep going csv.Configuration.HeaderValidated = null; csv.Configuration.MissingFieldFound = null; csv.Configuration.ReadingExceptionOccurred = exception => { if (exception is TypeConverterException) { // Some text in the Value column, instead of a number // In most datasets, Value is null and we're supplied a null reason to indicate why we don't have a value // However, some datasets include the null reason in the value column and cause type conversion errors // We'll just skip these rows return(false); } return(true); }; var map = new RecordMap() .Map(r => r.Selector, dataset.Selector ?? "Territorial Authority") .Map(r => r.Date, dataset.Date); map.Map(r => r.Measures) .ConvertUsing(rr => dataset .Measure .Select((measure, index) => new { measure, index }) .ToDictionary( mi => mi.measure.Name, mi => new ColumnValue { Index = mi.index, Column = mi.measure.Name, Separator = mi.measure.Separator ?? " — ", Value = rr.GetField <string>(mi.measure.Name) }) ); map.Map(r => r.Categories) .ConvertUsing(rr => dataset .Category .Select((category, index) => new { category, index }) .ToDictionary(ci => ci.category.Name, ci => new ColumnValue { Index = ci.index, Column = ci.category.Name, Separator = ci.category.Separator ?? " — ", Value = rr.GetField <string>(ci.category.Name) }) ); map .Map(r => r.Value, dataset.Value ?? "Value") .Map(r => r.ValueUnit, dataset.ValueUnit ?? "Value Unit") .Map(r => r.ValueLabel, dataset.ValueLabel ?? "Value Label") .Map(r => r.NullReason, dataset.NullReason ?? "Null Reason"); csv.Configuration.RegisterClassMap(map); return(dataset.ToRecords(csv, term)); } }
public void InitializeRecordMap(RecordMap recordMap, System.Data.Common.DbTransaction transaction) { throw new Exception("The method or operation is not implemented."); }
public TradeStats(Rylobot bot) { Bot = bot; Records = new RecordMap(); Predictors = new Dictionary <string, Predictor>(); }