public byte[] Serialize <TStruct>(ref TStruct item)
{
return(UseLZ4 ? LZ4MessagePackSerializer.Serialize(item, Resolver) :
MessagePackSerializer.Serialize(item, Resolver));
}
public static Task <bool> SetLz4Async <T>(this IDatabase db, string key, T value, TimeSpan?expiry = null, When when = When.Always, CommandFlags flags = CommandFlags.None) where T : class
{
return(db.StringSetAsync(key, LZ4MessagePackSerializer.Serialize(value, ContractlessStandardResolver.Instance), expiry, when, flags));
}
T Convert <T>(T value)
{
var resolver = new WithImmutableDefaultResolver();
return(LZ4MessagePackSerializer.Deserialize <T>(LZ4MessagePackSerializer.Serialize(value, resolver), resolver));
}
public IEnumerator DownloadSubs()
{
if (dlpending)
{
yield break;
}
var cache = Dir.cache + "hsubs2.msgpack";
if (File.Exists(cache))
{
dict = LZ4MessagePackSerializer.Deserialize <Dictionary <string, KeyValuePair <int, string> > >(File.ReadAllBytes(cache));
print("Found cached hsubs");
}
var furl = cfg.sheet + "/export?exportFormat=csv&gid=" + cfg.gid + "&range=" + cfg.range;
print("Downloading subs from " + furl);
var dl = new WWW(furl);
while (!dl.isDone)
{
yield return(dl);
}
if (dl.error != null)
{
if (downloading != null)
{
downloading = "Failed: " + dl.error;
yield return(new WaitForSeconds(5));
}
else
{
print(dl.error);
}
downloading = null;
dlpending = false;
yield break;
}
print($"Parsing {dl.text.Length} characters");
int cnt = 0;
int nrow = 0;
foreach (var row in CSV.ParseCSV(dl.text))
{
nrow++;
string en = null;
int idx = 0;
string sound = null;
foreach (var cell in row)
{
if (idx == 0)
{
sound = cell.ToLower();
}
if (idx == 2)
{
en = cell;
}
idx++;
}
if (sound != null && en != null && sound.Length < 64)
{
cnt++;
dict[sound] = new KeyValuePair <int, string>(nrow, en);
}
}
print($"Done. {cnt} lines found.");
if (cnt > 60000)
{
File.WriteAllBytes(cache, LZ4MessagePackSerializer.Serialize(dict));
}
dlpending = false;
downloading = null;
}
public static byte[] ToSchematicRepresentation <TState, TInput>(this Schematic <TState, TInput> schematic)
{
return(LZ4MessagePackSerializer.Serialize(schematic, ContractlessStandardResolver.Instance));
}
public void Archive_Store_Succeeded()
{
//Arrange
//Copy current archive for testing
string archive = Path.Combine(CurrentDirectory.FullName, "Store.Archived.Test.zip");
//Check if file already exists
if (File.Exists(archive))
{
File.Delete(archive);
}
var ntick1 = new Tick(new TickerSymbol("Testing", "Testing", CurrencyType.AUD), DataSource.IEX)
{
TimeZone = TimeZone.Utc,
AskSize = 12,
AskPrice = 123.1231m,
AskSource = DataSource.Binance,
BidSize = 231213,
BidPrice = 1231.123123m,
BidSource = DataSource.Binance,
DataType = DataType.Tick,
Depth = 0,
EndTime = DateTime.MaxValue,
IsBackfilling = false,
Occured = DateTime.MaxValue,
Price = 12313.123123m,
Size = 12313,
Source = DataSource.Binance,
TradePrice = 123123.1231m
};
var ntick2 = new Tick(new TickerSymbol("Testing", "Testing", CurrencyType.AUD), DataSource.IEX)
{
TimeZone = TimeZone.Utc,
AskSize = 12,
AskPrice = 123.1231m,
AskSource = DataSource.Binance,
BidSize = 231213,
BidPrice = 1231.123123m,
BidSource = DataSource.Binance,
DataType = DataType.Tick,
Depth = 0,
EndTime = DateTime.MaxValue,
IsBackfilling = false,
Occured = DateTime.MaxValue,
Price = 12313.123123m,
Size = 12313,
Source = DataSource.Binance,
TradePrice = 123123.1231m
};
var input = new Dictionary <string, byte[]>()
{
{ "File1.DAT", LZ4MessagePackSerializer.Serialize(ntick1) },
{ "File2.DAT", LZ4MessagePackSerializer.Serialize(ntick2) }
};
//Act
var result = Archive.Store(archive, input);
//Assert
result.Should().Be(2);
File.Exists(archive).Should().BeTrue();
Archive.IsValidArchive(archive).Should().BeTrue();
}
/// <inheritdoc />
public async Task <ICollection <MachineStatus <TState, TInput> > > BulkCreateMachinesAsync(
Schematic <TState, TInput> schematic,
IEnumerable <IDictionary <string, string> > metadata,
CancellationToken cancellationToken = default)
{
if (schematic == null)
{
throw new ArgumentNullException(nameof(schematic));
}
var schematicBytes = LZ4MessagePackSerializer.Serialize(
obj: schematic,
resolver: ContractlessStandardResolver.Instance);
var stateJson = MessagePackSerializer.ToJson(
obj: schematic.InitialState,
resolver: ContractlessStandardResolver.Instance);
const long commitNumber = 0L;
var updatedTime = DateTimeOffset.UtcNow;
var records = new List <Machine>();
var machineStatuses = new List <MachineStatus <TState, TInput> >();
foreach (var dictionary in metadata)
{
var machineId = Guid.NewGuid().ToString();
List <MetadataEntry> metadataEntries = null;
if (dictionary != null)
{
metadataEntries = dictionary.Select(kvp => new MetadataEntry
{
Key = kvp.Key,
Value = kvp.Value
}).ToList();
}
records.Add(new Machine
{
MachineId = machineId,
SchematicBytes = schematicBytes,
StateJson = stateJson,
CommitNumber = commitNumber,
UpdatedTime = updatedTime,
MetadataEntries = metadataEntries
});
machineStatuses.Add(new MachineStatus <TState, TInput>
{
MachineId = machineId,
Schematic = schematic.Clone(),
State = schematic.InitialState,
Metadata = dictionary,
CommitNumber = commitNumber,
UpdatedTime = updatedTime,
StateBag = new Dictionary <string, string>(0)
});
}
await DbContext.AddRangeAsync(records, cancellationToken);
await DbContext.SaveChangesAsync(cancellationToken);
return(machineStatuses);
}
public ActionResult DataUpload([FromQuery] string token, MilvanethProtocol data)
{
if (!_token.TryDecode(token, out var payload))
{
return(StatusCode(401));
}
if (!(data?.Context).Check() || !(data?.Data is PackedResult result) || !result.Check())
{
return(StatusCode(400));
}
try
{
if (payload.ValidTo < _time.UtcNow)
{
return(StatusCode(511));
}
_auth.EnsureToken(payload, TokenPurpose.AccessToken, GlobalOperation.DATA_UPLOAD, 0, out var account);
// virtual deduction for token validation operation
var karmaBefore = account.Karma;
account.Karma -= 20;
switch (result.Type)
{
case PackedResultType.Inventory:
var inventory = (InventoryResult)result.Result;
if (inventory.Context != data.Context.CharacterId)
{
break;
}
_repo.Character.Commit(account.AccountId, new CharacterData
{
CharacterId = inventory.Context,
ServiceId = data.Context.ServiceId,
AccountId = account.AccountId,
Inventory = LZ4MessagePackSerializer.Serialize(data.Data)
}, false);
account.Karma += 20;
break;
case PackedResultType.InventoryNetwork:
var inventoryNetwork = (InventoryResult)result.Result;
_repo.Retainer.Commit(account.AccountId, new RetainerData
{
RetainerId = inventoryNetwork.Context,
Character = data.Context.CharacterId,
World = data.Context.World,
Inventory = LZ4MessagePackSerializer.Serialize(data.Data)
}, false);
account.Karma += 20;
break;
case PackedResultType.Artisan:
var artisan = (ArtisanResult)result.Result;
_repo.Character.CommitRange(account.AccountId, artisan.ArtisanList.Select(x => new CharacterData
{
CharacterId = x.CharacterId,
CharacterName = x.IsValid ? x.CharacterName : null,
HomeWorld = x.IsValid && !x.FromMemory ? data.Context.World : 0,
}), false);
account.Karma += 20 + artisan.ArtisanList.Count(x => x.IsValid) * 10 + artisan.ArtisanList.Count(x => !x.IsValid) * 3;
break;
case PackedResultType.MarketHistory:
var marketHistory = (MarketHistoryResult)result.Result;
_repo.History.CommitRange(data.Context.CharacterId,
marketHistory.HistoryItems.Select(x => x.ToDb(result.ReportTime, data.Context.World)),
false);
account.Karma += 20 + 30;
break;
case PackedResultType.MarketListing:
var marketListing = (MarketListingResult)result.Result;
var artisanFk = marketListing.ListingItems.Select(x => x.ArtisanId);
var ownerFk = marketListing.ListingItems.Select(x => new { x.OwnerId, x.PlayerName }).ToImmutableList();
_repo.Character.CommitRange(account.AccountId,
ownerFk.Where(x => x.OwnerId != 0).GroupBy(x => x.OwnerId).Select(y =>
{
var x = y.First();
return(new CharacterData
{
CharacterId = x.OwnerId,
HomeWorld = data.Context.World,
CharacterName = x.PlayerName
});
}),
false);
_repo.Character.CommitRange(account.AccountId,
artisanFk.Where(x => x > 0 && ownerFk.All(y => y.OwnerId != x)).Distinct().Select(x => new CharacterData {
CharacterId = x
}),
true);
_repo.Retainer.CommitRange(account.AccountId,
marketListing.ListingItems.GroupBy(x => x.RetainerId).Select(y =>
{
var x = y.First();
return(new RetainerData
{
RetainerId = x.RetainerId, Character = x.OwnerId, RetainerName = x.RetainerName, Location = x.RetainerLocation, World = data.Context.World
});
}),
true);
_repo.Listing.CommitRange(data.Context.CharacterId,
marketListing.ListingItems.Select(x => x.ToDb(result.ReportTime, data.Context.World)),
false);
foreach (var x in marketListing.ListingItems.GroupBy(x => x.ItemId))
{
_repo.Overview.Commit(data.Context.CharacterId,
new OverviewData
{
ReportTime = result.ReportTime,
World = data.Context.World,
ItemId = x.Key,
OpenListing = (short)x.Count(),
}, false);
}
account.Karma += 20 + 100;
break;
case PackedResultType.MarketOverview:
var marketOverview = (MarketOverviewResult)result.Result;
_repo.Overview.CommitRange(data.Context.CharacterId,
marketOverview.ResultItems.Select(x => x.ToDb(result.ReportTime, data.Context.World)),
false);
account.Karma += 20 + 30 + marketOverview.ResultItems.Count;
break;
case PackedResultType.RetainerHistory:
// todo
break;
case PackedResultType.RetainerList:
var retainerList = (RetainerInfoResult)result.Result;
_repo.Retainer.CommitRange(account.AccountId,
retainerList.RetainerInfo.Select(x => x.ToDb(data.Context.CharacterId, data.Context.World)), false);
account.Karma += 20 + retainerList.RetainerInfo.Count * 4;
break;
case PackedResultType.RetainerUpdate:
#region NotFinishedCode
// todo
// 目前有两种方案，直接更新和Copy-on-Update，前者的主要问题在于并发条件下Time-Bucket体系可能会出现问题，后者则在于性能开销，故目前暂不对此数据进行处理
//var retainerUpdate = (RetainerUpdateResult)result.Result;
//foreach (var item in retainerUpdate.UpdateItems)
//{
// var record = _context.ListingData
// .Where(x => x.RetainerId == item.RetainerId && x.ContainerId == (short)item.ContainerId && x.SlotId == item.ContainerSlot)
// .OrderByDescending(x => x.ReportTime).Take(1);
// if (!record.Any())
// {
// var retInfo =
// _context.RetainerData.SingleOrDefault(x => x.RetainerId == item.RetainerId);
// var record2 = _context.ListingData.Where(x =>
// x.World == data.Context.World && x.ItemId == item.ItemInfo.ItemId &&
// x.ReportTime <= result.ReportTime)
// .OrderByDescending(x => x.ReportTime).Take(1);
// _repo.Commit(data.Context.CharacterId, new ListingData
// {
// BucketId = record2.Any() ? record2.First().BucketId : Guid.Empty,
// ReportTime = result.ReportTime,
// World = data.Context.World,
// ReporterId = data.Context.CharacterId,
// ListingId = 0,
// RetainerId = item.RetainerId,
// OwnerId = data.Context.CharacterId,
// ArtisanId = item.ItemInfo.ArtisanId,
// UnitPrice = item.NewPrice,
// TotalTax = 0,
// Quantity = item.ItemInfo.Amount,
// ItemId = item.ItemInfo.ItemId,
// UpdateTime = Helper.DateTimeToUnixTimeStamp(result.ReportTime),
// ContainerId = (short)item.ContainerId,
// SlotId = (short)item.ContainerSlot,
// Condition = (short)item.ItemInfo.Durability,
// SpiritBond = (short)item.ItemInfo.SpiritBond,
// Materia1 = item.ItemInfo.Materia1,
// Materia2 = item.ItemInfo.Materia2,
// Materia3 = item.ItemInfo.Materia3,
// Materia4 = item.ItemInfo.Materia4,
// Materia5 = item.ItemInfo.Materia5,
// RetainerName = retInfo?.RetainerName,
// PlayerName = item.ContainerId == InventoryContainerId.HIRE_LISTING ? null : _context.CharacterData.SingleOrDefault(x => x.CharacterId == data.Context.CharacterId)?.CharacterName,
// IsHq = item.ItemInfo.IsHq,
// MateriaCount = 0, //todo
// OnMannequin = item.ContainerId != InventoryContainerId.HIRE_LISTING,
// RetainerLoc = 0,//todo
// DyeId = item.ItemInfo.DyeId
// }, true);
// account.Karma += 25;
// continue;
// }
// var recordEntity = record.First();
// if (recordEntity.ReportTime >= result.ReportTime)
// {
// account.Karma -= 25;
// continue;
// }
// recordEntity.
//}
//account.Karma += 20 + retainerUpdate.UpdateItems.Count * 25;
#endregion
account.Karma += 20;
break;
case PackedResultType.Status:
var status = (StatusResult)result.Result;
if (status.CharacterId != data.Context.CharacterId)
{
break;
}
if (account.PlayedCharacter == null)
{
account.PlayedCharacter = new long[0];
}
if (!account.PlayedCharacter.Contains(status.CharacterId))
{
var temp = new long[account.PlayedCharacter.Length + 1];
Array.Copy(account.PlayedCharacter, temp, account.PlayedCharacter.Length);
temp[temp.Length - 1] = status.CharacterId;
account.PlayedCharacter = temp;
}
_repo.Character.Commit(account.AccountId, new CharacterData
{
CharacterId = status.CharacterId,
CharacterName = status.CharacterName,
ServiceId = data.Context.ServiceId,
AccountId = account.AccountId,
HomeWorld = status.CharacterHomeWorld,
JobLevels = status.LevelInfo.ToDb(),
GilHold = status.CharaInfo.GilHold
}, false);
account.Karma += 20 + 40;
break;
case PackedResultType.LobbyService:
var lobbyService = (LobbyServiceResult)result.Result;
if (lobbyService.ServiceId != data.Context.ServiceId)
{
break;
}
if (account.RelatedService == null)
{
account.RelatedService = new long[0];
}
if (!account.RelatedService.Contains(lobbyService.ServiceId))
{
var temp = new long[account.RelatedService.Length + 1];
Array.Copy(account.RelatedService, temp, account.RelatedService.Length);
temp[temp.Length - 1] = lobbyService.ServiceId;
account.RelatedService = temp;
}
account.Karma += 20;
break;
case PackedResultType.LobbyCharacter:
var lobbyCharacter = (LobbyCharacterResult)result.Result;
if (!DataChecker.CheckOnlineCharacterBinding(data.Context.ServiceId,
lobbyCharacter.CharacterItems))
{
#warning api availability is not checked.
account.Karma -= 180;
break;
}
if (account.RelatedService == null)
{
account.RelatedService = new long[0];
}
if (!account.RelatedService.Contains(data.Context.ServiceId))
{
var temp = new long[account.RelatedService.Length + 1];
Array.Copy(account.RelatedService, temp, account.RelatedService.Length);
temp[temp.Length - 1] = data.Context.ServiceId;
account.RelatedService = temp;
}
_repo.Character.CommitRange(account.AccountId, lobbyCharacter.CharacterItems.Select(x => x.ToDb(data.Context.ServiceId)), false);
account.Karma += 20 + 20 * lobbyCharacter.CharacterItems.Count;
break;
default:
// do nothing
break;
}
_context.KarmaLog.Add(new KarmaLog
{
ReportTime = _time.UtcNow,
AccountId = account.AccountId,
Reason = GlobalOperation.DATA_UPLOAD + (int)result.Type,
Before = karmaBefore,
After = account.Karma
});
_context.AccountData.Update(account);
_context.SaveChanges();
return(StatusCode(200));
}
catch (Exception e)
{
Log.Error(e, "Error in DATA/UPLOAD");
return(StatusCode(500));
}
}
/// <summary>
/// 序列化二进制
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="obj"></param>
/// <returns></returns>
public static byte[] Serializer <T>(T obj)
{
Init();
return(LZ4MessagePackSerializer.Serialize(obj));
}
public byte[] MessagePackWithTypeLz4()
{
return(LZ4MessagePackSerializer.Serialize(_testData));
}
/// <summary>
/// Serialize and compress any object into binary value using MessagePack + LZ4.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="value"></param>
/// <returns></returns>
public static byte[] SerializeObject <T>(T value)
{
return(LZ4MessagePackSerializer.Serialize(value, ContractlessStandardResolver.Instance));
}
static void Main(string[] args)
{
var timer = new Stopwatch();
var logic = new RedisLogic();
var dataGen = new RandomData();
//logic.SetStringData("basicCall", "Hello world");
//var x = logic.GetData<string>("basicCall");
//Console.WriteLine(x);
//x = logic.GetData<string>("basicCall");
//Console.WriteLine(x);
//var y = logic.GetAllKeys();
//Console.WriteLine(string.Join(", ", y));
var data = dataGen.GetRandomData(10);
var saveData = (from d in data
select JsonConvert.SerializeObject(d)).ToList();
//timer.Start();
//logic.SetListData("people", saveData);
//timer.Stop();
//Console.WriteLine($"Adding {saveData.Count} rows took {timer.ElapsedMilliseconds}ms.");
//timer.Restart();
//var z = logic.GetListData("people");
//timer.Stop();
//Console.WriteLine($"Getting {z.Count} rows took {timer.ElapsedMilliseconds}ms.");
////Check speed of geting list data
//for (int i = 0; i < 10; i++)
//{
// data = dataGen.GetRandomData(2000);
// saveData = (from d in data
// select JsonConvert.SerializeObject(d)).ToList();
// logic.SetListData("people", saveData);
// timer.Restart();
// z = logic.GetListData("people");
// timer.Stop();
// Console.WriteLine($"Getting {z.Count} rows took {timer.ElapsedMilliseconds}ms.");
//}
//var u = new List<string>();
////Check speed of geting hash data
//for (int i = 0; i < 10; i++)
//{
// var junk = dataGen.RandomHashData(2000);
// logic.SetHashData("junk", junk);
// timer.Restart();
// u = logic.GetHashValues("junk");
// timer.Stop();
// Console.WriteLine($"Getting {u.Count} hash values took {timer.ElapsedMilliseconds}ms.");
//}
//timer.Restart();
//var q = logic.GetHashKeys("junk");
//timer.Stop();
//Console.WriteLine($"Getting {q.Count} hash keys took {timer.ElapsedMilliseconds}ms.");
//timer.Restart();
//var e = logic.GetSingleHashValue("junk", q.First());
//timer.Stop();
//Console.WriteLine($"Getting specific hash value took {timer.ElapsedMilliseconds}ms.");
//saveData.Clear();
//Testing pipelinging speed
//for (int i = 0; i < 1; i++)
//{
// data = dataGen.GetRandomData(2000);
// saveData.AddRange((from d in data
// select JsonConvert.SerializeObject(d)).ToList());
//}
//timer.Restart();
//logic.SetListData("pipe", saveData.GetRange(0, 3));
//timer.Stop();
//Console.WriteLine($"Adding {saveData.Count} rows took {timer.ElapsedMilliseconds}ms.");
//logic.SetListPipeline("reg", saveData);
//logic.SetListPipelineAsParallel("para", saveData);
//var reg = new List<long>();
//var asParallels = new List<long>();
//for (int i = 0; i < 10; i++)
//{
// timer.Restart();
// logic.SetListPipeline("reg", saveData);
// timer.Stop();
// reg.Add(timer.ElapsedMilliseconds);
// timer.Restart();
// logic.SetListPipelineAsParallel("para", saveData);
// timer.Stop();
// asParallels.Add(timer.ElapsedMilliseconds);
//}
//Console.WriteLine($"Adding {saveData.Count} rows regular took {reg.Average()}ms.");
//Console.WriteLine($"Adding {saveData.Count} rows asParallel took {asParallels.Average()}ms.");
//timer.Restart();
//var w = logic.GetListData("pipe");
//timer.Stop();
//Console.WriteLine($"Getting {w.Count} rows pipelined took {timer.ElapsedMilliseconds}ms.");
var seraldata = new List <PersonData>();
Parallel.For(0, 10, i =>
{
seraldata.AddRange(dataGen.RandomPersonData(2000));
});
BinaryFormatter bf = new BinaryFormatter();
MemoryStream ms = new MemoryStream();
byte[] Array;
bf.Serialize(ms, seraldata);
Array = ms.ToArray();
int startSize = Array.Length;
var l4Time = new List <long>();
var l4Size = new List <long>();
var msgTime = new List <long>();
var msgSize = new List <long>();
for (int i = 0; i < 100; i++)
{
timer.Restart();
var lv4 = LZ4MessagePackSerializer.Serialize(seraldata);
timer.Stop();
l4Time.Add(timer.ElapsedMilliseconds);
l4Size.Add(lv4.LongLength);
timer.Restart();
var megPak = MessagePackSerializer.Serialize(seraldata);
timer.Stop();
msgTime.Add(timer.ElapsedMilliseconds);
msgSize.Add(megPak.LongLength);
}
Console.WriteLine($"Stating size {(startSize / 1024) / 1024} mb");
Console.WriteLine($"Lz4 serialization took {l4Time.Average()}ms and a size of {(l4Size.Average() / 1024) / 1024} mb");
Console.WriteLine($"Messagepack serialization took {msgTime.Average()}ms and a size of {(msgSize.Average() / 1024) / 1024} mb");
Console.WriteLine($"lz4 took { msgTime.Average() - l4Time.Average()} less time.");
Console.WriteLine($"Lz4 is {((msgSize.Average() - l4Size.Average()) / 1024) / 1024} smaller.");
Console.WriteLine($"Lz4 shunk in size by {(l4Size.Average() / startSize) * 100} %");
Console.WriteLine($"MsgPack shunk in size by {(msgSize.Average() / startSize) * 100} %");
timer.Restart();
var ser = seraldata.AsParallel().Select((x, i) => new { Index = i, Value = MessagePackSerializer.Serialize(x) }).ToList();
timer.Stop();
Console.WriteLine($"Serializing {ser.Count} rows in plinq took {timer.ElapsedMilliseconds}ms.");
timer.Restart();
ser = seraldata.Select((x, i) => new { Index = i, Value = MessagePackSerializer.Serialize(x) }).ToList();
timer.Stop();
Console.WriteLine($"Serializing {ser.Count} rows in linq took {timer.ElapsedMilliseconds}ms.");
timer.Restart();
logic.SetHastPipeline("serial", seraldata);
timer.Stop();
Console.WriteLine($"Setting {seraldata.Count} rows using serialization took {timer.ElapsedMilliseconds}ms.");
timer.Restart();
logic.SetHashPipeline("pipehash", seraldata);
timer.Stop();
Console.WriteLine($"Setting {seraldata.Count} rows hash took {timer.ElapsedMilliseconds}ms.");
timer.Restart();
logic.SetHastAsParallelPipeline("serialMulti", seraldata);
timer.Stop();
Console.WriteLine($"Setting {seraldata.Count} rows parallazied serialization took {timer.ElapsedMilliseconds}ms.");
timer.Restart();
var serializedData = logic.GetHashSerialValues("serial");
var deserial = serializedData.AsParallel().Select(x => MessagePackSerializer.Deserialize <PersonData>(x));
timer.Stop();
Console.WriteLine($"Deserializing {seraldata.Count} rows plinq took {timer.ElapsedMilliseconds}ms.");
timer.Restart();
var sdata = logic.GetHashPersonValues("serialMulti");
timer.Stop();
Console.WriteLine($"Deserializing {sdata.Count} rows inline took {timer.ElapsedMilliseconds}ms.");
timer.Restart();
//cleanup
logic.DeleteData("basicCall");
logic.DeleteData("people");
logic.DeleteData("junk");
logic.DeleteData("pipe");
logic.DeleteData("para");
logic.DeleteData("reg");
logic.DeleteData("serial");
logic.DeleteData("pipehash");
logic.DeleteData("serialMulti");
logic.DeleteData("parallel");
timer.Stop();
Console.WriteLine($"Clean up took {timer.ElapsedMilliseconds}ms.");
Console.WriteLine($"Press enter to exit.");
var t = Console.ReadLine();
}
private void butProvest_Click(object sender, EventArgs e)
{
// TODO allow file import
// todo make async
byte[] content = Encoding.UTF8.GetBytes(textBox_main.Text);
Keypair chosenKeypair = ucKeychain_mainTab.SelectedItems.FirstOrDefault();
IArmorProvider armorProvider = new HitaiArmorProvider();
Message message;
PasswordInputDialog passwordDialog;
Signature signature;
bool correctSignature;
switch (comboBox_actions.SelectedIndex)
{
case 0: // šifrovat
if (chosenKeypair == null)
{
MessageBox.Show("Musíte vybrat klíč adresáta.");
return;
}
message = AsymmetricEncryptionController.Encrypt(chosenKeypair, content);
byte[] armor =
armorProvider.ToArmor(LZ4MessagePackSerializer.Serialize(message),
ArmorType.Message);
string result = Encoding.UTF8.GetString(armor);
Clipboard.SetText(result);
MessageBox.Show("Výsledek byl zkopírován do schránky.");
break;
case 1: // dešifrovat
Type armorClass = ArmorRecognizer.RecognizeArmor(content);
armorProvider = (IArmorProvider)Activator.CreateInstance(armorClass);
(byte[] bytes, ArmorType armorType) = armorProvider.FromArmor(content);
// TODO nechat rozpoznat akci podle ArmorType
if (armorType != ArmorType.Message)
{
MessageBox.Show("Obsah není zprávou.");
return;
}
message = LZ4MessagePackSerializer.Deserialize <Message>(bytes);
Keypair recipient =
Keychain.Keys.FirstOrDefault(x => x.ShortId == message.RecipientId);
if (recipient == null)
{
MessageBox.Show("Nebyl nalezen odpovídající soukromý klíč.");
return;
}
passwordDialog = new PasswordInputDialog("Dešifrování");
if (passwordDialog.ShowDialog() != DialogResult.OK)
{
return;
}
byte[] data;
try {
data = AsymmetricEncryptionController.Decrypt(message,
passwordDialog.Password,
recipient);
}
catch (Exception ex) {
while (ex is AggregateException)
{
ex = ex.InnerException;
}
if (!(ex is CryptographicException))
{
throw;
}
MessageBox.Show("Nesprávné heslo.");
return;
}
string clearText = Encoding.UTF8.GetString(data);
textBox_main.Text = clearText;
break;
case 2: // podepsat
if (!chosenKeypair.IsPrivate)
{
MessageBox.Show("Podepisující klíč musí být soukromý.");
return;
}
passwordDialog = new PasswordInputDialog("Podepisování");
if (passwordDialog.ShowDialog() != DialogResult.OK)
{
return;
}
try {
signature = AsymmetricEncryptionController.Sign(
Encoding.UTF8.GetBytes(textBox_main.Text),
passwordDialog.Password, chosenKeypair);
}
catch (Exception ex) {
while (ex is AggregateException)
{
ex = ex.InnerException;
}
if (!(ex is CryptographicException))
{
throw;
}
MessageBox.Show("Nesprávné heslo.");
return;
}
Clipboard.SetText(Encoding.UTF8.GetString(armorProvider.ToArmor(
LZ4MessagePackSerializer.Serialize(signature),
ArmorType.Signature)));
MessageBox.Show("Výsledek byl zkopírován do schránky");
break;
case 3: // ověřit
armorProvider =
(IArmorProvider)Activator.CreateInstance(
ArmorRecognizer.RecognizeArmor(content));
(byte[] bytes2, ArmorType armorType2) = armorProvider.FromArmor(content);
if (armorType2 != ArmorType.Signature)
{
MessageBox.Show("Vstup neobsahuje podpis.");
return;
}
signature = LZ4MessagePackSerializer.Deserialize <Signature>(bytes2);
correctSignature = AsymmetricEncryptionController.Verify(signature, Keychain);
string clearText2;
try {
clearText2 = signature.GetCleartext();
}
catch {
clearText2 = null;
}
MessageBox.Show((correctSignature ? "Správný podpis" : "Nesprávný podpis") +
(string.IsNullOrEmpty(clearText2)
? "."
: " pro zprávu:\n" + clearText2));
break;
case 4: // šifrovat a podepsat
var chooseKeyDialog =
new ChooseKeyDialog("Vyberte klíč, kterým chcete zprávu podepsat.")
{
Keychain = Keychain
};
if (chooseKeyDialog.ShowDialog() != DialogResult.OK)
{
return;
}
message = AsymmetricEncryptionController.Encrypt(chosenKeypair, content);
passwordDialog = new PasswordInputDialog("Podepisování");
if (passwordDialog.ShowDialog() != DialogResult.OK)
{
return;
}
try {
signature = AsymmetricEncryptionController.Sign(message,
passwordDialog.Password, chooseKeyDialog.ChosenKeypair);
}
catch (Exception ex) {
while (ex is AggregateException)
{
ex = ex.InnerException;
}
if (!(ex is CryptographicException))
{
throw;
}
MessageBox.Show("Nesprávné heslo.");
return;
}
result = Encoding.UTF8.GetString(armorProvider.ToArmor(
LZ4MessagePackSerializer.Serialize(signature),
ArmorType.SignedMessage));
Clipboard.SetText(result);
MessageBox.Show("Výsledek byl zkopírován do schránky");
break;
case 5: // dešifrovat a ověřit
armorProvider =
(IArmorProvider)Activator.CreateInstance(
ArmorRecognizer.RecognizeArmor(content));
(byte[] bytes3, ArmorType armorType3) = armorProvider.FromArmor(content);
if (armorType3 != ArmorType.SignedMessage)
{
MessageBox.Show("Vstup neobsahuje podepsanou zprávu.");
return;
}
signature = LZ4MessagePackSerializer.Deserialize <Signature>(bytes3);
correctSignature = AsymmetricEncryptionController.Verify(signature, Keychain);
if (!correctSignature)
{
MessageBox.Show(
"Podpis není správný. Zpráva nebude dešifrována z bezpečnostních důvodů.");
return;
}
if (!signature.ContainsMessage())
{
MessageBox.Show(
"Podpis neobsahuje zašifrovanou zprávu. Chyba.");
return;
}
passwordDialog = new PasswordInputDialog("Dešifrování");
if (passwordDialog.ShowDialog() != DialogResult.OK)
{
return;
}
try {
result = Encoding.UTF8.GetString(
AsymmetricEncryptionController.Decrypt(signature.GetMessage(),
passwordDialog.Password, Keychain));
}
catch (Exception ex) {
while (ex is AggregateException)
{
ex = ex.InnerException;
}
if (!(ex is CryptographicException))
{
throw;
}
MessageBox.Show("Nesprávné heslo.");
return;
}
MessageBox.Show(
"Podpis byl úspěšně ověřen a dešifrovaná zpráva bude zobrazena.");
textBox_main.Text = result;
break;
default:
throw new NotImplementedException();
}
}
private static async Task SendResponse(IMessage message, WebSocket webSocket)
{
await webSocket.SendAsync(new ArraySegment <byte>(LZ4MessagePackSerializer.Serialize(message)), WebSocketMessageType.Binary, false,
CancellationToken.None);
}
public void TestSmall()
{
MessagePackBinary.GetMessagePackType(LZ4MessagePackSerializer.Serialize(100), 0).Is(MessagePackType.Integer);
MessagePackBinary.GetMessagePackType(LZ4MessagePackSerializer.Serialize("test"), 0).Is(MessagePackType.String);
MessagePackBinary.GetMessagePackType(LZ4MessagePackSerializer.Serialize(false), 0).Is(MessagePackType.Boolean);
}
public byte[] Serialize <T>(T data)
{
return(LZ4MessagePackSerializer.Serialize(data));
}
static void Benchmark <T>(T target)
{
const int Iteration = 10000; // 10000
var jsonSerializer = new JsonSerializer();
var msgpack = MsgPack.Serialization.SerializationContext.Default;
msgpack.GetSerializer <T>().PackSingleObject(target);
MessagePack.MessagePackSerializer.Serialize(target);
LZ4MessagePackSerializer.Serialize(target);
ZeroFormatter.ZeroFormatterSerializer.Serialize(target);
ProtoBuf.Serializer.Serialize(new MemoryStream(), target);
jsonSerializer.Serialize(new JsonTextWriter(new StringWriter()), target);
Console.WriteLine(typeof(T).Name + " serialization test");
Console.WriteLine();
Console.WriteLine("Serialize::");
byte[] data = null;
byte[] data0 = null;
byte[] data1 = null;
byte[] data2 = null;
byte[] data3 = null;
byte[] dataJson = null;
byte[] dataGzipJson = null;
using (new Measure("MsgPack-Cli"))
{
for (int i = 0; i < Iteration; i++)
{
data = msgpack.GetSerializer <T>().PackSingleObject(target);
}
}
using (new Measure("MessagePack-CSharp"))
{
for (int i = 0; i < Iteration; i++)
{
data0 = MessagePack.MessagePackSerializer.Serialize(target);
}
}
using (new Measure("MessagePack(LZ4)"))
{
for (int i = 0; i < Iteration; i++)
{
data3 = LZ4MessagePackSerializer.Serialize(target);
}
}
using (new Measure("ZeroFormatter"))
{
for (int i = 0; i < Iteration; i++)
{
data1 = ZeroFormatter.ZeroFormatterSerializer.Serialize(target);
}
}
using (new Measure("JsonNet"))
{
for (int i = 0; i < Iteration; i++)
{
using (var ms = new MemoryStream())
using (var sw = new StreamWriter(ms, Encoding.UTF8, 1024, true))
using (var jw = new JsonTextWriter(sw))
{
jsonSerializer.Serialize(jw, target);
}
}
}
using (new Measure("JsonNet+Gzip"))
{
for (int i = 0; i < Iteration; i++)
{
using (var ms = new MemoryStream())
using (var gzip = new GZipStream(ms, CompressionLevel.Fastest))
using (var sw = new StreamWriter(gzip, Encoding.UTF8, 1024, true))
using (var jw = new JsonTextWriter(sw))
{
jsonSerializer.Serialize(jw, target);
}
}
}
using (new Measure("protobuf-net"))
{
for (int i = 0; i < Iteration; i++)
{
using (var ms = new MemoryStream())
{
ProtoBuf.Serializer.Serialize(ms, target);
}
}
}
using (var ms = new MemoryStream())
{
ProtoBuf.Serializer.Serialize(ms, target);
data2 = ms.ToArray();
}
using (var ms = new MemoryStream())
{
using (var sw = new StreamWriter(ms, Encoding.UTF8, 1024, true))
using (var jw = new JsonTextWriter(sw))
{
jsonSerializer.Serialize(jw, target);
}
dataJson = ms.ToArray();
}
using (var ms = new MemoryStream())
{
using (var gzip = new GZipStream(ms, CompressionLevel.Fastest))
using (var sw = new StreamWriter(gzip, Encoding.UTF8, 1024, true))
using (var jw = new JsonTextWriter(sw))
{
jsonSerializer.Serialize(jw, target);
}
dataGzipJson = ms.ToArray();
}
msgpack.GetSerializer <T>().UnpackSingleObject(data);
MessagePack.MessagePackSerializer.Deserialize <T>(data0);
//ZeroFormatterSerializer.Deserialize<T>(data1);
ProtoBuf.Serializer.Deserialize <T>(new MemoryStream(data2));
LZ4MessagePackSerializer.Deserialize <T>(data3);
jsonSerializer.Deserialize <T>(new JsonTextReader(new StreamReader(new MemoryStream(dataJson))));
Console.WriteLine();
Console.WriteLine("Deserialize::");
using (new Measure("MsgPack-Cli"))
{
for (int i = 0; i < Iteration; i++)
{
msgpack.GetSerializer <T>().UnpackSingleObject(data);
}
}
using (new Measure("MessagePack-CSharp"))
{
for (int i = 0; i < Iteration; i++)
{
MessagePack.MessagePackSerializer.Deserialize <T>(data0);
}
}
using (new Measure("MessagePack(LZ4)"))
{
for (int i = 0; i < Iteration; i++)
{
LZ4MessagePackSerializer.Deserialize <T>(data3);
}
}
using (new Measure("ZeroFormatter"))
{
for (int i = 0; i < Iteration; i++)
{
ZeroFormatterSerializer.Deserialize <T>(data1);
}
}
using (new Measure("JsonNet"))
{
for (int i = 0; i < Iteration; i++)
{
using (var ms = new MemoryStream(dataJson))
using (var sr = new StreamReader(ms, Encoding.UTF8))
using (var jr = new JsonTextReader(sr))
{
jsonSerializer.Deserialize <T>(jr);
}
}
}
using (new Measure("JsonNet+Gzip"))
{
for (int i = 0; i < Iteration; i++)
{
using (var ms = new MemoryStream(dataGzipJson))
using (var gzip = new GZipStream(ms, CompressionMode.Decompress))
using (var sr = new StreamReader(gzip, Encoding.UTF8))
using (var jr = new JsonTextReader(sr))
{
jsonSerializer.Deserialize <T>(jr);
}
}
}
using (new Measure("protobuf-net"))
{
for (int i = 0; i < Iteration; i++)
{
using (var ms = new MemoryStream(data2))
{
ProtoBuf.Serializer.Deserialize <T>(ms);
}
}
}
Console.WriteLine();
Console.WriteLine("FileSize::");
var label = "";
label = "MsgPack-Cli"; Console.WriteLine($"{label,20} {data.Length} Byte");
label = "MessagePack-CSharp"; Console.WriteLine($"{label,20} {data0.Length} Byte");
label = "MessagePack(LZ4)"; Console.WriteLine($"{label,20} {data3.Length} Byte");
label = "ZeroFormatter"; Console.WriteLine($"{label,20} {data1.Length} Byte");
label = "protobuf-net"; Console.WriteLine($"{label,20} {data2.Length} Byte");
label = "JsonNet"; Console.WriteLine($"{label,20} {dataJson.Length} Byte");
label = "JsonNet+GZip"; Console.WriteLine($"{label,20} {dataGzipJson.Length} Byte");
Console.WriteLine();
Console.WriteLine();
}
public Task SerializeAsync <T>(Stream stream, T data)
{
LZ4MessagePackSerializer.Serialize(stream, data);
return(Task.CompletedTask);
}
/// <inheritdoc />
public async Task <MachineStatus <TState, TInput> > CreateMachineAsync(
Schematic <TState, TInput> schematic,
string machineId,
IDictionary <string, string> metadata,
CancellationToken cancellationToken = default)
{
if (schematic == null)
{
throw new ArgumentNullException(nameof(schematic));
}
var id = machineId ?? Guid.NewGuid().ToString();
var schematicBytes = LZ4MessagePackSerializer.Serialize(
obj: schematic,
resolver: ContractlessStandardResolver.Instance);
var stateJson = MessagePackSerializer.ToJson(
obj: schematic.InitialState,
resolver: ContractlessStandardResolver.Instance);
const long commitNumber = 0L;
var updatedTime = DateTimeOffset.UtcNow;
var record = new Machine
{
MachineId = id,
SchematicBytes = schematicBytes,
StateJson = stateJson,
CommitNumber = commitNumber,
UpdatedTime = updatedTime
};
if (metadata != null)
{
record.MetadataEntries = metadata.Select(kvp => new MetadataEntry
{
Key = kvp.Key,
Value = kvp.Value
}).ToList();
}
DbContext.Machines.Add(record);
try
{
await DbContext.SaveChangesAsync(cancellationToken);
}
catch (DbUpdateException dbEx)
when(dbEx.InnerException is SqlException sqlEx && sqlEx.Number == 2627)
{
throw new MachineAlreadyExistException(record.MachineId);
}
return(new MachineStatus <TState, TInput>
{
MachineId = id,
Schematic = schematic.Clone(),
State = schematic.InitialState,
Metadata = metadata,
CommitNumber = commitNumber,
UpdatedTime = updatedTime,
StateBag = new Dictionary <string, string>(0)
});
}
public static byte[] ToCompressBinary <T>(this T obj)
{
return(LZ4MessagePackSerializer.Serialize(obj));
}
/// <summary> /// Serializes this instance. /// </summary> /// <returns></returns> public byte[] Serialize() => LZ4MessagePackSerializer.Serialize(this);
public static async Task SetAsync <T>(this IDistributedCache distributedCache, string key, T value, DistributedCacheEntryOptions options, CancellationToken token = default(CancellationToken))
{
var bytes = LZ4MessagePackSerializer.Serialize(value, ContractlessStandardResolver.Instance);
await distributedCache.SetAsync(key, bytes, options, token);
}
public static void SavePreset(bool[] pScenes = null, string name = "preset name")
{
Traverse trav = Traverse.Create(phIBL);
ReflectionProbe probe = trav.Field("probeComponent").GetValue <ReflectionProbe>();
int selectedUserLut = trav.Field("selectedUserLut").GetValue <int>();
PHIBL.PostProcessing.Utilities.PostProcessingController PPCtrl_obj = trav.Field("PPCtrl").GetValue <PHIBL.PostProcessing.Utilities.PostProcessingController>();
PresetInfo preset = new PresetInfo
{
name = name,
scenes = pScenes,
profile = phIBL.Snapshot(),
nipples = trav.Field("nippleSSS").GetValue <float>(),
shadowDistance = QualitySettings.shadowDistance,
reflectionBounces = RenderSettings.reflectionBounces,
probeResolution = probe.resolution,
probeIntensity = probe.intensity,
enabledLUT = PPCtrl_obj.enableUserLut,
selectedLUT = selectedUserLut,
contributionLUT = PPCtrl_obj.userLut.contribution,
enableDithering = PPCtrl_obj.enableDither
};
File.WriteAllBytes(Directory.GetCurrentDirectory() + "\\Plugins\\PHIBL_PresetLoad\\presets\\" + name + ".preset", LZ4MessagePackSerializer.Serialize(preset, CustomCompositeResolver.Instance));
SetupPresets();
Console.WriteLine("[PHIBL_PresetLoad] Saved preset: " + name);
}
public byte[] GetParamByte()
{
return(LZ4MessagePackSerializer.Serialize(dictParameterSet));
}
public Task Serialize <T>(T message, Stream stream)
{
LZ4MessagePackSerializer.Serialize(stream, message, _resolver);
return(Task.CompletedTask);
}
protected override void RunActualProcess(ScenarioSliceParameters slice)
{
var dbProfileExport = Services.SqlConnectionPreparer.GetDatabaseConnection(Stage.ProfileGeneration, slice, DatabaseCode.HouseProfiles);
var saLoad = SaveableEntry <Prosumer> .GetSaveableEntry(dbProfileExport, SaveableEntryTableType.HouseLoad, Services.Logger);
var saGeneration = SaveableEntry <Prosumer> .GetSaveableEntry(dbProfileExport, SaveableEntryTableType.HouseGeneration, Services.Logger);
var dbArchiving = Services.SqlConnectionPreparer.GetDatabaseConnection(Stage.ProfileAnalysis, slice, DatabaseCode.Smartgrid);
dbArchiving.RecreateTable <SmartGridInformation>();
var saArchiveEntry = SaveableEntry <ArchiveEntry> .GetSaveableEntry(dbArchiving, SaveableEntryTableType.Smartgrid, Services.Logger);
int count = 0;
var smartSlice = slice.CopyThisSlice();
smartSlice.SmartGridEnabled = true;
RowCollection prosumerCollection = new RowCollection("Prosumers", "Prosumers");
Stopwatch sw = Stopwatch.StartNew();
const string tkfield = "Trafokreis";
Stopwatch swIdx = Stopwatch.StartNew();
saLoad.CreateIndexIfNotExists(tkfield);
saGeneration.CreateIndexIfNotExists(tkfield);
Info("Creating the index took " + swIdx.Elapsed);
var trafos = saLoad.SelectSingleDistinctField <string>(tkfield);
Info("Reading trafokreise took " + sw.Elapsed);
Dictionary <string, Profile> trafokreiseLoad = new Dictionary <string, Profile>();
Dictionary <string, Profile> trafokreiseGeneration = new Dictionary <string, Profile>();
ChangableProfile cityload = ChangableProfile.FromProfile(Profile.MakeConstantProfile(0, "Sum", Profile.ProfileResolution.QuarterHour));
ChangableProfile cityGeneration =
ChangableProfile.FromProfile(Profile.MakeConstantProfile(0, "Sum", Profile.ProfileResolution.QuarterHour));
ChangableProfile smartCityLoad =
ChangableProfile.FromProfile(Profile.MakeConstantProfile(0, "Sum", Profile.ProfileResolution.QuarterHour));
ChangableProfile smartCityGeneration =
ChangableProfile.FromProfile(Profile.MakeConstantProfile(0, "Sum", Profile.ProfileResolution.QuarterHour));
RowCollection smartGridPointValues = new RowCollection("SmartGridHASummaries", "SmartGridHASummaries");
SmartGridInformation smartGridInformation = new SmartGridInformation();
foreach (var trafo in trafos)
{
//if (count > 500) {
//continue;
//}
Dictionary <string, List <Prosumer> > prosumersByHa = new Dictionary <string, List <Prosumer> >();
ChangableProfile trafoLoadSum =
ChangableProfile.FromProfile(Profile.MakeConstantProfile(0, "Sum", Profile.ProfileResolution.QuarterHour));
foreach (var prosumer in saLoad.ReadSubsetOfTableDBAsEnumerable(tkfield, trafo))
{
RowBuilder rb = RowBuilder.Start("Name", prosumer.Name);
rb.Add("Energy", prosumer.Profile?.EnergySum());
rb.Add("HA", prosumer.HausanschlussKey);
rb.Add("Trafokreis", prosumer.TrafoKreis);
prosumerCollection.Add(rb);
count++;
if (count % 50 == 0)
{
Info("Processing Prosumers Load: " + count + " " + sw.Elapsed);
}
if (!string.IsNullOrWhiteSpace(prosumer.HausanschlussKey))
{
if (!prosumersByHa.ContainsKey(prosumer.HausanschlussKey))
{
prosumersByHa.Add(prosumer.HausanschlussKey, new List <Prosumer>());
}
prosumersByHa[prosumer.HausanschlussKey].Add(prosumer);
}
trafoLoadSum.Add(prosumer.Profile ?? throw new FlaException("empty profile"));
}
AnalysisKey keyload = new AnalysisKey(trafo, null, SumType.ByTrafokreis, GenerationOrLoad.Load, null, null, null);
ArchiveEntry aeload = new ArchiveEntry("Trafokreis " + trafo, keyload, trafoLoadSum.ToProfile(), GenerationOrLoad.Load, trafo);
saArchiveEntry.AddRow(aeload);
trafokreiseLoad.Add(trafo, trafoLoadSum.ToProfile());
var trafoGenerationSum = Profile.MakeConstantProfile(0, "Sum", Profile.ProfileResolution.QuarterHour);
foreach (var prosumer in saGeneration.ReadSubsetOfTableDBAsEnumerable(tkfield, trafo))
{
RowBuilder rb = RowBuilder.Start("Name", prosumer.Name);
rb.Add("Energy", prosumer.Profile?.EnergySum());
rb.Add("HA", prosumer.HausanschlussKey);
rb.Add("Trafokreis", prosumer.TrafoKreis);
prosumerCollection.Add(rb);
if (count % 50 == 0)
{
Info("Processing Prosumers Generation: " + count + " " + sw.Elapsed);
}
if (!string.IsNullOrWhiteSpace(prosumer.HausanschlussKey))
{
if (!prosumersByHa.ContainsKey(prosumer.HausanschlussKey))
{
prosumersByHa.Add(prosumer.HausanschlussKey, new List <Prosumer>());
}
prosumersByHa[prosumer.HausanschlussKey].Add(prosumer);
}
//var powerLimited = prosumer.Profile?.LimitToPercentageOfMax(0.5)?? throw new FlaException("blub");
//if(powerLimited.Values.Count ==0) { throw new FlaException("huch?");}
trafoGenerationSum = trafoGenerationSum.Add(prosumer.Profile ?? throw new FlaException(), "Sum");
}
AnalysisKey key = new AnalysisKey(trafo, null, SumType.ByTrafokreis, GenerationOrLoad.Generation, null, null, null);
ArchiveEntry ae = new ArchiveEntry("Trafokreis " + trafo, key, trafoGenerationSum, GenerationOrLoad.Generation, trafo);
saArchiveEntry.AddRow(ae);
trafokreiseGeneration.Add(trafo, trafoGenerationSum);
cityload.Add(trafoLoadSum);
cityGeneration.Add(trafoGenerationSum);
//if (count > 16000) {
ApplySmartGridstuff(prosumersByHa, trafo, smartCityLoad, smartCityGeneration, smartGridPointValues, smartGridInformation, smartSlice);
//}
}
var addedSmart = smartCityLoad.ToProfile().Add(smartCityGeneration.ToProfile(), "Netto-Last (smart)");
var addedSerializeFn = MakeAndRegisterFullFilename("addedProfile.lz4", slice);
dbArchiving.BeginTransaction();
dbArchiving.Save(smartGridInformation);
dbArchiving.CompleteTransaction();
SaveCityProfile(cityload, saArchiveEntry, SummedLoadType.CityLoad, GenerationOrLoad.Load);
SaveCityProfile(cityGeneration, saArchiveEntry, SummedLoadType.CityGeneration, GenerationOrLoad.Generation);
SaveCityProfile(smartCityGeneration, saArchiveEntry, SummedLoadType.SmartCityGeneration, GenerationOrLoad.Generation);
SaveCityProfile(smartCityLoad, saArchiveEntry, SummedLoadType.SmartCityLoad, GenerationOrLoad.Load);
saArchiveEntry.MakeCleanTableForListOfFields(false);
saArchiveEntry.SaveDictionaryToDatabase(MyLogger);
FileStream fs = new FileStream(addedSerializeFn, FileMode.Create);
var added = cityload.ToProfile().Subtract(cityGeneration.ToProfile(), "Netto-Last (konventionell)");
var lz4Arr = LZ4MessagePackSerializer.Serialize(added);
fs.Write(lz4Arr, 0, lz4Arr.Length);
fs.Close();
MakePlotlyTrafostationBoxPlots("Boxplot_Load_OhneMV.html", trafokreiseLoad, slice, true);
MakePlotlyTrafostationBoxPlots("Boxplot_Gen_OhneMV.html", trafokreiseGeneration, slice, true);
var fn = MakeAndRegisterFullFilename("ProsumerDump.xlsx", slice);
XlsxDumper.DumpProfilesToExcel(fn,
slice.DstYear,
15,
new RowWorksheetContent(prosumerCollection),
new RowWorksheetContent(smartGridPointValues));
var fnProf = MakeAndRegisterFullFilename("SmartGridProfiles_distributedStorage.xlsx", slice);
RowCollection rc = new RowCollection("StatusInfo", "Status");
double avgReductionFactor = smartGridInformation.SummedReductionFactor / smartGridInformation.NumberOfReductionFactors;
rc.Add(RowBuilder.Start("Total storage size", smartGridInformation.TotalStorageSize)
.Add("Number of Prosumers", smartGridInformation.NumberOfProsumers)
.Add("Citywide Reduction", avgReductionFactor)
.Add("MinimumLoadBefore", added.MinPower())
.Add("MinLoadSmart", addedSmart.MinPower())
.Add("MaxLoadBefore", added.MaxPower())
.Add("MaxLoadSmart", addedSmart.MaxPower())
);
XlsxDumper.DumpProfilesToExcel(fnProf,
slice.DstYear,
15,
new RowWorksheetContent(rc),
new ProfileWorksheetContent("load", "Last [MW]", 240, cityload.ToProfile()),
new ProfileWorksheetContent("generation", "Erzeugung [MW]", 240, cityGeneration.ToProfile()),
new ProfileWorksheetContent("added", "Netto-Last [kW]", 240, added),
new ProfileWorksheetContent("smartload", "Last (smart) [MW]", 240, smartCityLoad.ToProfile()),
new ProfileWorksheetContent("smartgeneration", "Erzeugung (smart) [MW]", 240, smartCityGeneration.ToProfile()),
new ProfileWorksheetContent("smartadded", "Netto-Last [kW]", 240, added, addedSmart));
SaveToArchiveDirectory(fnProf, RelativeDirectory.Report, smartSlice);
SaveToPublicationDirectory(fnProf, slice, "4.5");
SaveToPublicationDirectory(fnProf, slice, "5");
}
public byte[] Serialize <T>(T obj)
{
return(LZ4MessagePackSerializer.Serialize(obj));
}
public IEnumerator DownloadSubs()
{
string cache = Path.Combine(Utility.PluginsDirectory, "hsubs.msgpack");
if (File.Exists(cache))
{
subtitlesDict = LZ4MessagePackSerializer.Deserialize <Dictionary <string, string> >(File.ReadAllBytes(cache));
Logger.Log(LogLevel.Info, "Found cached hsubs");
}
Logger.Log(LogLevel.Info, "Downloading subs from " + SHEET_KEY);
var dl = new WWW($"https://docs.google.com/spreadsheets/d/{SHEET_KEY}/export?format=csv");
while (!dl.isDone)
{
yield return(dl);
}
if (dl.error != null)
{
Logger.Log(LogLevel.Warning, "Failed to fetch latest subtitles. Going to use cached ones.");
yield break;
}
Logger.Log(LogLevel.Info, $"Downloaded {dl.bytesDownloaded} bytes. Parsing...");
int cnt = 0;
foreach (IEnumerable <string> row in ParseCSV(dl.text))
{
int idx = 0;
string sound = null;
string tl = null;
foreach (string cell in row)
{
if (idx == 0)
{
sound = cell.ToLower();
}
if (idx == 2)
{
tl = cell;
}
idx++;
}
if (sound != null && tl != null && sound.Length < 64)
{
cnt++;
subtitlesDict[sound] = tl;
}
}
Logger.Log(LogLevel.Info, $"Done parsing subtitles: {cnt} lines found.");
if (cnt > 60000)
{
File.WriteAllBytes(cache, LZ4MessagePackSerializer.Serialize(subtitlesDict));
}
else
{
Logger.Log(LogLevel.Warning, "The amount of lines is suspiciously low (defaced sheet?); not caching.");
}
}
public Task WriteAsync(T message)
{
var bytes = LZ4MessagePackSerializer.Serialize(message, resolver);
return(inner.WriteAsync(bytes));
}
public byte[] Serialize <T>(T value)
{
return((EnableLZ4Compression)
? LZ4MessagePackSerializer.Serialize(value, Resolver)
: MessagePackSerializer.Serialize(value, Resolver));
}