public string Encode(FileToken token)
{
var pseudoIdBys = NetBitConverter.GetBytes(token.PseudoId);
var fileIdBys = NetBitConverter.GetBytes(token.FileId);
var ownerIdBys = NetBitConverter.GetBytes(token.FileOwnerId);
var mimeBys = NetBitConverter.GetBytes(token.MimeId);
var expireTimeBys = GetBytes(token.ExpireTime);
var fileCreateTimeBys = GetBytes(token.FileCreateTime);
var lstLen = 1 + pseudoIdBys.Length + fileIdBys.Length + ownerIdBys.Length + mimeBys.Length + expireTimeBys.Length + fileCreateTimeBys.Length;
var mdatLst = new List <byte>(lstLen);
mdatLst.Add(CurrentVersion);
mdatLst.AddRange(pseudoIdBys);
mdatLst.AddRange(fileIdBys);
mdatLst.AddRange(ownerIdBys);
mdatLst.AddRange(mimeBys);
mdatLst.AddRange(expireTimeBys);
mdatLst.AddRange(fileCreateTimeBys);
var mdatBys = mdatLst.ToArray();
//签名
var signBys = ArrayUtil.Addition(_appSecretBytes, mdatBys);
var hashBys = Md5(signBys);
//编码成字符串
var encBys = ArrayUtil.Addition(hashBys, mdatBys);
return(_urlDataCodec.Encode(encBys));
}
public string Encode(OwnerToken token)
{
var ownerTypeBys = NetBitConverter.GetBytes(token.OwnerType);
var ownerIdBys = NetBitConverter.GetBytes(token.OwnerId);
var expireTimeBys = GetBytes(token.ExpireTime);
var lstLen = 1 + ownerTypeBys.Length + ownerIdBys.Length + expireTimeBys.Length;
var mdatLst = new List <byte>(lstLen);
mdatLst.Add(CurrentVersion);
mdatLst.AddRange(ownerTypeBys);
mdatLst.AddRange(ownerIdBys);
mdatLst.AddRange(expireTimeBys);
var mdatBys = mdatLst.ToArray();
//签名
var signBys = ArrayUtil.Addition(_appSecretBytes, mdatBys);
var hashBys = Sha1(signBys);
//编码成字符串
var encBys = ArrayUtil.Addition(hashBys, mdatBys);
return(_urlDataCodec.Encode(encBys));
}
async ValueTask ReadChunks(long chunkTableOffset, int chunkCount)
{
if (ChunkReader is null)
{
throw new InvalidOperationException();
}
var chunkTable = new byte[(chunkCount + 1) * (4 + sizeof(long))];
if (await ChunkReader.ReadAtAsync(chunkTableOffset, chunkTable).ConfigureAwait(false) != chunkTable.Length)
{
return;
}
_chunks = Enumerable.Range(0, chunkCount + 1).Select(i => new Chunk
{
Name = (i < chunkCount) ? Encoding.ASCII.GetString(chunkTable, 12 * i, 4) : null,
Position = NetBitConverter.ToInt64(chunkTable, 12 * i + 4)
}).ToArray();
for (int i = 0; i < chunkCount; i++)
{
_chunks[i].Length = _chunks[i + 1].Position - _chunks[i].Position;
}
FanOut = await ReadFanOutAsync().ConfigureAwait(false);
}
protected virtual async ValueTask <uint[]?> ReadFanOutAsync()
{
byte[] fanOut = new byte[256 * sizeof(int)];
if (await ReadFromChunkAsync("OIDF", 0, fanOut).ConfigureAwait(false) != fanOut.Length)
{
return(null);
}
return(Enumerable.Range(0, 256).Select(i => NetBitConverter.ToUInt32(fanOut, sizeof(int) * i)).ToArray());
}
public FileToken Decode(string tokenStr)
{
var encBys = _urlDataCodec.Decode(tokenStr);
//校验签名
var hashLen = 16;
var mdatBys = new byte[encBys.Length - hashLen];
Array.Copy(encBys, hashLen, mdatBys, 0, mdatBys.Length);
var signBys = ArrayUtil.Addition(_appSecretBytes, mdatBys);
var hashBys = Md5(signBys);
if (!ArrayUtil.Equals(hashBys, 0, encBys, 0, hashLen))
{
throw new InvalidDataException("bad sign");
}
if (mdatBys[0] != CurrentVersion)
{
throw new NotSupportedException("bad token version");
}
//解析成对象
var index = 1; //忽略版本
var pseudoId = NetBitConverter.ToUInt32(mdatBys, index);
index += 4;
var fileId = NetBitConverter.ToInt32(mdatBys, index);
index += 4;
var ownerId = NetBitConverter.ToInt32(mdatBys, index);
index += 4;
var mimeId = NetBitConverter.ToUInt32(mdatBys, index);
index += 4;
var expireTime = ToDateTime(mdatBys, index);
index += sizeof(long);
var fileCreateTime = ToDateTime(mdatBys, index);
return(new FileToken
{
PseudoId = pseudoId,
FileId = fileId,
FileOwnerId = ownerId,
MimeId = mimeId,
ExpireTime = expireTime,
FileCreateTime = fileCreateTime
});
}
async ValueTask InitAsync()
{
if (_ver == 0)
{
_fIdx ??= FileBucket.OpenRead(Path.ChangeExtension(PackFile, ".idx"));
byte[] header = new byte[8];
long fanOutOffset = -1;
if (header.Length == await _fIdx.ReadAtAsync(0, header).ConfigureAwait(false))
{
var index = new byte[] { 255, (byte)'t', (byte)'O', (byte)'c', 0, 0, 0, 2 };
if (header.SequenceEqual(index))
{
// We have a v2 header.
fanOutOffset = 8;
_ver = 2;
}
else if (header.Take(4).SequenceEqual(index.Take(4)))
{
// We have an unsupported future header
_ver = -1;
_fIdx.Dispose();
_fIdx = null;
return;
}
else
{
// We have a v0/v1 header, which is no header
fanOutOffset = 0;
_ver = 1;
}
}
if (_fanOut == null && _ver > 0)
{
byte[] fanOut = new byte[4 * 256];
if (fanOut.Length == await _fIdx.ReadAtAsync(fanOutOffset, fanOut).ConfigureAwait(false))
{
_fanOut = new uint[256];
for (int i = 0; i < 256; i++)
{
_fanOut[i] = NetBitConverter.ToUInt32(fanOut, i * 4);
}
}
_hasBitmap = File.Exists(Path.ChangeExtension(PackFile, ".bitmap"));
}
}
}
unsafe TRead LoadStruct(byte[] bytes)
{
object r;
fixed(byte *pData = &bytes[0])
{
r = Marshal.PtrToStructure <TRead>((IntPtr)pData);
}
foreach (var f in typeof(TRead).GetFields())
{
if (f.GetCustomAttributes <NetworkOrderAttribute>()?.Any() ?? false)
{
var dd = f.GetValue(r);
if (dd is int di)
{
dd = NetBitConverter.FromNetwork(di);
}
else if (dd is long dl)
{
dd = NetBitConverter.FromNetwork(dl);
}
else if (dd is short ds)
{
dd = NetBitConverter.FromNetwork(ds);
}
else if (dd is uint dui)
{
dd = NetBitConverter.FromNetwork(dui);
}
else if (dd is ulong dul)
{
dd = NetBitConverter.FromNetwork(dul);
}
else if (dd is short dus)
{
dd = NetBitConverter.FromNetwork(dus);
}
else
{
continue;
}
f.SetValue(r, dd);
}
}
return((TRead)r);
}
public async Task Read4BitmapsXor()
{
string bmpFile = FindResource("*.bitmap") ?? throw new InvalidOperationException("Bitmap not found");
var fb = FileBucket.OpenRead(bmpFile);
var headers = await fb.ReadFullAsync(32); // Skip headers
uint count = NetBitConverter.ToUInt32(headers, 8);
Assert.AreEqual(106u, count);
List <GitEwahBitmapBucket> buckets = new List <GitEwahBitmapBucket>();
for (int i = 0; i < 4; i++)
{
buckets.Add(new GitEwahBitmapBucket(fb.Duplicate(false)));
await fb.ReadNetworkUInt32Async(); // Bitlength
uint u2 = await fb.ReadNetworkUInt32Async(); // Compressed length
for (uint n = 0; n < u2; n++)
{
await fb.ReadNetworkUInt64Async();
}
await fb.ReadNetworkUInt32Async(); // Last RLW start
}
var allXor = new BitwiseXorBucket(new BitwiseXorBucket(buckets[0], buckets[1]), new BitwiseXorBucket(buckets[2], buckets[3]));
int maxBits = buckets.Max(x => x.ReadBitLengthAsync().AsTask().Result);
Assert.AreEqual(2369, maxBits);
var bb = await allXor.ReadFullAsync((maxBits + 7) / 8);
for (int i = 0; i < bb.Length - 1; i++)
{
Assert.AreEqual((byte)0xFF, bb[i]);
}
}
public OwnerToken Decode(string tokenStr)
{
var encBys = _urlDataCodec.Decode(tokenStr);
//校验签名
var hashLen = 20;
var mdatBys = new byte[encBys.Length - hashLen];
Array.Copy(encBys, hashLen, mdatBys, 0, mdatBys.Length);
var signBys = ArrayUtil.Addition(_appSecretBytes, mdatBys);
var hashBys = Sha1(signBys);
if (!ArrayUtil.Equals(hashBys, 0, encBys, 0, hashLen))
{
throw new InvalidDataException("bad sign");
}
if (mdatBys[0] != CurrentVersion)
{
throw new NotSupportedException("bad token version");
}
//解析成对象
var index = 1; //忽略版本
var ownerType = NetBitConverter.ToInt32(mdatBys, index);
index += 4;
var ownerId = NetBitConverter.ToInt32(mdatBys, index);
index += 4;
var expireTime = ToDateTime(mdatBys, index);
return(new OwnerToken
{
OwnerType = ownerType,
OwnerId = ownerId,
ExpireTime = expireTime
});
}
public string GetFileDirectoryPath(uint pseudoId, DateTime fileCreateTime, int fileId)
{
var pIdBys = NetBitConverter.GetBytes(pseudoId);
//每个文件有一个独立的目录，用于存放各种尺寸的文件
//fileDirName = FileTableIndex_FileId
var fileDirName = $"{pseudoId % _dbOption.CurrentValue.FileTableCount}_{fileId}";
//dirPath = Root/Year/Month/Day/pseudoId[0]+pseudoId[1]
var dirPath = Path.Combine(
_option.CurrentValue.RootPath,
fileCreateTime.Year.ToString(),
fileCreateTime.Month.ToString("D2"),
fileCreateTime.Day.ToString("D2"),
pIdBys[0].ToString("x") + pIdBys[1].ToString("x"),
fileDirName
);
if (!Directory.Exists(dirPath))
{
Directory.CreateDirectory(dirPath);
}
return(dirPath);
}
private async ValueTask <long> ReadCommitTimeOffset(uint index)
{
var offsetData = new byte[sizeof(int)];
if (offsetData.Length != await ReadFromChunkAsync("GDA2", index * sizeof(int), offsetData).ConfigureAwait(false))
{
return(long.MinValue);
}
int v = NetBitConverter.ToInt32(offsetData, 0);
if (v >= 0)
{
return(v);
}
offsetData = new byte[sizeof(long)];
if (offsetData.Length != await ReadFromChunkAsync("GDO2", (~v) * sizeof(long), offsetData).ConfigureAwait(false))
{
return(long.MinValue);
}
return(NetBitConverter.ToInt64(offsetData, 0));
}
internal override async ValueTask <IGitCommitGraphInfo?> GetCommitInfo(GitId id)
{
await InitAsync().ConfigureAwait(false);
var(success, index) = await TryFindIdAsync(id).ConfigureAwait(false);
if (success)
{
int hashLength = GitId.HashLength(IdType);
int commitDataSz = hashLength + 2 * sizeof(uint) + sizeof(ulong);
byte[] commitData = new byte[commitDataSz];
if (commitDataSz != await ReadFromChunkAsync("CDAT", index * commitDataSz, commitData).ConfigureAwait(false))
{
return(null);
}
// commitData now contains the root hash, 2 parent indexes and the topological level
uint parent0 = NetBitConverter.ToUInt32(commitData, hashLength);
uint parent1 = NetBitConverter.ToUInt32(commitData, hashLength + sizeof(uint));
ulong chainLevel = NetBitConverter.ToUInt64(commitData, hashLength + 2 * sizeof(uint));
Task <GitId>[] parents;
if (parent0 == 0x70000000)
{
return(new GitCommitGraphInfo(Array.Empty <GitId>(), chainLevel, _haveV2 ? await ReadCommitTimeOffset(index).ConfigureAwait(false) : long.MinValue));
}
else if (parent1 == 0x70000000)
{
parents = new[] { GetOidAsync((int)parent0).AsTask() }
}
;
else if (parent1 >= 0x80000000)
{
var extraParents = new byte[sizeof(uint) * 256];
int len = await ReadFromChunkAsync("EDGE", 4 *(parent1 & 0x7FFFFFFF), extraParents).ConfigureAwait(false) / sizeof(uint);
if (len == 0 || len >= 256)
{
return(null); // Handle as if not exists in chain. Should never happen
}
int?stopAfter = null;
parents = new[] { GetOidAsync((int)parent0).AsTask() }.Concat(
Enumerable.Range(0, len)
.Select(i => NetBitConverter.ToUInt32(extraParents, i * sizeof(uint)))
.TakeWhile((v, i) => { if (i > stopAfter)
{
return(false);
}
else if ((v & 0x80000000) != 0)
{
stopAfter = i;
}
; return(true); })
.Select(v => GetOidAsync((int)(v & 0x7FFFFFFF)).AsTask())).ToArray();
}
else
{
parents = new[] { GetOidAsync((int)parent0).AsTask(), GetOidAsync((int)parent1).AsTask() }
};
IEnumerable <Task> waits;
Task <long>? v2 = null;
if (_haveV2)
{
v2 = ReadCommitTimeOffset(index).AsTask();
waits = parents.Concat(new Task[] { v2 });
}
else
{
waits = parents;
}
await Task.WhenAll(waits).ConfigureAwait(false);
long offset = v2 != null ? await v2.ConfigureAwait(false) : long.MinValue;
return(new GitCommitGraphInfo(parents.Select(x => x.Result).ToArray(), chainLevel, offset));
}
return(null);
}
private static DateTime ToDateTime(byte[] bytes, int startIndex)
{
var l = NetBitConverter.ToInt64(bytes, startIndex);
return(DateTime.FromBinary(l));
}
private static byte[] GetBytes(DateTime dt)
{
return(NetBitConverter.GetBytes(dt.ToBinary()));
}
public async Task ReadBitmap()
{
string bmpFile = FindResource("*.bitmap") ?? throw new InvalidOperationException("Bitmap not found");
var fb = FileBucket.OpenRead(bmpFile);
var headers = await fb.ReadFullAsync(32); // Skip headers
uint count = NetBitConverter.ToUInt32(headers, 8);
Assert.AreEqual(106u, count);
//BitArray
var bitLengths = new int[4];
{
var c = fb.Duplicate(false);
Assert.AreEqual(32, c.Position);
for (int i = 0; i < 4; i++)
{
bitLengths[i] = (int)await c.ReadNetworkUInt32Async();
uint u2 = await c.ReadNetworkUInt32Async();
List <ulong> w = new List <ulong>();
for (uint n = 0; n < u2; n++)
{
w.Add(await c.ReadNetworkUInt64Async());
}
await c.ReadNetworkUInt32Async();
TestContext.WriteLine($"EWAH {i}: {bitLengths[i]}\t{u2}");
foreach (var v in w)
{
TestContext.Write($"{v:X16} ");
}
TestContext.WriteLine();
}
//GC.KeepAlive(u1 + u2 + u3);
}
for (int i = 0; i < 4; i++)
{
using var ewah = new GitEwahBitmapBucket(fb.NoClose(true));
Assert.AreEqual(0L, ewah.Position);
int expectedBytes = (int)(8 * ((bitLengths[i] + 63) / 64));
long?p = await ewah.ReadRemainingBytesAsync();
int peekLen = ewah.Peek().Length;
Assert.IsTrue(peekLen > 0, "Can peek something");
Assert.IsTrue(peekLen <= expectedBytes, "No overshoot");
Assert.AreEqual(expectedBytes, (int)p, "ReadRemaining returned expected value");
var bb = await ewah.ReadFullAsync(65536);
Assert.AreEqual(expectedBytes, bb.Length, $"Read {bb.Length}, expected {bitLengths[i]} bits, what would be {(bitLengths[i] + 7) / 8} bytes, or {expectedBytes} bytes when reading longs");
StringBuilder sb = new StringBuilder();
for (int ii = 0; ii < bb.Length; ii++)
{
sb.Append(bb[ii].ToString("x2"));
}
TestContext.WriteLine();
int removeAfter = 2 * ((bitLengths[i] + 7) / 8);
if (removeAfter < sb.Length)
{
sb.Remove(removeAfter, sb.Length - removeAfter);
}
TestContext.WriteLine(sb.ToString());
}
}
public async Task CompressionTest(BucketCompressionAlgorithm alg)
{
bool overshoot = false;
var baseStream = new MemoryStream();
for (int i = 0; i < 10; i++)
{
baseStream.Write(Guid.NewGuid().ToByteArray(), 0, 16);
}
for (int i = 0; i < 10; i++)
{
baseStream.Write(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0 }, 0, 8);
}
var baseData = baseStream.ToArray();
Bucket compressed;
switch (alg)
{
case BucketCompressionAlgorithm.ZLib:
#if !NET6_0_OR_GREATER
compressed = baseData.AsBucket().Compress(alg);
#else
{
var ms = new MemoryStream();
var zs = new System.IO.Compression.ZLibStream(ms, System.IO.Compression.CompressionLevel.Optimal);
zs.Write(baseData, 0, baseData.Length);
zs.Close();
compressed = ms.ToArray().AsBucket();
}
#endif
break;
case BucketCompressionAlgorithm.GZip:
{
var ms = new MemoryStream();
var zs = new System.IO.Compression.GZipStream(ms, System.IO.Compression.CompressionLevel.Optimal);
zs.Write(baseData, 0, baseData.Length);
zs.Close();
compressed = ms.ToArray().AsBucket();
break;
}
case BucketCompressionAlgorithm.Deflate:
{
var ms = new MemoryStream();
var zs = new System.IO.Compression.DeflateStream(ms, System.IO.Compression.CompressionLevel.Optimal);
zs.Write(baseData, 0, baseData.Length);
zs.Close();
compressed = ms.ToArray().AsBucket();
break;
}
#if NET6_0_OR_GREATER
case BucketCompressionAlgorithm.Brotli:
{
var ms = new MemoryStream();
var zs = new System.IO.Compression.BrotliStream(ms, System.IO.Compression.CompressionLevel.Optimal);
zs.Write(baseData, 0, baseData.Length);
zs.Close();
compressed = ms.ToArray().AsBucket();
overshoot = true;
break;
}
#endif
default:
throw new InvalidOperationException();
}
var finishData = overshoot ? Array.Empty <byte>() : Guid.NewGuid().ToByteArray();
var compressedData = await compressed.Append(finishData.AsBucket()).ToArrayAsync();
ushort firstTwo = NetBitConverter.ToUInt16(compressedData, 0);
switch (alg)
{
case BucketCompressionAlgorithm.GZip:
Assert.AreEqual(0x1F8B, firstTwo, $"Got 0x{firstTwo:x4}");
break;
case BucketCompressionAlgorithm.ZLib:
bool isZlib = new ushort[] { 0x7801, 0x789C, 0x78da }.Contains(firstTwo);
Assert.IsTrue(isZlib, $"Got 0x{firstTwo:x4}");
break;
case BucketCompressionAlgorithm.Deflate:
// FirstTwo can be anything
break;
}
var inner = compressedData.AsBucket();
var bb = await inner.Decompress(alg).ReadFullAsync(4096);
Assert.AreEqual(baseData.Length, bb.Length);
var decompressed = bb.ToArray();
Assert.IsTrue(decompressed.SequenceEqual(baseData), "Same data after decompression");
bb = await inner.ReadFullAsync(4096);
Assert.AreEqual(finishData.Length, bb.Length);
Assert.IsTrue(bb.ToArray().SequenceEqual(finishData));
var r = await baseData.AsBucket().Compress(alg).ToArrayAsync();
Stream rs;
switch (alg)
{
case BucketCompressionAlgorithm.ZLib:
rs = r.AsBucket().Decompress(BucketCompressionAlgorithm.ZLib).AsStream();
break;
case BucketCompressionAlgorithm.GZip:
rs = new System.IO.Compression.GZipStream(new MemoryStream(r), System.IO.Compression.CompressionMode.Decompress);
break;
case BucketCompressionAlgorithm.Deflate:
rs = new System.IO.Compression.DeflateStream(new MemoryStream(r), System.IO.Compression.CompressionMode.Decompress);
break;
#if NET6_0_OR_GREATER
case BucketCompressionAlgorithm.Brotli:
rs = new System.IO.Compression.BrotliStream(new MemoryStream(r), System.IO.Compression.CompressionMode.Decompress);
break;
#endif
default:
throw new InvalidOperationException();
}
byte[] resultBytes = new byte[4096];
Assert.AreEqual(baseData.Length, rs.Read(resultBytes, 0, resultBytes.Length));
}
private static byte[] GetBytes(DateTime dt)
{
return(NetBitConverter.GetBytes(DateTimeUtil.DateTimeToUnixTimestamp(dt)));
}
private static DateTime ToDateTime(byte[] bytes, int startIndex)
{
var l = NetBitConverter.ToInt64(bytes, startIndex);
return(DateTimeUtil.UnixTimestampToDateTime(l));
}
private async ValueTask <bool> RefillAsync(bool allowWait)
{
if (_state <= ewah_state.start && !allowWait && Inner.Peek().IsEmpty)
{
return(false);
}
if (_lengthBits is null)
{
var bb = await Inner.ReadFullAsync(4 + 4).ConfigureAwait(false);
_lengthBits = NetBitConverter.ToUInt32(bb, 0);
_compressedSize = NetBitConverter.ToInt32(bb, 4);
_left = _compressedSize;
_state = ewah_state.start;
}
int peekLength = Inner.Peek().Length / sizeof(ulong);
_wpos = 0;
switch (_state)
{
case ewah_state.start:
ulong curOp = await Inner.ReadNetworkUInt64Async().ConfigureAwait(false);
_repBit = (curOp & 1UL) != 0;
_repCount = (uint)(curOp >> 1);
_rawCount = (int)(curOp >> 33);
_left--;
peekLength--;
_state = ewah_state.same;
goto case ewah_state.same;
case ewah_state.same:
byte val = _repBit ? (byte)0xFF : (byte)0;
while (_repCount > 0 && _wpos + 8 < _buffer.Length)
{
_buffer[_wpos++] = val;
_buffer[_wpos++] = val;
_buffer[_wpos++] = val;
_buffer[_wpos++] = val;
_buffer[_wpos++] = val;
_buffer[_wpos++] = val;
_buffer[_wpos++] = val;
_buffer[_wpos++] = val;
_repCount--;
}
if (_repCount > 0)
{
_readable = new BucketBytes(_buffer, 0, _wpos);
return(true);
}
_state = ewah_state.raw;
goto case ewah_state.raw;
case ewah_state.raw:
while (_rawCount > 0)
{
if ((_wpos > 8 && peekLength < 8) || (_wpos + 8 >= _buffer.Length))
{
// Avoid new reads if we already have something. Return result
_readable = new BucketBytes(_buffer, 0, _wpos);
return(true);
}
var bb = await Inner.ReadFullAsync(sizeof(ulong)).ConfigureAwait(false);
if (bb.Length != sizeof(ulong))
{
throw new BucketEofException(Inner);
}
peekLength--;
_left--;
_rawCount--;
for (int i = bb.Length - 1; i >= 0; i--)
{
_buffer[_wpos++] = bb[i];
}
}
if (_left == 0)
{
_state = ewah_state.footer;
_readable = new BucketBytes(_buffer, 0, _wpos);
return(true);
}
_state = ewah_state.start;
goto case ewah_state.start;
case ewah_state.footer:
await Inner.ReadNetworkUInt32Async().ConfigureAwait(false);
_state = ewah_state.done;
goto case ewah_state.done;
case ewah_state.done:
default:
return(false);
}
}
