/// <summary>
/// 从存储PartCF加载分区集
/// </summary>
internal static async ValueTask <ulong[]> LoadPartitions(byte appId, EntityModel model, KeyPredicate?[] predicates)
{
IntPtr beginKeyPtr;
IntPtr endKeyPtr;
int keySize;
unsafe
{
int *varSizes = stackalloc int[model.SysStoreOptions.PartitionKeys.Length];
keySize = CalcPartitionKeySize(predicates, model, varSizes);
var bk = stackalloc byte[keySize];
var ek = stackalloc byte[keySize];
WritePartitionKeyRange(predicates, appId, model, bk, ek, varSizes);
beginKeyPtr = new IntPtr(bk);
endKeyPtr = new IntPtr(ek);
}
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = beginKeyPtr,
BeginKeySize = new IntPtr(keySize),
EndKeyPtr = endKeyPtr,
EndKeySize = new IntPtr(keySize),
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = KeyUtil.PARTCF_INDEX
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var scanRes = await StoreApi.Api.ReadIndexByScanAsync(reqPtr);
if (scanRes == null)
{
return(null);
}
var raftGroupIds = new ulong[scanRes.Length];
unsafe
{
int index = 0;
scanRes.ForEachRow((kp, ks, vp, vs) =>
{
//TODO:***进一步过滤未附加的Key的谓词
ulong *groupIdPtr = (ulong *)vp.ToPointer();
raftGroupIds[index] = *groupIdPtr;
index++;
});
scanRes.Dispose();
}
return(raftGroupIds);
}
internal static async ValueTask <ValueTuple <string, string>[]> LoadViewRoutes()
{
byte beginKey = KeyUtil.METACF_VIEW_ROUTER_PREFIX;
IntPtr beginKeyPtr;
unsafe { beginKeyPtr = new IntPtr(&beginKey); }
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = beginKeyPtr,
BeginKeySize = new IntPtr(1),
EndKeyPtr = IntPtr.Zero,
EndKeySize = IntPtr.Zero,
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = -1
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var scanRes = await HostApi.ReadIndexByScanAsync(reqPtr);
if (scanRes == null || scanRes.Length == 0)
{
return(null);
}
var routes = new ValueTuple <string, string> [scanRes.Length];
var index = 0;
scanRes.ForEachRow((kp, ks, vp, vs) =>
{
unsafe
{
string key = new string((sbyte *)kp.ToPointer(), 1, ks - 1, System.Text.Encoding.UTF8);
string value = null;
if (vs > 0)
{
value = new string((sbyte *)vp.ToPointer(), 0, vs, System.Text.Encoding.UTF8);
}
routes[index] = ValueTuple.Create(key, value);
}
index++;
});
scanRes.Dispose();
return(routes);
}
/// <summary>
/// 用于设计时加载所有Model
/// </summary>
internal static async ValueTask <ModelBase[]> LoadAllModelAsync()
{
byte key = KeyUtil.METACF_MODEL_PREFIX;
IntPtr keyPtr;
unsafe { keyPtr = new IntPtr(&key); }
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = keyPtr,
BeginKeySize = new IntPtr(1),
EndKeyPtr = IntPtr.Zero,
EndKeySize = IntPtr.Zero,
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = -1
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var res = await HostApi.ReadIndexByScanAsync(reqPtr);
var models = new ModelBase[res.Length];
try
{
int index = 0;
res.ForEachRow((kp, ks, vp, vs) =>
{
models[index] = (ModelBase)DeserializeModel(vp, vs);
models[index].AcceptChanges(); //TODO:check是否在这里AcceptChanges
index++;
});
}
catch (Exception) { throw; }
finally { res.Dispose(); }
return(models);
}
/// <summary>
/// 用于设计时加载所有Folder
/// </summary>
internal static async ValueTask <ModelFolder[]> LoadAllFolderAsync()
{
byte key = KeyUtil.METACF_FOLDER_PREFIX;
IntPtr keyPtr;
unsafe { keyPtr = new IntPtr(&key); }
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = keyPtr,
BeginKeySize = new IntPtr(1),
EndKeyPtr = IntPtr.Zero,
EndKeySize = IntPtr.Zero,
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = -1
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var res = await HostApi.ReadIndexByScanAsync(reqPtr);
var folders = new ModelFolder[res.Length];
try
{
int index = 0;
res.ForEachRow((kp, ks, vp, vs) =>
{
folders[index] = (ModelFolder)DeserializeModel(vp, vs);
index++;
});
}
catch (Exception) { throw; }
finally { res.Dispose(); }
return(folders);
}
/// <summary>
/// 用于设计时加载所有ApplicationModel
/// </summary>
internal static async ValueTask <ApplicationModel[]> LoadAllApplicationAsync()
{
byte appKey = KeyUtil.METACF_APP_PREFIX;
IntPtr appKeyPtr;
unsafe { appKeyPtr = new IntPtr(&appKey); }
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = appKeyPtr,
BeginKeySize = new IntPtr(1),
EndKeyPtr = IntPtr.Zero,
EndKeySize = IntPtr.Zero,
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = -1
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var res = await HostApi.ReadIndexByScanAsync(reqPtr);
var apps = new ApplicationModel[res.Length];
try
{
int index = 0;
res.ForEachRow((kp, ks, vp, vs) =>
{
apps[index] = (ApplicationModel)DeserializeModel(vp, vs, APP_DATA_OFFSET);
index++;
});
}
catch (Exception) { throw; }
finally { res.Dispose(); }
return(apps);
}
/// <summary>
/// 执行分区扫描，全局表与分区表通用
/// </summary>
private async ValueTask <IScanResponse> ExecPartScanAsync(ulong groupId, uint pskip, uint ptake)
{
//TODO:暂重复序列化，考虑复制序列化结果
IntPtr filterPtr = IntPtr.Zero;
if (!Expression.IsNull(filter))
{
filterPtr = ModelStore.SerializeModel(filter, out _);
}
//开始处理NativeApi所需参数
//暂没有CreateTime谓词使用前缀匹配方式
IntPtr keyPtr;
int keySize = KeyUtil.ENTITY_KEY_SIZE - 10;
unsafe
{
byte *bk = stackalloc byte[keySize];
EntityId.WriteRaftGroupId(bk, groupId);
keyPtr = new IntPtr(bk);
}
var req = new ClrScanRequire
{
RaftGroupId = groupId,
BeginKeyPtr = keyPtr,
BeginKeySize = new IntPtr(keySize),
EndKeyPtr = IntPtr.Zero,
EndKeySize = IntPtr.Zero,
FilterPtr = filterPtr,
Skip = pskip,
Take = ptake,
DataCF = -1,
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
return(await StoreApi.Api.ReadIndexByScanAsync(reqPtr));
}
private static async Task ScanParts(Action <IntPtr, int, IntPtr, int> action)
{
IntPtr beginKeyPtr;
IntPtr endKeyPtr;
int keySize = 4;
uint beginKey = 0;
uint endKey = 0xFFFFFFFF;
unsafe
{
beginKeyPtr = new IntPtr(&beginKey);
endKeyPtr = new IntPtr(&endKey);
}
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = beginKeyPtr,
BeginKeySize = new IntPtr(keySize),
EndKeyPtr = endKeyPtr,
EndKeySize = new IntPtr(keySize),
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = KeyUtil.PARTCF_INDEX
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var scanRes = await StoreApi.Api.ReadIndexByScanAsync(reqPtr);
if (scanRes == null)
{
return;
}
scanRes.ForEachRow(action);
scanRes.Dispose();
}
private static async Task ScanNodes(Action <IntPtr, int, IntPtr, int> action)
{
IntPtr bkPtr;
IntPtr ekPtr;
byte bkey = 1;
byte ekey = 2;
unsafe
{
bkPtr = new IntPtr(&bkey);
ekPtr = new IntPtr(&ekey);
}
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = bkPtr,
BeginKeySize = new IntPtr(1),
EndKeyPtr = ekPtr,
EndKeySize = new IntPtr(1),
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = KeyUtil.METACF_INDEX
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var scanRes = await StoreApi.Api.ReadIndexByScanAsync(reqPtr);
if (scanRes == null)
{
return;
}
scanRes.ForEachRow(action);
scanRes.Dispose();
}
/// <summary>
/// 列出指定目录下的子目录及文件信息
/// </summary>
/// <returns>目录不存在返回null</returns>
/// <param name="appId">App identifier.</param>
/// <param name="path">必须'/'开头，但不能'/'结尾</param>
public static async ValueTask <BlobObject[]> ListAsync(byte appId, string path /*, int skip, int take*/)
{
//TODO:***暂简单实现，忽略分页
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException(nameof(path));
}
if (!path.StartsWith('/') || (path.Length > 1 && path.EndsWith('/')))
{
throw new ArgumentException("路径格式错误", nameof(path));
}
var list = new List <BlobObject>();
ulong metaRaftGroupId = (ulong)appId << IdUtil.RAFTGROUPID_APPID_OFFSET
| (ulong)IdUtil.RAFT_TYPE_BLOB_META << (IdUtil.RAFTGROUPID_FLAGS_OFFFSET + IdUtil.RAFTGROUPID_FLAGS_TYPE_OFFSET);
#region ----先查询目录是否存在----
byte[] pathData = System.Text.Encoding.UTF8.GetBytes(path);
if (path.Length > 1) //注意: 非根目录替换last '/' 0x2F to 0x00
{
for (int i = pathData.Length - 1; i >= 0; i--)
{
if (pathData[i] == 0x2F)
{
pathData[i] = 0;
break;
}
}
}
int keySize = pathData.Length + 2;
IntPtr keyPtr;
unsafe
{
byte *kp = stackalloc byte[keySize];
kp[0] = KEY_TYPE_PATH;
kp[1] = appId;
Marshal.Copy(pathData, 0, new IntPtr(kp + 2), pathData.Length);
keyPtr = new IntPtr(kp);
}
var pathMetaData = await StoreApi.Api.ReadIndexByGetAsync(metaRaftGroupId, keyPtr, (uint)keySize);
if (pathMetaData == null)
{
return(null);
}
BlobPath curPath = new BlobPath();
curPath.ReadFrom(pathMetaData.DataPtr);
pathMetaData.Dispose();
#endregion
#region ----查询当前目录下子目录----
string pathPrefix = path.Length == 1 ? path : path + "/";
byte[] childPathPrefixData = System.Text.Encoding.UTF8.GetBytes(pathPrefix);
childPathPrefixData[childPathPrefixData.Length - 1] = 0; //注意替换last '/' 0x2F to 0x00
keySize = childPathPrefixData.Length + 2;
IntPtr beginKeyPtr;
IntPtr endKeyPtr;
IntPtr filterPtr = IntPtr.Zero;
unsafe
{
byte *bk = stackalloc byte[keySize];
bk[0] = KEY_TYPE_PATH; //TypeFlag = Path
bk[1] = appId;
Marshal.Copy(childPathPrefixData, 0, new IntPtr(bk + 2), childPathPrefixData.Length);
beginKeyPtr = new IntPtr(bk);
byte *ek = stackalloc byte[keySize];
ek[0] = KEY_TYPE_PATH;
ek[1] = appId;
Marshal.Copy(childPathPrefixData, 0, new IntPtr(ek + 2), childPathPrefixData.Length);
ek[childPathPrefixData.Length + 1] = 1;
endKeyPtr = new IntPtr(ek);
}
//Log.Debug($"扫描子目录BeginKey={StringHelper.ToHexString(beginKeyPtr, keySize)}");
//Log.Debug($"扫描子目录EndKey={StringHelper.ToHexString(endKeyPtr, keySize)}");
var req = new ClrScanRequire
{
RaftGroupId = metaRaftGroupId,
BeginKeyPtr = beginKeyPtr,
BeginKeySize = new IntPtr(keySize),
EndKeyPtr = endKeyPtr,
EndKeySize = new IntPtr(keySize),
FilterPtr = filterPtr,
Skip = 0,
Take = uint.MaxValue,
DataCF = -1
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var scanRes = await StoreApi.Api.ReadIndexByScanAsync(reqPtr);
if (scanRes != null)
{
scanRes.ForEachRow((kp, ks, vp, vs) =>
{
BlobObject bo = new BlobObject();
bo.ReadFrom(kp, ks, vp, vs);
list.Add(bo);
});
scanRes.Dispose();
}
#endregion
#region ----查询当前目录下文件----
if (curPath.Chunks != null && curPath.Chunks.Length > 0)
{
keySize = 19;
unsafe
{
byte *bk = stackalloc byte[keySize];
bk[0] = KEY_TYPE_FILE;
bk[1] = appId;
Guid *idPtr = (Guid *)(bk + 2);
* idPtr = curPath.Id;
bk[18] = 0;
beginKeyPtr = new IntPtr(bk);
byte *ek = stackalloc byte[keySize];
ek[0] = KEY_TYPE_FILE;
ek[1] = appId;
idPtr = (Guid *)(ek + 2);
*idPtr = curPath.Id;
ek[18] = 1;
endKeyPtr = new IntPtr(ek);
}
var req2 = new ClrScanRequire
{
RaftGroupId = metaRaftGroupId,
BeginKeyPtr = beginKeyPtr,
BeginKeySize = new IntPtr(keySize),
EndKeyPtr = endKeyPtr,
EndKeySize = new IntPtr(keySize),
FilterPtr = filterPtr,
Skip = 0,
Take = uint.MaxValue,
DataCF = -1
};
IntPtr reqPtr2;
unsafe { reqPtr2 = new IntPtr(&req2); }
scanRes = await StoreApi.Api.ReadIndexByScanAsync(reqPtr2);
if (scanRes != null)
{
scanRes.ForEachRow((kp, ks, vp, vs) =>
{
BlobObject bo = new BlobObject();
bo.ReadFrom(kp, ks, vp, vs);
list.Add(bo);
});
scanRes.Dispose();
}
}
#endregion
return(list.ToArray());
}
/// <summary>
/// 用于设计时加载变更添加的索引的构建状态
/// </summary>
internal static async ValueTask LoadIndexBuildingStatesAsync(ApplicationModel app, EntityModel model)
{
if (!model.SysStoreOptions.HasIndexes || model.SysStoreOptions.SchemaVersion == 0)
{
return;
}
var addedIndexes = model.SysStoreOptions.Indexes.Where(t => t.State == EntityIndexState.Building);
if (!addedIndexes.Any())
{
return;
}
//先初始化为Ready状态
foreach (var item in addedIndexes)
{
item.State = EntityIndexState.Ready;
}
IntPtr bkPtr;
unsafe
{
byte *bk = stackalloc byte[5];
bk[0] = KeyUtil.METACF_INDEX_STATE_PREFIX;
uint *tiPtr = (uint *)(bk + 1);
* tiPtr = KeyUtil.EncodeTableId(app.StoreId, model.TableId);
bkPtr = new IntPtr(bk);
}
var req = new ClrScanRequire
{
RaftGroupId = KeyUtil.META_RAFTGROUP_ID,
BeginKeyPtr = bkPtr,
BeginKeySize = new IntPtr(5),
EndKeyPtr = IntPtr.Zero,
EndKeySize = IntPtr.Zero,
FilterPtr = IntPtr.Zero,
Skip = 0,
Take = uint.MaxValue,
DataCF = -1
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
var res = await HostApi.ReadIndexByScanAsync(reqPtr);
if (res == null || res.Length == 0)
{
return;
}
byte indexId = 0;
unsafe
{
res.ForEachRow((kp, ks, vp, vs) =>
{
var keyPtr = (byte *)kp.ToPointer();
var valPtr = (byte *)vp.ToPointer();
indexId = keyPtr[5];
var index = model.SysStoreOptions.Indexes.SingleOrDefault/*Single*/ (t => t.IndexId == indexId);
if (index != null)
{
index.State = (EntityIndexState)valPtr[0];
}
else
{
Log.Warn($"查询索引构建状态发现不存在的索引{indexId}");
}
});
}
res.Dispose();
}
/// <summary>
/// 执行本地索引扫描，分区表与非分区表通用
/// </summary>
private async ValueTask <IScanResponse> ExecLocalIndexScanAsync(
EntityIndexModel indexModel, ulong groupId, uint pskip, uint ptake, bool toIndexTarget)
{
//TODO:***将不能编入Key的谓词作为Filter条件
//开始处理NativeApi所需参数
IntPtr bkPtr;
IntPtr ekPtr;
int bkSize = KeyUtil.INDEXCF_PREFIX_SIZE;
int ekSize = KeyUtil.INDEXCF_PREFIX_SIZE;
IntPtr filterPtr = IntPtr.Zero;
unsafe
{
int *varSizes = stackalloc int[indexModel.Fields.Length];
if (_keys != null)
{
bkSize = ekSize = _keys.CalcKeySize(varSizes);
}
byte *bk = stackalloc byte[bkSize];
bkPtr = new IntPtr(bk);
byte *ek = stackalloc byte[ekSize];
ekPtr = new IntPtr(ek);
if (_keys == null)
{
KeyUtil.WriteIndexKeyPrefix(bk, groupId, indexModel.IndexId);
ekPtr = IntPtr.Zero;
ekSize = 0;
}
else
{
_keys.WriteKeyRange(groupId, indexModel, bk, ek, varSizes);
if (_keys.IsOnlyEqual()) //只有相等性判断使用前缀匹配
{
ekPtr = IntPtr.Zero;
ekSize = 0;
}
}
}
if (!Expression.IsNull(filter))
{
filterPtr = ModelStore.SerializeModel(filter, out _);
}
var req = new ClrScanRequire
{
RaftGroupId = groupId,
BeginKeyPtr = bkPtr,
BeginKeySize = new IntPtr(bkSize),
EndKeyPtr = ekPtr,
EndKeySize = new IntPtr(ekSize),
FilterPtr = filterPtr,
Skip = pskip,
Take = ptake,
DataCF = KeyUtil.INDEXCF_INDEX,
ToIndexTarget = toIndexTarget
};
IntPtr reqPtr;
unsafe { reqPtr = new IntPtr(&req); }
return(await StoreApi.Api.ReadIndexByScanAsync(reqPtr));
}
