/// <summary>
/// Read file at path, split the contents in chunks and store them together with a StreamChunk.
/// </summary>
public static ChunkHash GenerateChunk(string path, Repo repo)
{
StreamChunk message = new StreamChunk();
using (Stream stream = new FileStream(path, FileMode.Open)) {
BinaryReader br = new BinaryReader(stream);
message.Size = (ulong)stream.Length;
while (true)
{
byte[] data = br.ReadBytes(4096);
if (data.Length == 0)
{
break;
}
Chunk c = new Chunk(data);
ChunkHash ch = repo.WriteChunk(c);
message.Chunks.Add(ch.bytes);
}
}
byte[] messageBytes = StreamChunk.SerializeToBytes(message);
Chunk messageChunk = new Chunk(messageBytes);
ChunkHash messageHash = repo.WriteChunk(messageChunk);
return(messageHash);
}
public override void StoreMessage(string prefix, ChunkHash id)
{
string path = Path.Combine (messageRoot, Path.GetFileName (prefix));
Directory.CreateDirectory(path);
path = Path.Combine (path, id.ToString ());
File.WriteAllBytes (path, new byte[0]);
}
public static ChunkHash GenerateChunk(string path, Repo repo)
{
string fullPath = Path.GetFullPath(path);
TreeChunk tree = new TreeChunk();
//Subdirectories
string[] dirs = Directory.GetDirectories(fullPath);
foreach (string d in dirs)
{
TreeFile df = new TreeFile();
df.Name = Path.GetFileName(d);
df.TreeChunkHash = TreeChunk.GenerateChunk(d, repo).bytes;
tree.Directories.Add(df);
}
//Files
string[] files = Directory.GetFiles(fullPath);
foreach (string f in files)
{
TreeFile ff = new TreeFile();
ff.Name = Path.GetFileName(f);
ff.TreeChunkHash = StreamChunk.GenerateChunk(f, repo).bytes;
tree.Files.Add(ff);
}
Chunk treeChunk = new Chunk(TreeChunk.SerializeToBytes(tree));
ChunkHash ch = repo.WriteChunk(treeChunk);
return(ch);
}
public override ChunkHash WriteChunk(Chunk chunk)
{
ChunkHash hash = base.WriteChunk(chunk);
this.RouteMessage.Chunks.Add(hash.bytes);
return(hash);
}
public override void StoreMessage(string prefix, ChunkHash chunkHash)
{
foreach (Repo r in repos)
{
r.StoreMessage(prefix, chunkHash);
}
}
void ProcessStoreMessage()
{
RequestStoreMessage request = RequestStoreMessage.Deserialize(ProtocolParser.ReadBytes(input));
ReplyStoreMessage reply = new ReplyStoreMessage();
localRepo.StoreMessage(request.Prefix, ChunkHash.FromHashBytes(request.ChunkHash));
ProtocolParser.WriteBytes(output, ReplyStoreMessage.SerializeToBytes(reply));
}
public Chunk(byte[] buffer)
{
//Hash data
this.Data = buffer;
this.ClearHash = ClearHash.ComputeHash (buffer);
this.ChunkHash = ChunkHash.FromHashBytes (this.ClearHash.bytes);
}
/// <summary>
/// Verify the ClearID.ClearHash against chunk data.
/// The data is assumed to be decrypted, otherwise the verification will fail.
/// </summary>
public bool Verify(ChunkHash ch)
{
if (ch.Equals(ChunkHash) == false)
{
return(false);
}
return(true);
}
public override void StoreMessage(string prefix, ChunkHash id)
{
string path = Path.Combine(messageRoot, Path.GetFileName(prefix));
Directory.CreateDirectory(path);
path = Path.Combine(path, id.ToString());
File.WriteAllBytes(path, new byte[0]);
}
public ChunkHash ReadChunkHash()
{
ChunkHash ch = new ChunkHash();
ch.ChunkId = BitConverter.ToInt32(msg, s(1));
Buffer.BlockCopy(msg, s(2), ch.Hash, 0, msg.Length - s(2));
return(ch);
}
public Chunk(byte[] buffer)
{
//Hash data
this.Data = buffer;
this.ClearHash = ClearHash.ComputeHash(buffer);
this.ChunkHash = ChunkHash.FromHashBytes(this.ClearHash.bytes);
}
public static void Main(string[] args, KeyStorage keyStorage)
{
//Usage
if (args.Length != 4)
{
throw new HelpException("Missing arguments");
}
string sourcePath = args [1];
string repoPath = args [2];
string receipientName = args [3];
//Source
if (Directory.Exists(sourcePath) == false)
{
throw new HelpException("Source directory not found: " + sourcePath);
}
//Repo
Repo repo = Repo.Create(repoPath);
//Sender and Recipient keys
PrivateKey senderKey = keyStorage.DefaultKey;
PublicKey recipientKey = keyStorage.GetPublic(receipientName);
//Prepare Route message recording of ChunkID
RouteRepo rr = new RouteRepo(repo);
//Prepare Encryption
EncryptedRepo er = new EncryptedRepo(rr, null);
er.AddKey(recipientKey);
Console.Write("Generating Tree...");
//Send Tree
ChunkHash tree = TreeChunk.GenerateChunk(sourcePath, er);
//TreeMessage
TreeMessage tm = new TreeMessage(tree, Path.GetDirectoryName(sourcePath));
Chunk tmc = Message.ToChunk(tm, senderKey);
ChunkHash tmch = er.WriteChunk(tmc);
er.StoreMessage("file", tmch);
//RouteMessage
RouteMessage rm = rr.RouteMessage;
rm.MessageChunkHash = tmch.bytes;
rm.To = receipientName;
//Store unencrypted RouteMessage
Chunk rmChunk = Message.ToChunk(rm);
repo.WriteChunk(rmChunk);
repo.StoreMessage("route", rmChunk.ChunkHash);
Console.WriteLine("RouteMessage Stored");
}
public override void StoreMessage(string prefix, ChunkHash chunkHash)
{
RequestStoreMessage msg = new RequestStoreMessage();
msg.Prefix = prefix;
msg.ChunkHash = chunkHash.bytes;
SendMessage(msg);
ReplyStoreMessage.Deserialize(ProtocolParser.ReadBytes(input));
}
public override Chunk ReadChunk(ChunkHash id)
{
Chunk chunk = base.ReadChunk (id);
try {
return Decrypt (chunk);
} catch (Exception) {
return null;
}
}
public override Chunk ReadChunk(ChunkHash id)
{
Chunk chunk = base.ReadChunk(id);
try {
return(Decrypt(chunk));
} catch (Exception) {
return(null);
}
}
public override ChunkHash WriteChunk(Chunk chunk)
{
ChunkHash ch = null;
foreach (Repo r in repos)
{
ch = r.WriteChunk(chunk);
}
return(ch);
}
void ProcessReadChunk()
{
RequestReadChunk request = RequestReadChunk.Deserialize(ProtocolParser.ReadBytes(input));
ReplyReadChunk reply = new ReplyReadChunk();
Chunk c = localRepo.ReadChunk(ChunkHash.FromHashBytes(request.ChunkHash));
reply.ChunkData = c.Data;
ProtocolParser.WriteBytes(output, ReplyReadChunk.SerializeToBytes(reply));
}
public override ChunkHash GetCustomHash(CustomID id)
{
string idPath = Path.Combine(idRoot, id.ToString() + ".id");
if (File.Exists(idPath) == false)
{
return(null);
}
byte[] hash = File.ReadAllBytes(idPath);
return(ChunkHash.FromHashBytes(hash));
}
public override Chunk ReadChunk(ChunkHash chunkHash)
{
foreach (Repo r in repos)
{
Chunk b = r.ReadChunk(chunkHash);
if (b != null)
{
return(b);
}
}
return(null);
}
public static void Main(string[] args, KeyStorage keyStorage)
{
if (args.Length != 4)
{
throw new HelpException("Missing arguments");
}
//Storage
Repo repo = Repo.Create(args [1]);
repo = new EncryptedRepo(repo, keyStorage);
//Find message
Chunk chunk = null;
if (args [2].Length == 64)
{
ChunkHash id = ChunkHash.FromHashBytes(Hash.FromString(args [2]).bytes);
chunk = repo.ReadChunk(id);
}
else
{
ICollection <ChunkHash> messages = repo.GetMessageList("file");
foreach (ChunkHash bh in messages)
{
if (bh.ToString().StartsWith(args [2]))
{
chunk = repo.ReadChunk(bh);
}
}
}
Message message = Message.FromChunk(chunk, keyStorage);
if (message == null)
{
Console.Error.WriteLine("Message not found");
return;
}
TreeMessage tm = message as TreeMessage;
if (tm == null)
{
Console.Error.WriteLine("Not a TreeMessage: " + message.GetType().Name);
return;
}
Console.WriteLine("Found TreeMessage " + tm.Name);
string targetPath = Path.Combine(args [3], tm.Name);
TreeChunk.Extract(repo, ChunkHash.FromHashBytes(tm.TreeChunkHash), targetPath);
}
public static void Extract(Repo store, ChunkHash cid, string targetPath)
{
Directory.CreateDirectory (targetPath);
Chunk c = store.ReadChunk (cid);
TreeChunk tree = TreeChunk.Deserialize (c.Data);
foreach (TreeFile file in tree.Files) {
StreamChunk.Extract (store, ChunkHash.FromHashBytes (file.TreeChunkHash), Path.Combine (targetPath, file.Name));
}
foreach (TreeFile subdir in tree.Directories) {
TreeChunk.Extract (store, ChunkHash.FromHashBytes (subdir.TreeChunkHash), Path.Combine (targetPath, subdir.Name));
}
}
public override List <ChunkHash> GetMessageList(string prefix)
{
string path = Path.Combine(messageRoot, Path.GetFileName(prefix));
List <ChunkHash> list = new List <ChunkHash> ();
string[] files = Directory.GetFiles(path);
foreach (string file in files)
{
string name = Path.GetFileName(file);
list.Add(ChunkHash.FromString(name));
}
return(list);
}
public override Chunk ReadChunk(ChunkHash chunkHash)
{
//Read Data
string dataPath = GetPath (chunkHash);
Chunk chunk = new Chunk (File.ReadAllBytes (dataPath));
//Verify Hash
if (chunk.ChunkHash.Equals (chunkHash) == false)
throw new InvalidDataException ("Hash mismatch: " + chunkHash);
//Read keys
chunk.ClearHash = ClearHash.FromHashBytes (chunkHash.bytes);
return chunk;
}
public static void Extract(Repo store, ChunkHash fileHash, string targetPath)
{
Chunk chunk = store.ReadChunk (fileHash);
StreamChunk streamChunk = StreamChunk.Deserialize<StreamChunk> (chunk.Data);
using (FileStream file = File.Open(targetPath, FileMode.Create)) {
foreach (byte[] hashBytes in streamChunk.Chunks) {
Chunk fc = store.ReadChunk (ChunkHash.FromHashBytes (hashBytes));
file.Write (fc.Data, 0, fc.Data.Length);
}
//Verify length
if (file.Length != (long)streamChunk.Size)
throw new InvalidDataException ("Invalid file length");
}
}
public void HandleNewChunkHash(C2SMessageReader mr)
{
if (!ServerCommunicationCenter.commCenter.ClientsEndPoint.ContainsKey(mr.ClientEP()))
{
return;
}
ChunkHash ch = mr.ReadChunkHash();
int id = mr.ReadRegisteringFileId();
Debug.WriteLine("hash for:" + id);
if (!ServerObjectGraph.sog.FileRegistrations.ContainsKey(id))
{
return;
}
ServerObjectGraph.sog.FileRegistrations[id].HashArrived(ch);
}
public override List <ChunkHash> GetMessageList(string prefix)
{
RequestMessageList msg = new RequestMessageList(prefix);
SendMessage(msg);
ReplyMessageList reply = ReplyMessageList.Deserialize(ProtocolParser.ReadBytes(input));
List <ChunkHash> list = new List <ChunkHash> ();
foreach (byte[] hash in reply.ChunkHash)
{
list.Add(ChunkHash.FromHashBytes(hash));
}
return(list);
}
public override ChunkHash GetCustomHash(CustomID customID)
{
RequestCustomHash msg = new RequestCustomHash();
msg.CustomID = customID.bytes;
SendMessage(msg);
byte[] rbytes = ProtocolParser.ReadBytes(input);
//Console.WriteLine("Got reply " + rbytes.Length);
ReplyCustomHash reply = ReplyCustomHash.Deserialize(rbytes);
//Console.WriteLine("Got reply " + reply);
return(ChunkHash.FromHashBytes(reply.ChunkHash));
}
public override Chunk ReadChunk(ChunkHash chunkHash)
{
//Read Data
string dataPath = GetPath(chunkHash);
Chunk chunk = new Chunk(File.ReadAllBytes(dataPath));
//Verify Hash
if (chunk.ChunkHash.Equals(chunkHash) == false)
{
throw new InvalidDataException("Hash mismatch: " + chunkHash);
}
//Read keys
chunk.ClearHash = ClearHash.FromHashBytes(chunkHash.bytes);
return(chunk);
}
public static void Extract(Repo store, ChunkHash cid, string targetPath)
{
Directory.CreateDirectory(targetPath);
Chunk c = store.ReadChunk(cid);
TreeChunk tree = TreeChunk.Deserialize(c.Data);
foreach (TreeFile file in tree.Files)
{
StreamChunk.Extract(store, ChunkHash.FromHashBytes(file.TreeChunkHash), Path.Combine(targetPath, file.Name));
}
foreach (TreeFile subdir in tree.Directories)
{
TreeChunk.Extract(store, ChunkHash.FromHashBytes(subdir.TreeChunkHash), Path.Combine(targetPath, subdir.Name));
}
}
public override Chunk ReadChunk(ChunkHash chunkHash)
{
RequestReadChunk msg = new RequestReadChunk();
msg.ChunkHash = chunkHash.bytes;
SendMessage(msg);
ReplyReadChunk reply = ReplyReadChunk.Deserialize(ProtocolParser.ReadBytes(input));
Chunk c = new Chunk(reply.ChunkData);
//Verify Hash
if (c.ChunkHash.Equals(chunkHash) == false)
{
throw new InvalidDataException("Hash mismatch: " + chunkHash);
}
return(c);
}
public static void Extract(Repo store, ChunkHash fileHash, string targetPath)
{
Chunk chunk = store.ReadChunk(fileHash);
StreamChunk streamChunk = StreamChunk.Deserialize <StreamChunk> (chunk.Data);
using (FileStream file = File.Open(targetPath, FileMode.Create)) {
foreach (byte[] hashBytes in streamChunk.Chunks)
{
Chunk fc = store.ReadChunk(ChunkHash.FromHashBytes(hashBytes));
file.Write(fc.Data, 0, fc.Data.Length);
}
//Verify length
if (file.Length != (long)streamChunk.Size)
{
throw new InvalidDataException("Invalid file length");
}
}
}
private string GetPath(ChunkHash hash)
{
string hex = hash.ToHex();
#if DEBUG
if (hex.Length != 64) //32 bytes
{
throw new InvalidDataException();
}
#endif
string path = dataRoot;
//FAT32 has a file per folder limit of slightly below 16 bits
//Total 32 bytes = 64 characters hex
path = Path.Combine(path, hex.Substring(0, 2));
path = Path.Combine(path, hex.Substring(0, 4));
path = Path.Combine(path, hex.Substring(0, 6));
path = Path.Combine(path, hex);
return(path);
}
void ProcessGetCustomHash()
{
RequestCustomHash request = RequestCustomHash.Deserialize(ProtocolParser.ReadBytes(input));
ReplyCustomHash reply = new ReplyCustomHash();
ChunkHash ch = localRepo.GetCustomHash(CustomID.FromBytes(request.CustomID));
if (ch == null)
{
reply.ChunkHash = null;
}
else
{
reply.ChunkHash = ch.bytes;
}
//Console.Error.WriteLine("PipeServer: Sending: " + reply);
byte[] rbytes = ReplyCustomHash.SerializeToBytes(reply);
//Console.Error.WriteLine("PipeServer: Sending: " + rbytes.Length + ", " + BitConverter.ToString(rbytes));
ProtocolParser.WriteBytes(output, rbytes);
}
public override ChunkHash WriteChunk(Chunk chunk)
{
if (recipientKeys.Count == 0)
{
throw new InvalidOperationException("EncryptedRepo must have at least one key");
}
//Encrypt
Chunk encryptedChunk = Encrypt(chunk);
//Reuse already existsing CustomID
if (chunk.CustomID != null)
{
ChunkHash hash = GetCustomHash(chunk.CustomID);
if (hash != null)
{
return(hash);
}
}
return(base.WriteChunk(encryptedChunk));
}
public override Chunk ReadChunk(ChunkHash chunkHash)
{
Chunk encrypted = base.ReadChunk(chunkHash);
return(new Chunk(Decrypt(encrypted.Data)));
}
public override Chunk ReadChunk(ChunkHash chunkHash)
{
Chunk encrypted = base.ReadChunk (chunkHash);
return new Chunk (Decrypt (encrypted.Data));
}
public TreeMessage(ChunkHash tree, string name)
{
this.TreeChunkHash = tree.bytes;
this.Name = name;
}
public override int GetHashCode()
{
return(ChunkHash.GetHashCode());
}
private string GetPath(ChunkHash hash)
{
string hex = hash.ToHex ();
#if DEBUG
if(hex.Length != 64) //32 bytes
throw new InvalidDataException();
#endif
string path = dataRoot;
//FAT32 has a file per folder limit of slightly below 16 bits
//Total 32 bytes = 64 characters hex
path = Path.Combine (path, hex.Substring (0, 2));
path = Path.Combine (path, hex.Substring (0, 4));
path = Path.Combine (path, hex.Substring (0, 6));
path = Path.Combine (path, hex);
return path;
}
/// <summary>
/// Verify the ClearID.ClearHash against chunk data.
/// The data is assumed to be decrypted, otherwise the verification will fail.
/// </summary>
public bool Verify(ChunkHash ch)
{
if (ch.Equals (ChunkHash) == false)
return false;
return true;
}
