/// <summary>
/// Register the standard hash algorithms for IPFS.
/// </summary>
/// <seealso href="https://github.com/multiformats/multicodec/blob/master/table.csv"/>
static HashingAlgorithm()
{
Register("sha1", 0x11, 20, () => SHA1.Create());
Register("sha2-256", 0x12, 32, () => SHA256.Create());
Register("sha2-512", 0x13, 64, () => SHA512.Create());
Register("dbl-sha2-256", 0x56, 32, () => new DoubleSha256());
Register("keccak-224", 0x1A, 224 / 8, () => new KeccakManaged(224));
Register("keccak-256", 0x1B, 256 / 8, () => new KeccakManaged(256));
Register("keccak-384", 0x1C, 384 / 8, () => new KeccakManaged(384));
Register("keccak-512", 0x1D, 512 / 8, () => new KeccakManaged(512));
Register("sha3-224", 0x17, 224 / 8, () => new BouncyDigest(new BC.Sha3Digest(224)));
Register("sha3-256", 0x16, 256 / 8, () => new BouncyDigest(new BC.Sha3Digest(256)));
Register("sha3-384", 0x15, 384 / 8, () => new BouncyDigest(new BC.Sha3Digest(384)));
Register("sha3-512", 0x14, 512 / 8, () => new BouncyDigest(new BC.Sha3Digest(512)));
Register("shake-128", 0x18, 128 / 8, () => new BouncyDigest(new BC.ShakeDigest(128)));
Register("shake-256", 0x19, 256 / 8, () => new BouncyDigest(new BC.ShakeDigest(256)));
Register("blake2b-160", 0xb214, 160 / 8, () => new BouncyDigest(new BC.Blake2bDigest(160)));
Register("blake2b-256", 0xb220, 256 / 8, () => new BouncyDigest(new BC.Blake2bDigest(256)));
Register("blake2b-384", 0xb230, 384 / 8, () => new BouncyDigest(new BC.Blake2bDigest(384)));
Register("blake2b-512", 0xb240, 512 / 8, () => new BouncyDigest(new BC.Blake2bDigest(512)));
Register("blake2s-128", 0xb250, 128 / 8, () => new BouncyDigest(new BC.Blake2sDigest(128)));
Register("blake2s-160", 0xb254, 160 / 8, () => new BouncyDigest(new BC.Blake2sDigest(160)));
Register("blake2s-224", 0xb25c, 224 / 8, () => new BouncyDigest(new BC.Blake2sDigest(224)));
Register("blake2s-256", 0xb260, 256 / 8, () => new BouncyDigest(new BC.Blake2sDigest(256)));
Register("md4", 0xd4, 128 / 8, () => new BouncyDigest(new BC.MD4Digest()));
Register("md5", 0xd5, 128 / 8, () => MD5.Create());
Register("murmur3-128", 0x22, 128 / 8, () => MurmurHash.Create128(managed: true, preference: AlgorithmPreference.Auto));
RegisterAlias("murmur3", "murmur3-128");
Register("identity", 0, 0, () => new IdentityHash());
RegisterAlias("id", "identity");
}
public void PushRecords(IEnumerable <DatasourceRecord> records)
{
//TODO: determine how the queues will be named
var sender = _factory.CreateMessageSender(_config.ReceiverQueue.QueueName);
var recList = records.ToList(); //avoid reenumerations
using (var ms = _recordSerializer.Serialize(recList))
{
string hashString;
using (var hasher = MurmurHash.Create128())
{
var hash = hasher.ComputeHash(ms);
hashString = Encoding.UTF8.GetString(hash);
}
ms.Position = 0;
var msg = new BrokeredMessage(ms, false)
{
MessageId = hashString
};
var sw = new Stopwatch();
sw.Start();
sender.Send(msg);
sw.Stop();
Log.DebugFormat("Pushed {0} records to topic/queue in {1}ms", recList.Count, sw.ElapsedMilliseconds);
}
}
public static string GetHashAsString(string key)
{
try
{
if (String.IsNullOrEmpty(key))
{
throw new Exception("Incorrect key");
}
HashAlgorithm murmur128 = MurmurHash.Create128(0, true, AlgorithmPreference.X86);
var hash = murmur128.ComputeHash(Encoding.ASCII.GetBytes(key));
if (hash.Length < 16)
{
throw new Exception("Hash Length must be more than 16 byte");
}
var builder = new StringBuilder(16);
for (int i = 0; i < hash.Length; i++)
{
builder.Append(hash[i].ToString("x2"));
}
return(builder.ToString().ToUpper());
}
catch { throw; }
}
public ExistenceStatCollector(string memberName, Configuration configuration, out bool cancel)
{
MemberName = memberName;
_byteConverter = ByteConverterHelpers.GetByteConverter <T>();
_murmurHash = MurmurHash.Create128(managed: true, preference: AlgorithmPreference.X64);
cancel = false;
}
public static byte[] CreateHash(byte[] input)
{
if (input == null)
{
return(null);
}
using HashAlgorithm murmur128 = MurmurHash.Create128();
return(murmur128.ComputeHash(input));
}
/// <summary>
/// MurmurHash
/// </summary>
/// <param name="k"></param>
/// <returns></returns>
public static byte[] ComputeMurmur(string k)
{
Murmur128 murmur = MurmurHash.Create128((uint)Environment.TickCount);
byte[] digest = murmur.ComputeHash(Encoding.UTF8.GetBytes(k));
murmur.Clear();
murmur.Dispose();
return(digest);
}
public ulong Hash(string key, uint seed)
{
Murmur128 murmur128 = MurmurHash.Create128(seed: seed, preference: AlgorithmPreference.X64);
byte[] keyToBytes = Encoding.ASCII.GetBytes(key);
byte[] seedResult = murmur128.ComputeHash(keyToBytes);
return(BitConverter.ToUInt64(seedResult, 0));
}
public static string CreateHash(string input)
{
if (input == null)
{
return(null);
}
using HashAlgorithm murmur128 = MurmurHash.Create128();
byte[] hashOutput = murmur128.ComputeHash(Encoding.UTF8.GetBytes(input));
return(Convert.ToBase64String(hashOutput));
}
/// <summary>
/// Generate a non-cryptographic string hash by MurMur algorithm
/// </summary>
/// <param name="stringToHash"></param>
/// <returns></returns>
public static string GenerateStringHash(string stringToHash)
{
if (stringToHash == null)
{
throw new ArgumentNullException(nameof(stringToHash));
}
var hash = MurmurHash.Create128();
var hashArray = hash.ComputeHash(Encoding.UTF8.GetBytes(stringToHash));
return(ConvertBytesToString(hashArray));
}
public CloudFile(byte[] data, string extension)
{
HashAlgorithm murmur128 = MurmurHash.Create128(managed: false);
using (var ms = new MemoryStream(data))
{
var streamHash = murmur128.ComputeHash(ms);
Hash = streamHash.ToHashString();
Extension = extension;
Data = data;
}
}
public ulong GetHashCode(byte[] bytes)
{
Murmur128 murmurHash;
if (!pool.TryPop(out murmurHash))
{
murmurHash = MurmurHash.Create128(managed: true, preference: AlgorithmPreference.X64);
}
byte[] result = murmurHash.ComputeHash(bytes);
pool.Push(murmurHash);
return(BitConverter.ToUInt64(result, 0));
}
/// <summary>
/// The constructor calculates all needed values for the Bloom Filter.
/// </summary>
/// <param name="wordCount"></param>
/// <param name="errorProbability"></param>
public BloomFilter(int wordCount, double errorProbability)
{
// Bloom Filter Size and the NumberOfHashFunctions is calculated by the wikipedia article.
// Source: https://en.wikipedia.org/wiki/Bloom_filter#Optimal_number_of_hash_functions
FilterSize = (int)Math.Ceiling(-1 * (wordCount * Math.Log(errorProbability) / Math.Pow(Math.Log(2), 2)));
NumberOfHashFunctions = (int)Math.Ceiling(-1 * Math.Log(errorProbability, 2));
bloomFilter = new byte[FilterSize];
// Setup the list of HashFunctions with the calculated FilterSize.
// Here, the Murmur library, which was mentioned in the task description, was used.
for (var i = 0; i < NumberOfHashFunctions; i++)
{
hashFunctions.Add(MurmurHash.Create128((uint)i));
}
}
/// <summary>
///Returns the original key if already less than 1024 bytes, or hashes, then returns
/// </summary>
/// <param name="key">The original key</param>
/// <param name="prefix">Optional prefix - won't be hashed</param>
/// <returns></returns>
public string Calculate(string key, string prefix = null)
{
if (prefix == null)
{
prefix = string.Empty;
}
if (key.Length < 256)
{
return(key);
}
using (var murmur128 = MurmurHash.Create128(managed: false))
return(string.Concat(prefix,
Convert.ToBase64String(murmur128.ComputeHash(GenerateStreamFromString(key)))));
}
public void returns_hashed_key_if_passed_in_string_is_greater_than_130()
{
var key = "";
for (var i = 0; i < 10000; i++)
{
key += Guid.NewGuid().ToString();
}
var result = _calculateKey.Calculate(key);
HashAlgorithm murmur128 = MurmurHash.Create128(managed: false);
var expectedKey = Convert.ToBase64String(murmur128.ComputeHash(CalculateKey.GenerateStreamFromString(key)));
Assert.That(result, Is.EqualTo(expectedKey));
}
private long GetRingPosition(long key)
{
using (Murmur128 hashAlgorithm = MurmurHash.Create128())
{
byte[] hash = hashAlgorithm.ComputeHash(BitConverter.GetBytes(key));
long total = 0;
for (int startIndex = 0; startIndex < hash.Length; startIndex += 8)
{
unchecked
{
total += BitConverter.ToInt64(hash, startIndex);
}
}
return(total);
}
}
//将传入的字符串通过murmurhash函数生成32位长的字符串
private static string GetHashAsString(string stringToHash)
{
Murmur128 urlHash = MurmurHash.Create128(managed: false);
byte[] urlbyte = System.Text.Encoding.UTF8.GetBytes(stringToHash);
byte[] hash = urlHash.ComputeHash(urlbyte);
//以下代码也可以用 BitConverter.ToString(hash)代替
var builder = new StringBuilder(16);
for (int i = 0; i < hash.Length; i++)
{
builder.Append(hash[i].ToString("x2"));
}
return(builder.ToString());
}
public long GetKey(string key)
{
if (key == null)
{
throw new ArgumentNullException("key");
}
using (var hashAlgorithm = MurmurHash.Create128())
{
byte[] keyBuffer = Encoding.UTF8.GetBytes(key);
var hash = hashAlgorithm.ComputeHash(keyBuffer);
long total = 0;
for (int startIndex = 0; startIndex < hash.Length; startIndex += 8)
{
unchecked
{
total += BitConverter.ToInt64(hash, startIndex);
}
}
return(total);
}
}
public static ulong Hash(string value, uint seed = 0)
{
var bytes = MurmurHash.Create128(seed).ComputeHash(Encoding.UTF8.GetBytes(value));
return(BitConverter.ToUInt64(bytes, 0));
}
protected static ulong Hash64(byte[] value)
{
var hashalg = MurmurHash.Create128(managed: false);
return(hashalg.ComputeHash64(value));
}
public HyperLogLog(byte b) : base(b)
{
_hashAlgorithm = MurmurHash.Create128();
}
public static string CreateHash(FileStream stream)
{
HashAlgorithm murmur128 = MurmurHash.Create128();
return(Convert.ToBase64String(murmur128.ComputeHash(stream)));
}
public HyperLogLog(SerializationInfo info, StreamingContext context) : base(info, context)
{
_hashAlgorithm = MurmurHash.Create128();
}
public MURMUR3_128()
: base(HashType.MURMUR3_128, "murmur3-128", 16)
{
_factory = () => MurmurHash.Create128(managed: false, preference: AlgorithmPreference.X64);
}
static bool FTP_to_SFTP()
{
long alreadyCopied = 0;
byte[] buffer = new byte[BufferSize];
try
{
using (var _ftpClient = new FtpClient(FtpProtocol.Ftp, FTP_Server, FTP_Port, FTP_Credentials))
{
_sftpClient.Connect();
var filesToCopy = _ftpClient.ListEntries(FTP_PublishedFilesDirectory);
long totalSize = filesToCopy.Sum(_ => _.Size ?? 0);
int totalFiles = filesToCopy.Count();
int copiedFiles = 0;
foreach (var fileToCopy in filesToCopy)
{
var sftpFileName = $"{SFTP_PublishedFilesDirectory}/{Path.GetFileName(fileToCopy.Name.Replace("-", ""))}";
HashAlgorithm murmur128 = MurmurHash.Create128(managed: false);
string ftpHash = null;
string sftpHash = null;
long ftpSize = fileToCopy.Size.Value;
long sftpSize = 0;
string tempFile = Path.GetTempFileName();
using (var ftpFileStream = _ftpClient.Retr(fileToCopy.Path))
using (var tempFileStream = File.Open(tempFile, FileMode.Create, FileAccess.Write, FileShare.None))
{
int readBytes = ftpFileStream.Read(buffer, 0, buffer.Length);
while (readBytes > 0)
{
tempFileStream.Write(buffer, 0, readBytes);
readBytes = ftpFileStream.Read(buffer, 0, buffer.Length);
}
}
Log.WriteLine($"Copy {fileToCopy.Name} from FTP to {tempFile}");
using (var tempFileStream = File.Open(tempFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
ftpHash = ToHashString(murmur128.ComputeHash(tempFileStream));
}
if (_sftpClient.Exists(sftpFileName))
{
sftpSize = _sftpClient.Get(sftpFileName).Length;
using (var sftpFileStream = _sftpClient.OpenRead(sftpFileName))
{
sftpHash = ToHashString(murmur128.ComputeHash(sftpFileStream));
}
}
if (ftpHash == sftpHash)
{
alreadyCopied += fileToCopy.Size.Value;
copiedFiles++;
ConsoleProgress.Write("({1}/{2}) {0}%", alreadyCopied * 100 / totalSize, copiedFiles, totalFiles);
File.Delete(tempFile);
continue;
}
Log.WriteLine($"{fileToCopy.Name} - {ftpHash} ({ftpSize}) - {sftpHash} ({(string.IsNullOrEmpty(sftpHash) ? 0 : sftpSize)})");
using (var tempFileStream = File.Open(tempFile, FileMode.Open, FileAccess.Read, FileShare.Read))
using (var sftpFileStream = _sftpClient.Open(sftpFileName, FileMode.Create, FileAccess.Write))
{
int readBytes = tempFileStream.Read(buffer, 0, buffer.Length);
while (readBytes > 0)
{
sftpFileStream.Write(buffer, 0, readBytes);
alreadyCopied += readBytes;
ConsoleProgress.Write("({1}/{2}) {0}%", alreadyCopied * 100 / totalSize, copiedFiles, totalFiles);
readBytes = tempFileStream.Read(buffer, 0, buffer.Length);
}
}
File.Delete(tempFile);
copiedFiles++;
sftpSize = _sftpClient.Get(sftpFileName).Length;
using (var sftpFileStream = _sftpClient.OpenRead(sftpFileName))
{
sftpHash = ToHashString(murmur128.ComputeHash(sftpFileStream));
}
Log.WriteLine($"Verify {fileToCopy.Name} - {ftpHash} ({ftpSize}) - {sftpHash} ({sftpSize})");
}
_sftpClient.Disconnect();
return(true);
}
}
catch (Exception e)
{
Log.WriteLine(e.Message);
return(false);
}
}
public async Task Upload(IMarkdownFunctionSettings settings, int clientId, int userId, int scenarioId, int partitionId, int partitionCount, List <SmCalcProduct> results)
{
var seededHash = MurmurHash.Create128(seed: (uint)clientId);
var entities = Build(seededHash, partitionId, partitionCount, results, userId);
await _recommendationProductRepository.Write(clientId : clientId, scenarioId : scenarioId, partitionId : partitionId, entities : entities);
}
public async Task Upload(IMarkdownFunctionSettings settings, SmRecommendationProductSummary product, int revisionId, List <SmCalcRecommendation> recommendations, int userId)
{
var seededHash = MurmurHash.Create128(seed: (uint)product.ClientId);
var entities = Build(seededHash, product, recommendations, userId);
await _recommendationRepository.Write(clientId : product.ClientId, scenarioId : product.ScenarioId, revisionId : revisionId, partitionId : product.PartitionNumber, productId : product.ProductId, entities : entities);
}
static void Main(string[] args)
{
ConsoleInterop.DisableQuickEdit();
Console.CursorVisible = false;
if (ConfigurationManager.AppSettings.AllKeys.Contains(SFTP_ServerKey))
{
SFTP_Server = ConfigurationManager.AppSettings[SFTP_ServerKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(SFTP_PortKey))
{
SFTP_Port = int.Parse(ConfigurationManager.AppSettings[SFTP_PortKey]);
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(SFTP_UserKey))
{
SFTP_User = ConfigurationManager.AppSettings[SFTP_UserKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(SFTP_PasswordKey))
{
SFTP_Password = ConfigurationManager.AppSettings[SFTP_PasswordKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(SFTP_PublishedFilesDirectoryKey))
{
SFTP_PublishedFilesDirectory = ConfigurationManager.AppSettings[SFTP_PublishedFilesDirectoryKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(SFTP_UploadedFilesDirectoryKey))
{
SFTP_UploadedFilesDirectory = ConfigurationManager.AppSettings[SFTP_UploadedFilesDirectoryKey];
}
SFTP_Credentials = new NetworkCredential(SFTP_User, SFTP_Password);
_sftpClient = new SftpClient(SFTP_Server, SFTP_Port, SFTP_Credentials.UserName, SFTP_Credentials.Password);
if (ConfigurationManager.AppSettings.AllKeys.Contains(FTP_ServerKey))
{
FTP_Server = ConfigurationManager.AppSettings[FTP_ServerKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(FTP_PortKey))
{
FTP_Port = int.Parse(ConfigurationManager.AppSettings[FTP_PortKey]);
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(FTP_UserKey))
{
FTP_User = ConfigurationManager.AppSettings[FTP_UserKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(FTP_PasswordKey))
{
FTP_Password = ConfigurationManager.AppSettings[FTP_PasswordKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(FTP_PublishedFilesDirectoryKey))
{
FTP_PublishedFilesDirectory = ConfigurationManager.AppSettings[FTP_PublishedFilesDirectoryKey];
}
if (ConfigurationManager.AppSettings.AllKeys.Contains(FTP_UploadedFilesDirectoryKey))
{
FTP_UploadedFilesDirectory = ConfigurationManager.AppSettings[FTP_UploadedFilesDirectoryKey];
}
FTP_Credentials = new NetworkCredential(FTP_User, FTP_Password);
Console.Write("Add project dispositions if not exist : ");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
ExecuteScript(sqlConn, "ProjectDispositionMemorized");
}
Console.WriteLine("OK\n");
Console.WriteLine("Add IsDeleted to Audit if not exists");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
ExecuteCommandFormat <int>(sqlConn, $"EXEC AddColumnIfNotExists 'Audit', 'IsDeleted', '[BIT] NOT NULL DEFAULT((0))';");
sqlConn.Close();
}
Console.WriteLine("OK\n");
Console.WriteLine("Add LinkedInspectionId to InspectionStep if not exists");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
bool exists = false;
using (SqlDataReader reader = ExecuteCommandFormat <SqlDataReader>(sqlConn, $"SELECT column_id FROM sys.columns WHERE NAME = 'LinkedInspectionId' AND object_id = OBJECT_ID('InspectionStep');"))
{
exists = reader.HasRows;
reader.Close();
sqlConn.Close();
}
Console.WriteLine($"LinkedInspectionId exists : {(exists ? "Yes\n" : "No")}");
if (!exists)
{
Console.Write("Clear all inspections : ");
ExecuteScript(sqlConn, "ClearAllInspections");
Console.WriteLine("OK");
Console.Write("Add LinkedInspectionId to InspectionStep : ");
ExecuteScript(sqlConn, "AddInspectionStep_LinkedInspectionStep");
Console.WriteLine("OK\n");
}
}
Console.WriteLine("Add QualificationReason table if not exists");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
bool exists = false;
using (SqlDataReader reader = ExecuteCommandFormat <SqlDataReader>(sqlConn, $"SELECT * FROM sys.tables WHERE NAME = 'QualificationReason';"))
{
exists = reader.HasRows;
reader.Close();
sqlConn.Close();
}
Console.WriteLine($"QualificationReason table exists : {(exists ? "Yes\n" : "No")}");
if (!exists)
{
Console.Write("Add QualificationReason table : ");
ExecuteScript(sqlConn, "QualificationReason");
Console.WriteLine("OK\n");
}
}
Console.WriteLine("Add IsDeleted and AnomalyOrigin if not exist");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
ExecuteScript(sqlConn, "IsDeleted_AnomalyOrigin");
}
Console.WriteLine("OK\n");
Console.WriteLine("Extract thumbnails");
string thumbnailsDir = @"C:\Utils\Thumbnails";
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
Console.Write("Configure OLE : ");
ExecuteScript(sqlConn, "ConfigureOle", true, "master");
Console.WriteLine("OK");
WindowsIdentity currentIdentity = WindowsIdentity.GetCurrent();
Console.Write($"Configure Role Server bulkadmin for {currentIdentity.Name} : ");
ExecuteCommandFormat <int>(sqlConn, "ALTER SERVER ROLE [bulkadmin] ADD MEMBER [{0}];", currentIdentity.Name);
Console.WriteLine("OK");
}
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
Console.Write("Add ThumbnailHash to Action : ");
ExecuteScript(sqlConn, "ActionThumbnailHash");
Console.WriteLine("OK");
Console.Write("Add Action_ExportThumbnail procedure : ");
ExecuteScript(sqlConn, "Action_ExportThumbnail");
Console.WriteLine("OK");
Console.Write("Add Action_ExportAllThumbnail procedure : ");
ExecuteScript(sqlConn, "Action_ExportAllThumbnail");
Console.WriteLine("OK");
Console.Write($"Create thumbnails directory ({thumbnailsDir}) : ");
Directory.CreateDirectory(thumbnailsDir);
Console.WriteLine("OK");
Console.Write($"Extract thumbnails : ");
ExecuteCommandFormat <int>(sqlConn, "EXECUTE dbo.[Action_ExportAllThumbnail] N'{0}', N'{1}';", thumbnailsDir, ".jpg");
Console.WriteLine("OK");
Console.Write($"List exported thumbnails : ");
var thumbnails = Directory.EnumerateFiles(thumbnailsDir);
Console.WriteLine("OK");
Console.Write($"Compute hashes and rename thumbnails : ");
Dictionary <int, (string hash, string extension)> hashThumbnailsDict = new Dictionary <int, (string hash, string extension)>();
foreach (string thumbnail in thumbnails.Where(_ => int.TryParse(Path.GetFileNameWithoutExtension(_), out int tmpResult)))
{
HashAlgorithm murmur128 = MurmurHash.Create128(managed: false);
string newName;
using (var fileStream = File.OpenRead(thumbnail))
{
newName = ToHashString(murmur128.ComputeHash(fileStream));
}
hashThumbnailsDict.Add(int.Parse(Path.GetFileNameWithoutExtension(thumbnail)), (newName, Path.GetExtension(thumbnail)));
newName = $"{newName}{Path.GetExtension(thumbnail)}";
if (File.Exists(Path.Combine(thumbnailsDir, newName)))
{
File.Delete(thumbnail);
}
else
{
File.Move(thumbnail, Path.Combine(thumbnailsDir, newName));
}
}
Console.WriteLine("OK");
Console.Write($"Insert thumbnails into CloudFile : ");
var cloudFiles = new Dictionary <string, string>();
foreach (var kv in hashThumbnailsDict)
{
if (!cloudFiles.ContainsKey(kv.Value.hash))
{
cloudFiles.Add(kv.Value.hash, kv.Value.extension);
}
}
foreach (var kv in cloudFiles)
{
ExecuteCommandFormat <int>(sqlConn, "INSERT INTO [dbo].[CloudFile] ([Hash],[Extension]) VALUES ('{0}','{1}');",
kv.Key,
kv.Value);
}
Console.WriteLine("OK");
Console.Write($"Update CloudFile links in Action : ");
foreach (var kv in hashThumbnailsDict)
{
ExecuteCommandFormat <int>(sqlConn, "UPDATE [dbo].[Action] SET [ThumbnailHash] = '{0}' WHERE [ActionId] = {1};",
kv.Value.hash,
kv.Key);
}
Console.WriteLine("OK");
Console.Write($"Delete Thumbnail from Action : ");
ExecuteCommandFormat <int>(sqlConn, "ALTER TABLE [dbo].[Action] DROP COLUMN [Thumbnail];");
Console.WriteLine("OK");
Console.Write($"Delete Action_ExportAllThumbnail procedure : ");
ExecuteCommandFormat <int>(sqlConn, "DROP PROCEDURE [dbo].Action_ExportAllThumbnail;");
Console.WriteLine("OK");
Console.Write($"Delete Action_ExportThumbnail procedure : ");
ExecuteCommandFormat <int>(sqlConn, "DROP PROCEDURE [dbo].Action_ExportThumbnail;");
Console.WriteLine("OK\n");
}
Console.Write("Copy thumbnails to SFTP : ");
var thumbnailsToCopy = Directory.EnumerateFiles(thumbnailsDir);
long totalSize = thumbnailsToCopy.Sum(_ => (new FileInfo(_)).Length);
long alreadyCopied = 0;
byte[] buffer = new byte[BufferSize];
try
{
_sftpClient.Connect();
foreach (var thumbnailToCopy in thumbnailsToCopy)
{
using (var localFileStream = File.OpenRead(thumbnailToCopy))
using (var remoteStream = _sftpClient.OpenWrite($"{SFTP_PublishedFilesDirectory}/{Path.GetFileName(thumbnailToCopy)}"))
{
int readBytes = localFileStream.Read(buffer, 0, buffer.Length);
while (readBytes > 0)
{
remoteStream.Write(buffer, 0, readBytes);
alreadyCopied += readBytes;
ConsoleProgress.Write("{0}%", alreadyCopied * 100 / totalSize);
readBytes = localFileStream.Read(buffer, 0, buffer.Length);
}
}
}
_sftpClient.Disconnect();
ConsoleProgress.Finish("OK\n");
}
catch (Exception e)
{
Console.WriteLine("FAIL");
Console.WriteLine(e.Message);
Console.WriteLine("\nFinish.\nPress a key to exit...");
Console.ReadKey();
return;
}
Console.WriteLine("Add CloudFile links if not exist");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
ExecuteScript(sqlConn, "CloudFile_Links");
}
Console.WriteLine("OK\n");
Console.WriteLine("Migrate videos of projects to videos of processes");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
Console.Write("Insert video resources : ");
ExecuteScript(sqlConn, "VideoResources");
Console.WriteLine("OK");
Console.Write("Add VideoSync table if not exists : ");
ExecuteScript(sqlConn, "VideoSync");
Console.WriteLine("OK");
Console.Write("Update Video if necessary : ");
ExecuteScript(sqlConn, "VideoProjectToProcess");
Console.WriteLine("OK\n");
}
Console.WriteLine("Fix hashes");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
Console.Write("Part 1 : ");
ExecuteScript(sqlConn, "FixHashes1");
Console.WriteLine("OK");
string constraintName = null;
Console.Write("Update PublishedFile.Hash : ");
using (SqlDataReader reader = ExecuteCommandFormat <SqlDataReader>(sqlConn, $"SELECT c.[name] FROM[sys].[key_constraints] c INNER JOIN[sys].[objects] o ON c.[parent_object_id] = o.[object_id] WHERE c.[type] = 'PK' AND o.[name] = 'PublishedFile';"))
{
if (reader.HasRows)
{
reader.Read();
constraintName = reader.GetString(0);
}
reader.Close();
sqlConn.Close();
}
if (!string.IsNullOrEmpty(constraintName))
{
ExecuteCommandFormat <int>(sqlConn, "ALTER TABLE [dbo].[PublishedFile] DROP CONSTRAINT {0};", constraintName);
}
ExecuteCommandFormat <int>(sqlConn, "ALTER TABLE [dbo].[PublishedFile] ALTER COLUMN [Hash] NCHAR(32) NOT NULL;");
ExecuteCommandFormat <int>(sqlConn, "ALTER TABLE [dbo].[PublishedFile] ADD CONSTRAINT PK_PublishedFile PRIMARY KEY (Hash);");
Console.WriteLine("OK");
constraintName = null;
Console.Write("Update CutVideo.Hash : ");
using (SqlDataReader reader = ExecuteCommandFormat <SqlDataReader>(sqlConn, $"SELECT c.[name] FROM[sys].[key_constraints] c INNER JOIN[sys].[objects] o ON c.[parent_object_id] = o.[object_id] WHERE c.[type] = 'PK' AND o.[name] = 'CutVideo';"))
{
if (reader.HasRows)
{
reader.Read();
constraintName = reader.GetString(0);
}
reader.Close();
sqlConn.Close();
}
if (!string.IsNullOrEmpty(constraintName))
{
ExecuteCommandFormat <int>(sqlConn, "ALTER TABLE [dbo].[CutVideo] DROP CONSTRAINT {0};", constraintName);
}
ExecuteCommandFormat <int>(sqlConn, "ALTER TABLE [dbo].[CutVideo] ALTER COLUMN [Hash] NCHAR(32) NOT NULL;");
ExecuteCommandFormat <int>(sqlConn, "ALTER TABLE [dbo].[CutVideo] ADD CONSTRAINT PK_CutVideo PRIMARY KEY (Hash);");
Console.WriteLine("OK");
Console.Write("Part 2 : ");
ExecuteScript(sqlConn, "FixHashes2");
Console.WriteLine("OK\n");
}
Console.WriteLine("Add Timeslot if not exist");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
ExecuteScript(sqlConn, "Timeslot");
}
Console.WriteLine("OK\n");
Console.WriteLine("Add InspectionSchedule if not exist");
using (var sqlConn = new SqlConnection(GetConnectionString()))
{
ExecuteScript(sqlConn, "InspectionSchedule");
}
Console.WriteLine("OK\n");
Console.Write("Copy FTP files to SFTP : ");
if (FTP_to_SFTP())
{
ConsoleProgress.Finish("OK\n");
}
else
{
ConsoleProgress.Finish("FAIL\n");
}
Console.WriteLine("\nFinish.\nPress a key to exit...");
Console.ReadKey();
}
public static void Run()
{
var str = "asdasdasd";
{
var hash = MurmurHash3Factory.Instance.Create(new MurmurHash3Config
{
HashSizeInBits = 128
}).ComputeHash(Encoding.UTF8.GetBytes(str)).Hash;
Console.WriteLine(string.Join(" ", hash));
Console.WriteLine(BitConverter.ToInt32(hash, 0));
}
{
var hash = MurmurHash.Create128().ComputeHash(Encoding.UTF8.GetBytes(str));
Console.WriteLine(string.Join(" ", hash));
Console.WriteLine(BitConverter.ToInt32(hash, 0));
}
{
var code = MurMurHash3.Hash(str);
Console.WriteLine(string.Join(" ", BitConverter.GetBytes(code)));
Console.WriteLine(code);
}
// 碰撞率
var count = 5000000;
var datas = new List <string>();
for (var i = 0; i < count; i++)
{
datas.Add(Guid.NewGuid().ToString("N"));
}
{
var sw = new Stopwatch();
sw.Start();
var list = new List <int>();
// for (int i = 0; i < count; i++)
// list.Add(i.ToString().GetHashCode());
foreach (var item in datas)
{
list.Add(item.GetHashCode());
}
sw.Stop();
Console.WriteLine($"GetHashCode {count - list.Distinct().Count()}/{count}\t耗时{sw.ElapsedMilliseconds}ms");
}
{
var murmurHash3 = MurmurHash3Factory.Instance.Create(new MurmurHash3Config
{
HashSizeInBits = 128,
Seed = 0xe17a1465
});
var sw = new Stopwatch();
sw.Start();
var list = new List <long>();
// for (int i = 0; i < count; i++)
// {
// var hash = murmurHash3.ComputeHash(Encoding.UTF8.GetBytes(i.ToString())).Hash;
// list.Add(BitConverter.ToInt32(hash, 0));
// }
foreach (var item in datas)
{
var hash = murmurHash3.ComputeHash(Encoding.UTF8.GetBytes(item)).Hash;
list.Add(BitConverter.ToInt64(hash, 0));
}
sw.Stop();
Console.WriteLine($"System.Data.HashFunction.MurmurHash {count - list.Distinct().Count()}/{count}\t耗时{sw.ElapsedMilliseconds}ms");
}
{
var murmurHash3 = MurmurHash.Create128(0xe17a1465);
var sw = new Stopwatch();
sw.Start();
var list = new List <long>();
// for (int i = 0; i < count; i++)
// {
// var hash = murmurHash3.ComputeHash(Encoding.UTF8.GetBytes(i.ToString()));
// list.Add(BitConverter.ToInt32(hash, 0));
// }
foreach (var item in datas)
{
var hash = murmurHash3.ComputeHash(Encoding.UTF8.GetBytes(item));
list.Add(BitConverter.ToInt64(hash, 0));
}
sw.Stop();
Console.WriteLine($"Murmur-net {count - list.Distinct().Count()}/{count}\t耗时{sw.ElapsedMilliseconds}ms");
}
{
var sw = new Stopwatch();
sw.Start();
var list = new List <int>();
foreach (var item in datas)
{
list.Add(MurMurHash3.Hash(item));
}
sw.Stop();
Console.WriteLine($"MurMurHash3 {count - list.Distinct().Count()}/{count}\t耗时{sw.ElapsedMilliseconds}ms");
}
}
public void Configuration(IAppBuilder app)
{
// Dependency injection settings
var container = new UnityContainer();
var config = WebApiConfig.Register(container);
app.Use <LoggingMiddleware>();
// For more information on how to configure your application, visit http://go.microsoft.com/fwlink/?LinkID=316888
//enable cors origin requestsit
//app.UseCors(Microsoft.Owin.Cors.CorsOptions.AllowAll);
/*var myProvider = new AuthorizationServerProvider(config.DependencyResolver as IServiceBus);
* OAuthAuthorizationServerOptions options = new OAuthAuthorizationServerOptions
* {
* AllowInsecureHttp = true,
* TokenEndpointPath = new PathString("/token"),
* AccessTokenExpireTimeSpan = TimeSpan.FromDays(1), // OAuth token valid 1 day
* Provider = myProvider,
* RefreshTokenProvider = new SimpleRefreshTokenProvider()
* };
* app.UseOAuthAuthorizationServer(options);
* app.UseOAuthBearerAuthentication(new OAuthBearerAuthenticationOptions());*/
app.UseJwtBearerAuthentication(new JwtBearerAuthenticationOptions
{
//AllowedAudiences = new[] { "KL²", "KL² Web Admin", "KL² Tablet" },
TokenValidationParameters = new TokenValidationParameters
{
IssuerSigningKey = JwtTokenProvider._jwtSecret.ToSymmetricSecurityKey(),
ValidIssuer = JwtTokenProvider._appDomain,
ValidateAudience = false,
ValidateLifetime = true,
ClockSkew = TimeSpan.FromMinutes(0)
}
});
app.UseWebApi(config);
// Branch the pipeline here for requests that start with "/signalr"
app.Map("/signalr", map =>
{
// Setup the CORS middleware to run before SignalR.
// By default this will allow all origins. You can
// configure the set of origins and/or http verbs by
// providing a cors options with a different policy.
map.UseCors(CorsOptions.AllowAll);
map.RunSignalR(SignalRConfig.Register(container));
});
//app.MapSignalR(SignalRConfig.Register(container));
var corsPolicy = new System.Web.Cors.CorsPolicy
{
AllowAnyHeader = true,
AllowAnyMethod = true,
AllowAnyOrigin = true
};
// ExposedHeaders has a private setter for some reason so one must use reflection to set it.
corsPolicy.GetType()
.GetProperty(nameof(corsPolicy.ExposedHeaders))
.SetValue(corsPolicy, tusdotnet.Helpers.CorsHelper.GetExposedHeaders());
app.UseCors(new CorsOptions
{
PolicyProvider = new CorsPolicyProvider
{
PolicyResolver = context => Task.FromResult(corsPolicy)
}
});
app.UseTus(context => new DefaultTusConfiguration
{
Store = ((UnityResolver)config.DependencyResolver).Resolve <IFileProvider>().GetTusStore(),
UrlPath = "/files",
Events = new Events
{
OnFileCompleteAsync = async ctx =>
{
if (ctx.Store is ITusReadableStore readableStore)
{
var fileProvider = ((UnityResolver)config.DependencyResolver).Resolve <IFileProvider>();
var file = await ctx.GetFileAsync();
if (file == null)
{
return;
}
//await readableStore.GetFileAsync(ctx.FileId, ctx.CancellationToken);
var metadata = await file.GetMetadataAsync(ctx.CancellationToken);
string filename = metadata?.ContainsKey("filename") == true ? metadata["filename"].GetString(System.Text.Encoding.UTF8) : file.Id;
if (metadata?.ContainsKey("computeHash") == true)
{
HashAlgorithm murmur128 = MurmurHash.Create128(managed: false);
using (var ms = await fileProvider.OpenRead(file.Id, DirectoryEnum.Uploaded))
{
var streamHash = murmur128.ComputeHash(ms).ToHashString();
filename = $"{streamHash}{Path.GetExtension(filename)}";
}
}
// On doit déplacer le fichier vers PublishedFiles en le renommant
await fileProvider.Complete(file.Id, filename);
// TODO : ajouter une metadata indiquant un traitement avec FFMPEG (encodage vidéo, traitement image)
// Supprimer les fichiers temp
var tempFiles = await fileProvider.EnumerateFiles(DirectoryEnum.Uploaded, $"{file.Id}.*");
foreach (string tmpFile in tempFiles)
{
await fileProvider.Delete(tmpFile, DirectoryEnum.Uploaded);
}
fileProvider.WriteAllText($"{fileProvider.UploadedFilesDirectory}/{file.Id}.name", filename);
}
}
}
});
private static string GetMurmur3HashString(string value)
=> BitConverter.ToString(MurmurHash.Create128().ComputeHash(Encoding.UTF8.GetBytes(value))).Replace("-", "");
