private uint GetHashVersion()
{
using (var stream = new MemoryStream())
{
var writer = new StreamWriter(stream);
foreach (var candidate in SerializableMemberCandidates.OrderBy(item => item.Name))
{
writer.Write(candidate.Name);
writer.Write(VersionStreamSeparator);
writer.Write(candidate.Type.GetPrettyName());
writer.Write(VersionStreamSeparator);
}
foreach (var enumValue in EnumValues)
{
writer.Write(enumValue);
writer.Write(VersionStreamSeparator);
}
writer.Flush();
stream.Position = 0;
return(MurMurHash3.Hash(stream));
}
}
private void InitializeViaNikseBmp(NikseBitmap nbmp)
{
Width = nbmp.Width;
Height = nbmp.Height;
_colors = new byte[Width * Height];
var numberOfColoredPixels = 0;
for (int y = 0; y < Height; y++)
{
for (int x = 0; x < Width; x++)
{
var alpha = nbmp.GetAlpha(x, y);
if (alpha < 100)
{
_colors[Width * y + x] = 0;
}
else
{
_colors[Width * y + x] = 1;
numberOfColoredPixels++;
}
}
}
NumberOfColoredPixels = numberOfColoredPixels;
Hash = MurMurHash3.Hash(_colors);
}
static void TestHash(JObject config)
{
var values = config["Values"].ToObject <string[]>();
var outputFile = config["OutputFile"].Value <string>();
File.AppendAllLines(outputFile, values.Select(v => MurMurHash3.ComputeIdHash(v).ToString()));
}
public void PerformingAHash_SmallValueTests(string text)
{
// to exercise all cases of chunking the stream
var initialHash = MurMurHash3.Hash(text);
var secondHash = MurMurHash3.Hash(text);
Assert.AreEqual(initialHash, secondHash);
}
public BinaryOcrBitmap(int width, int height)
{
Width = width;
Height = height;
_colors = new byte[Width * Height];
Hash = MurMurHash3.Hash(_colors);
CalcuateNumberOfColoredPixels();
}
public void PerformingAHash_AreDifferentValues(string text1, string text2)
{
// to exercise all cases of chunking the stream
var firstHash = MurMurHash3.Hash(text1);
var secondHash = MurMurHash3.Hash(text2);
Assert.AreNotEqual(firstHash, secondHash);
}
public void Murmur3Tests()
{
Assert.AreEqual(455139366U, MurMurHash3.Hash(GetBytes("asdf")));
Assert.AreEqual(3902511862U, MurMurHash3.Hash(GetBytes("abcde")));
Assert.AreEqual(1635893381U, MurMurHash3.Hash(GetBytes("abcdef")));
Assert.AreEqual(2285673222U, MurMurHash3.Hash(GetBytes("abcdefg")));
Assert.AreEqual(774705101U, MurMurHash3.Hash(GetBytes("hello world!")));
}
/// <summary>
/// Utility function that returns an integer checksum of a string.
/// Used to compare file bodies to know whether they've been hotloaded
/// or not.
/// </summary>
public static int Checksum(string body)
{
byte[] input = System.Text.Encoding.UTF8.GetBytes(body);
using (MemoryStream stream = new MemoryStream(input)) {
int hash = MurMurHash3.Hash(stream);
return(hash);
}
}
public void TestGetMurMurHash3Hash()
{
var hash1 = MurMurHash3.Hash("fasdfsafsafsaf");
var hash2 = MurMurHash3.Hash("afasdfsafsafsaf");
var hash3 = MurMurHash3.Hash("afasdfsafsafsaf");
Assert.NotEqual(hash1, hash2);
Assert.Equal(hash2, hash3);
}
protected DomainCommand(
string aggregateRootId,
Type aggregateRootType,
int version = 0) : this()
{
Version = version;
AggregateRootId = aggregateRootId;
AggregateRootType = aggregateRootType;
PartitionKey = MurMurHash3.Hash(aggregateRootType.FullName);
Version = version;
}
public void TestMethod2()
{
MurMurHash3 mm3 = new MurMurHash3();
for (int i = 0; i < integers.Count; i++)
{
int res = (int)((ulong)mm3.MurmurHash3_x64_64((int)integers[i]) >> 32);
int comp = (int)intMurmur3Hashes[i];
Assert.AreEqual(res, comp);
}
}
protected void Initialize()
{
if (AggregateRootType == null || _initialized)
{
return;
}
Topic = AggregateRootType.FullName;
PartitionKey = MurMurHash3.Hash(AggregateRootType.FullName);
_initialized = true;
}
public void PerformingAHash_YieldsAStaticHashValue()
{
var text = @"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's
standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book
It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It
was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with
desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.";
var initialHash = MurMurHash3.Hash(text);
var secondHash = MurMurHash3.Hash(text);
Assert.AreEqual(initialHash, secondHash);
}
private void InitializeViaNikseBmp(NikseBitmap nbmp)
{
Width = nbmp.Width;
Height = nbmp.Height;
_colors = new byte[Width * Height];
for (int y = 0; y < Height; y++)
{
for (int x = 0; x < Width; x++)
{
this.SetPixel(x, y, nbmp.GetPixel(x, y));
}
}
Hash = MurMurHash3.Hash(_colors);
CalcuateNumberOfColoredPixels();
}
private static void CheckHash(string hashStr)
{
uint hash = Crc32.Compute(hashStr);
if (!hashes.ContainsKey(hash))
{
hashes.Add(hash, hashStr);
}
uint mmhash = MurMurHash3.Hash(hashStr);
if (!mmhashes.ContainsKey(mmhash))
{
mmhashes.Add(mmhash, hashStr);
}
}
public void Add(String key, T o)
{
var bucket = Math.Abs(MurMurHash3.Hash(new MemoryStream(Encoding.UTF8.GetBytes(key)))) % Buckets;
if (bucket != 0 && bucket != 1)
{
throw new Exception("temp exception hit ...");
}
List <T> vals = null;
if (!objs.TryGetValue(bucket, out vals))
{
objs[bucket] = vals = new List <T>();
}
vals.Add(o);
}
} // end addTimeStamp()
/// <summary>
/// Compute the hash of the key in the buffer. Use the Murmurhash3
/// algorithm to compute the hash. If not all of the values have been
/// added to the key (i.e. if the buffer is not full), then throw an
/// exception.
/// </summary>
public void computHashes()
{
// Check all the values for the key have been added
if (this.current_size != this.buffer_size)
{
throw new KineticaException("The RecordKey buffer is not full; check that all the relevant values have been added.");
}
// Hash the value
MurMurHash3.LongPair murmur = new MurMurHash3.LongPair();
MurMurHash3.murmurhash3_x64_128(this.buffer, 0, (uint)this.buffer_size, 10, out murmur);
// Save the hash value
this.routingHash = murmur.val1;
this.hash_code = (int)(this.routingHash ^ ((this.routingHash >> 32) & 0x0000ffffL));
} // end computHashes
private static ResultParameter GetVersionResultParameter(Assembly assembly)
{
string hash = " - ";
using (var file = new FileStream(assembly.Location, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 4 * 4096))
{
byte[] hashValue = BitConverter.GetBytes(MurMurHash3.Hash(file));
for (int i = 0; i < 4; i++)
{
hash += String.Format("{0:x2}", hashValue[i]);
}
}
System.Reflection.AssemblyName name = assembly.GetName();
return(new ResultParameter("Version", name.Name, name.Version.ToString() + hash));
}
} // end addDouble
/// <summary>
/// Add a string to the buffer. Hash the string value and add it
/// as a long internally.
/// </summary>
/// <param name="value">The string value to be added. Can be null.</param>
public void addString(string value)
{
// Handle nulls
if (value == null)
{
this.addLong(0L);
return;
}
// Hash the value
MurMurHash3.LongPair murmur = new MurMurHash3.LongPair();
System.Text.Encoding encoding = new System.Text.UTF8Encoding();
byte[] input = encoding.GetBytes(value);
MurMurHash3.murmurhash3_x64_128(input, 0, (uint)input.Length, 10, out murmur);
// Add the hashed value to the buffer
this.addLong(murmur.val1);
} // end addString
public void TestMurmur3()
{
var rnd = new Random(100);
byte[] buffer = new byte[100];
for (int i = 0; i < 10; i++)
{
rnd.NextBytes(buffer);
var hello = MurMurHash3.Hash(buffer);
Assert.AreEqual(getMurMur(i), hello);
}
var test2 = MurMurHash3.Hash("H3wlo World4!!!!");
Assert.AreEqual(1251584510, test2);
}
public void TestMethod1()
{
MurMurHash3 mm3 = new MurMurHash3();
for (int i = 0; i < byteStream.Count; i++)
{
sbyte[] s = byteStream[i].ToArray();
if (i == 15)
{
System.Diagnostics.Trace.WriteLine("stop");
}
long res = mm3.MurmurHash3_x64_64(s, (uint)s.Length, 9001);
long res2 = (long)((ulong)res >> 32);
System.Diagnostics.Trace.WriteLine(i + " Computed " + mm3.MurmurHash3_x64_64(s, (uint)s.Length, 9001).ToString("x")
+ " expected " + byteStreamMurMur3Hashes[i].ToString("x"));
Assert.AreEqual((long)((ulong)res >> 32), byteStreamMurMur3Hashes[i]);
}
}
public HttpPackageRepository(string url)
{
url = url.Trim();
if (Regex.IsMatch(url, "http(s)?://"))
{
this.Url = url;
}
else
{
this.Url = "http://" + url;
}
// Trim end to fix redirection. E.g. 'packages.opentap.io/' redirects to 'packages.opentap.io'.
this.Url = this.Url.TrimEnd('/');
defaultUrl = this.Url;
this.Url = CheckUrlRedirect(this.Url);
// Get the users Uniquely generated id
var id = GetUserId();
string installDir = ExecutorClient.ExeDir;
UpdateId = String.Format("{0:X8}{1:X8}", MurMurHash3.Hash(id), MurMurHash3.Hash(installDir));
}
uint IHashAlgo.Hash(string data)
{
return(MurMurHash3.Hash(Encoding.UTF8.GetBytes(data)));
}
private void UpdateHash()
{
hashCode = MurMurHash3.Hash(type, targetMask);
}
static void Main(string[] args)
{
try
{
var s2 = Stopwatch.StartNew();
for (int i = 0; i < 1000000; i++)
{
var stream = new MemoryStream(Encoding.UTF8.GetBytes("this is a blooming small test"));
var z22 = MurMurHash3.Hash(stream);
// var mm1 = z22 % 100;
}
s2.Stop();
var q24 = s2.ElapsedMilliseconds;
Container c = null;
var reg = new MyRegistry(args.Any() == false ? "appsettings.json" : args[0]);
c = new Container(reg);
reg.For <IContainer>().Use(c);
var databasePath = reg.DataDirectory;
var db = new SQLiteConnection(databasePath + "\\client.mdb");
db.CreateTable <ClientWithBucket>();
bool clearBucketState = false;
if (clearBucketState)
{
db.Delete <ClientWithBucket>("IFuzzyMatcher");
}
var clientFactory = new ClientFactory(c);
var bucketMax = clientFactory.GetClientBuckets <IFuzzyMatcher>().Max();
var bucketMin = clientFactory.GetClientBuckets <IFuzzyMatcher>().Min();
if (bucketMin != 0)
{
throw new Exception($"Minimum bucket is not zero it is - {bucketMin} - should be zero");
}
if (db.Find <ClientWithBucket>("IFuzzyMatcher") == null)
{
db.Insert(new ClientWithBucket {
BucketCount = bucketMax, ClientName = "IFuzzyMatcher"
});
}
else
{
var ppp = db.Find <ClientWithBucket>("IFuzzyMatcher");
if (ppp.BucketCount != bucketMax)
{
throw new Exception($"We have (dynamic) bucketMax for 'IFuzzyMatcher = {bucketMax} - but last recorded run bucketMax was - {ppp.BucketCount}... cannot continue - need to match bucketCount - or rebuild");
}
}
var multiplexer = new Multiplexer <FuzzyWordEntry>(bucketMax + 1 /*number of buckets*/);
CsvReader rdr = new CsvReader(new StreamReader(@"C:\home\colin\as\input\Retail-Large.csv"));
var records = rdr.GetRecords <Retail>();
records.Take(100000)
.Do(x => multiplexer.Add(x.Name, new FuzzyWordEntry {
DocId = Int32.Parse(x.Id), Phrase = x.Name
}));
List <Task> tasks = new List <Task>();
multiplexer.GetBuckets().Do(x =>
{
tasks.Add(new Task(() => clientFactory.GetClient <IFuzzyMatcher>(x.Item1).AddEntry(x.Item2)));
tasks.Last().Start();
});
Task.WaitAll(tasks.ToArray());
foreach (var bucket in clientFactory.GetClientBuckets <IFuzzyMatcher>())
{
var z = clientFactory.GetClient <IFuzzyMatcher>(bucket);
if (false)
{
}
else
{
var q = z.FuzzyQuery(new List <string>(new[]
{ "aleshia tomkiewicz", "daniel towers", "morna dick", "colin dick" }));
foreach (var g in q)
{
Console.WriteLine(g.Query);
foreach (var n in g.Detail)
{
Console.WriteLine($" {n.Candidate} - {n.Score} - {n.PhraseId}");
}
}
}
}
Console.ReadLine();
L.CloseLog();
}
catch (Exception e)
{
L.Exception(e);
}
}
private int GetCacheKey(string aggregateRootId, Type aggregateRootType)
{
var fullName = string.Format(CacheKeyMask, aggregateRootType.FullName, aggregateRootId);
return(MurMurHash3.Hash(fullName));
}
private int Hash(string key)
{
return(MurMurHash3.Hash(key));
}
/// <summary>
/// Utility function that returns an integer checksum of a stream.
/// Used to compare file bodies to know whether they've been hotloaded
/// or not.
/// </summary>
public static int Checksum(Stream stream)
{
return(MurMurHash3.Hash(stream));
}
uint IHashAlgo.Hash(byte[] data)
{
return(MurMurHash3.Hash(data));
}
protected RootParsingObject CreateJson(MarketInfo market_info)
{
//var market_info = new MarketInfo();
var root = new RootParsingObject();
var listings = new List <Listing>();
//market_info.ftpFolder = AfewStoreParser.NAME;
//market_info.website = AfewStoreParser.SITEURL;
//market_info.currency = AfewStoreParser.CURRENCY;
//market_info.start_parse_date = DateTime.Now;
//market_info.end_parse_date = DateTime.Now;
//market_info.delivery_to_usa = AfewStoreParser.DELIVERY_TO_USA;
//market_info.photo_parameters.is_watermark_image = false;
//market_info.photo_parameters.background_color = "white";
//market_info.currently_language = "en";
int i = 0;
foreach (var sneaker in catalog.sneakers)
{
var listing = new Listing();
//id murmurhash
Encoding encoding = new UTF8Encoding();
if (sneaker.link != null)
{
byte[] input = encoding.GetBytes(sneaker.link);
using (MemoryStream stream = new MemoryStream(input))
{
listing.id = MurMurHash3.Hash(stream);
if (listing.id < 0)
{
listing.id = listing.id * -1;
}
}
}
else
{
byte[] input = encoding.GetBytes(sneaker.title);
using (MemoryStream stream = new MemoryStream(input))
{
listing.id = MurMurHash3.Hash(stream);
if (listing.id < 0)
{
listing.id = listing.id * -1;
}
}
}
listing.url = sneaker.link;
listing.sku = sneaker.sku;
listing.title = sneaker.title;
listing.brand = sneaker.brand;
listing.colorbrand = sneaker.color;
listing.category = Helper.ConvertCategoryRusToEng(sneaker.category);
//if (String.IsNullOrWhiteSpace(listing.category)) Program.Logger.Warn("Wrong category: " + sneaker.category + " sku: " +sneaker.sku);
listing.sex = Helper.ConvertSexRusToEng(sneaker.sex);
listing.price = sneaker.price;
listing.old_price = sneaker.oldPrice;
listing.images = sneaker.images;
listing.sizes = Helper.GetSizeListUs(sneaker.sizes);
i++;
listing.position = i;
listings.Add(listing);
}
root.market_info = market_info;
root.listings = listings;
return(root);
//throw new NotImplementedException();
}
