1 /* 2 * lookup3.c, by Bob Jenkins, May 2006, Public Domain. 3 * 4 * These are functions for producing 32-bit hashes for hash table lookup. 5 * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final_() 6 * are externally useful functions. Routines to test the hash are included 7 * if SELF_TEST is defined. You can use this free for any purpose. It's in 8 * the public domain. It has no warranty. 9 * 10 * You probably want to use hashlittle(). hashlittle() and hashbig() 11 * hash byte arrays. hashlittle() is is faster than hashbig() on 12 * little-endian machines. Intel and AMD are little-endian machines. 13 * On second thought, you probably want hashlittle2(), which is identical to 14 * hashlittle() except it returns two 32-bit hashes for the price of one. 15 * You could implement hashbig2() if you wanted but I haven't bothered here. 16 * 17 * If you want to find a hash of, say, exactly 7 integers, do 18 * a = i1; b = i2; c = i3; 19 * mixin (mix!("a", "b", "c")); 20 * a += i4; b += i5; c += i6; 21 * mixin (mix!("a", "b", "c")); 22 * a += i7; 23 * final_(a, b, c); 24 * then use c as the hash value. If you have a variable length array of 25 * 4-byte integers to hash, use hashword(). If you have a byte array (like 26 * a character string), use hashlittle(). If you have several byte arrays, or 27 * a mix of things, see the comments above hashlittle(). 28 * 29 * Why is this so big? I read 12 bytes at a time into 3 4-byte integers, 30 * then mix those integers. This is fast (you can do a lot more thorough 31 * mixing with (12 * 3) instructions on 3 integers than you can with 3 instructions 32 * on 1 byte), but shoehorning those bytes into integers efficiently is messy. 33 */ 34 /** 35 * License: MIT 36 */ 37 module jansson_d.lookup3; 38 39 40 package: 41 42 /* Detect Valgrind or AddressSanitizer */ 43 version (VALGRIND) { 44 version = NO_MASKING_TRICK; 45 } else { 46 //#if defined(__has_feature) /* Clang */ 47 //#if __has_feature(address_sanitizer) /* is ASAN enabled? */ 48 //version = NO_MASKING_TRICK; 49 //#endif 50 //#else 51 //#if defined(__SANITIZE_ADDRESS__) /* GCC 4.8.x, is ASAN enabled? */ 52 //version = NO_MASKING_TRICK; 53 //#endif 54 //#endif 55 } 56 57 /* 58 * My best guess at if you are big-endian or little-endian. This may 59 * need adjustment. 60 */ 61 version (LittleEndian) { 62 enum HASH_LITTLE_ENDIAN = 1; 63 enum HASH_BIG_ENDIAN = 0; 64 } else version (BigEndian) { 65 enum HASH_LITTLE_ENDIAN = 0; 66 enum HASH_BIG_ENDIAN = 1; 67 } else { 68 enum HASH_LITTLE_ENDIAN = 0; 69 enum HASH_BIG_ENDIAN = 0; 70 } 71 72 template hashsize(string n) 73 { 74 enum hashsize = "(cast(size_t)(1) << (" ~ n ~ "))"; 75 } 76 77 template hashmask(string n) 78 { 79 enum hashmask = "(mixin (jansson_d.lookup3.hashsize!(\"" ~ n ~ "\")) - 1)"; 80 } 81 82 template rot(string x, string k) 83 { 84 enum rot = "(((" ~ x ~ ") << (" ~ k ~ ")) | ((" ~ x ~ ") >> (32 - (" ~ k ~ "))))"; 85 } 86 87 /* 88 * mix -- mix 3 32-bit values reversibly. 89 * 90 * This is reversible, so any information in (a, b, c) before mix() is 91 * still in (a, b, c) after mix(). 92 * 93 * If four pairs of (a, b, c) inputs are run through mix(), or through 94 * mix() in reverse, there are at least 32 bits of the output that 95 * are sometimes the same for one pair and different for another pair. 96 * This was tested for: 97 * * pairs that differed by one bit, by two bits, in any combination 98 * of top bits of (a, b, c), or in any combination of bottom bits of 99 * (a, b, c). 100 * * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed 101 * the output delta to a Gray code (a^(a>>1)) so a string of 1's (as 102 * is commonly produced by subtraction) look like a single 1-bit 103 * difference. 104 * * the base values were pseudorandom, all zero but one bit set, or 105 * all zero plus a counter that starts at zero. 106 * 107 * Some k values for my "a-=c; a^=rot(c, k); c+=b;" arrangement that 108 * satisfy this are 109 * 4 6 8 16 19 4 110 * 9 15 3 18 27 15 111 * 14 9 3 7 17 3 112 * Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing 113 * for "differ" defined as + with a one-bit base and a two-bit delta. I 114 * used http://burtleburtle.net/bob/hash/avalanche.html to choose 115 * the operations, constants, and arrangements of the variables. 116 * 117 * This does not achieve avalanche. There are input bits of (a, b, c) 118 * that fail to affect some output bits of (a, b, c), especially of a. The 119 * most thoroughly mixed value is c, but it doesn't really even achieve 120 * avalanche in c. 121 * 122 * This allows some parallelism. Read-after-writes are good at doubling 123 * the number of bits affected, so the goal of mixing pulls in the opposite 124 * direction as the goal of parallelism. I did what I could. Rotates 125 * seem to cost as much as shifts on every machine I could lay my hands 126 * on, and rotates are much kinder to the top and bottom bits, so I used 127 * rotates. 128 */ 129 template mix(string a, string b, string c) 130 { 131 enum mix = "{ a -= c; a ^= mixin (jansson_d.lookup3.rot!(\"" ~ c ~ "\", \"4\")); c += b; b -= a; b ^= mixin (jansson_d.lookup3.rot!(\"" ~ a ~ "\", \"6\")); a += c; c -= b; c ^= mixin (jansson_d.lookup3.rot!(\"" ~ b ~ "\", \"8\")); b += a; a -= c; a ^= mixin (jansson_d.lookup3.rot!(\"" ~ c ~ "\", \"16\")); c += b; b -= a; b ^= mixin (jansson_d.lookup3.rot!(\"" ~ a ~ "\", \"19\")); a += c; c -= b; c ^= mixin (jansson_d.lookup3.rot!(\"" ~ b ~ "\", \"4\")); b += a; }"; 132 } 133 134 /* 135 * final_ -- final mixing of 3 32-bit values (a, b, c) into c 136 * 137 * Pairs of (a, b, c) values differing in only a few bits will usually 138 * produce values of c that look totally different. This was tested for 139 * * pairs that differed by one bit, by two bits, in any combination 140 * of top bits of (a, b, c), or in any combination of bottom bits of 141 * (a, b, c). 142 * * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed 143 * the output delta to a Gray code (a^(a>>1)) so a string of 1's (as 144 * is commonly produced by subtraction) look like a single 1-bit 145 * difference. 146 * * the base values were pseudorandom, all zero but one bit set, or 147 * all zero plus a counter that starts at zero. 148 * 149 * These constants passed: 150 * 14 11 25 16 4 14 24 151 * 12 14 25 16 4 14 24 152 * and these came close: 153 * 4 8 15 26 3 22 24 154 * 10 8 15 26 3 22 24 155 * 11 8 15 26 3 22 24 156 */ 157 template final_(string a, string b, string c) 158 { 159 enum final_ = "{ c ^= b; c -= mixin (jansson_d.lookup3.rot!(\"" ~ b ~ "\", \"14\")); a ^= c; a -= mixin (jansson_d.lookup3.rot!(\"" ~ c ~ "\", \"11\")); b ^= a; b -= mixin (jansson_d.lookup3.rot!(\"" ~ a ~ "\", \"25\")); c ^= b; c -= mixin (jansson_d.lookup3.rot!(\"" ~ b ~ "\", \"16\")); a ^= c; a -= mixin (jansson_d.lookup3.rot!(\"" ~ c ~ "\", \"4\")); b ^= a; b -= mixin (jansson_d.lookup3.rot!(\"" ~ a ~ "\", \"14\")); c ^= b; c -= mixin (jansson_d.lookup3.rot!(\"" ~ b ~ "\", \"24\")); }"; 160 } 161 162 //#define final_(a, b, c) { c ^= b; c -= rot(b, 14); a ^= c; a -= rot(c, 11); b ^= a; b -= rot(a, 25); c ^= b; c -= rot(b, 16); a ^= c; a -= rot(c, 4); b ^= a; b -= rot(a, 14); c ^= b; c -= rot(b, 24); } 163 164 /* 165 * hashlittle() -- hash a variable-length key into a 32-bit value 166 * 167 * The best hash table sizes are powers of 2. There is no need to do 168 * mod a prime (mod is sooo slow!). If you need less than 32 bits, 169 * use a bitmask. For example, if you need only 10 bits, do 170 * h = (h & hashmask(10)); 171 * In which case, the hash table should have hashsize(10) elements. 172 * 173 * If you are hashing n strings cast(ubyte**)(k), do it like this: 174 * for (i = 0, h = 0; i < n; ++i) { h = hashlittle(k[i], len[i], h); } 175 * 176 * By Bob Jenkins, 2006. bob_jenkins@burtleburtle.net. You may use this 177 * code any way you wish, private, educational, or commercial. It's free. 178 * 179 * Use for hash table lookup, or anything where one collision in 2^^32 is 180 * acceptable. Do NOT use for cryptographic purposes. 181 */ 182 /** 183 * hash a variable-length key into a 32-bit value 184 * 185 * Params: 186 * key = the key (the unaligned variable-length array of bytes) 187 * length_ = the length of the key, counting by bytes 188 * initval = can be any 4-byte value 189 * 190 * Returns: a 32-bit value. Every bit of the key affects every bit of the return value. Two keys differing by one or two bits will have totally different hash values. 191 */ 192 pure nothrow @nogc @live 193 package uint hashlittle(scope const void* key, size_t length_, uint initval) 194 195 in 196 { 197 assert(key != null); 198 } 199 200 do 201 { 202 /* needed for Mac Powerbook G4 */ 203 union u_ 204 { 205 const (void)* ptr_; 206 size_t i; 207 } 208 209 /* Set up the internal state */ 210 uint c = 0xDEADBEEF + (cast(uint)(length_)) + initval; 211 uint b = c; 212 uint a = c; 213 214 u_ u = void; 215 u.ptr_ = key; 216 217 if ((.HASH_LITTLE_ENDIAN) && ((u.i & 0x03) == 0)) { 218 /* read 32-bit chunks */ 219 const (uint)* k = cast(const (uint)*)(key); 220 221 /*------ all but last block: aligned reads and affect 32 bits of (a, b, c) */ 222 while (length_ > 12) { 223 a += k[0]; 224 b += k[1]; 225 c += k[2]; 226 mixin (.mix!("a", "b", "c")); 227 length_ -= 12; 228 k += 3; 229 } 230 231 /*----------------------------- handle the last (probably partial) block */ 232 /* 233 * "k[2]&0xFFFFFF" actually reads beyond the end of the string, but 234 * then masks off the part it's not allowed to read. Because the 235 * string is aligned, the masked-off tail is in the same word as the 236 * rest of the string. Every machine with memory protection I've seen 237 * does it on word boundaries, so is OK with this. But VALGRIND will 238 * still catch it and complain. The masking trick does make the hash 239 * noticeably faster for short strings (like English words). 240 */ 241 version (NO_MASKING_TRICK) { 242 const ubyte* k8 = cast(const (ubyte)*)(k); 243 244 switch (length_) { 245 case 12: 246 c += k[2]; 247 b += k[1]; 248 a += k[0]; 249 250 break; 251 252 case 11: 253 c += cast(uint)(k8[10]) << 16; 254 255 /* fall through */ 256 goto case; 257 258 case 10: 259 c += cast(uint)(k8[9]) << 8; 260 261 /* fall through */ 262 goto case; 263 264 case 9 : 265 c += k8[8]; 266 267 /* fall through */ 268 goto case; 269 270 case 8 : 271 b += k[1]; 272 a += k[0]; 273 274 break; 275 276 case 7 : 277 b += cast(uint)(k8[6]) << 16; 278 279 /* fall through */ 280 goto case; 281 282 case 6 : 283 b += cast(uint)(k8[5]) << 8; 284 285 /* fall through */ 286 goto case; 287 288 case 5 : 289 b += k8[4]; 290 291 /* fall through */ 292 goto case; 293 294 case 4 : 295 a += k[0]; 296 297 break; 298 299 case 3 : 300 a += cast(uint)(k8[2]) << 16; 301 302 /* fall through */ 303 goto case; 304 305 case 2 : 306 a += cast(uint)(k8[1]) << 8; 307 308 /* fall through */ 309 goto case; 310 311 case 1 : 312 a += k8[0]; 313 314 break; 315 316 case 0 : 317 return c; 318 } 319 } else { 320 switch (length_) { 321 case 12: 322 c += k[2]; 323 b += k[1]; 324 a += k[0]; 325 326 break; 327 328 case 11: 329 c += k[2] & 0xFFFFFF; 330 b += k[1]; 331 a += k[0]; 332 333 break; 334 335 case 10: 336 c += k[2] & 0xFFFF; 337 b += k[1]; 338 a += k[0]; 339 340 break; 341 342 case 9 : 343 c += k[2] & 0xFF; 344 b += k[1]; 345 a += k[0]; 346 347 break; 348 349 case 8 : 350 b += k[1]; 351 a += k[0]; 352 353 break; 354 355 case 7 : 356 b += k[1] & 0xFFFFFF; 357 a += k[0]; 358 359 break; 360 361 case 6 : 362 b += k[1] & 0xFFFF; 363 a += k[0]; 364 365 break; 366 367 case 5 : 368 b += k[1] & 0xFF; 369 a += k[0]; 370 371 break; 372 373 case 4 : 374 a += k[0]; 375 376 break; 377 378 case 3 : 379 a += k[0] & 0xFFFFFF; 380 381 break; 382 383 case 2 : 384 a += k[0] & 0xFFFF; 385 386 break; 387 388 case 1 : 389 a += k[0] & 0xFF; 390 391 break; 392 393 case 0 : 394 /* zero length strings require no mixing */ 395 return c; 396 397 default: 398 break; 399 } 400 } 401 } else if ((.HASH_LITTLE_ENDIAN) && ((u.i & 0x01) == 0)) { 402 /* read 16-bit chunks */ 403 const (ushort)* k = cast(const (ushort)*)(key); 404 405 /*--------------- all but last block: aligned reads and different mixing */ 406 while (length_ > 12) { 407 a += k[0] + (cast(uint)(k[1]) << 16); 408 b += k[2] + (cast(uint)(k[3]) << 16); 409 c += k[4] + (cast(uint)(k[5]) << 16); 410 mixin (.mix!("a", "b", "c")); 411 length_ -= 12; 412 k += 6; 413 } 414 415 /*----------------------------- handle the last (probably partial) block */ 416 const ubyte* k8 = cast(const ubyte*)(k); 417 418 switch (length_) { 419 case 12: 420 c += k[4] + (cast(uint)(k[5]) << 16); 421 b += k[2] + (cast(uint)(k[3]) << 16); 422 a += k[0] + (cast(uint)(k[1]) << 16); 423 424 break; 425 426 case 11: 427 c += cast(uint)(k8[10]) << 16; 428 429 /* fall through */ 430 goto case; 431 432 case 10: 433 c += k[4]; 434 b += k[2] + (cast(uint)(k[3]) << 16); 435 a += k[0] + (cast(uint)(k[1]) << 16); 436 437 break; 438 439 case 9 : 440 c += k8[8]; 441 442 /* fall through */ 443 goto case; 444 445 case 8 : 446 b += k[2] + (cast(uint)(k[3]) << 16); 447 a += k[0] + (cast(uint)(k[1]) << 16); 448 449 break; 450 451 case 7 : 452 b += cast(uint)(k8[6]) << 16; 453 454 /* fall through */ 455 goto case; 456 457 case 6 : 458 b += k[2]; 459 a += k[0] + (cast(uint)(k[1]) << 16); 460 461 break; 462 463 case 5 : 464 b += k8[4]; 465 466 /* fall through */ 467 goto case; 468 469 case 4 : 470 a += k[0] + (cast(uint)(k[1]) << 16); 471 472 break; 473 474 case 3 : 475 a += cast(uint)(k8[2]) << 16; 476 477 /* fall through */ 478 goto case; 479 480 case 2 : 481 a += k[0]; 482 483 break; 484 485 case 1 : 486 a += k8[0]; 487 488 break; 489 490 case 0 : 491 /* zero length requires no mixing */ 492 return c; 493 494 default: 495 break; 496 } 497 } else { /* need to read the key one byte at a time */ 498 const (ubyte)* k = cast(const ubyte*)(key); 499 500 /*--------------- all but the last block: affect some 32 bits of (a, b, c) */ 501 while (length_ > 12) { 502 a += k[0]; 503 a += cast(uint)(k[1]) << 8; 504 a += cast(uint)(k[2]) << 16; 505 a += cast(uint)(k[3]) << 24; 506 b += k[4]; 507 b += cast(uint)(k[5]) << 8; 508 b += cast(uint)(k[6]) << 16; 509 b += cast(uint)(k[7]) << 24; 510 c += k[8]; 511 c += cast(uint)(k[9]) << 8; 512 c += cast(uint)(k[10]) << 16; 513 c += cast(uint)(k[11]) << 24; 514 mixin (.mix!("a", "b", "c")); 515 length_ -= 12; 516 k += 12; 517 } 518 519 /*-------------------------------- last block: affect all 32 bits of (c) */ 520 switch (length_) { /* all the case statements fall through */ 521 case 12: 522 c += cast(uint)(k[11]) << 24; 523 524 /* fall through */ 525 goto case; 526 527 case 11: 528 c += cast(uint)(k[10]) << 16; 529 530 /* fall through */ 531 goto case; 532 533 case 10: 534 c += cast(uint)(k[9]) << 8; 535 536 /* fall through */ 537 goto case; 538 539 case 9 : 540 c += k[8]; 541 542 /* fall through */ 543 goto case; 544 545 case 8 : 546 b += cast(uint)(k[7]) << 24; 547 548 /* fall through */ 549 goto case; 550 551 case 7 : 552 b += cast(uint)(k[6]) << 16; 553 554 /* fall through */ 555 goto case; 556 557 case 6 : 558 b += cast(uint)(k[5]) << 8; 559 560 /* fall through */ 561 goto case; 562 563 case 5 : 564 b += k[4]; 565 566 /* fall through */ 567 goto case; 568 569 case 4 : 570 a += cast(uint)(k[3]) << 24; 571 572 /* fall through */ 573 goto case; 574 575 case 3 : 576 a += cast(uint)(k[2]) << 16; 577 578 /* fall through */ 579 goto case; 580 581 case 2 : 582 a += cast(uint)(k[1]) << 8; 583 584 /* fall through */ 585 goto case; 586 587 case 1 : 588 a += k[0]; 589 590 break; 591 592 case 0 : 593 return c; 594 595 default: 596 break; 597 } 598 } 599 600 mixin (.final_!("a", "b", "c")); 601 602 return c; 603 }