001/* 002 * Copyright (C) 2011 The Guava Authors 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 005 * in compliance with the License. You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software distributed under the License 010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 011 * or implied. See the License for the specific language governing permissions and limitations under 012 * the License. 013 */ 014 015package com.google.common.hash; 016 017import static com.google.common.base.Preconditions.checkArgument; 018import static com.google.common.base.Preconditions.checkNotNull; 019 020import com.google.common.annotations.Beta; 021import com.google.common.annotations.VisibleForTesting; 022import com.google.common.base.Objects; 023import com.google.common.base.Predicate; 024import com.google.common.hash.BloomFilterStrategies.LockFreeBitArray; 025import com.google.common.math.DoubleMath; 026import com.google.common.math.LongMath; 027import com.google.common.primitives.SignedBytes; 028import com.google.common.primitives.UnsignedBytes; 029import com.google.errorprone.annotations.CanIgnoreReturnValue; 030import java.io.DataInputStream; 031import java.io.DataOutputStream; 032import java.io.IOException; 033import java.io.InputStream; 034import java.io.InvalidObjectException; 035import java.io.ObjectInputStream; 036import java.io.OutputStream; 037import java.io.Serializable; 038import java.math.RoundingMode; 039import java.util.stream.Collector; 040import javax.annotation.CheckForNull; 041import org.checkerframework.checker.nullness.qual.Nullable; 042 043/** 044 * A Bloom filter for instances of {@code T}. A Bloom filter offers an approximate containment test 045 * with one-sided error: if it claims that an element is contained in it, this might be in error, 046 * but if it claims that an element is <i>not</i> contained in it, then this is definitely true. 047 * 048 * <p>If you are unfamiliar with Bloom filters, this nice <a 049 * href="http://llimllib.github.io/bloomfilter-tutorial/">tutorial</a> may help you understand how 050 * they work. 051 * 052 * <p>The false positive probability ({@code FPP}) of a Bloom filter is defined as the probability 053 * that {@linkplain #mightContain(Object)} will erroneously return {@code true} for an object that 054 * has not actually been put in the {@code BloomFilter}. 055 * 056 * <p>Bloom filters are serializable. They also support a more compact serial representation via the 057 * {@link #writeTo} and {@link #readFrom} methods. Both serialized forms will continue to be 058 * supported by future versions of this library. However, serial forms generated by newer versions 059 * of the code may not be readable by older versions of the code (e.g., a serialized Bloom filter 060 * generated today may <i>not</i> be readable by a binary that was compiled 6 months ago). 061 * 062 * <p>As of Guava 23.0, this class is thread-safe and lock-free. It internally uses atomics and 063 * compare-and-swap to ensure correctness when multiple threads are used to access it. 064 * 065 * @param <T> the type of instances that the {@code BloomFilter} accepts 066 * @author Dimitris Andreou 067 * @author Kevin Bourrillion 068 * @since 11.0 (thread-safe since 23.0) 069 */ 070@Beta 071@ElementTypesAreNonnullByDefault 072public final class BloomFilter<T extends @Nullable Object> implements Predicate<T>, Serializable { 073 /** 074 * A strategy to translate T instances, to {@code numHashFunctions} bit indexes. 075 * 076 * <p>Implementations should be collections of pure functions (i.e. stateless). 077 */ 078 interface Strategy extends java.io.Serializable { 079 080 /** 081 * Sets {@code numHashFunctions} bits of the given bit array, by hashing a user element. 082 * 083 * <p>Returns whether any bits changed as a result of this operation. 084 */ 085 <T extends @Nullable Object> boolean put( 086 @ParametricNullness T object, 087 Funnel<? super T> funnel, 088 int numHashFunctions, 089 LockFreeBitArray bits); 090 091 /** 092 * Queries {@code numHashFunctions} bits of the given bit array, by hashing a user element; 093 * returns {@code true} if and only if all selected bits are set. 094 */ 095 <T extends @Nullable Object> boolean mightContain( 096 @ParametricNullness T object, 097 Funnel<? super T> funnel, 098 int numHashFunctions, 099 LockFreeBitArray bits); 100 101 /** 102 * Identifier used to encode this strategy, when marshalled as part of a BloomFilter. Only 103 * values in the [-128, 127] range are valid for the compact serial form. Non-negative values 104 * are reserved for enums defined in BloomFilterStrategies; negative values are reserved for any 105 * custom, stateful strategy we may define (e.g. any kind of strategy that would depend on user 106 * input). 107 */ 108 int ordinal(); 109 } 110 111 /** The bit set of the BloomFilter (not necessarily power of 2!) */ 112 private final LockFreeBitArray bits; 113 114 /** Number of hashes per element */ 115 private final int numHashFunctions; 116 117 /** The funnel to translate Ts to bytes */ 118 private final Funnel<? super T> funnel; 119 120 /** The strategy we employ to map an element T to {@code numHashFunctions} bit indexes. */ 121 private final Strategy strategy; 122 123 /** Creates a BloomFilter. */ 124 private BloomFilter( 125 LockFreeBitArray bits, int numHashFunctions, Funnel<? super T> funnel, Strategy strategy) { 126 checkArgument(numHashFunctions > 0, "numHashFunctions (%s) must be > 0", numHashFunctions); 127 checkArgument( 128 numHashFunctions <= 255, "numHashFunctions (%s) must be <= 255", numHashFunctions); 129 this.bits = checkNotNull(bits); 130 this.numHashFunctions = numHashFunctions; 131 this.funnel = checkNotNull(funnel); 132 this.strategy = checkNotNull(strategy); 133 } 134 135 /** 136 * Creates a new {@code BloomFilter} that's a copy of this instance. The new instance is equal to 137 * this instance but shares no mutable state. 138 * 139 * @since 12.0 140 */ 141 public BloomFilter<T> copy() { 142 return new BloomFilter<T>(bits.copy(), numHashFunctions, funnel, strategy); 143 } 144 145 /** 146 * Returns {@code true} if the element <i>might</i> have been put in this Bloom filter, {@code 147 * false} if this is <i>definitely</i> not the case. 148 */ 149 public boolean mightContain(@ParametricNullness T object) { 150 return strategy.mightContain(object, funnel, numHashFunctions, bits); 151 } 152 153 /** 154 * @deprecated Provided only to satisfy the {@link Predicate} interface; use {@link #mightContain} 155 * instead. 156 */ 157 @Deprecated 158 @Override 159 public boolean apply(@ParametricNullness T input) { 160 return mightContain(input); 161 } 162 163 /** 164 * Puts an element into this {@code BloomFilter}. Ensures that subsequent invocations of {@link 165 * #mightContain(Object)} with the same element will always return {@code true}. 166 * 167 * @return true if the Bloom filter's bits changed as a result of this operation. If the bits 168 * changed, this is <i>definitely</i> the first time {@code object} has been added to the 169 * filter. If the bits haven't changed, this <i>might</i> be the first time {@code object} has 170 * been added to the filter. Note that {@code put(t)} always returns the <i>opposite</i> 171 * result to what {@code mightContain(t)} would have returned at the time it is called. 172 * @since 12.0 (present in 11.0 with {@code void} return type}) 173 */ 174 @CanIgnoreReturnValue 175 public boolean put(@ParametricNullness T object) { 176 return strategy.put(object, funnel, numHashFunctions, bits); 177 } 178 179 /** 180 * Returns the probability that {@linkplain #mightContain(Object)} will erroneously return {@code 181 * true} for an object that has not actually been put in the {@code BloomFilter}. 182 * 183 * <p>Ideally, this number should be close to the {@code fpp} parameter passed in {@linkplain 184 * #create(Funnel, int, double)}, or smaller. If it is significantly higher, it is usually the 185 * case that too many elements (more than expected) have been put in the {@code BloomFilter}, 186 * degenerating it. 187 * 188 * @since 14.0 (since 11.0 as expectedFalsePositiveProbability()) 189 */ 190 public double expectedFpp() { 191 return Math.pow((double) bits.bitCount() / bitSize(), numHashFunctions); 192 } 193 194 /** 195 * Returns an estimate for the total number of distinct elements that have been added to this 196 * Bloom filter. This approximation is reasonably accurate if it does not exceed the value of 197 * {@code expectedInsertions} that was used when constructing the filter. 198 * 199 * @since 22.0 200 */ 201 public long approximateElementCount() { 202 long bitSize = bits.bitSize(); 203 long bitCount = bits.bitCount(); 204 205 /** 206 * Each insertion is expected to reduce the # of clear bits by a factor of 207 * `numHashFunctions/bitSize`. So, after n insertions, expected bitCount is `bitSize * (1 - (1 - 208 * numHashFunctions/bitSize)^n)`. Solving that for n, and approximating `ln x` as `x - 1` when x 209 * is close to 1 (why?), gives the following formula. 210 */ 211 double fractionOfBitsSet = (double) bitCount / bitSize; 212 return DoubleMath.roundToLong( 213 -Math.log1p(-fractionOfBitsSet) * bitSize / numHashFunctions, RoundingMode.HALF_UP); 214 } 215 216 /** Returns the number of bits in the underlying bit array. */ 217 @VisibleForTesting 218 long bitSize() { 219 return bits.bitSize(); 220 } 221 222 /** 223 * Determines whether a given Bloom filter is compatible with this Bloom filter. For two Bloom 224 * filters to be compatible, they must: 225 * 226 * <ul> 227 * <li>not be the same instance 228 * <li>have the same number of hash functions 229 * <li>have the same bit size 230 * <li>have the same strategy 231 * <li>have equal funnels 232 * </ul> 233 * 234 * @param that The Bloom filter to check for compatibility. 235 * @since 15.0 236 */ 237 public boolean isCompatible(BloomFilter<T> that) { 238 checkNotNull(that); 239 return this != that 240 && this.numHashFunctions == that.numHashFunctions 241 && this.bitSize() == that.bitSize() 242 && this.strategy.equals(that.strategy) 243 && this.funnel.equals(that.funnel); 244 } 245 246 /** 247 * Combines this Bloom filter with another Bloom filter by performing a bitwise OR of the 248 * underlying data. The mutations happen to <b>this</b> instance. Callers must ensure the Bloom 249 * filters are appropriately sized to avoid saturating them. 250 * 251 * @param that The Bloom filter to combine this Bloom filter with. It is not mutated. 252 * @throws IllegalArgumentException if {@code isCompatible(that) == false} 253 * @since 15.0 254 */ 255 public void putAll(BloomFilter<T> that) { 256 checkNotNull(that); 257 checkArgument(this != that, "Cannot combine a BloomFilter with itself."); 258 checkArgument( 259 this.numHashFunctions == that.numHashFunctions, 260 "BloomFilters must have the same number of hash functions (%s != %s)", 261 this.numHashFunctions, 262 that.numHashFunctions); 263 checkArgument( 264 this.bitSize() == that.bitSize(), 265 "BloomFilters must have the same size underlying bit arrays (%s != %s)", 266 this.bitSize(), 267 that.bitSize()); 268 checkArgument( 269 this.strategy.equals(that.strategy), 270 "BloomFilters must have equal strategies (%s != %s)", 271 this.strategy, 272 that.strategy); 273 checkArgument( 274 this.funnel.equals(that.funnel), 275 "BloomFilters must have equal funnels (%s != %s)", 276 this.funnel, 277 that.funnel); 278 this.bits.putAll(that.bits); 279 } 280 281 @Override 282 public boolean equals(@CheckForNull Object object) { 283 if (object == this) { 284 return true; 285 } 286 if (object instanceof BloomFilter) { 287 BloomFilter<?> that = (BloomFilter<?>) object; 288 return this.numHashFunctions == that.numHashFunctions 289 && this.funnel.equals(that.funnel) 290 && this.bits.equals(that.bits) 291 && this.strategy.equals(that.strategy); 292 } 293 return false; 294 } 295 296 @Override 297 public int hashCode() { 298 return Objects.hashCode(numHashFunctions, funnel, strategy, bits); 299 } 300 301 /** 302 * Returns a {@code Collector} expecting the specified number of insertions, and yielding a {@link 303 * BloomFilter} with false positive probability 3%. 304 * 305 * <p>Note that if the {@code Collector} receives significantly more elements than specified, the 306 * resulting {@code BloomFilter} will suffer a sharp deterioration of its false positive 307 * probability. 308 * 309 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 310 * is. 311 * 312 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 313 * ensuring proper serialization and deserialization, which is important since {@link #equals} 314 * also relies on object identity of funnels. 315 * 316 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 317 * @param expectedInsertions the number of expected insertions to the constructed {@code 318 * BloomFilter}; must be positive 319 * @return a {@code Collector} generating a {@code BloomFilter} of the received elements 320 * @since 23.0 321 */ 322 public static <T extends @Nullable Object> Collector<T, ?, BloomFilter<T>> toBloomFilter( 323 Funnel<? super T> funnel, long expectedInsertions) { 324 return toBloomFilter(funnel, expectedInsertions, 0.03); 325 } 326 327 /** 328 * Returns a {@code Collector} expecting the specified number of insertions, and yielding a {@link 329 * BloomFilter} with the specified expected false positive probability. 330 * 331 * <p>Note that if the {@code Collector} receives significantly more elements than specified, the 332 * resulting {@code BloomFilter} will suffer a sharp deterioration of its false positive 333 * probability. 334 * 335 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 336 * is. 337 * 338 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 339 * ensuring proper serialization and deserialization, which is important since {@link #equals} 340 * also relies on object identity of funnels. 341 * 342 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 343 * @param expectedInsertions the number of expected insertions to the constructed {@code 344 * BloomFilter}; must be positive 345 * @param fpp the desired false positive probability (must be positive and less than 1.0) 346 * @return a {@code Collector} generating a {@code BloomFilter} of the received elements 347 * @since 23.0 348 */ 349 public static <T extends @Nullable Object> Collector<T, ?, BloomFilter<T>> toBloomFilter( 350 Funnel<? super T> funnel, long expectedInsertions, double fpp) { 351 checkNotNull(funnel); 352 checkArgument( 353 expectedInsertions >= 0, "Expected insertions (%s) must be >= 0", expectedInsertions); 354 checkArgument(fpp > 0.0, "False positive probability (%s) must be > 0.0", fpp); 355 checkArgument(fpp < 1.0, "False positive probability (%s) must be < 1.0", fpp); 356 return Collector.of( 357 () -> BloomFilter.create(funnel, expectedInsertions, fpp), 358 BloomFilter::put, 359 (bf1, bf2) -> { 360 bf1.putAll(bf2); 361 return bf1; 362 }, 363 Collector.Characteristics.UNORDERED, 364 Collector.Characteristics.CONCURRENT); 365 } 366 367 /** 368 * Creates a {@link BloomFilter} with the expected number of insertions and expected false 369 * positive probability. 370 * 371 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 372 * will result in its saturation, and a sharp deterioration of its false positive probability. 373 * 374 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 375 * is. 376 * 377 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 378 * ensuring proper serialization and deserialization, which is important since {@link #equals} 379 * also relies on object identity of funnels. 380 * 381 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 382 * @param expectedInsertions the number of expected insertions to the constructed {@code 383 * BloomFilter}; must be positive 384 * @param fpp the desired false positive probability (must be positive and less than 1.0) 385 * @return a {@code BloomFilter} 386 */ 387 public static <T extends @Nullable Object> BloomFilter<T> create( 388 Funnel<? super T> funnel, int expectedInsertions, double fpp) { 389 return create(funnel, (long) expectedInsertions, fpp); 390 } 391 392 /** 393 * Creates a {@link BloomFilter} with the expected number of insertions and expected false 394 * positive probability. 395 * 396 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 397 * will result in its saturation, and a sharp deterioration of its false positive probability. 398 * 399 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 400 * is. 401 * 402 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 403 * ensuring proper serialization and deserialization, which is important since {@link #equals} 404 * also relies on object identity of funnels. 405 * 406 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 407 * @param expectedInsertions the number of expected insertions to the constructed {@code 408 * BloomFilter}; must be positive 409 * @param fpp the desired false positive probability (must be positive and less than 1.0) 410 * @return a {@code BloomFilter} 411 * @since 19.0 412 */ 413 public static <T extends @Nullable Object> BloomFilter<T> create( 414 Funnel<? super T> funnel, long expectedInsertions, double fpp) { 415 return create(funnel, expectedInsertions, fpp, BloomFilterStrategies.MURMUR128_MITZ_64); 416 } 417 418 @VisibleForTesting 419 static <T extends @Nullable Object> BloomFilter<T> create( 420 Funnel<? super T> funnel, long expectedInsertions, double fpp, Strategy strategy) { 421 checkNotNull(funnel); 422 checkArgument( 423 expectedInsertions >= 0, "Expected insertions (%s) must be >= 0", expectedInsertions); 424 checkArgument(fpp > 0.0, "False positive probability (%s) must be > 0.0", fpp); 425 checkArgument(fpp < 1.0, "False positive probability (%s) must be < 1.0", fpp); 426 checkNotNull(strategy); 427 428 if (expectedInsertions == 0) { 429 expectedInsertions = 1; 430 } 431 /* 432 * TODO(user): Put a warning in the javadoc about tiny fpp values, since the resulting size 433 * is proportional to -log(p), but there is not much of a point after all, e.g. 434 * optimalM(1000, 0.0000000000000001) = 76680 which is less than 10kb. Who cares! 435 */ 436 long numBits = optimalNumOfBits(expectedInsertions, fpp); 437 int numHashFunctions = optimalNumOfHashFunctions(expectedInsertions, numBits); 438 try { 439 return new BloomFilter<T>(new LockFreeBitArray(numBits), numHashFunctions, funnel, strategy); 440 } catch (IllegalArgumentException e) { 441 throw new IllegalArgumentException("Could not create BloomFilter of " + numBits + " bits", e); 442 } 443 } 444 445 /** 446 * Creates a {@link BloomFilter} with the expected number of insertions and a default expected 447 * false positive probability of 3%. 448 * 449 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 450 * will result in its saturation, and a sharp deterioration of its false positive probability. 451 * 452 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 453 * is. 454 * 455 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 456 * ensuring proper serialization and deserialization, which is important since {@link #equals} 457 * also relies on object identity of funnels. 458 * 459 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 460 * @param expectedInsertions the number of expected insertions to the constructed {@code 461 * BloomFilter}; must be positive 462 * @return a {@code BloomFilter} 463 */ 464 public static <T extends @Nullable Object> BloomFilter<T> create( 465 Funnel<? super T> funnel, int expectedInsertions) { 466 return create(funnel, (long) expectedInsertions); 467 } 468 469 /** 470 * Creates a {@link BloomFilter} with the expected number of insertions and a default expected 471 * false positive probability of 3%. 472 * 473 * <p>Note that overflowing a {@code BloomFilter} with significantly more elements than specified, 474 * will result in its saturation, and a sharp deterioration of its false positive probability. 475 * 476 * <p>The constructed {@code BloomFilter} will be serializable if the provided {@code Funnel<T>} 477 * is. 478 * 479 * <p>It is recommended that the funnel be implemented as a Java enum. This has the benefit of 480 * ensuring proper serialization and deserialization, which is important since {@link #equals} 481 * also relies on object identity of funnels. 482 * 483 * @param funnel the funnel of T's that the constructed {@code BloomFilter} will use 484 * @param expectedInsertions the number of expected insertions to the constructed {@code 485 * BloomFilter}; must be positive 486 * @return a {@code BloomFilter} 487 * @since 19.0 488 */ 489 public static <T extends @Nullable Object> BloomFilter<T> create( 490 Funnel<? super T> funnel, long expectedInsertions) { 491 return create(funnel, expectedInsertions, 0.03); // FYI, for 3%, we always get 5 hash functions 492 } 493 494 // Cheat sheet: 495 // 496 // m: total bits 497 // n: expected insertions 498 // b: m/n, bits per insertion 499 // p: expected false positive probability 500 // 501 // 1) Optimal k = b * ln2 502 // 2) p = (1 - e ^ (-kn/m))^k 503 // 3) For optimal k: p = 2 ^ (-k) ~= 0.6185^b 504 // 4) For optimal k: m = -nlnp / ((ln2) ^ 2) 505 506 /** 507 * Computes the optimal k (number of hashes per element inserted in Bloom filter), given the 508 * expected insertions and total number of bits in the Bloom filter. 509 * 510 * <p>See http://en.wikipedia.org/wiki/File:Bloom_filter_fp_probability.svg for the formula. 511 * 512 * @param n expected insertions (must be positive) 513 * @param m total number of bits in Bloom filter (must be positive) 514 */ 515 @VisibleForTesting 516 static int optimalNumOfHashFunctions(long n, long m) { 517 // (m / n) * log(2), but avoid truncation due to division! 518 return Math.max(1, (int) Math.round((double) m / n * Math.log(2))); 519 } 520 521 /** 522 * Computes m (total bits of Bloom filter) which is expected to achieve, for the specified 523 * expected insertions, the required false positive probability. 524 * 525 * <p>See http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives for the 526 * formula. 527 * 528 * @param n expected insertions (must be positive) 529 * @param p false positive rate (must be 0 < p < 1) 530 */ 531 @VisibleForTesting 532 static long optimalNumOfBits(long n, double p) { 533 if (p == 0) { 534 p = Double.MIN_VALUE; 535 } 536 return (long) (-n * Math.log(p) / (Math.log(2) * Math.log(2))); 537 } 538 539 private Object writeReplace() { 540 return new SerialForm<T>(this); 541 } 542 543 private void readObject(ObjectInputStream stream) throws InvalidObjectException { 544 throw new InvalidObjectException("Use SerializedForm"); 545 } 546 547 private static class SerialForm<T extends @Nullable Object> implements Serializable { 548 final long[] data; 549 final int numHashFunctions; 550 final Funnel<? super T> funnel; 551 final Strategy strategy; 552 553 SerialForm(BloomFilter<T> bf) { 554 this.data = LockFreeBitArray.toPlainArray(bf.bits.data); 555 this.numHashFunctions = bf.numHashFunctions; 556 this.funnel = bf.funnel; 557 this.strategy = bf.strategy; 558 } 559 560 Object readResolve() { 561 return new BloomFilter<T>(new LockFreeBitArray(data), numHashFunctions, funnel, strategy); 562 } 563 564 private static final long serialVersionUID = 1; 565 } 566 567 /** 568 * Writes this {@code BloomFilter} to an output stream, with a custom format (not Java 569 * serialization). This has been measured to save at least 400 bytes compared to regular 570 * serialization. 571 * 572 * <p>Use {@linkplain #readFrom(InputStream, Funnel)} to reconstruct the written BloomFilter. 573 */ 574 public void writeTo(OutputStream out) throws IOException { 575 // Serial form: 576 // 1 signed byte for the strategy 577 // 1 unsigned byte for the number of hash functions 578 // 1 big endian int, the number of longs in our bitset 579 // N big endian longs of our bitset 580 DataOutputStream dout = new DataOutputStream(out); 581 dout.writeByte(SignedBytes.checkedCast(strategy.ordinal())); 582 dout.writeByte(UnsignedBytes.checkedCast(numHashFunctions)); // note: checked at the c'tor 583 dout.writeInt(bits.data.length()); 584 for (int i = 0; i < bits.data.length(); i++) { 585 dout.writeLong(bits.data.get(i)); 586 } 587 } 588 589 /** 590 * Reads a byte stream, which was written by {@linkplain #writeTo(OutputStream)}, into a {@code 591 * BloomFilter}. 592 * 593 * <p>The {@code Funnel} to be used is not encoded in the stream, so it must be provided here. 594 * <b>Warning:</b> the funnel provided <b>must</b> behave identically to the one used to populate 595 * the original Bloom filter! 596 * 597 * @throws IOException if the InputStream throws an {@code IOException}, or if its data does not 598 * appear to be a BloomFilter serialized using the {@linkplain #writeTo(OutputStream)} method. 599 */ 600 public static <T extends @Nullable Object> BloomFilter<T> readFrom( 601 InputStream in, Funnel<? super T> funnel) throws IOException { 602 checkNotNull(in, "InputStream"); 603 checkNotNull(funnel, "Funnel"); 604 int strategyOrdinal = -1; 605 int numHashFunctions = -1; 606 int dataLength = -1; 607 try { 608 DataInputStream din = new DataInputStream(in); 609 // currently this assumes there is no negative ordinal; will have to be updated if we 610 // add non-stateless strategies (for which we've reserved negative ordinals; see 611 // Strategy.ordinal()). 612 strategyOrdinal = din.readByte(); 613 numHashFunctions = UnsignedBytes.toInt(din.readByte()); 614 dataLength = din.readInt(); 615 616 Strategy strategy = BloomFilterStrategies.values()[strategyOrdinal]; 617 618 LockFreeBitArray dataArray = new LockFreeBitArray(LongMath.checkedMultiply(dataLength, 64L)); 619 for (int i = 0; i < dataLength; i++) { 620 dataArray.putData(i, din.readLong()); 621 } 622 623 return new BloomFilter<T>(dataArray, numHashFunctions, funnel, strategy); 624 } catch (RuntimeException e) { 625 String message = 626 "Unable to deserialize BloomFilter from InputStream." 627 + " strategyOrdinal: " 628 + strategyOrdinal 629 + " numHashFunctions: " 630 + numHashFunctions 631 + " dataLength: " 632 + dataLength; 633 throw new IOException(message, e); 634 } 635 } 636}