001/* 002 * Copyright (C) 2012 The Guava Authors 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016 017package com.google.common.io; 018 019import static com.google.common.base.Preconditions.checkArgument; 020import static com.google.common.base.Preconditions.checkNotNull; 021 022import com.google.common.collect.ImmutableList; 023import com.google.common.hash.Funnels; 024import com.google.common.hash.HashCode; 025import com.google.common.hash.HashFunction; 026import com.google.common.hash.Hasher; 027 028import java.io.BufferedInputStream; 029import java.io.ByteArrayInputStream; 030import java.io.IOException; 031import java.io.InputStream; 032import java.io.InputStreamReader; 033import java.io.OutputStream; 034import java.io.Reader; 035import java.nio.charset.Charset; 036import java.util.Arrays; 037import java.util.Iterator; 038 039/** 040 * A readable source of bytes, such as a file. Unlike an {@link InputStream}, a 041 * {@code ByteSource} is not an open, stateful stream for input that can be read and closed. 042 * Instead, it is an immutable <i>supplier</i> of {@code InputStream} instances. 043 * 044 * <p>{@code ByteSource} provides two kinds of methods: 045 * <ul> 046 * <li><b>Methods that return a stream:</b> These methods should return a <i>new</i>, independent 047 * instance each time they are called. The caller is responsible for ensuring that the returned 048 * stream is closed. 049 * <li><b>Convenience methods:</b> These are implementations of common operations that are 050 * typically implemented by opening a stream using one of the methods in the first category, doing 051 * something and finally closing the stream that was opened. 052 * </ul> 053 * 054 * @since 14.0 055 * @author Colin Decker 056 */ 057public abstract class ByteSource implements InputSupplier<InputStream> { 058 059 private static final int BUF_SIZE = 0x1000; // 4K 060 061 /** 062 * Returns a {@link CharSource} view of this byte source that decodes bytes read from this source 063 * as characters using the given {@link Charset}. 064 */ 065 public CharSource asCharSource(Charset charset) { 066 return new AsCharSource(charset); 067 } 068 069 /** 070 * Opens a new {@link InputStream} for reading from this source. This method should return a new, 071 * independent stream each time it is called. 072 * 073 * <p>The caller is responsible for ensuring that the returned stream is closed. 074 * 075 * @throws IOException if an I/O error occurs in the process of opening the stream 076 */ 077 public abstract InputStream openStream() throws IOException; 078 079 /** 080 * This method is a temporary method provided for easing migration from suppliers to sources and 081 * sinks. 082 * 083 * @since 15.0 084 * @deprecated This method is only provided for temporary compatibility with the 085 * {@link InputSupplier} interface and should not be called directly. Use {@link #openStream} 086 * instead. 087 */ 088 @Override 089 @Deprecated 090 public final InputStream getInput() throws IOException { 091 return openStream(); 092 } 093 094 /** 095 * Opens a new buffered {@link InputStream} for reading from this source. The returned stream is 096 * not required to be a {@link BufferedInputStream} in order to allow implementations to simply 097 * delegate to {@link #openStream()} when the stream returned by that method does not benefit 098 * from additional buffering (for example, a {@code ByteArrayInputStream}). This method should 099 * return a new, independent stream each time it is called. 100 * 101 * <p>The caller is responsible for ensuring that the returned stream is closed. 102 * 103 * @throws IOException if an I/O error occurs in the process of opening the stream 104 * @since 15.0 (in 14.0 with return type {@link BufferedInputStream}) 105 */ 106 public InputStream openBufferedStream() throws IOException { 107 InputStream in = openStream(); 108 return (in instanceof BufferedInputStream) 109 ? (BufferedInputStream) in 110 : new BufferedInputStream(in); 111 } 112 113 /** 114 * Returns a view of a slice of this byte source that is at most {@code length} bytes long 115 * starting at the given {@code offset}. 116 * 117 * @throws IllegalArgumentException if {@code offset} or {@code length} is negative 118 */ 119 public ByteSource slice(long offset, long length) { 120 return new SlicedByteSource(offset, length); 121 } 122 123 /** 124 * Returns whether the source has zero bytes. The default implementation is to open a stream and 125 * check for EOF. 126 * 127 * @throws IOException if an I/O error occurs 128 * @since 15.0 129 */ 130 public boolean isEmpty() throws IOException { 131 Closer closer = Closer.create(); 132 try { 133 InputStream in = closer.register(openStream()); 134 return in.read() == -1; 135 } catch (Throwable e) { 136 throw closer.rethrow(e); 137 } finally { 138 closer.close(); 139 } 140 } 141 142 /** 143 * Returns the size of this source in bytes. For most implementations, this is a heavyweight 144 * operation that will open a stream, read (or {@link InputStream#skip(long) skip}, if possible) 145 * to the end of the stream and return the total number of bytes that were read. 146 * 147 * <p>For some sources, such as a file, this method may use a more efficient implementation. Note 148 * that in such cases, it is <i>possible</i> that this method will return a different number of 149 * bytes than would be returned by reading all of the bytes (for example, some special files may 150 * return a size of 0 despite actually having content when read). 151 * 152 * <p>In either case, if this is a mutable source such as a file, the size it returns may not be 153 * the same number of bytes a subsequent read would return. 154 * 155 * @throws IOException if an I/O error occurs in the process of reading the size of this source 156 */ 157 public long size() throws IOException { 158 Closer closer = Closer.create(); 159 try { 160 InputStream in = closer.register(openStream()); 161 return countBySkipping(in); 162 } catch (IOException e) { 163 // skip may not be supported... at any rate, try reading 164 } finally { 165 closer.close(); 166 } 167 168 closer = Closer.create(); 169 try { 170 InputStream in = closer.register(openStream()); 171 return countByReading(in); 172 } catch (Throwable e) { 173 throw closer.rethrow(e); 174 } finally { 175 closer.close(); 176 } 177 } 178 179 /** 180 * Counts the bytes in the given input stream using skip if possible. Returns SKIP_FAILED if the 181 * first call to skip threw, in which case skip may just not be supported. 182 */ 183 private long countBySkipping(InputStream in) throws IOException { 184 long count = 0; 185 while (true) { 186 // don't try to skip more than available() 187 // things may work really wrong with FileInputStream otherwise 188 long skipped = in.skip(Math.min(in.available(), Integer.MAX_VALUE)); 189 if (skipped <= 0) { 190 if (in.read() == -1) { 191 return count; 192 } else if (count == 0 && in.available() == 0) { 193 // if available is still zero after reading a single byte, it 194 // will probably always be zero, so we should countByReading 195 throw new IOException(); 196 } 197 count++; 198 } else { 199 count += skipped; 200 } 201 } 202 } 203 204 private static final byte[] countBuffer = new byte[BUF_SIZE]; 205 206 private long countByReading(InputStream in) throws IOException { 207 long count = 0; 208 long read; 209 while ((read = in.read(countBuffer)) != -1) { 210 count += read; 211 } 212 return count; 213 } 214 215 /** 216 * Copies the contents of this byte source to the given {@code OutputStream}. Does not close 217 * {@code output}. 218 * 219 * @throws IOException if an I/O error occurs in the process of reading from this source or 220 * writing to {@code output} 221 */ 222 public long copyTo(OutputStream output) throws IOException { 223 checkNotNull(output); 224 225 Closer closer = Closer.create(); 226 try { 227 InputStream in = closer.register(openStream()); 228 return ByteStreams.copy(in, output); 229 } catch (Throwable e) { 230 throw closer.rethrow(e); 231 } finally { 232 closer.close(); 233 } 234 } 235 236 /** 237 * Copies the contents of this byte source to the given {@code ByteSink}. 238 * 239 * @throws IOException if an I/O error occurs in the process of reading from this source or 240 * writing to {@code sink} 241 */ 242 public long copyTo(ByteSink sink) throws IOException { 243 checkNotNull(sink); 244 245 Closer closer = Closer.create(); 246 try { 247 InputStream in = closer.register(openStream()); 248 OutputStream out = closer.register(sink.openStream()); 249 return ByteStreams.copy(in, out); 250 } catch (Throwable e) { 251 throw closer.rethrow(e); 252 } finally { 253 closer.close(); 254 } 255 } 256 257 /** 258 * Reads the full contents of this byte source as a byte array. 259 * 260 * @throws IOException if an I/O error occurs in the process of reading from this source 261 */ 262 public byte[] read() throws IOException { 263 Closer closer = Closer.create(); 264 try { 265 InputStream in = closer.register(openStream()); 266 return ByteStreams.toByteArray(in); 267 } catch (Throwable e) { 268 throw closer.rethrow(e); 269 } finally { 270 closer.close(); 271 } 272 } 273 274 /** 275 * Hashes the contents of this byte source using the given hash function. 276 * 277 * @throws IOException if an I/O error occurs in the process of reading from this source 278 */ 279 public HashCode hash(HashFunction hashFunction) throws IOException { 280 Hasher hasher = hashFunction.newHasher(); 281 copyTo(Funnels.asOutputStream(hasher)); 282 return hasher.hash(); 283 } 284 285 /** 286 * Checks that the contents of this byte source are equal to the contents of the given byte 287 * source. 288 * 289 * @throws IOException if an I/O error occurs in the process of reading from this source or 290 * {@code other} 291 */ 292 public boolean contentEquals(ByteSource other) throws IOException { 293 checkNotNull(other); 294 295 byte[] buf1 = new byte[BUF_SIZE]; 296 byte[] buf2 = new byte[BUF_SIZE]; 297 298 Closer closer = Closer.create(); 299 try { 300 InputStream in1 = closer.register(openStream()); 301 InputStream in2 = closer.register(other.openStream()); 302 while (true) { 303 int read1 = ByteStreams.read(in1, buf1, 0, BUF_SIZE); 304 int read2 = ByteStreams.read(in2, buf2, 0, BUF_SIZE); 305 if (read1 != read2 || !Arrays.equals(buf1, buf2)) { 306 return false; 307 } else if (read1 != BUF_SIZE) { 308 return true; 309 } 310 } 311 } catch (Throwable e) { 312 throw closer.rethrow(e); 313 } finally { 314 closer.close(); 315 } 316 } 317 318 /** 319 * Concatenates multiple {@link ByteSource} instances into a single source. Streams returned from 320 * the source will contain the concatenated data from the streams of the underlying sources. 321 * 322 * <p>Only one underlying stream will be open at a time. Closing the concatenated stream will 323 * close the open underlying stream. 324 * 325 * @param sources the sources to concatenate 326 * @return a {@code ByteSource} containing the concatenated data 327 * @since 15.0 328 */ 329 public static ByteSource concat(Iterable<? extends ByteSource> sources) { 330 return new ConcatenatedByteSource(sources); 331 } 332 333 /** 334 * Concatenates multiple {@link ByteSource} instances into a single source. Streams returned from 335 * the source will contain the concatenated data from the streams of the underlying sources. 336 * 337 * <p>Only one underlying stream will be open at a time. Closing the concatenated stream will 338 * close the open underlying stream. 339 * 340 * <p>Note: The input {@code Iterator} will be copied to an {@code ImmutableList} when this 341 * method is called. This will fail if the iterator is infinite and may cause problems if the 342 * iterator eagerly fetches data for each source when iterated (rather than producing sources 343 * that only load data through their streams). Prefer using the {@link #concat(Iterable)} 344 * overload if possible. 345 * 346 * @param sources the sources to concatenate 347 * @return a {@code ByteSource} containing the concatenated data 348 * @throws NullPointerException if any of {@code sources} is {@code null} 349 * @since 15.0 350 */ 351 public static ByteSource concat(Iterator<? extends ByteSource> sources) { 352 return concat(ImmutableList.copyOf(sources)); 353 } 354 355 /** 356 * Concatenates multiple {@link ByteSource} instances into a single source. Streams returned from 357 * the source will contain the concatenated data from the streams of the underlying sources. 358 * 359 * <p>Only one underlying stream will be open at a time. Closing the concatenated stream will 360 * close the open underlying stream. 361 * 362 * @param sources the sources to concatenate 363 * @return a {@code ByteSource} containing the concatenated data 364 * @throws NullPointerException if any of {@code sources} is {@code null} 365 * @since 15.0 366 */ 367 public static ByteSource concat(ByteSource... sources) { 368 return concat(ImmutableList.copyOf(sources)); 369 } 370 371 /** 372 * Returns a view of the given byte array as a {@link ByteSource}. To view only a specific range 373 * in the array, use {@code ByteSource.wrap(b).slice(offset, length)}. 374 * 375 * @since 15.0 (since 14.0 as {@code ByteStreams.asByteSource(byte[])}). 376 */ 377 public static ByteSource wrap(byte[] b) { 378 return new ByteArrayByteSource(b); 379 } 380 381 /** 382 * Returns an immutable {@link ByteSource} that contains no bytes. 383 * 384 * @since 15.0 385 */ 386 public static ByteSource empty() { 387 return EmptyByteSource.INSTANCE; 388 } 389 390 /** 391 * A char source that reads bytes from this source and decodes them as characters using a 392 * charset. 393 */ 394 private final class AsCharSource extends CharSource { 395 396 private final Charset charset; 397 398 private AsCharSource(Charset charset) { 399 this.charset = checkNotNull(charset); 400 } 401 402 @Override 403 public Reader openStream() throws IOException { 404 return new InputStreamReader(ByteSource.this.openStream(), charset); 405 } 406 407 @Override 408 public String toString() { 409 return ByteSource.this.toString() + ".asCharSource(" + charset + ")"; 410 } 411 } 412 413 /** 414 * A view of a subsection of the containing byte source. 415 */ 416 private final class SlicedByteSource extends ByteSource { 417 418 private final long offset; 419 private final long length; 420 421 private SlicedByteSource(long offset, long length) { 422 checkArgument(offset >= 0, "offset (%s) may not be negative", offset); 423 checkArgument(length >= 0, "length (%s) may not be negative", length); 424 this.offset = offset; 425 this.length = length; 426 } 427 428 @Override 429 public InputStream openStream() throws IOException { 430 return sliceStream(ByteSource.this.openStream()); 431 } 432 433 @Override 434 public InputStream openBufferedStream() throws IOException { 435 return sliceStream(ByteSource.this.openBufferedStream()); 436 } 437 438 private InputStream sliceStream(InputStream in) throws IOException { 439 if (offset > 0) { 440 try { 441 ByteStreams.skipFully(in, offset); 442 } catch (Throwable e) { 443 Closer closer = Closer.create(); 444 closer.register(in); 445 try { 446 throw closer.rethrow(e); 447 } finally { 448 closer.close(); 449 } 450 } 451 } 452 return ByteStreams.limit(in, length); 453 } 454 455 @Override 456 public ByteSource slice(long offset, long length) { 457 checkArgument(offset >= 0, "offset (%s) may not be negative", offset); 458 checkArgument(length >= 0, "length (%s) may not be negative", length); 459 long maxLength = this.length - offset; 460 return ByteSource.this.slice(this.offset + offset, Math.min(length, maxLength)); 461 } 462 463 @Override 464 public boolean isEmpty() throws IOException { 465 return length == 0 || super.isEmpty(); 466 } 467 468 @Override 469 public String toString() { 470 return ByteSource.this.toString() + ".slice(" + offset + ", " + length + ")"; 471 } 472 } 473 474 private static class ByteArrayByteSource extends ByteSource { 475 476 protected final byte[] bytes; 477 478 protected ByteArrayByteSource(byte[] bytes) { 479 this.bytes = checkNotNull(bytes); 480 } 481 482 @Override 483 public InputStream openStream() { 484 return new ByteArrayInputStream(bytes); 485 } 486 487 @Override 488 public InputStream openBufferedStream() throws IOException { 489 return openStream(); 490 } 491 492 @Override 493 public boolean isEmpty() { 494 return bytes.length == 0; 495 } 496 497 @Override 498 public long size() { 499 return bytes.length; 500 } 501 502 @Override 503 public byte[] read() { 504 return bytes.clone(); 505 } 506 507 @Override 508 public long copyTo(OutputStream output) throws IOException { 509 output.write(bytes); 510 return bytes.length; 511 } 512 513 @Override 514 public HashCode hash(HashFunction hashFunction) throws IOException { 515 return hashFunction.hashBytes(bytes); 516 } 517 518 // TODO(user): Possibly override slice() 519 520 @Override 521 public String toString() { 522 return "ByteSource.wrap(" + BaseEncoding.base16().encode(bytes) + ")"; 523 } 524 } 525 526 private static final class EmptyByteSource extends ByteArrayByteSource { 527 528 private static final EmptyByteSource INSTANCE = new EmptyByteSource(); 529 530 private EmptyByteSource() { 531 super(new byte[0]); 532 } 533 534 @Override 535 public CharSource asCharSource(Charset charset) { 536 checkNotNull(charset); 537 return CharSource.empty(); 538 } 539 540 @Override 541 public byte[] read() { 542 return bytes; // length is 0, no need to clone 543 } 544 545 @Override 546 public String toString() { 547 return "ByteSource.empty()"; 548 } 549 } 550 551 private static final class ConcatenatedByteSource extends ByteSource { 552 553 private final Iterable<? extends ByteSource> sources; 554 555 ConcatenatedByteSource(Iterable<? extends ByteSource> sources) { 556 this.sources = checkNotNull(sources); 557 } 558 559 @Override 560 public InputStream openStream() throws IOException { 561 return new MultiInputStream(sources.iterator()); 562 } 563 564 @Override 565 public boolean isEmpty() throws IOException { 566 for (ByteSource source : sources) { 567 if (!source.isEmpty()) { 568 return false; 569 } 570 } 571 return true; 572 } 573 574 @Override 575 public long size() throws IOException { 576 long result = 0L; 577 for (ByteSource source : sources) { 578 result += source.size(); 579 } 580 return result; 581 } 582 583 @Override 584 public String toString() { 585 return "ByteSource.concat(" + sources + ")"; 586 } 587 } 588}