001/* 002 * Copyright (C) 2007 The Guava Authors 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016 017package com.google.common.io; 018 019import static com.google.common.base.Preconditions.checkArgument; 020import static com.google.common.base.Preconditions.checkNotNull; 021import static com.google.common.base.Preconditions.checkPositionIndex; 022 023import com.google.common.annotations.Beta; 024 025import java.io.ByteArrayInputStream; 026import java.io.ByteArrayOutputStream; 027import java.io.DataInput; 028import java.io.DataInputStream; 029import java.io.DataOutput; 030import java.io.DataOutputStream; 031import java.io.EOFException; 032import java.io.FilterInputStream; 033import java.io.IOException; 034import java.io.InputStream; 035import java.io.OutputStream; 036import java.nio.ByteBuffer; 037import java.nio.channels.FileChannel; 038import java.nio.channels.ReadableByteChannel; 039import java.nio.channels.WritableByteChannel; 040import java.util.Arrays; 041 042/** 043 * Provides utility methods for working with byte arrays and I/O streams. 044 * 045 * @author Chris Nokleberg 046 * @author Colin Decker 047 * @since 1.0 048 */ 049@Beta 050public final class ByteStreams { 051 private static final int BUF_SIZE = 8192; 052 /** 053 * There are three methods to implement {@link FileChannel#transferTo(long, long, 054 * WritableByteChannel)}: 055 * 056 * <ol> 057 * <li> Use sendfile(2) or equivalent. Requires that both the input channel and the output channel 058 * have their own file descriptors. Generally this only happens when both channels are files or 059 * sockets. This performs zero copies - the bytes never enter userspace.</li> 060 * <li> Use mmap(2) or equivalent. Requires that either the input channel or the output channel 061 * have file descriptors. Bytes are copied from the file into a kernel buffer, then directly 062 * into the other buffer (userspace). Note that if the file is very large, a naive 063 * implementation will effectively put the whole file in memory. On many systems with paging 064 * and virtual memory, this is not a problem - because it is mapped read-only, the kernel can 065 * always page it to disk "for free". However, on systems where killing processes happens all 066 * the time in normal conditions (i.e., android) the OS must make a tradeoff between paging 067 * memory and killing other processes - so allocating a gigantic buffer and then sequentially 068 * accessing it could result in other processes dying. This is solvable via madvise(2), but 069 * that obviously doesn't exist in java.</li> 070 * <li> Ordinary copy. Kernel copies bytes into a kernel buffer, from a kernel buffer into a 071 * userspace buffer (byte[] or ByteBuffer), then copies them from that buffer into the 072 * destination channel.</li> 073 * </ol> 074 * 075 * This value is intended to be large enough to make the overhead of system calls negligible, 076 * without being so large that it causes problems for systems with atypical memory management if 077 * approaches 2 or 3 are used. 078 */ 079 private static final int ZERO_COPY_CHUNK_SIZE = 512 * 1024; 080 081 private ByteStreams() {} 082 083 /** 084 * Copies all bytes from the input stream to the output stream. 085 * Does not close or flush either stream. 086 * 087 * @param from the input stream to read from 088 * @param to the output stream to write to 089 * @return the number of bytes copied 090 * @throws IOException if an I/O error occurs 091 */ 092 public static long copy(InputStream from, OutputStream to) 093 throws IOException { 094 checkNotNull(from); 095 checkNotNull(to); 096 byte[] buf = new byte[BUF_SIZE]; 097 long total = 0; 098 while (true) { 099 int r = from.read(buf); 100 if (r == -1) { 101 break; 102 } 103 to.write(buf, 0, r); 104 total += r; 105 } 106 return total; 107 } 108 109 /** 110 * Copies all bytes from the readable channel to the writable channel. 111 * Does not close or flush either channel. 112 * 113 * @param from the readable channel to read from 114 * @param to the writable channel to write to 115 * @return the number of bytes copied 116 * @throws IOException if an I/O error occurs 117 */ 118 public static long copy(ReadableByteChannel from, 119 WritableByteChannel to) throws IOException { 120 checkNotNull(from); 121 checkNotNull(to); 122 if (from instanceof FileChannel) { 123 FileChannel sourceChannel = (FileChannel) from; 124 long oldPosition = sourceChannel.position(); 125 long position = oldPosition; 126 long copied; 127 do { 128 copied = sourceChannel.transferTo(position, ZERO_COPY_CHUNK_SIZE, to); 129 position += copied; 130 sourceChannel.position(position); 131 } while (copied > 0 || position < sourceChannel.size()); 132 return position - oldPosition; 133 } 134 135 ByteBuffer buf = ByteBuffer.allocate(BUF_SIZE); 136 long total = 0; 137 while (from.read(buf) != -1) { 138 buf.flip(); 139 while (buf.hasRemaining()) { 140 total += to.write(buf); 141 } 142 buf.clear(); 143 } 144 return total; 145 } 146 147 /** 148 * Reads all bytes from an input stream into a byte array. 149 * Does not close the stream. 150 * 151 * @param in the input stream to read from 152 * @return a byte array containing all the bytes from the stream 153 * @throws IOException if an I/O error occurs 154 */ 155 public static byte[] toByteArray(InputStream in) throws IOException { 156 ByteArrayOutputStream out = new ByteArrayOutputStream(); 157 copy(in, out); 158 return out.toByteArray(); 159 } 160 161 /** 162 * Reads all bytes from an input stream into a byte array. The given 163 * expected size is used to create an initial byte array, but if the actual 164 * number of bytes read from the stream differs, the correct result will be 165 * returned anyway. 166 */ 167 static byte[] toByteArray( 168 InputStream in, int expectedSize) throws IOException { 169 byte[] bytes = new byte[expectedSize]; 170 int remaining = expectedSize; 171 172 while (remaining > 0) { 173 int off = expectedSize - remaining; 174 int read = in.read(bytes, off, remaining); 175 if (read == -1) { 176 // end of stream before reading expectedSize bytes 177 // just return the bytes read so far 178 return Arrays.copyOf(bytes, off); 179 } 180 remaining -= read; 181 } 182 183 // bytes is now full 184 int b = in.read(); 185 if (b == -1) { 186 return bytes; 187 } 188 189 // the stream was longer, so read the rest normally 190 FastByteArrayOutputStream out = new FastByteArrayOutputStream(); 191 out.write(b); // write the byte we read when testing for end of stream 192 copy(in, out); 193 194 byte[] result = new byte[bytes.length + out.size()]; 195 System.arraycopy(bytes, 0, result, 0, bytes.length); 196 out.writeTo(result, bytes.length); 197 return result; 198 } 199 200 /** 201 * BAOS that provides limited access to its internal byte array. 202 */ 203 private static final class FastByteArrayOutputStream 204 extends ByteArrayOutputStream { 205 /** 206 * Writes the contents of the internal buffer to the given array starting 207 * at the given offset. Assumes the array has space to hold count bytes. 208 */ 209 void writeTo(byte[] b, int off) { 210 System.arraycopy(buf, 0, b, off, count); 211 } 212 } 213 214 /** 215 * Returns a new {@link ByteArrayDataInput} instance to read from the {@code 216 * bytes} array from the beginning. 217 */ 218 public static ByteArrayDataInput newDataInput(byte[] bytes) { 219 return newDataInput(new ByteArrayInputStream(bytes)); 220 } 221 222 /** 223 * Returns a new {@link ByteArrayDataInput} instance to read from the {@code 224 * bytes} array, starting at the given position. 225 * 226 * @throws IndexOutOfBoundsException if {@code start} is negative or greater 227 * than the length of the array 228 */ 229 public static ByteArrayDataInput newDataInput(byte[] bytes, int start) { 230 checkPositionIndex(start, bytes.length); 231 return newDataInput( 232 new ByteArrayInputStream(bytes, start, bytes.length - start)); 233 } 234 235 /** 236 * Returns a new {@link ByteArrayDataInput} instance to read from the given 237 * {@code ByteArrayInputStream}. The given input stream is not reset before 238 * being read from by the returned {@code ByteArrayDataInput}. 239 * 240 * @since 17.0 241 */ 242 public static ByteArrayDataInput newDataInput( 243 ByteArrayInputStream byteArrayInputStream) { 244 return new ByteArrayDataInputStream(checkNotNull(byteArrayInputStream)); 245 } 246 247 private static class ByteArrayDataInputStream implements ByteArrayDataInput { 248 final DataInput input; 249 250 ByteArrayDataInputStream(ByteArrayInputStream byteArrayInputStream) { 251 this.input = new DataInputStream(byteArrayInputStream); 252 } 253 254 @Override public void readFully(byte b[]) { 255 try { 256 input.readFully(b); 257 } catch (IOException e) { 258 throw new IllegalStateException(e); 259 } 260 } 261 262 @Override public void readFully(byte b[], int off, int len) { 263 try { 264 input.readFully(b, off, len); 265 } catch (IOException e) { 266 throw new IllegalStateException(e); 267 } 268 } 269 270 @Override public int skipBytes(int n) { 271 try { 272 return input.skipBytes(n); 273 } catch (IOException e) { 274 throw new IllegalStateException(e); 275 } 276 } 277 278 @Override public boolean readBoolean() { 279 try { 280 return input.readBoolean(); 281 } catch (IOException e) { 282 throw new IllegalStateException(e); 283 } 284 } 285 286 @Override public byte readByte() { 287 try { 288 return input.readByte(); 289 } catch (EOFException e) { 290 throw new IllegalStateException(e); 291 } catch (IOException impossible) { 292 throw new AssertionError(impossible); 293 } 294 } 295 296 @Override public int readUnsignedByte() { 297 try { 298 return input.readUnsignedByte(); 299 } catch (IOException e) { 300 throw new IllegalStateException(e); 301 } 302 } 303 304 @Override public short readShort() { 305 try { 306 return input.readShort(); 307 } catch (IOException e) { 308 throw new IllegalStateException(e); 309 } 310 } 311 312 @Override public int readUnsignedShort() { 313 try { 314 return input.readUnsignedShort(); 315 } catch (IOException e) { 316 throw new IllegalStateException(e); 317 } 318 } 319 320 @Override public char readChar() { 321 try { 322 return input.readChar(); 323 } catch (IOException e) { 324 throw new IllegalStateException(e); 325 } 326 } 327 328 @Override public int readInt() { 329 try { 330 return input.readInt(); 331 } catch (IOException e) { 332 throw new IllegalStateException(e); 333 } 334 } 335 336 @Override public long readLong() { 337 try { 338 return input.readLong(); 339 } catch (IOException e) { 340 throw new IllegalStateException(e); 341 } 342 } 343 344 @Override public float readFloat() { 345 try { 346 return input.readFloat(); 347 } catch (IOException e) { 348 throw new IllegalStateException(e); 349 } 350 } 351 352 @Override public double readDouble() { 353 try { 354 return input.readDouble(); 355 } catch (IOException e) { 356 throw new IllegalStateException(e); 357 } 358 } 359 360 @Override public String readLine() { 361 try { 362 return input.readLine(); 363 } catch (IOException e) { 364 throw new IllegalStateException(e); 365 } 366 } 367 368 @Override public String readUTF() { 369 try { 370 return input.readUTF(); 371 } catch (IOException e) { 372 throw new IllegalStateException(e); 373 } 374 } 375 } 376 377 /** 378 * Returns a new {@link ByteArrayDataOutput} instance with a default size. 379 */ 380 public static ByteArrayDataOutput newDataOutput() { 381 return newDataOutput(new ByteArrayOutputStream()); 382 } 383 384 /** 385 * Returns a new {@link ByteArrayDataOutput} instance sized to hold 386 * {@code size} bytes before resizing. 387 * 388 * @throws IllegalArgumentException if {@code size} is negative 389 */ 390 public static ByteArrayDataOutput newDataOutput(int size) { 391 // When called at high frequency, boxing size generates too much garbage, 392 // so avoid doing that if we can. 393 if (size < 0) { 394 throw new IllegalArgumentException(String.format("Invalid size: %s", size)); 395 } 396 return newDataOutput(new ByteArrayOutputStream(size)); 397 } 398 399 /** 400 * Returns a new {@link ByteArrayDataOutput} instance which writes to the 401 * given {@code ByteArrayOutputStream}. The given output stream is not reset 402 * before being written to by the returned {@code ByteArrayDataOutput} and 403 * new data will be appended to any existing content. 404 * 405 * <p>Note that if the given output stream was not empty or is modified after 406 * the {@code ByteArrayDataOutput} is created, the contract for 407 * {@link ByteArrayDataOutput#toByteArray} will not be honored (the bytes 408 * returned in the byte array may not be exactly what was written via calls to 409 * {@code ByteArrayDataOutput}). 410 * 411 * @since 17.0 412 */ 413 public static ByteArrayDataOutput newDataOutput( 414 ByteArrayOutputStream byteArrayOutputSteam) { 415 return new ByteArrayDataOutputStream(checkNotNull(byteArrayOutputSteam)); 416 } 417 418 @SuppressWarnings("deprecation") // for writeBytes 419 private static class ByteArrayDataOutputStream 420 implements ByteArrayDataOutput { 421 422 final DataOutput output; 423 final ByteArrayOutputStream byteArrayOutputSteam; 424 425 ByteArrayDataOutputStream(ByteArrayOutputStream byteArrayOutputSteam) { 426 this.byteArrayOutputSteam = byteArrayOutputSteam; 427 output = new DataOutputStream(byteArrayOutputSteam); 428 } 429 430 @Override public void write(int b) { 431 try { 432 output.write(b); 433 } catch (IOException impossible) { 434 throw new AssertionError(impossible); 435 } 436 } 437 438 @Override public void write(byte[] b) { 439 try { 440 output.write(b); 441 } catch (IOException impossible) { 442 throw new AssertionError(impossible); 443 } 444 } 445 446 @Override public void write(byte[] b, int off, int len) { 447 try { 448 output.write(b, off, len); 449 } catch (IOException impossible) { 450 throw new AssertionError(impossible); 451 } 452 } 453 454 @Override public void writeBoolean(boolean v) { 455 try { 456 output.writeBoolean(v); 457 } catch (IOException impossible) { 458 throw new AssertionError(impossible); 459 } 460 } 461 462 @Override public void writeByte(int v) { 463 try { 464 output.writeByte(v); 465 } catch (IOException impossible) { 466 throw new AssertionError(impossible); 467 } 468 } 469 470 @Override public void writeBytes(String s) { 471 try { 472 output.writeBytes(s); 473 } catch (IOException impossible) { 474 throw new AssertionError(impossible); 475 } 476 } 477 478 @Override public void writeChar(int v) { 479 try { 480 output.writeChar(v); 481 } catch (IOException impossible) { 482 throw new AssertionError(impossible); 483 } 484 } 485 486 @Override public void writeChars(String s) { 487 try { 488 output.writeChars(s); 489 } catch (IOException impossible) { 490 throw new AssertionError(impossible); 491 } 492 } 493 494 @Override public void writeDouble(double v) { 495 try { 496 output.writeDouble(v); 497 } catch (IOException impossible) { 498 throw new AssertionError(impossible); 499 } 500 } 501 502 @Override public void writeFloat(float v) { 503 try { 504 output.writeFloat(v); 505 } catch (IOException impossible) { 506 throw new AssertionError(impossible); 507 } 508 } 509 510 @Override public void writeInt(int v) { 511 try { 512 output.writeInt(v); 513 } catch (IOException impossible) { 514 throw new AssertionError(impossible); 515 } 516 } 517 518 @Override public void writeLong(long v) { 519 try { 520 output.writeLong(v); 521 } catch (IOException impossible) { 522 throw new AssertionError(impossible); 523 } 524 } 525 526 @Override public void writeShort(int v) { 527 try { 528 output.writeShort(v); 529 } catch (IOException impossible) { 530 throw new AssertionError(impossible); 531 } 532 } 533 534 @Override public void writeUTF(String s) { 535 try { 536 output.writeUTF(s); 537 } catch (IOException impossible) { 538 throw new AssertionError(impossible); 539 } 540 } 541 542 @Override public byte[] toByteArray() { 543 return byteArrayOutputSteam.toByteArray(); 544 } 545 } 546 547 private static final OutputStream NULL_OUTPUT_STREAM = 548 new OutputStream() { 549 /** Discards the specified byte. */ 550 @Override public void write(int b) { 551 } 552 /** Discards the specified byte array. */ 553 @Override public void write(byte[] b) { 554 checkNotNull(b); 555 } 556 /** Discards the specified byte array. */ 557 @Override public void write(byte[] b, int off, int len) { 558 checkNotNull(b); 559 } 560 561 @Override 562 public String toString() { 563 return "ByteStreams.nullOutputStream()"; 564 } 565 }; 566 567 /** 568 * Returns an {@link OutputStream} that simply discards written bytes. 569 * 570 * @since 14.0 (since 1.0 as com.google.common.io.NullOutputStream) 571 */ 572 public static OutputStream nullOutputStream() { 573 return NULL_OUTPUT_STREAM; 574 } 575 576 /** 577 * Wraps a {@link InputStream}, limiting the number of bytes which can be 578 * read. 579 * 580 * @param in the input stream to be wrapped 581 * @param limit the maximum number of bytes to be read 582 * @return a length-limited {@link InputStream} 583 * @since 14.0 (since 1.0 as com.google.common.io.LimitInputStream) 584 */ 585 public static InputStream limit(InputStream in, long limit) { 586 return new LimitedInputStream(in, limit); 587 } 588 589 private static final class LimitedInputStream extends FilterInputStream { 590 591 private long left; 592 private long mark = -1; 593 594 LimitedInputStream(InputStream in, long limit) { 595 super(in); 596 checkNotNull(in); 597 checkArgument(limit >= 0, "limit must be non-negative"); 598 left = limit; 599 } 600 601 @Override public int available() throws IOException { 602 return (int) Math.min(in.available(), left); 603 } 604 605 // it's okay to mark even if mark isn't supported, as reset won't work 606 @Override public synchronized void mark(int readLimit) { 607 in.mark(readLimit); 608 mark = left; 609 } 610 611 @Override public int read() throws IOException { 612 if (left == 0) { 613 return -1; 614 } 615 616 int result = in.read(); 617 if (result != -1) { 618 --left; 619 } 620 return result; 621 } 622 623 @Override public int read(byte[] b, int off, int len) throws IOException { 624 if (left == 0) { 625 return -1; 626 } 627 628 len = (int) Math.min(len, left); 629 int result = in.read(b, off, len); 630 if (result != -1) { 631 left -= result; 632 } 633 return result; 634 } 635 636 @Override public synchronized void reset() throws IOException { 637 if (!in.markSupported()) { 638 throw new IOException("Mark not supported"); 639 } 640 if (mark == -1) { 641 throw new IOException("Mark not set"); 642 } 643 644 in.reset(); 645 left = mark; 646 } 647 648 @Override public long skip(long n) throws IOException { 649 n = Math.min(n, left); 650 long skipped = in.skip(n); 651 left -= skipped; 652 return skipped; 653 } 654 } 655 656 /** 657 * Attempts to read enough bytes from the stream to fill the given byte array, 658 * with the same behavior as {@link DataInput#readFully(byte[])}. 659 * Does not close the stream. 660 * 661 * @param in the input stream to read from. 662 * @param b the buffer into which the data is read. 663 * @throws EOFException if this stream reaches the end before reading all 664 * the bytes. 665 * @throws IOException if an I/O error occurs. 666 */ 667 public static void readFully(InputStream in, byte[] b) throws IOException { 668 readFully(in, b, 0, b.length); 669 } 670 671 /** 672 * Attempts to read {@code len} bytes from the stream into the given array 673 * starting at {@code off}, with the same behavior as 674 * {@link DataInput#readFully(byte[], int, int)}. Does not close the 675 * stream. 676 * 677 * @param in the input stream to read from. 678 * @param b the buffer into which the data is read. 679 * @param off an int specifying the offset into the data. 680 * @param len an int specifying the number of bytes to read. 681 * @throws EOFException if this stream reaches the end before reading all 682 * the bytes. 683 * @throws IOException if an I/O error occurs. 684 */ 685 public static void readFully( 686 InputStream in, byte[] b, int off, int len) throws IOException { 687 int read = read(in, b, off, len); 688 if (read != len) { 689 throw new EOFException("reached end of stream after reading " 690 + read + " bytes; " + len + " bytes expected"); 691 } 692 } 693 694 /** 695 * Discards {@code n} bytes of data from the input stream. This method 696 * will block until the full amount has been skipped. Does not close the 697 * stream. 698 * 699 * @param in the input stream to read from 700 * @param n the number of bytes to skip 701 * @throws EOFException if this stream reaches the end before skipping all 702 * the bytes 703 * @throws IOException if an I/O error occurs, or the stream does not 704 * support skipping 705 */ 706 public static void skipFully(InputStream in, long n) throws IOException { 707 long toSkip = n; 708 while (n > 0) { 709 long amt = in.skip(n); 710 if (amt == 0) { 711 // Force a blocking read to avoid infinite loop 712 if (in.read() == -1) { 713 long skipped = toSkip - n; 714 throw new EOFException("reached end of stream after skipping " 715 + skipped + " bytes; " + toSkip + " bytes expected"); 716 } 717 n--; 718 } else { 719 n -= amt; 720 } 721 } 722 } 723 724 /** 725 * Process the bytes of the given input stream using the given processor. 726 * 727 * @param input the input stream to process 728 * @param processor the object to which to pass the bytes of the stream 729 * @return the result of the byte processor 730 * @throws IOException if an I/O error occurs 731 * @since 14.0 732 */ 733 public static <T> T readBytes( 734 InputStream input, ByteProcessor<T> processor) throws IOException { 735 checkNotNull(input); 736 checkNotNull(processor); 737 738 byte[] buf = new byte[BUF_SIZE]; 739 int read; 740 do { 741 read = input.read(buf); 742 } while (read != -1 && processor.processBytes(buf, 0, read)); 743 return processor.getResult(); 744 } 745 746 /** 747 * Reads some bytes from an input stream and stores them into the buffer array 748 * {@code b}. This method blocks until {@code len} bytes of input data have 749 * been read into the array, or end of file is detected. The number of bytes 750 * read is returned, possibly zero. Does not close the stream. 751 * 752 * <p>A caller can detect EOF if the number of bytes read is less than 753 * {@code len}. All subsequent calls on the same stream will return zero. 754 * 755 * <p>If {@code b} is null, a {@code NullPointerException} is thrown. If 756 * {@code off} is negative, or {@code len} is negative, or {@code off+len} is 757 * greater than the length of the array {@code b}, then an 758 * {@code IndexOutOfBoundsException} is thrown. If {@code len} is zero, then 759 * no bytes are read. Otherwise, the first byte read is stored into element 760 * {@code b[off]}, the next one into {@code b[off+1]}, and so on. The number 761 * of bytes read is, at most, equal to {@code len}. 762 * 763 * @param in the input stream to read from 764 * @param b the buffer into which the data is read 765 * @param off an int specifying the offset into the data 766 * @param len an int specifying the number of bytes to read 767 * @return the number of bytes read 768 * @throws IOException if an I/O error occurs 769 */ 770 public static int read(InputStream in, byte[] b, int off, int len) 771 throws IOException { 772 checkNotNull(in); 773 checkNotNull(b); 774 if (len < 0) { 775 throw new IndexOutOfBoundsException("len is negative"); 776 } 777 int total = 0; 778 while (total < len) { 779 int result = in.read(b, off + total, len - total); 780 if (result == -1) { 781 break; 782 } 783 total += result; 784 } 785 return total; 786 } 787}