001 /*
002 * Copyright (C) 2007 Google Inc.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017 package com.google.common.io;
018
019 import com.google.common.annotations.Beta;
020 import com.google.common.base.Preconditions;
021
022 import java.io.ByteArrayInputStream;
023 import java.io.ByteArrayOutputStream;
024 import java.io.DataInput;
025 import java.io.DataInputStream;
026 import java.io.DataOutput;
027 import java.io.DataOutputStream;
028 import java.io.EOFException;
029 import java.io.IOException;
030 import java.io.InputStream;
031 import java.io.OutputStream;
032 import java.nio.ByteBuffer;
033 import java.nio.channels.ReadableByteChannel;
034 import java.nio.channels.WritableByteChannel;
035 import java.security.MessageDigest;
036 import java.util.Arrays;
037 import java.util.zip.Checksum;
038
039 /**
040 * Provides utility methods for working with byte arrays and I/O streams.
041 *
042 * <p>All method parameters must be non-null unless documented otherwise.
043 *
044 * @author Chris Nokleberg
045 * @since 1
046 */
047 @Beta
048 public final class ByteStreams {
049 private static final int BUF_SIZE = 0x1000; // 4K
050
051 private ByteStreams() {}
052
053 /**
054 * Returns a factory that will supply instances of
055 * {@link ByteArrayInputStream} that read from the given byte array.
056 *
057 * @param b the input buffer
058 * @return the factory
059 */
060 public static InputSupplier<ByteArrayInputStream> newInputStreamSupplier(
061 byte[] b) {
062 return newInputStreamSupplier(b, 0, b.length);
063 }
064
065 /**
066 * Returns a factory that will supply instances of
067 * {@link ByteArrayInputStream} that read from the given byte array.
068 *
069 * @param b the input buffer
070 * @param off the offset in the buffer of the first byte to read
071 * @param len the maximum number of bytes to read from the buffer
072 * @return the factory
073 */
074 public static InputSupplier<ByteArrayInputStream> newInputStreamSupplier(
075 final byte[] b, final int off, final int len) {
076 return new InputSupplier<ByteArrayInputStream>() {
077 public ByteArrayInputStream getInput() {
078 return new ByteArrayInputStream(b, off, len);
079 }
080 };
081 }
082
083 /**
084 * Writes a byte array to an output stream from the given supplier.
085 *
086 * @param from the bytes to write
087 * @param to the output supplier
088 * @throws IOException if an I/O error occurs
089 */
090 public static void write(byte[] from,
091 OutputSupplier<? extends OutputStream> to) throws IOException {
092 Preconditions.checkNotNull(from);
093 boolean threw = true;
094 OutputStream out = to.getOutput();
095 try {
096 out.write(from);
097 threw = false;
098 } finally {
099 Closeables.close(out, threw);
100 }
101 }
102
103 /**
104 * Opens input and output streams from the given suppliers, copies all
105 * bytes from the input to the output, and closes the streams.
106 *
107 * @param from the input factory
108 * @param to the output factory
109 * @return the number of bytes copied
110 * @throws IOException if an I/O error occurs
111 */
112 public static long copy(InputSupplier<? extends InputStream> from,
113 OutputSupplier<? extends OutputStream> to) throws IOException {
114 boolean threw = true;
115 InputStream in = from.getInput();
116 try {
117 OutputStream out = to.getOutput();
118 try {
119 long count = copy(in, out);
120 threw = false;
121 return count;
122 } finally {
123 Closeables.close(out, threw);
124 }
125 } finally {
126 Closeables.close(in, threw);
127 }
128 }
129
130 /**
131 * Opens an input stream from the supplier, copies all bytes from the
132 * input to the output, and closes the input stream. Does not close
133 * or flush the output stream.
134 *
135 * @param from the input factory
136 * @param to the output stream to write to
137 * @return the number of bytes copied
138 * @throws IOException if an I/O error occurs
139 */
140 public static long copy(InputSupplier<? extends InputStream> from,
141 OutputStream to) throws IOException {
142 boolean threw = true;
143 InputStream in = from.getInput();
144 try {
145 long count = copy(in, to);
146 threw = false;
147 return count;
148 } finally {
149 Closeables.close(in, threw);
150 }
151 }
152
153 /**
154 * Copies all bytes from the input stream to the output stream.
155 * Does not close or flush either stream.
156 *
157 * @param from the input stream to read from
158 * @param to the output stream to write to
159 * @return the number of bytes copied
160 * @throws IOException if an I/O error occurs
161 */
162 public static long copy(InputStream from, OutputStream to)
163 throws IOException {
164 byte[] buf = new byte[BUF_SIZE];
165 long total = 0;
166 while (true) {
167 int r = from.read(buf);
168 if (r == -1) {
169 break;
170 }
171 to.write(buf, 0, r);
172 total += r;
173 }
174 return total;
175 }
176
177 /**
178 * Copies all bytes from the readable channel to the writable channel.
179 * Does not close or flush either channel.
180 *
181 * @param from the readable channel to read from
182 * @param to the writable channel to write to
183 * @return the number of bytes copied
184 * @throws IOException if an I/O error occurs
185 */
186 public static long copy(ReadableByteChannel from,
187 WritableByteChannel to) throws IOException {
188 ByteBuffer buf = ByteBuffer.allocate(BUF_SIZE);
189 long total = 0;
190 while (from.read(buf) != -1) {
191 buf.flip();
192 while (buf.hasRemaining()) {
193 total += to.write(buf);
194 }
195 buf.clear();
196 }
197 return total;
198 }
199
200 /**
201 * Reads all bytes from an input stream into a byte array.
202 * Does not close the stream.
203 *
204 * @param in the input stream to read from
205 * @return a byte array containing all the bytes from the stream
206 * @throws IOException if an I/O error occurs
207 */
208 public static byte[] toByteArray(InputStream in) throws IOException {
209 ByteArrayOutputStream out = new ByteArrayOutputStream();
210 copy(in, out);
211 return out.toByteArray();
212 }
213
214 /**
215 * Returns the data from a {@link InputStream} factory as a byte array.
216 *
217 * @param supplier the factory
218 * @throws IOException if an I/O error occurs
219 */
220 public static byte[] toByteArray(
221 InputSupplier<? extends InputStream> supplier) throws IOException {
222 boolean threw = true;
223 InputStream in = supplier.getInput();
224 try {
225 byte[] result = toByteArray(in);
226 threw = false;
227 return result;
228 } finally {
229 Closeables.close(in, threw);
230 }
231 }
232
233 /**
234 * Returns a new {@link ByteArrayDataInput} instance to read from the {@code
235 * bytes} array from the beginning.
236 */
237 public static ByteArrayDataInput newDataInput(byte[] bytes) {
238 return new ByteArrayDataInputStream(bytes);
239 }
240
241 /**
242 * Returns a new {@link ByteArrayDataInput} instance to read from the {@code
243 * bytes} array, starting at the given position.
244 *
245 * @throws IndexOutOfBoundsException if {@code start} is negative or greater
246 * than the length of the array
247 */
248 public static ByteArrayDataInput newDataInput(byte[] bytes, int start) {
249 Preconditions.checkPositionIndex(start, bytes.length);
250 return new ByteArrayDataInputStream(bytes, start);
251 }
252
253 private static class ByteArrayDataInputStream implements ByteArrayDataInput {
254 final DataInput input;
255
256 ByteArrayDataInputStream(byte[] bytes) {
257 this.input = new DataInputStream(new ByteArrayInputStream(bytes));
258 }
259
260 ByteArrayDataInputStream(byte[] bytes, int start) {
261 this.input = new DataInputStream(
262 new ByteArrayInputStream(bytes, start, bytes.length - start));
263 }
264
265 @Override public void readFully(byte b[]) {
266 try {
267 input.readFully(b);
268 } catch (IOException e) {
269 throw new IllegalStateException(e);
270 }
271 }
272
273 @Override public void readFully(byte b[], int off, int len) {
274 try {
275 input.readFully(b, off, len);
276 } catch (IOException e) {
277 throw new IllegalStateException(e);
278 }
279 }
280
281 @Override public int skipBytes(int n) {
282 try {
283 return input.skipBytes(n);
284 } catch (IOException e) {
285 throw new IllegalStateException(e);
286 }
287 }
288
289 @Override public boolean readBoolean() {
290 try {
291 return input.readBoolean();
292 } catch (IOException e) {
293 throw new IllegalStateException(e);
294 }
295 }
296
297 @Override public byte readByte() {
298 try {
299 return input.readByte();
300 } catch (EOFException e) {
301 throw new IllegalStateException(e);
302 } catch (IOException impossible) {
303 throw new AssertionError(impossible);
304 }
305 }
306
307 @Override public int readUnsignedByte() {
308 try {
309 return input.readUnsignedByte();
310 } catch (IOException e) {
311 throw new IllegalStateException(e);
312 }
313 }
314
315 @Override public short readShort() {
316 try {
317 return input.readShort();
318 } catch (IOException e) {
319 throw new IllegalStateException(e);
320 }
321 }
322
323 @Override public int readUnsignedShort() {
324 try {
325 return input.readUnsignedShort();
326 } catch (IOException e) {
327 throw new IllegalStateException(e);
328 }
329 }
330
331 @Override public char readChar() {
332 try {
333 return input.readChar();
334 } catch (IOException e) {
335 throw new IllegalStateException(e);
336 }
337 }
338
339 @Override public int readInt() {
340 try {
341 return input.readInt();
342 } catch (IOException e) {
343 throw new IllegalStateException(e);
344 }
345 }
346
347 @Override public long readLong() {
348 try {
349 return input.readLong();
350 } catch (IOException e) {
351 throw new IllegalStateException(e);
352 }
353 }
354
355 @Override public float readFloat() {
356 try {
357 return input.readFloat();
358 } catch (IOException e) {
359 throw new IllegalStateException(e);
360 }
361 }
362
363 @Override public double readDouble() {
364 try {
365 return input.readDouble();
366 } catch (IOException e) {
367 throw new IllegalStateException(e);
368 }
369 }
370
371 @Override public String readLine() {
372 try {
373 return input.readLine();
374 } catch (IOException e) {
375 throw new IllegalStateException(e);
376 }
377 }
378
379 @Override public String readUTF() {
380 try {
381 return input.readUTF();
382 } catch (IOException e) {
383 throw new IllegalStateException(e);
384 }
385 }
386 }
387
388 /**
389 * Returns a new {@link ByteArrayDataOutput} instance with a default size.
390 */
391 public static ByteArrayDataOutput newDataOutput() {
392 return new ByteArrayDataOutputStream();
393 }
394
395 /**
396 * Returns a new {@link ByteArrayDataOutput} instance sized to hold
397 * {@code size} bytes before resizing.
398 *
399 * @throws IllegalArgumentException if {@code size} is negative
400 */
401 public static ByteArrayDataOutput newDataOutput(int size) {
402 Preconditions.checkArgument(size >= 0, "Invalid size: %s", size);
403 return new ByteArrayDataOutputStream(size);
404 }
405
406 @SuppressWarnings("deprecation") // for writeBytes
407 private static class ByteArrayDataOutputStream
408 implements ByteArrayDataOutput {
409
410 final DataOutput output;
411 final ByteArrayOutputStream byteArrayOutputSteam;
412
413 ByteArrayDataOutputStream() {
414 this(new ByteArrayOutputStream());
415 }
416
417 ByteArrayDataOutputStream(int size) {
418 this(new ByteArrayOutputStream(size));
419 }
420
421 ByteArrayDataOutputStream(ByteArrayOutputStream byteArrayOutputSteam) {
422 this.byteArrayOutputSteam = byteArrayOutputSteam;
423 output = new DataOutputStream(byteArrayOutputSteam);
424 }
425
426 @Override public void write(int b) {
427 try {
428 output.write(b);
429 } catch (IOException impossible) {
430 throw new AssertionError(impossible);
431 }
432 }
433
434 @Override public void write(byte[] b) {
435 try {
436 output.write(b);
437 } catch (IOException impossible) {
438 throw new AssertionError(impossible);
439 }
440 }
441
442 @Override public void write(byte[] b, int off, int len) {
443 try {
444 output.write(b, off, len);
445 } catch (IOException impossible) {
446 throw new AssertionError(impossible);
447 }
448 }
449
450 @Override public void writeBoolean(boolean v) {
451 try {
452 output.writeBoolean(v);
453 } catch (IOException impossible) {
454 throw new AssertionError(impossible);
455 }
456 }
457
458 @Override public void writeByte(int v) {
459 try {
460 output.writeByte(v);
461 } catch (IOException impossible) {
462 throw new AssertionError(impossible);
463 }
464 }
465
466 @Override public void writeBytes(String s) {
467 try {
468 output.writeBytes(s);
469 } catch (IOException impossible) {
470 throw new AssertionError(impossible);
471 }
472 }
473
474 @Override public void writeChar(int v) {
475 try {
476 output.writeChar(v);
477 } catch (IOException impossible) {
478 throw new AssertionError(impossible);
479 }
480 }
481
482 @Override public void writeChars(String s) {
483 try {
484 output.writeChars(s);
485 } catch (IOException impossible) {
486 throw new AssertionError(impossible);
487 }
488 }
489
490 @Override public void writeDouble(double v) {
491 try {
492 output.writeDouble(v);
493 } catch (IOException impossible) {
494 throw new AssertionError(impossible);
495 }
496 }
497
498 @Override public void writeFloat(float v) {
499 try {
500 output.writeFloat(v);
501 } catch (IOException impossible) {
502 throw new AssertionError(impossible);
503 }
504 }
505
506 @Override public void writeInt(int v) {
507 try {
508 output.writeInt(v);
509 } catch (IOException impossible) {
510 throw new AssertionError(impossible);
511 }
512 }
513
514 @Override public void writeLong(long v) {
515 try {
516 output.writeLong(v);
517 } catch (IOException impossible) {
518 throw new AssertionError(impossible);
519 }
520 }
521
522 @Override public void writeShort(int v) {
523 try {
524 output.writeShort(v);
525 } catch (IOException impossible) {
526 throw new AssertionError(impossible);
527 }
528 }
529
530 @Override public void writeUTF(String s) {
531 try {
532 output.writeUTF(s);
533 } catch (IOException impossible) {
534 throw new AssertionError(impossible);
535 }
536 }
537
538 @Override public byte[] toByteArray() {
539 return byteArrayOutputSteam.toByteArray();
540 }
541
542 }
543
544 // TODO: Not all streams support skipping.
545 /** Returns the length of a supplied input stream, in bytes. */
546 public static long length(InputSupplier<? extends InputStream> supplier)
547 throws IOException {
548 long count = 0;
549 boolean threw = true;
550 InputStream in = supplier.getInput();
551 try {
552 while (true) {
553 // We skip only Integer.MAX_VALUE due to JDK overflow bugs.
554 long amt = in.skip(Integer.MAX_VALUE);
555 if (amt == 0) {
556 if (in.read() == -1) {
557 threw = false;
558 return count;
559 }
560 count++;
561 } else {
562 count += amt;
563 }
564 }
565 } finally {
566 Closeables.close(in, threw);
567 }
568 }
569
570 /**
571 * Returns true if the supplied input streams contain the same bytes.
572 *
573 * @throws IOException if an I/O error occurs
574 */
575 public static boolean equal(InputSupplier<? extends InputStream> supplier1,
576 InputSupplier<? extends InputStream> supplier2) throws IOException {
577 byte[] buf1 = new byte[BUF_SIZE];
578 byte[] buf2 = new byte[BUF_SIZE];
579
580 boolean threw = true;
581 InputStream in1 = supplier1.getInput();
582 try {
583 InputStream in2 = supplier2.getInput();
584 try {
585 while (true) {
586 int read1 = read(in1, buf1, 0, BUF_SIZE);
587 int read2 = read(in2, buf2, 0, BUF_SIZE);
588 if (read1 != read2 || !Arrays.equals(buf1, buf2)) {
589 threw = false;
590 return false;
591 } else if (read1 != BUF_SIZE) {
592 threw = false;
593 return true;
594 }
595 }
596 } finally {
597 Closeables.close(in2, threw);
598 }
599 } finally {
600 Closeables.close(in1, threw);
601 }
602 }
603
604 /**
605 * Attempts to read enough bytes from the stream to fill the given byte array,
606 * with the same behavior as {@link DataInput#readFully(byte[])}.
607 * Does not close the stream.
608 *
609 * @param in the input stream to read from.
610 * @param b the buffer into which the data is read.
611 * @throws EOFException if this stream reaches the end before reading all
612 * the bytes.
613 * @throws IOException if an I/O error occurs.
614 */
615 public static void readFully(InputStream in, byte[] b) throws IOException {
616 readFully(in, b, 0, b.length);
617 }
618
619 /**
620 * Attempts to read {@code len} bytes from the stream into the given array
621 * starting at {@code off}, with the same behavior as
622 * {@link DataInput#readFully(byte[], int, int)}. Does not close the
623 * stream.
624 *
625 * @param in the input stream to read from.
626 * @param b the buffer into which the data is read.
627 * @param off an int specifying the offset into the data.
628 * @param len an int specifying the number of bytes to read.
629 * @throws EOFException if this stream reaches the end before reading all
630 * the bytes.
631 * @throws IOException if an I/O error occurs.
632 */
633 public static void readFully(InputStream in, byte[] b, int off, int len)
634 throws IOException {
635 if (read(in, b, off, len) != len) {
636 throw new EOFException();
637 }
638 }
639
640 /**
641 * Discards {@code n} bytes of data from the input stream. This method
642 * will block until the full amount has been skipped. Does not close the
643 * stream.
644 *
645 * @param in the input stream to read from
646 * @param n the number of bytes to skip
647 * @throws EOFException if this stream reaches the end before skipping all
648 * the bytes
649 * @throws IOException if an I/O error occurs, or the stream does not
650 * support skipping
651 */
652 public static void skipFully(InputStream in, long n) throws IOException {
653 while (n > 0) {
654 long amt = in.skip(n);
655 if (amt == 0) {
656 // Force a blocking read to avoid infinite loop
657 if (in.read() == -1) {
658 throw new EOFException();
659 }
660 n--;
661 } else {
662 n -= amt;
663 }
664 }
665 }
666
667 /**
668 * Process the bytes of a supplied stream
669 *
670 * @param supplier the input stream factory
671 * @param processor the object to which to pass the bytes of the stream
672 * @return the result of the byte processor
673 * @throws IOException if an I/O error occurs
674 */
675 public static <T> T readBytes(InputSupplier<? extends InputStream> supplier,
676 ByteProcessor<T> processor) throws IOException {
677 byte[] buf = new byte[BUF_SIZE];
678 boolean threw = true;
679 InputStream in = supplier.getInput();
680 try {
681 int amt;
682 do {
683 amt = in.read(buf);
684 if (amt == -1) {
685 threw = false;
686 break;
687 }
688 } while (processor.processBytes(buf, 0, amt));
689 return processor.getResult();
690 } finally {
691 Closeables.close(in, threw);
692 }
693 }
694
695 /**
696 * Computes and returns the checksum value for a supplied input stream.
697 * The checksum object is reset when this method returns successfully.
698 *
699 * @param supplier the input stream factory
700 * @param checksum the checksum object
701 * @return the result of {@link Checksum#getValue} after updating the
702 * checksum object with all of the bytes in the stream
703 * @throws IOException if an I/O error occurs
704 */
705 public static long getChecksum(InputSupplier<? extends InputStream> supplier,
706 final Checksum checksum) throws IOException {
707 return readBytes(supplier, new ByteProcessor<Long>() {
708 public boolean processBytes(byte[] buf, int off, int len) {
709 checksum.update(buf, off, len);
710 return true;
711 }
712
713 public Long getResult() {
714 long result = checksum.getValue();
715 checksum.reset();
716 return result;
717 }
718 });
719 }
720
721 /**
722 * Computes and returns the digest value for a supplied input stream.
723 * The digest object is reset when this method returns successfully.
724 *
725 * @param supplier the input stream factory
726 * @param md the digest object
727 * @return the result of {@link MessageDigest#digest()} after updating the
728 * digest object with all of the bytes in the stream
729 * @throws IOException if an I/O error occurs
730 */
731 public static byte[] getDigest(InputSupplier<? extends InputStream> supplier,
732 final MessageDigest md) throws IOException {
733 return readBytes(supplier, new ByteProcessor<byte[]>() {
734 public boolean processBytes(byte[] buf, int off, int len) {
735 md.update(buf, off, len);
736 return true;
737 }
738
739 public byte[] getResult() {
740 return md.digest();
741 }
742 });
743 }
744
745 /**
746 * Reads some bytes from an input stream and stores them into the buffer array
747 * {@code b}. This method blocks until {@code len} bytes of input data have
748 * been read into the array, or end of file is detected. The number of bytes
749 * read is returned, possibly zero. Does not close the stream.
750 *
751 * <p>A caller can detect EOF if the number of bytes read is less than
752 * {@code len}. All subsequent calls on the same stream will return zero.
753 *
754 * <p>If {@code b} is null, a {@code NullPointerException} is thrown. If
755 * {@code off} is negative, or {@code len} is negative, or {@code off+len} is
756 * greater than the length of the array {@code b}, then an
757 * {@code IndexOutOfBoundsException} is thrown. If {@code len} is zero, then
758 * no bytes are read. Otherwise, the first byte read is stored into element
759 * {@code b[off]}, the next one into {@code b[off+1]}, and so on. The number
760 * of bytes read is, at most, equal to {@code len}.
761 *
762 * @param in the input stream to read from
763 * @param b the buffer into which the data is read
764 * @param off an int specifying the offset into the data
765 * @param len an int specifying the number of bytes to read
766 * @return the number of bytes read
767 * @throws IOException if an I/O error occurs
768 */
769 public static int read(InputStream in, byte[] b, int off, int len)
770 throws IOException {
771 if (len < 0) {
772 throw new IndexOutOfBoundsException("len is negative");
773 }
774 int total = 0;
775 while (total < len) {
776 int result = in.read(b, off + total, len - total);
777 if (result == -1) {
778 break;
779 }
780 total += result;
781 }
782 return total;
783 }
784
785 /**
786 * Returns an {@link InputSupplier} that returns input streams from the
787 * an underlying supplier, where each stream starts at the given
788 * offset and is limited to the specified number of bytes.
789 *
790 * @param supplier the supplier from which to get the raw streams
791 * @param offset the offset in bytes into the underlying stream where
792 * the returned streams will start
793 * @param length the maximum length of the returned streams
794 * @throws IllegalArgumentException if offset or length are negative
795 */
796 public static InputSupplier<InputStream> slice(
797 final InputSupplier<? extends InputStream> supplier,
798 final long offset,
799 final long length) {
800 Preconditions.checkNotNull(supplier);
801 Preconditions.checkArgument(offset >= 0, "offset is negative");
802 Preconditions.checkArgument(length >= 0, "length is negative");
803 return new InputSupplier<InputStream>() {
804 @Override public InputStream getInput() throws IOException {
805 InputStream in = supplier.getInput();
806 if (offset > 0) {
807 try {
808 skipFully(in, offset);
809 } catch (IOException e) {
810 Closeables.closeQuietly(in);
811 throw e;
812 }
813 }
814 return new LimitInputStream(in, length);
815 }
816 };
817 }
818
819 /**
820 * Joins multiple {@link InputStream} suppliers into a single supplier.
821 * Streams returned from the supplier will contain the concatenated data from
822 * the streams of the underlying suppliers.
823 *
824 * <p>Only one underlying input stream will be open at a time. Closing the
825 * joined stream will close the open underlying stream.
826 *
827 * <p>Reading from the joined stream will throw a {@link NullPointerException}
828 * if any of the suppliers are null or return null.
829 *
830 * @param suppliers the suppliers to concatenate
831 * @return a supplier that will return a stream containing the concatenated
832 * stream data
833 */
834 public static InputSupplier<InputStream> join(
835 final Iterable<? extends InputSupplier<? extends InputStream>> suppliers) {
836 return new InputSupplier<InputStream>() {
837 @Override public InputStream getInput() throws IOException {
838 return new MultiInputStream(suppliers.iterator());
839 }
840 };
841 }
842
843 /** Varargs form of {@link #join(Iterable)}. */
844 public static InputSupplier<InputStream> join(
845 InputSupplier<? extends InputStream>... suppliers) {
846 return join(Arrays.asList(suppliers));
847 }
848 }