    001/*
    002 *  Licensed to the Apache Software Foundation (ASF) under one or more
    003 *  contributor license agreements.  See the NOTICE file distributed with
    004 *  this work for additional information regarding copyright ownership.
    005 *  The ASF licenses this file to You under the Apache License, Version 2.0
    006 *  (the "License"); you may not use this file except in compliance with
    007 *  the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 *  Unless required by applicable law or agreed to in writing, software
    012 *  distributed under the License is distributed on an "AS IS" BASIS,
    013 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 *  See the License for the specific language governing permissions and
    015 *  limitations under the License.
    016 */
    017package org.apache.commons.compress.archivers.zip;
    018
    019
    020import java.io.Closeable;
    021import java.io.File;
    022import java.io.FileNotFoundException;
    023import java.io.IOException;
    024import java.io.InputStream;
    025import java.nio.file.Path;
    026import java.util.Iterator;
    027import java.util.Queue;
    028import java.util.concurrent.ConcurrentLinkedQueue;
    029import java.util.concurrent.atomic.AtomicBoolean;
    030import java.util.zip.Deflater;
    031
    032import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore;
    033import org.apache.commons.compress.parallel.ScatterGatherBackingStore;
    034import org.apache.commons.compress.utils.BoundedInputStream;
    035
    036/**
    037 * A ZIP output stream that is optimized for multi-threaded scatter/gather construction of ZIP files.
    038 * <p>
    039 * The internal data format of the entries used by this class are entirely private to this class and are not part of any public api whatsoever.
    040 * </p>
    041 * <p>
    042 * It is possible to extend this class to support different kinds of backing storage, the default implementation only supports file-based backing.
    043 * </p>
    044 * <p>
    045 * Thread safety: This class supports multiple threads. But the "writeTo" method must be called by the thread that originally created the
    046 * {@link ZipArchiveEntry}.
    047 * </p>
    048 *
    049 * @since 1.10
    050 */
    051public class ScatterZipOutputStream implements Closeable {
    052
    053    private static class CompressedEntry {
    054        final ZipArchiveEntryRequest zipArchiveEntryRequest;
    055        final long crc;
    056        final long compressedSize;
    057        final long size;
    058
    059        public CompressedEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest, final long crc, final long compressedSize, final long size) {
    060            this.zipArchiveEntryRequest = zipArchiveEntryRequest;
    061            this.crc = crc;
    062            this.compressedSize = compressedSize;
    063            this.size = size;
    064        }
    065
    066        /**
    067         * Updates the original {@link ZipArchiveEntry} with sizes/crc.
    068         * Do not use this method from threads that did not create the instance itself!
    069         * @return the zipArchiveEntry that is the basis for this request.
    070         */
    071
    072        public ZipArchiveEntry transferToArchiveEntry() {
    073            final ZipArchiveEntry entry = zipArchiveEntryRequest.getZipArchiveEntry();
    074            entry.setCompressedSize(compressedSize);
    075            entry.setSize(size);
    076            entry.setCrc(crc);
    077            entry.setMethod(zipArchiveEntryRequest.getMethod());
    078            return entry;
    079        }
    080    }
    081
    082    public static class ZipEntryWriter implements Closeable {
    083        private final Iterator<CompressedEntry> itemsIterator;
    084        private final InputStream itemsIteratorData;
    085
    086        public ZipEntryWriter(final ScatterZipOutputStream scatter) throws IOException {
    087            scatter.backingStore.closeForWriting();
    088            itemsIterator = scatter.items.iterator();
    089            itemsIteratorData = scatter.backingStore.getInputStream();
    090        }
    091
    092        @Override
    093        public void close() throws IOException {
    094            if (itemsIteratorData != null) {
    095                itemsIteratorData.close();
    096            }
    097        }
    098
    099        public void writeNextZipEntry(final ZipArchiveOutputStream target) throws IOException {
    100            final CompressedEntry compressedEntry = itemsIterator.next();
    101            try (final BoundedInputStream rawStream = new BoundedInputStream(itemsIteratorData, compressedEntry.compressedSize)) {
    102                target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
    103            }
    104        }
    105    }
    106
    107    /**
    108     * Creates a {@link ScatterZipOutputStream} with default compression level that is backed by a file
    109     *
    110     * @param file The file to offload compressed data into.
    111     * @return A ScatterZipOutputStream that is ready for use.
    112     * @throws FileNotFoundException if the file cannot be found
    113     */
    114    public static ScatterZipOutputStream fileBased(final File file) throws FileNotFoundException {
    115        return pathBased(file.toPath(), Deflater.DEFAULT_COMPRESSION);
    116    }
    117
    118    /**
    119     * Creates a {@link ScatterZipOutputStream} that is backed by a file
    120     *
    121     * @param file             The file to offload compressed data into.
    122     * @param compressionLevel The compression level to use, @see #Deflater
    123     * @return A  ScatterZipOutputStream that is ready for use.
    124     * @throws FileNotFoundException if the file cannot be found
    125     */
    126    public static ScatterZipOutputStream fileBased(final File file, final int compressionLevel) throws FileNotFoundException {
    127        return pathBased(file.toPath(), compressionLevel);
    128    }
    129
    130    /**
    131     * Creates a {@link ScatterZipOutputStream} with default compression level that is backed by a file
    132     * @param path The path to offload compressed data into.
    133     * @return A ScatterZipOutputStream that is ready for use.
    134     * @throws FileNotFoundException if the path cannot be found
    135     * @since 1.22
    136     */
    137    public static ScatterZipOutputStream pathBased(final Path path) throws FileNotFoundException {
    138        return pathBased(path, Deflater.DEFAULT_COMPRESSION);
    139    }
    140
    141    /**
    142     * Creates a {@link ScatterZipOutputStream} that is backed by a file
    143     * @param path The path to offload compressed data into.
    144     * @param compressionLevel The compression level to use, @see #Deflater
    145     * @return A ScatterZipOutputStream that is ready for use.
    146     * @throws FileNotFoundException if the path cannot be found
    147     * @since 1.22
    148     */
    149    public static ScatterZipOutputStream pathBased(final Path path, final int compressionLevel) throws FileNotFoundException {
    150        final ScatterGatherBackingStore bs = new FileBasedScatterGatherBackingStore(path);
    151        // lifecycle is bound to the ScatterZipOutputStream returned
    152        final StreamCompressor sc = StreamCompressor.create(compressionLevel, bs); //NOSONAR
    153        return new ScatterZipOutputStream(bs, sc);
    154    }
    155
    156    private final Queue<CompressedEntry> items = new ConcurrentLinkedQueue<>();
    157
    158    private final ScatterGatherBackingStore backingStore;
    159
    160    private final StreamCompressor streamCompressor;
    161
    162    private final AtomicBoolean isClosed = new AtomicBoolean();
    163
    164    private ZipEntryWriter zipEntryWriter;
    165
    166    public ScatterZipOutputStream(final ScatterGatherBackingStore backingStore,
    167                                  final StreamCompressor streamCompressor) {
    168        this.backingStore = backingStore;
    169        this.streamCompressor = streamCompressor;
    170    }
    171
    172    /**
    173     * Adds an archive entry to this scatter stream.
    174     *
    175     * @param zipArchiveEntryRequest The entry to write.
    176     * @throws IOException    If writing fails
    177     */
    178    public void addArchiveEntry(final ZipArchiveEntryRequest zipArchiveEntryRequest) throws IOException {
    179        try (final InputStream payloadStream = zipArchiveEntryRequest.getPayloadStream()) {
    180            streamCompressor.deflate(payloadStream, zipArchiveEntryRequest.getMethod());
    181        }
    182        items.add(new CompressedEntry(zipArchiveEntryRequest, streamCompressor.getCrc32(),
    183                                      streamCompressor.getBytesWrittenForLastEntry(), streamCompressor.getBytesRead()));
    184    }
    185
    186    /**
    187     * Closes this stream, freeing all resources involved in the creation of this stream.
    188     * @throws IOException If closing fails
    189     */
    190    @Override
    191    public void close() throws IOException {
    192        if (!isClosed.compareAndSet(false, true)) {
    193            return;
    194        }
    195        try {
    196            if (zipEntryWriter != null) {
    197                zipEntryWriter.close();
    198            }
    199            backingStore.close();
    200        } finally {
    201            streamCompressor.close();
    202        }
    203    }
    204
    205    /**
    206     * Writes the contents of this scatter stream to a target archive.
    207     *
    208     * @param target The archive to receive the contents of this {@link ScatterZipOutputStream}.
    209     * @throws IOException If writing fails
    210     * @see #zipEntryWriter()
    211     */
    212    public void writeTo(final ZipArchiveOutputStream target) throws IOException {
    213        backingStore.closeForWriting();
    214        try (final InputStream data = backingStore.getInputStream()) {
    215            for (final CompressedEntry compressedEntry : items) {
    216                try (final BoundedInputStream rawStream = new BoundedInputStream(data,
    217                        compressedEntry.compressedSize)) {
    218                    target.addRawArchiveEntry(compressedEntry.transferToArchiveEntry(), rawStream);
    219                }
    220            }
    221        }
    222    }
    223
    224    /**
    225     * Gets a ZIP entry writer for this scatter stream.
    226     * @throws IOException If getting scatter stream input stream
    227     * @return the ZipEntryWriter created on first call of the method
    228     */
    229    public ZipEntryWriter zipEntryWriter() throws IOException {
    230        if (zipEntryWriter == null) {
    231            zipEntryWriter = new ZipEntryWriter(this);
    232        }
    233        return zipEntryWriter;
    234    }
    235}