diff --git a/project.clj b/project.clj index 0bd10e71..bbf5b9eb 100644 --- a/project.clj +++ b/project.clj @@ -11,9 +11,7 @@ [[org.clojure/tools.reader "1.3.7"] [com.taoensso/encore "3.77.0"] [org.tukaani/xz "1.9"] - [io.airlift/aircompressor "0.25"] - [org.iq80.snappy/snappy "0.4"] - [org.xerial.snappy/snappy-java "1.1.10.5"]] + [io.airlift/aircompressor "0.25"]] :profiles {;; :default [:base :system :user :provided :dev] diff --git a/src/taoensso/nippy.clj b/src/taoensso/nippy.clj index 27046512..8ffd5016 100644 --- a/src/taoensso/nippy.clj +++ b/src/taoensso/nippy.clj @@ -337,7 +337,7 @@ compression/lz4-compressor compression/lz4hc-compressor #_compression/lzo-compressor - compression/snappy-compressor + #_compression/snappy-compressor ; Can be unsafe compression/lzma2-compressor encryption/encrypt @@ -1704,7 +1704,7 @@ (defn- get-auto-compressor [compressor-id] (case compressor-id nil nil - :snappy snappy-compressor + :snappy compression/snappy-compressor :lzma2 lzma2-compressor :lz4 lz4-compressor :no-header (throw (ex-info ":auto not supported on headerless data." {})) @@ -1806,7 +1806,7 @@ (catch Exception e (ex-fn e))))) - ;; Hackish + can actually segfault JVM due to Snappy bug, + ;; Hacky + can actually segfault JVM due to Snappy bug, ;; Ref. - no better alternatives, unfortunately thaw-v1-data (fn [data-ba ex-fn] diff --git a/src/taoensso/nippy/compression.clj b/src/taoensso/nippy/compression.clj index 801d35f2..5c27191e 100644 --- a/src/taoensso/nippy/compression.clj +++ b/src/taoensso/nippy/compression.clj @@ -116,8 +116,6 @@ (compress [_ ba] (airlift-compress @airlift-lzo-compressor_ ba true)) (decompress [_ ba] (airlift-decompress @airlift-lzo-decompressor_ ba nil)))) -;; Using `io.airlift/aircompressor`, vulnerable to https://github.com/airlift/aircompressor/issues/183 -#_ (do (enc/def* ^:private airlift-snappy-compressor_ (enc/thread-local (io.airlift.compress.snappy.SnappyCompressor.))) (enc/def* ^:private airlift-snappy-decompressor_ (enc/thread-local (io.airlift.compress.snappy.SnappyDecompressor.))) @@ -129,21 +127,6 @@ (when-not prepend-size? (io.airlift.compress.snappy.SnappyDecompressor/getUncompressedLength ba 0)))))) -;; Using `org.iq80.snappy/snappy`, vulnerable to https://github.com/airlift/aircompressor/issues/183 -#_ -(deftype SnappyCompressor [_] - ICompressor - (header-id [_] :snappy) - (compress [_ ba] (org.iq80.snappy.Snappy/compress ba)) - (decompress [_ ba] (org.iq80.snappy.Snappy/uncompress ba 0 (alength ^bytes ba)))) - -;; Using `org.xerial.snappy/snappy-java`, some compatibility issues due to JNI -(deftype SnappyCompressor [_] - ICompressor - (header-id [_] :snappy) - (compress [_ ba] (org.xerial.snappy.Snappy/compress ba)) - (decompress [_ ba] (org.xerial.snappy.Snappy/uncompress ba))) - ;;;; LZMA2 (deftype LZMA2Compressor [compression-level] @@ -208,16 +191,6 @@ See `taoensso.nippy-benchmarks` for detailed comparative benchmarks." (LZOCompressor.)) -(def snappy-compressor - "Default `Snappy` compressor: - - Compression ratio: `C` (0.58 on reference benchmark). - - Compression speed: `A+` (206 msecs on reference benchmark). - - Decompression speed: `B` (134 msecs on reference benchmark). - - Good general-purpose compressor, favours speed. - See `taoensso.nippy-benchmarks` for detailed comparative benchmarks." - (SnappyCompressor. false)) - (def lzma2-compressor "Default `LZMA2` compressor: - Compression ratio: `A+` (0.4 on reference benchmark). @@ -232,3 +205,9 @@ "Different LZ4 modes no longer supported, prefer `lz4-compressor`." {:deprecated "vX.Y.Z (YYYY-MM-DD)"} (LZ4Compressor.)) + +(enc/def* ^:no-doc snappy-compressor + "Snappy compressor no longer recommended, prefer `lz4-compressor`. + Decompression can be unsafe against untrusted data!" + {:deprecated "vX.Y.Z (YYYY-MM-DD)"} + (SnappyCompressor. false)) diff --git a/test/taoensso/nippy_tests.clj b/test/taoensso/nippy_tests.clj index db923cfa..06bf095d 100644 --- a/test/taoensso/nippy_tests.clj +++ b/test/taoensso/nippy_tests.clj @@ -426,13 +426,13 @@ (doseq [c [compr/zstd-compressor compr/lz4-compressor compr/lzo-compressor - compr/snappy-compressor + #_compr/snappy-compressor ; Ref. compr/lzma2-compressor]] (dotimes [_ 2e4] (is (nil? (enc/catching (compr/decompress c (crypto/rand-bytes 1024)))) - "Decompression never core dumps, even against invalid data")))) + "Decompression never crashes JVM, even against invalid data")))) ;;;; Benchmarks