Fixes #12612 - Use Compression classes for client decoding. (#12613)

* Now using Compression classes for client response decoding.
* Removed oej.client.GZIPContentDecoder, now using only oeh.http.GZIPContentDecoder where necessary.
* Updated OSGi dependencies, now that jetty-client depends on jetty-compression-common.
* Now using ServiceLoader to load the Compression implementations.
* Updated tests that required client jars in a web application.
* Deprecated gzip.mod in favor of compression-gzip.mod.
* Added distribution tests.
* Removed EncoderSink.canEncode(), since it is too late to skip encoding.
* Fixed GzipEncoderSink, to finish deflating when last=true.
* Enhanced logic in CompressionResponse, to skip by status code, content-type, content-encoding and empty content.
* Added tests for CompressionHandler in ee11.
* Updated documentation and migration guide.
* Implemented support for quality values in Accept-Encoding.
* Added module compression-all and correspondent distribution test.

Signed-off-by: Simone Bordet <simone.bordet@gmail.com>

documentation/jetty/modules/code/examples/pom.xml

@@ -70,6 +70,10 @@
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-util-ajax</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty.compression</groupId>
+      <artifactId>jetty-compression-server</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty.ee11</groupId>
       <artifactId>jetty-ee11-servlet</artifactId>

documentation/jetty/modules/code/examples/src/main/java/org/eclipse/jetty/docs/programming/client/http/HTTPClientDocs.java

@@ -426,6 +426,20 @@ public void outputStreamRequestContent() throws Exception
         // end::outputStreamRequestContent[]
     }
 
+    public void removeDecoders() throws Exception
+    {
+        // tag::removeDecoders[]
+        HttpClient httpClient = new HttpClient();
+
+        // Starting HttpClient will discover response content decoders
+        // implementations from the module-path or class-path via ServiceLoader.
+        httpClient.start();
+
+        // Remove all response content decoders.
+        httpClient.getContentDecoderFactories().clear();
+        // end::removeDecoders[]
+    }
+
     public void futureResponseListener() throws Exception
     {
         HttpClient httpClient = new HttpClient();

documentation/jetty/modules/code/examples/src/main/java/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java

@@ -33,6 +33,8 @@
 import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory;
 import org.eclipse.jetty.client.ContentResponse;
 import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.compression.server.CompressionConfig;
+import org.eclipse.jetty.compression.server.CompressionHandler;
 import org.eclipse.jetty.ee11.servlet.DefaultServlet;
 import org.eclipse.jetty.ee11.servlet.ResourceServlet;
 import org.eclipse.jetty.ee11.servlet.ServletContextHandler;
@@ -1400,6 +1402,85 @@ public boolean handle(Request request, Response response, Callback callback)
         // end::contextGzipHandler[]
     }
 
+    public void serverCompressionHandler() throws Exception
+    {
+        // tag::serverCompressionHandler[]
+        Server server = new Server();
+        Connector connector = new ServerConnector(server);
+        server.addConnector(connector);
+
+        // Create and configure CompressionHandler.
+        CompressionHandler compressionHandler = new CompressionHandler();
+        server.setHandler(compressionHandler);
+
+        CompressionConfig compressionConfig = CompressionConfig.builder()
+            // Do not compress these URI paths.
+            .compressExcludePath("/uncompressed")
+            // Also compress POST responses.
+            .compressIncludeMethod("POST")
+            // Do not compress these mime types.
+            .compressExcludeMimeType("font/ttf")
+            .build();
+        // Map the request URI path spec '/*' with the compression configuration.
+        // You can map different path specs with different compression configurations.
+        compressionHandler.putConfiguration("/*", compressionConfig);
+
+        // Create a ContextHandlerCollection to manage contexts.
+        ContextHandlerCollection contexts = new ContextHandlerCollection();
+        compressionHandler.setHandler(contexts);
+
+        server.start();
+        // end::serverCompressionHandler[]
+    }
+
+    public void contextCompressionHandler() throws Exception
+    {
+        class ShopHandler extends Handler.Abstract
+        {
+            @Override
+            public boolean handle(Request request, Response response, Callback callback)
+            {
+                // Implement the shop, remembering to complete the callback.
+                return true;
+            }
+        }
+
+        class RESTHandler extends Handler.Abstract
+        {
+            @Override
+            public boolean handle(Request request, Response response, Callback callback)
+            {
+                // Implement the REST APIs, remembering to complete the callback.
+                return true;
+            }
+        }
+
+        // tag::contextCompressionHandler[]
+        Server server = new Server();
+        ServerConnector connector = new ServerConnector(server);
+        server.addConnector(connector);
+
+        // Create a ContextHandlerCollection to hold contexts.
+        ContextHandlerCollection contextCollection = new ContextHandlerCollection();
+        // Link the ContextHandlerCollection to the Server.
+        server.setHandler(contextCollection);
+
+        // Create the context for the shop web application wrapped with CompressionHandler so only the shop will do compression.
+        CompressionHandler shopCompressionHandler = new CompressionHandler(new ContextHandler(new ShopHandler(), "/shop"));
+
+        // Add it to ContextHandlerCollection.
+        contextCollection.addHandler(shopCompressionHandler);
+
+        // Create the context for the API web application.
+        ContextHandler apiContext = new ContextHandler(new RESTHandler(), "/api");
+
+        // Add it to ContextHandlerCollection.
+        contextCollection.addHandler(apiContext);
+
+        server.start();
+        // end::contextCompressionHandler[]
+    }
+
     public void rewriteHandler() throws Exception
     {
         // tag::rewriteHandler[]

documentation/jetty/modules/operations-guide/pages/modules/standard.adoc

@@ -54,6 +54,57 @@ This property allows you to cap the max heap memory retained by the pool.
 `jetty.byteBufferPool.maxDirectMemory`::
 This property allows you to cap the max direct memory retained by the pool.
 
+[[compression]]
+== Compression Modules
+
+The compression modules allow you to configure server-wide request decompression and response compression by installing the `org.eclipse.jetty.compression.server.CompressionHandler` at the root of the `Handler` tree (see also xref:programming-guide:server/http.adoc#handler-use-compression[this section] for further details).
+
+The supported algorithms are the following:
+
+* brotli, via the <<compression-brotli,compression-brotli>> module
+* gzip, via the <<compression-gzip,compression-gzip>> module
+* zstandard, via the <<compression-zstandard,compression-zstandard>> module
+
+You can explicitly enable one or more of the compression modules, or all of them with the <<compression-all,compression-all>> module.
+
+[[compression-all]]
+=== Module `compression-all`
+
+This module enables request decompression and response compression for all the currently supported algorithms listed in <<compression,this section>>.
+
+[[compression-brotli]]
+=== Module `compression-brotli`
+
+This module enables request decompression and response compression with the link:https://github.com/google/brotli[brotli] algorithm.
+
+The module properties are:
+
+----
+include::{jetty-home}/modules/compression-brotli.mod[tags=documentation]
+----
+
+[[compression-gzip]]
+=== Module `compression-gzip`
+
+This module enables request decompression and response compression with the link:https://en.wikipedia.org/wiki/Gzip[gzip] algorithm.
+
+The module properties are:
+
+----
+include::{jetty-home}/modules/compression-gzip.mod[tags=documentation]
+----
+
+[[compression-zstandard]]
+=== Module `compression-zstandard`
+
+This module enables request decompression and response compression with the link:https://github.com/facebook/zstd[zstdandard] algorithm.
+
+The module properties are:
+
+----
+include::{jetty-home}/modules/compression-zstandard.mod[tags=documentation]
+----
+
 [[connectionlimit]]
 == Module `connectionlimit`
 

documentation/jetty/modules/programming-guide/pages/client/http.adoc

@@ -45,6 +45,7 @@ Out of the box features that you get with the Jetty HTTP client include:
 * Cookies support -- cookies sent by servers are stored and sent back to servers in matching requests.
 * Authentication support -- HTTP "Basic", "Digest" and "SPNEGO" authentications are supported, others are pluggable.
 * Forward proxy support -- HTTP proxying, SOCKS4 and SOCKS5 proxying.
+* Response content decoding -- response content compressed by the server is transparently decompressed.
 
 [[start]]
 == Starting HttpClient
@@ -114,7 +115,8 @@ A `HttpClient` instance can be thought as a browser instance, and it manages the
 * A `CookieStore` (see <<cookie,this section>>).
 * A `AuthenticationStore` (see <<authentication,this section>>).
 * A `ProxyConfiguration` (see <<proxy,this section>>).
-* A set of ``Destination``s
+* A set of ``ContentDecoder.Factory``s (see <<content-response,this section>>).
+* A set of ``Destination``s, where each `Destination` manages a <<connection-pool,pool of connections>>.
 
 A `Destination` is the client-side component that represents an _origin_ server, and manages a queue of requests for that origin, and a <<connection-pool,pool of connections>> to that origin.
 
@@ -441,6 +443,47 @@ include::code:example$src/main/java/org/eclipse/jetty/docs/programming/client/ht
 [[content-response]]
 === Response Content Handling
 
+Response content may be _encoded_ by the server, and this is signaled by the presence of the `Content-Encoding` response header.
+
+Typically, encoding the response content means that it is compressed by the server, to save bytes transmitted over the network.
+The most common encodings are:
+
+* `gzip`, using the link:https://en.wikipedia.org/wiki/Gzip[gzip] algorithm.
+* `br`, using the link:https://github.com/google/brotli[brotli] algorithm.
+* `zstd`, using the link:https://github.com/facebook/zstd[zstdandard] algorithm.
+
+Jetty's `HttpClient` supports by default the `gzip` encoding, but the support is pluggable and discovered via Java's `ServiceLoader` mechanism.
+
+In order to add support for brotli and zstandard, it is enough to add to your classpath the correspondent Jetty artifacts with the implementation, respectively:
+
+* `jetty-compression-brotli`
+* `jetty-compression-zstandard`
+
+The Maven artifact coordinates are the following:
+
+[,xml,subs=attributes+]
+----
+<dependencies>
+  <dependency>
+    <groupId>org.eclipse.jetty.compression</groupId>
+    <artifactId>jetty-compression-brotli</artifactId>
+    <version>{jetty-version}</version>
+  </dependency>
+  <dependency>
+    <groupId>org.eclipse.jetty.compression</groupId>
+    <artifactId>jetty-compression-zstandard</artifactId>
+    <version>{jetty-version}</version>
+  </dependency>
+</dependencies>
+----
+
+If you want to remove support for automatic response content decoding (for example in proxies), use the following code:
+
+[,java,indent=0]
+----
+include::code:example$src/main/java/org/eclipse/jetty/docs/programming/client/http/HTTPClientDocs.java[tags=removeDecoders]
+----
+
 Jetty's `HttpClient` allows applications to handle response content in different ways.
 
 You can buffer the response content in memory; this is done when using the <<blocking,blocking APIs>> and the content is buffered within a `ContentResponse` up to 2 MiB.

documentation/jetty/modules/programming-guide/pages/migration/12.0-to-12.1.adoc

@@ -16,6 +16,14 @@
 [[api-changes]]
 == APIs Changes
 
+=== `HttpClient`
+
+In Jetty 12.0.x, applications could configure response content decoding through `HttpClient.getContentDecoderFactories()`, and implement their own by implementing `org.eclipse.jetty.client.ContentDecoder`.
+
+In Jetty 12.1.x, applications can configure response content decoding through `HttpClient.getContentDecoderFactories()`.
+The decoding is based on the `org.eclipse.jetty.compression.Compression` classes.
+Applications can implement their own response content decoding by implementing a `Compression` subclass and the corresponding `DecoderSource`, based on the xref:arch/io.adoc#content-source[`Content.Source`] and xref:arch/io.adoc#content-source-chunk[`Content.Chunk`] APIs.
+
 === `IteratingCallback`
 
 Class `IteratingCallback` underwent refinements that changed the behavior of the `onCompleteFailure(Throwable)` method.

documentation/jetty/modules/programming-guide/pages/server/http.adoc

@@ -852,10 +852,111 @@ For example, you can specify to match only the specific HTTP request method `DEL
 
 Notable subclasses of `ConditionalHandler` are, for example, <<handler-use-qos,`QoSHandler`>> and <<handler-use-thread-limit,`ThreadLimitHandler`>>.
 
+[[handler-use-compression]]
+==== CompressionHandler
+
+`CompressionHandler` provides support for automatic decompression of compressed request content and automatic compression of response content.
+
+The Maven artifact coordinates are:
+
+[,xml,subs=attributes+]
+----
+<dependency>
+  <groupId>org.eclipse.jetty.compression</groupId>
+  <artifactId>jetty-compression-server</artifactId>
+  <version>{jetty-version}</version>
+</dependency>
+----
+
+The compression algorithms supported are:
+
+* link:https://github.com/google/brotli[brotli]; identified by `br` in `Accept-Encoding` and `Content-Encoding` HTTP headers.
+* link:https://en.wikipedia.org/wiki/Gzip[gzip]; identified by `gzip` in `Accept-Encoding` and `Content-Encoding` HTTP headers.
+* link:https://github.com/facebook/zstd[zstdandard]; identified by `zstd` in `Accept-Encoding` and `Content-Encoding` HTTP headers.
+
+`CompressionHandler` discovers compression algorithm implementations via Java's `ServiceLoader` mechanism, so it is enough to put the Jetty artifacts with the compression implementation in the module-path or class-path, respectively:
+
+* `jetty-compression-brotli`
+* `jetty-compression-gzip`
+* `jetty-compression-zstandard`
+
+The Maven artifact coordinates are the following:
+
+[,xml,subs=attributes+]
+----
+<dependencies>
+  <dependency>
+    <groupId>org.eclipse.jetty.compression</groupId>
+    <artifactId>jetty-compression-brotli</artifactId>
+    <version>{jetty-version}</version>
+  </dependency>
+  <dependency>
+    <groupId>org.eclipse.jetty.compression</groupId>
+    <artifactId>jetty-compression-gzip</artifactId>
+    <version>{jetty-version}</version>
+  </dependency>
+  <dependency>
+    <groupId>org.eclipse.jetty.compression</groupId>
+    <artifactId>jetty-compression-zstandard</artifactId>
+    <version>{jetty-version}</version>
+  </dependency>
+</dependencies>
+----
+
+`CompressionHandler` is a `Handler.Wrapper` that inspects the request and, if the conditions are met, it wraps the request and the response to eventually perform decompression of the request content or compression of the response content.
+The decompression/compression is not performed until the web application reads request content or writes response content.
+
+`CompressionHandler` comes with a default configuration, but can be explicitly configured through a `CompressionConfig` object that can be constructed via the builder pattern.
+
+Furthermore, `CompressionHandler` can be configured either at the server level, or at the context level.
+
+`CompressionHandler` can be configured at the server level in this way:
+
+[,java,indent=0]
+----
+include::code:example$src/main/java/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java[tags=serverCompressionHandler]
+----
+
+The `Handler` tree structure looks like the following:
+
+[,screen]
+----
+Server
+└── CompressionHandler
+    └── ContextHandlerCollection
+        ├── ContextHandler 1
+        :── ...
+        └── ContextHandler N
+----
+
+However, in less common cases, you can configure `CompressionHandler` on a per-context basis, for example because you want to configure `CompressionHandler` with different parameters for each context, or because you want only some contexts to have compression support:
+
+[,java,indent=0]
+----
+include::code:example$src/main/java/org/eclipse/jetty/docs/programming/server/http/HTTPServerDocs.java[tags=contextCompressionHandler]
+----
+
+The `Handler` tree structure looks like the following:
+
+[,screen]
+----
+Server
+└── ContextHandlerCollection
+    └── ContextHandlerCollection
+        ├── CompressionHandler
+        │   └── ContextHandler /shop
+        │       └── ShopHandler
+        └── ContextHandler /api
+            └── RESTHandler
+----
+
 [[handler-use-gzip]]
 ==== GzipHandler
 
-`GzipHandler` provides supports for automatic decompression of compressed request content and automatic compression of response content.
+`GzipHandler` provides supports for automatic decompression of gzip compressed request content and automatic gzip compression of response content.
+
+`GzipHandler` is now obsoleted by <<handler-use-compression,`CompressionHandler`>>, but still maintained for backwards compatibility.
+`CompressionHandler` currently supports `gzip` but also other compression algorithms such as `brotli` and `zstandard`.
 
 `GzipHandler` is a `Handler.Wrapper` that inspects the request and, if the request matches the `GzipHandler` configuration, just installs the required components to eventually perform decompression of the request content or compression of the response content.
 The decompression/compression is not performed until the web application reads request content or writes response content.
@@ -900,8 +1001,6 @@ Server
             └── RESTHandler
 ----
 
-// TODO: does ServletContextHandler really need a special configuration?
-
 [[handler-use-rewrite]]
 ==== RewriteHandler