Standardize pattern for exposing advanced configuration for compression streams

There is a desire to expose compression-algorithm specific configuration when constructing compression streams. The same options can be used for ZLibStream, DeflateStream, GZipStream. Named those options with Zlib prefix same as the native library name they use another prefix we considered is Deflate - the algorithm name, the prefix needed because we have existing CompressionLevel enum that used for all compression streams including BrotliStream and it cannot be used for fine tuning the compression level for specific stream

runtime/src/libraries/System.IO.Compression/ref/System.IO.Compression.cs

Lines 9 to 15 in d710379

    
           public enum CompressionLevel 
        
           { 
        
               Optimal = 0, 
        
               Fastest = 1, 
        
               NoCompression = 2, 
        
               SmallestSize = 3, 
        
           }

API Proposal

namespace System.IO.Compression
{
    public enum ZlibCompressionLevel
    {
        DefaultCompression = -1,
        NoCompression = 0,
        Level1 = 1,
        BestSpeed = Level1,
        Level2 = 2,
        Level3 = 3,
        Level4 = 4,
        Level5 = 5,
        Level6 = 6,
        Level7 = 7,
        Level8 = 8,
        Level9 = 9,
        BestCompression = Level9,
    }
    public enum ZlibCompressionStrategy
    {
        DefaultStrategy = 0,
        Filtered = 1,
        HuffmanOnly = 2,
        Rle = 3,
        Fixed = 4
    }
    public enum ZlibFlushMode
    {
        NoFlush = 0,
        PartialFlush = 1,
        SyncFlush = 2,
        FullFlush = 3,
        Finish = 4,
    }
    public sealed class ZLibStream : Stream
    {
        // CompressionMode.Compress
        public ZLibStream(Stream stream, ZlibCompressionLevel compressionLevel = ZlibCompressionLevel.DefaultCompression,
            ZlibCompressionStrategy strategy = ZlibCompressionStrategy.DefaultStrategy, bool leaveOpen = false);
        public ZlibFlushMode FlushMode { get; set; }
    }
    public partial class DeflateStream : Stream
    {
        public DeflateStream(Stream stream, ZlibCompressionLevel compressionLevel = ZlibCompressionLevel.DefaultCompression,
            ZlibCompressionStrategy strategy = ZlibCompressionStrategy.DefaultStrategy, bool leaveOpen = false);
        public ZlibFlushMode FlushMode { get; set; }
     }
     public partial class GZipStream : Stream
    {
        public GZipStream(Stream stream, ZlibCompressionLevel compressionLevel = ZlibCompressionLevel.DefaultCompression,
            ZlibCompressionStrategy strategy = ZlibCompressionStrategy.DefaultStrategy, bool leaveOpen = false) { }
        public ZlibFlushMode FlushMode { get; set; }
    }
    public enum BrotliCompressionQuality
    {
        NoCompression,
        Quality1,
        Quality2,
        Quality3,
        Quality4,
        Quality5,
        Quality6,
        Quality7,
        Quality8,
        Quality9,
        Quality10,
        Quality11
    }
    public sealed partial class BrotliStream : System.IO.Stream
    {
        public BrotliStream(Stream stream, BrotliCompressionQuality quality = BrotliCompressionQuality.Quality4, bool leaveOpen = false) { }
    }
}

ZlibCompressionLevel, ZlibCompressionStrategy are only for Compress mode.

Currently ZlibFlushMode.NoFlush used in normal Read/Write operations, ZlibFlushMode.SyncFlush used for Stream.Flush and ZlibFlushMode.Finish is used on dispose. The value set by the new FlushMode property will be used for normal Read/Write operations only.

The MemoryLevel and/or WindowBits options are omitted because there is no ask for them, we could add these and other options if/when they are requested.

API Usage

private MemoryStream CompressStream(Stream uncompressedStream)
{
    var compressorOutput = new MemoryStream();
    using (var compressionStream = new ZLibStream(compressorOutput, compressionLevel: ZlibCompressionLevel.Level5, strategy: ZlibCompressionStrategy.Filtered, leaveOpen: true))
    {
        compressionStream.FlushMode = ZlibFlushMode.NoFlush;
        var buffer = new byte[4096];
        int bytesRead;
        while ((bytesRead = uncompressedStream.Read(buffer, 0, buffer.Length)) > 0)
        {
            compressionStream.Write(buffer, 0, bytesRead);
        }
    }

    compressorOutput.Position = 0;
    return compressorOutput;
}

Alternative Designs

An alternative design would introduce an options type, maybe per each stream, like so:

namespace System.IO.Compression
{
    public class BrotliOptions
    {
        public readonly BrotliCompressionQualityQuality { get; }
        public readonly int WindowBits { get; }
        public BrotliOptions(BrotliCompressionQuality quality = BrotliCompressionQuality.Level4, int windowBits = 22) { }
    }
    public sealed class BrotliStream : Stream
    {
        public BrotliStream(Stream stream, BrotliOptions options, bool leaveOpen = false);
    }
    public struct ZLibOptions
    {
        public int WindowBits { get; }
        public ZlibMemoryLevel MemoryLevel { get; }
        public int CompressionLevel { get; }
        public int CompressionStrategy { get; }
        public ZlibOptions(ZlibCompressionLevel compressionLevel = ZlibCompressionLevel.DefaultCompression, 
             ZlibCompressionStrategy strategy = ZlibCompressionStrategy.DefaultStrategy,
             ZlibMemoryLevel memoryLevel = ZlibMemoryLevel.Level8, int windowBits = -1);
    }
    public sealed class ZLibStream : Stream
    {
        public ZLibStream(Stream stream, ZLibOptions options, bool leaveOpen = false);
    }
    public class DeflateStream : Stream
    {
        public DeflateStream(Stream stream, ZLibOptions options, bool leaveOpen = false);
    }
    public class GZipStream : Stream
    {
        public GZipStream(Stream stream, ZLibOptionsoptions, bool leaveOpen = false);
    }
}

We can then decide whether we want to be in the business of defining enums for the underlying types or whether we consider them passthru.

I personally like the Options approach; it's easier to pass around as part of configuration options, and IMO makes refactoring (e.g. to change the compression algorithm being used) easier if that becomes an issue.

The Options approach doesn't allow for setting the ZlibFlushMode property for each read/write. That was a requirement identified in the previous thread.

The Options approach doesn't allow for setting the ZlibFlushMode property for each read/write. That was a requirement identified in the previous thread.

This isn't necessarily true:

class ZLibOptions
{
    public static readonly int FlushFinish = 4;
    public int FlushMode { get; set; }
}

class ZLibStream : Stream
{
    public ZLibStream(Stream stream, ZLibOptions options, bool leaveOpen = false);
    public ZLibOptions Options { get; set; }
}
// ...
using var stream = new ZLibStream(myStream, options);
stream.Options.FlushMode = ZLibOptions.FlushFinish;
stream.Write(...);

However, couldn't the existing Flush API be augmented to allow you to specify the flush mode? I imagine, by default, you'd want Z_NO_FLUSH and then would want to specify something special in a separate step. I'm not sure if ZLib supports that sort of behaviour though

Ah yeah, no. Deflate.Flush is a separate operation from Stream.Flush.

@FiniteReality

FWIW, I'd say that current abstraction is broken (both in terms of correspondence with e.g. actual zlib functions, of POLA, and in simplicity of use) and adding a better abstraction would be a better idea. #2236 (comment) sums up my stance on this. As such, I'd say that providing an access to more low-level API would solve all the issues raised here (and in #2236, #16923, and probably some other places too).

What am I talking about precisely? Look at actual zlib. We of course can skip both the init/end as separate calls, but still: both deflate() and inflate() calls take z_stream_s & flush as param, which can be roughly translated to 3 OOP params: input stream, output stream, and call options. While some options are arguably useless in C# (alloc fn, dealloc fn), the architecture would be still the same. zlib provides two explicit methods for this. Using a single class for two different (albeit related) functionalities is akin to having a serializer being able to deserialize etc. In this case, the semantics are outright ugly:

DeflateStream compressStream = new( srcStream, CompressionLevel.Optimal );
DeflateStream decompressStream = new( srcStream, CompressionMode.Decompress );

ZlibStream compressStream = new( srcStream, CompressionLevel.Optimal );
ZlibStream decompressStream = new( srcStream, CompressionMode.Decompress );

This is neither intuitive or useful in any way. A proper way would be to have either (ZlibOptions would extend generic Options common for all (de)compressors):

ZlibDeflateStream compressStream = new( srcStream, zlibOptions );
ZlibInflateStream decompressStream = new( srcStream, zlibOptions );

or, if for some reason we want to have them bundled into a single class (I would advocate against it, but it would be still better than current approach):

ZlibStream compressStream = new( srcStream, Mode.Compress, zlibOptions );
ZlibStream decompressStream = new( srcStream, Mode.Decompress, zlibOptions );

Using an option pattern for flushing would be IMVHO the best option (sic!), since we can just set flush option as read-write (as @FiniteReality stated). I would keep the other options, especially those used in init or related to it (mode [compress/decompress], compression level, compression strategy, data type etc.) read-only. This way, we can have the best of all - access to low-level operations, easy backwards-compatible extensibility, clear abstractions. Switching the used headers etc. would be trivial this way.

e.g. https://github.com/Elskom/zlib.managed/tree/master/zlib.managed does this in the most straightforward way.

tl;dr please, for the sake of humanity and future generations, use options instead of wacky constructors.

I've landed here because the current API for managing Brotli compression only gives me:

public enum CompressionLevel
{
        Optimal = 0,
        Fastest = 1,
        NoCompression = 2
}

In one sample, optimal compressed to 700K in 53sec and fastest to 4,400K in 2.8sec. I'd like to find a middle ground between these two extremes for our application.

I'm not too hung up on how we end up tweaking the various knobs but do think it is important to have access to all (or near all) the knobs for a given compression algorithm.

From a cursory reading of this and related threads, I get the feeling that a dotnet-wide compression enum is an abstraction that is trying to cover too many use cases. The options style approach seems fine to me.

UPDATE
I just went through a blog post that suggested I do

Level = (CompressionLevel)5;

as a way that may affect the compression level. This did work but it was not clear how or why this would do what I wanted. It was only after going through dotnet source code that it made any sense. First to the BrotliStream:

and then into

So, in summary, it would have saved a lot of time if we simply exposed algorithm specific options to the programmer. The current behavior is unintuitive and limiting. A new options-based configuration API sounds expressive and intuitive.

@carlossanlop

~~@carlossanlop any objections marking this as api-ready-for-review? We can discuss this in the meeting if we want to do this at all.~~

Never mind; I just realized that I don't have a design proposal, just a vague direction to go after 😄

I think the best option is to overhaul System.IO.Compression and deprecate the current classes and enums inside for better designed compression types, or at least change them but to carefully do it to not break someome badly. This is why I think the best option is to deprecate them and start over on them. I will see what I can do on my end to fix a regression in my zlib that I did not have tests to catch and then propose them to replace the current zlib apis for .NET 7, I might also at the same time work on the other types in System.IO.Compression as well and work them all in a single proposal that could go into this issue. However it might take a few months to work on.

What about dictionary size or window length

Compresión level maybe would be better with an enum to know which options are available, same as brotli quality

I think node js can compress brotli with multiple threads, could this be possible on c# too?

What about dictionary size or window length

If you need them, please open a new issue (API poposal) with your user scenario, we are open to add more options to the new ***CompressionOptions types

I tried opening them and I couldn't do it If you can do it, please

I would like to be able to increase dictionary length and set custom brotli mode, also deflate doesn't got any custom compression options?

I tried opening them and I couldn't do it If you can do it, please

What prevented you from doing so?

I didn't have permission or something it gives me an error

If you could copy/paste the exact text of the error here, that would be extremely helpful.

I didn't have permission or something it gives me an error

No permission needed for creating an issue

Yes I tried and on PC and let me do it, sorry.

Navigation Menu

Repository navigation

Standardize pattern for exposing advanced configuration for compression streams #42820

API Proposal

API Usage

Alternative Designs

55 remaining items

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

	public enum CompressionLevel
	{
	Optimal = 0,
	Fastest = 1,
	NoCompression = 2,
	SmallestSize = 3,
	}

Standardize pattern for exposing advanced configuration for compression streams #42820

Description

API Proposal

API Usage

Alternative Designs

Activity

FiniteReality commented on Sep 29, 2020

JimBobSquarePants commented on Sep 29, 2020

FiniteReality commented on Sep 29, 2020

JimBobSquarePants commented on Sep 30, 2020

FyiurAmron commented on Dec 4, 2020

rog1039 commented on Jan 6, 2021

terrajobst commented on May 11, 2021

AraHaan commented on Oct 10, 2021

55 remaining items

Mrgaton commented on Jul 23, 2024

Mrgaton commented on Jul 23, 2024

Mrgaton commented on Jul 23, 2024

buyaa-n commented on Jul 29, 2024

Mrgaton commented on Jul 29, 2024

stephentoub commented on Jul 30, 2024

Mrgaton commented on Jul 30, 2024

masonwheeler commented on Jul 30, 2024

buyaa-n commented on Jul 30, 2024

Mrgaton commented on Jul 30, 2024

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions