version: 1.10

package flate

import "compress/flate"

Overview

Package flate implements the DEFLATE compressed data format, described in RFC

  1. The gzip and zlib packages implement access to DEFLATE-based file formats.


Example:

  1. // The dictionary is a string of bytes. When compressing some input data,
  2. // the compressor will attempt to substitute substrings with matches found
  3. // in the dictionary. As such, the dictionary should only contain substrings
  4. // that are expected to be found in the actual data stream.
  5. const dict = `<?xml version="1.0"?>` + `<book>` + `<data>` + `<meta name="` + `" content="`
  7. // The data to compress should (but is not required to) contain frequent
  8. // substrings that match those in the dictionary.
  9. const data = `<?xml version="1.0"?>
  10. <book>
  11.     <meta name="title" content="The Go Programming Language"/>
  12.     <meta name="authors" content="Alan Donovan and Brian Kernighan"/>
  13.     <meta name="published" content="2015-10-26"/>
  14.     <meta name="isbn" content="978-0134190440"/>
  15.     <data>...</data>
  16. </book>
  17. `
  19. var b bytes.Buffer
  21. // Compress the data using the specially crafted dictionary.
  22. zw, err := flate.NewWriterDict(&b, flate.DefaultCompression, []byte(dict))
  23. if err != nil {
  24.     log.Fatal(err)
  25. }
  26. if _, err := io.Copy(zw, strings.NewReader(data)); err != nil {
  27.     log.Fatal(err)
  28. }
  29. if err := zw.Close(); err != nil {
  30.     log.Fatal(err)
  31. }
  33. // The decompressor must use the same dictionary as the compressor.
  34. // Otherwise, the input may appear as corrupted.
  35. fmt.Println("Decompressed output using the dictionary:")
  36. zr := flate.NewReaderDict(bytes.NewReader(b.Bytes()), []byte(dict))
  37. if _, err := io.Copy(os.Stdout, zr); err != nil {
  38.     log.Fatal(err)
  39. }
  40. if err := zr.Close(); err != nil {
  41.     log.Fatal(err)
  42. }
  44. fmt.Println()
  46. // Substitute all of the bytes in the dictionary with a '#' to visually
  47. // demonstrate the approximate effectiveness of using a preset dictionary.
  48. fmt.Println("Substrings matched by the dictionary are marked with #:")
  49. hashDict := []byte(dict)
  50. for i := range hashDict {
  51.     hashDict[i] = '#'
  52. }
  53. zr = flate.NewReaderDict(&b, hashDict)
  54. if _, err := io.Copy(os.Stdout, zr); err != nil {
  55.     log.Fatal(err)
  56. }
  57. if err := zr.Close(); err != nil {
  58.     log.Fatal(err)
  59. }
  61. // Output:
  62. // Decompressed output using the dictionary:
  63. // <?xml version="1.0"?>
  64. // <book>
  65. //     <meta name="title" content="The Go Programming Language"/>
  66. //     <meta name="authors" content="Alan Donovan and Brian Kernighan"/>
  67. //     <meta name="published" content="2015-10-26"/>
  68. //     <meta name="isbn" content="978-0134190440"/>
  69. //     <data>...</data>
  70. // </book>
  71. //
  72. // Substrings matched by the dictionary are marked with #:
  73. // #####################
  74. // ######
  75. //     ############title###########The Go Programming Language"/#
  76. //     ############authors###########Alan Donovan and Brian Kernighan"/#
  77. //     ############published###########2015-10-26"/#
  78. //     ############isbn###########978-0134190440"/#
  79. //     ######...</#####
  80. // </#####


Example:

  1. proverbs := []string{
  2.     "Don't communicate by sharing memory, share memory by communicating.\n",
  3.     "Concurrency is not parallelism.\n",
  4.     "The bigger the interface, the weaker the abstraction.\n",
  5.     "Documentation is for users.\n",
  6. }
  8. var r strings.Reader
  9. var b bytes.Buffer
  10. buf := make([]byte, 32<<10)
  12. zw, err := flate.NewWriter(nil, flate.DefaultCompression)
  13. if err != nil {
  14.     log.Fatal(err)
  15. }
  16. zr := flate.NewReader(nil)
  18. for _, s := range proverbs {
  19.     r.Reset(s)
  20.     b.Reset()
  22.     // Reset the compressor and encode from some input stream.
  23.     zw.Reset(&b)
  24.     if _, err := io.CopyBuffer(zw, &r, buf); err != nil {
  25.         log.Fatal(err)
  26.     }
  27.     if err := zw.Close(); err != nil {
  28.         log.Fatal(err)
  29.     }
  31.     // Reset the decompressor and decode to some output stream.
  32.     if err := zr.(flate.Resetter).Reset(&b, nil); err != nil {
  33.         log.Fatal(err)
  34.     }
  35.     if _, err := io.CopyBuffer(os.Stdout, zr, buf); err != nil {
  36.         log.Fatal(err)
  37.     }
  38.     if err := zr.Close(); err != nil {
  39.         log.Fatal(err)
  40.     }
  41. }
  43. // Output:
  44. // Don't communicate by sharing memory, share memory by communicating.
  45. // Concurrency is not parallelism.
  46. // The bigger the interface, the weaker the abstraction.
  47. // Documentation is for users.


Example:

  1. var wg sync.WaitGroup
  2. defer wg.Wait()
  4. // Use io.Pipe to simulate a network connection.
  5. // A real network application should take care to properly close the
  6. // underlying connection.
  7. rp, wp := io.Pipe()
  9. // Start a goroutine to act as the transmitter.
  10. wg.Add(1)
  11. go func() {
  12.     defer wg.Done()
  14.     zw, err := flate.NewWriter(wp, flate.BestSpeed)
  15.     if err != nil {
  16.         log.Fatal(err)
  17.     }
  19.     b := make([]byte, 256)
  20.     for _, m := range strings.Fields("A long time ago in a galaxy far, far away...") {
  21.         // We use a simple framing format where the first byte is the
  22.         // message length, followed the message itself.
  23.         b[0] = uint8(copy(b[1:], m))
  25.         if _, err := zw.Write(b[:1+len(m)]); err != nil {
  26.             log.Fatal(err)
  27.         }
  29.         // Flush ensures that the receiver can read all data sent so far.
  30.         if err := zw.Flush(); err != nil {
  31.             log.Fatal(err)
  32.         }
  33.     }
  35.     if err := zw.Close(); err != nil {
  36.         log.Fatal(err)
  37.     }
  38. }()
  40. // Start a goroutine to act as the receiver.
  41. wg.Add(1)
  42. go func() {
  43.     defer wg.Done()
  45.     zr := flate.NewReader(rp)
  47.     b := make([]byte, 256)
  48.     for {
  49.         // Read the message length.
  50.         // This is guaranteed to return for every corresponding
  51.         // Flush and Close on the transmitter side.
  52.         if _, err := io.ReadFull(zr, b[:1]); err != nil {
  53.             if err == io.EOF {
  54.                 break // The transmitter closed the stream
  55.             }
  56.             log.Fatal(err)
  57.         }
  59.         // Read the message content.
  60.         n := int(b[0])
  61.         if _, err := io.ReadFull(zr, b[:n]); err != nil {
  62.             log.Fatal(err)
  63.         }
  65.         fmt.Printf("Received %d bytes: %s\n", n, b[:n])
  66.     }
  67.     fmt.Println()
  69.     if err := zr.Close(); err != nil {
  70.         log.Fatal(err)
  71.     }
  72. }()
  74. // Output:
  75. // Received 1 bytes: A
  76. // Received 4 bytes: long
  77. // Received 4 bytes: time
  78. // Received 3 bytes: ago
  79. // Received 2 bytes: in
  80. // Received 1 bytes: a
  81. // Received 6 bytes: galaxy
  82. // Received 4 bytes: far,
  83. // Received 3 bytes: far
  84. // Received 7 bytes: away...

Index

Examples

Package files

deflate.go deflatefast.go dict_decoder.go huffman_bit_writer.go huffman_code.go inflate.go token.go

Constants

  1. const (
  2.     NoCompression      = 0
  3.     BestSpeed          = 1
  4.     BestCompression    = 9
  5.     DefaultCompression = -1
  6.  
  7.     // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
  8.     // entropy encoding. This mode is useful in compressing data that has
  9.     // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
  10.     // that lacks an entropy encoder. Compression gains are achieved when
  11.     // certain bytes in the input stream occur more frequently than others.
  12.     //
  13.     // Note that HuffmanOnly produces a compressed output that is
  14.     // RFC 1951 compliant. That is, any valid DEFLATE decompressor will
  15.     // continue to be able to decompress this output.
  16.     HuffmanOnly = -2
  17. )

func NewReader

  1. func NewReader(r io.Reader) io.ReadCloser

NewReader returns a new ReadCloser that can be used to read the uncompressed
version of r. If r does not also implement io.ByteReader, the decompressor may
read more data than necessary from r. It is the caller’s responsibility to call
Close on the ReadCloser when finished reading.

The ReadCloser returned by NewReader also implements Resetter.

func NewReaderDict

  1. func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser

NewReaderDict is like NewReader but initializes the reader with a preset
dictionary. The returned Reader behaves as if the uncompressed data stream
started with the given dictionary, which has already been read. NewReaderDict is
typically used to read data compressed by NewWriterDict.

The ReadCloser returned by NewReader also implements Resetter.

type CorruptInputError

  1. type CorruptInputError int64

A CorruptInputError reports the presence of corrupt input at a given offset.

func (CorruptInputError) Error

  1. func (e CorruptInputError) Error() string

type InternalError

  1. type InternalError string

An InternalError reports an error in the flate code itself.

func (InternalError) Error

  1. func (e InternalError) Error() string

type ReadError

  1. type ReadError struct {
  2.     Offset int64 // byte offset where error occurred
  3.     Err    error // error returned by underlying Read
  4. }

A ReadError reports an error encountered while reading input.

Deprecated: No longer returned.

func (*ReadError) Error

  1. func (e *ReadError) Error() string

type Reader

  1. type Reader interface {
  2.     io.Reader
  3.     io.ByteReader
  4. }

The actual read interface needed by NewReader. If the passed in io.Reader does
not also have ReadByte, the NewReader will introduce its own buffering.

type Resetter

  1. type Resetter interface {
  2.     // Reset discards any buffered data and resets the Resetter as if it was
  3.     // newly initialized with the given reader.
  4.     Reset(r io.Reader, dict []byte) error
  5. }

Resetter resets a ReadCloser returned by NewReader or NewReaderDict to to switch
to a new underlying Reader. This permits reusing a ReadCloser instead of
allocating a new one.

type WriteError

  1. type WriteError struct {
  2.     Offset int64 // byte offset where error occurred
  3.     Err    error // error returned by underlying Write
  4. }

A WriteError reports an error encountered while writing output.

Deprecated: No longer returned.

func (*WriteError) Error

  1. func (e *WriteError) Error() string

type Writer

  1. type Writer struct {
  2.     // contains filtered or unexported fields
  3. }

A Writer takes data written to it and writes the compressed form of that data to
an underlying writer (see NewWriter).

func NewWriter

  1. func NewWriter(w io.Writer, level int) (*Writer, error)

NewWriter returns a new Writer compressing data at the given level. Following
zlib, levels range from 1 (BestSpeed) to 9 (BestCompression); higher levels
typically run slower but compress more. Level 0 (NoCompression) does not attempt
any compression; it only adds the necessary DEFLATE framing. Level -1
(DefaultCompression) uses the default compression level. Level -2 (HuffmanOnly)
will use Huffman compression only, giving a very fast compression for all types
of input, but sacrificing considerable compression efficiency.

If level is in the range [-2, 9] then the error returned will be nil. Otherwise
the error returned will be non-nil.

func NewWriterDict

  1. func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error)

NewWriterDict is like NewWriter but initializes the new Writer with a preset
dictionary. The returned Writer behaves as if the dictionary had been written to
it without producing any compressed output. The compressed data written to w can
only be decompressed by a Reader initialized with the same dictionary.

func (*Writer) Close

  1. func (w *Writer) Close() error

Close flushes and closes the writer.

func (*Writer) Flush

  1. func (w *Writer) Flush() error

Flush flushes any pending data to the underlying writer. It is useful mainly in
compressed network protocols, to ensure that a remote reader has enough data to
reconstruct a packet. Flush does not return until the data has been written.
Calling Flush when there is no pending data still causes the Writer to emit a
sync marker of at least 4 bytes. If the underlying writer returns an error,
Flush returns that error.

In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.

func (*Writer) Reset

  1. func (w *Writer) Reset(dst io.Writer)

Reset discards the writer’s state and makes it equivalent to the result of
NewWriter or NewWriterDict called with dst and w’s level and dictionary.

func (*Writer) Write

  1. func (w *Writer) Write(data []byte) (n int, err error)

Write writes data to w, which will eventually write the compressed form of data
to its underlying writer.