🔀 Merge pull request #446 from ruby/backport/v0.3-max_response_size

✨ Limit max_response_size (backport 0.3)

Author: nevans

lib/net/imap.rb

@@ -159,6 +159,10 @@ module Net
   #
   # Use paginated or limited versions of commands whenever possible.
   #
+  # Use #max_response_size to impose a limit on incoming server responses
+  # as they are being read.  <em>This is especially important for untrusted
+  # servers.</em>
+  #
   # Use #add_response_handler to handle responses after each one is received.
   # Use the +response_handlers+ argument to ::new to assign response handlers
   # before the receiver thread is started.
@@ -772,6 +776,40 @@ class IMAP < Protocol
     # Seconds to wait until an IDLE response is received.
     attr_reader :idle_response_timeout
 
+    # The maximum allowed server response size.  When +nil+, there is no limit
+    # on response size.
+    #
+    # The default value is _unlimited_ (after +v0.5.8+, the default is 512 MiB).
+    # A _much_ lower value should be used with untrusted servers (for example,
+    # when connecting to a user-provided hostname).  When using a lower limit,
+    # message bodies should be fetched in chunks rather than all at once.
+    #
+    # <em>Please Note:</em> this only limits the size per response.  It does
+    # not prevent a flood of individual responses and it does not limit how
+    # many unhandled responses may be stored on the responses hash.  See
+    # Net::IMAP@Unbounded+memory+use.
+    #
+    # Socket reads are limited to the maximum remaining bytes for the current
+    # response: max_response_size minus the bytes that have already been read.
+    # When the limit is reached, or reading a +literal+ _would_ go over the
+    # limit, ResponseTooLargeError is raised and the connection is closed.
+    # See also #socket_read_limit.
+    #
+    # Note that changes will not take effect immediately, because the receiver
+    # thread may already be waiting for the next response using the previous
+    # value.  Net::IMAP#noop can force a response and enforce the new setting
+    # immediately.
+    #
+    # ==== Versioned Defaults
+    #
+    # Net::IMAP#max_response_size <em>was added in +v0.2.5+ and +v0.3.9+ as an
+    # attr_accessor, and in +v0.4.20+ and +v0.5.7+ as a delegator to a config
+    # attribute.</em>
+    #
+    # * original: +nil+ <em>(no limit)</em>
+    # * +0.5+: 512 MiB
+    attr_accessor :max_response_size
+
     attr_accessor :client_thread # :nodoc:
 
     # Returns the debug mode.
@@ -2044,6 +2082,7 @@ def remove_response_handler(handler)
     #                     that the greeting is handled in the current thread,
     #                     but all other responses are handled in the receiver
     #                     thread.
+    # max_response_size:: See #max_response_size.
     #
     # The most common errors are:
     #
@@ -2074,6 +2113,7 @@ def initialize(host, port_or_options = {},
       @tagno = 0
       @open_timeout = options[:open_timeout] || 30
       @idle_response_timeout = options[:idle_response_timeout] || 5
+      @max_response_size     = options[:max_response_size]
       @parser = ResponseParser.new
       @sock = tcp_socket(@host, @port)
       @reader = ResponseReader.new(self, @sock)

lib/net/imap/errors.rb

@@ -11,6 +11,40 @@ class Error < StandardError
     class DataFormatError < Error
     end
 
+    # Error raised when the socket cannot be read, due to a configured limit.
+    class ResponseReadError < Error
+    end
+
+    # Error raised when a response is larger than IMAP#max_response_size.
+    class ResponseTooLargeError < ResponseReadError
+      attr_reader :bytes_read, :literal_size
+      attr_reader :max_response_size
+
+      def initialize(msg = nil, *args,
+                     bytes_read:        nil,
+                     literal_size:      nil,
+                     max_response_size: nil,
+                     **kwargs)
+        @bytes_read        = bytes_read
+        @literal_size      = literal_size
+        @max_response_size = max_response_size
+        msg ||= [
+          "Response size", response_size_msg, "exceeds max_response_size",
+          max_response_size && "(#{max_response_size}B)",
+        ].compact.join(" ")
+        return super(msg, *args) if kwargs.empty? # ruby 2.6 compatibility
+        super(msg, *args, **kwargs)
+      end
+
+      private
+
+      def response_size_msg
+        if bytes_read && literal_size
+          "(#{bytes_read}B read + #{literal_size}B literal)"
+        end
+      end
+    end
+
     # Error raised when a response from the server is non-parseable.
     class ResponseParseError < Error
     end

lib/net/imap/response_reader.rb

@@ -28,19 +28,48 @@ def read_response_buffer
 
       attr_reader :buff, :literal_size
 
+      def bytes_read;       buff.bytesize                         end
+      def empty?;           buff.empty?                           end
+      def done?;            line_done? && !get_literal_size       end
+      def line_done?;       buff.end_with?(CRLF)                  end
       def get_literal_size; /\{(\d+)\}\r\n\z/n =~ buff && $1.to_i end
 
       def read_line
-        buff << (@sock.gets(CRLF) or throw :eof)
+        buff << (@sock.gets(CRLF, read_limit) or throw :eof)
+        max_response_remaining! unless line_done?
       end
 
       def read_literal
+        # check before allocating memory for literal
+        max_response_remaining!
         literal = String.new(capacity: literal_size)
-        buff << (@sock.read(literal_size, literal) or throw :eof)
+        buff << (@sock.read(read_limit(literal_size), literal) or throw :eof)
       ensure
         @literal_size = nil
       end
 
+      def read_limit(limit = nil)
+        [limit, max_response_remaining!].compact.min
+      end
+
+      def max_response_size;      client.max_response_size                end
+      def max_response_remaining; max_response_size &.- bytes_read        end
+      def response_too_large?;    max_response_size &.< min_response_size end
+      def min_response_size;      bytes_read + min_response_remaining     end
+
+      def min_response_remaining
+        empty? ? 3 : done? ? 0 : (literal_size || 0) + 2
+      end
+
+      def max_response_remaining!
+        return max_response_remaining unless response_too_large?
+        raise ResponseTooLargeError.new(
+          max_response_size: max_response_size,
+          bytes_read:        bytes_read,
+          literal_size:      literal_size,
+        )
+      end
+
     end
   end
 end

test/net/imap/test_errors.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "net/imap"
+require "test/unit"
+
+class IMAPErrorsTest < Test::Unit::TestCase
+
+  test "ResponseTooLargeError" do
+    err = Net::IMAP::ResponseTooLargeError.new
+    assert_nil err.bytes_read
+    assert_nil err.literal_size
+    assert_nil err.max_response_size
+
+    err = Net::IMAP::ResponseTooLargeError.new("manually set message")
+    assert_equal "manually set message", err.message
+    assert_nil err.bytes_read
+    assert_nil err.literal_size
+    assert_nil err.max_response_size
+
+    err = Net::IMAP::ResponseTooLargeError.new(max_response_size: 1024)
+    assert_equal "Response size exceeds max_response_size (1024B)", err.message
+    assert_nil err.bytes_read
+    assert_nil err.literal_size
+    assert_equal 1024, err.max_response_size
+
+    err = Net::IMAP::ResponseTooLargeError.new(bytes_read:        1200,
+                                               max_response_size: 1200)
+    assert_equal 1200, err.bytes_read
+    assert_equal "Response size exceeds max_response_size (1200B)", err.message
+
+    err = Net::IMAP::ResponseTooLargeError.new(bytes_read:        800,
+                                               literal_size:      1000,
+                                               max_response_size: 1200)
+    assert_equal  800, err.bytes_read
+    assert_equal 1000, err.literal_size
+    assert_equal("Response size (800B read + 1000B literal) " \
+                 "exceeds max_response_size (1200B)", err.message)
+  end
+
+end

test/net/imap/test_imap_max_response_size.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "net/imap"
+require "test/unit"
+
+class IMAPMaxResponseSizeTest < Test::Unit::TestCase
+
+  def setup
+    @do_not_reverse_lookup = Socket.do_not_reverse_lookup
+    Socket.do_not_reverse_lookup = true
+    @threads = []
+  end
+
+  def teardown
+    if !@threads.empty?
+      assert_join_threads(@threads)
+    end
+  ensure
+    Socket.do_not_reverse_lookup = @do_not_reverse_lookup
+  end
+
+  test "#max_response_size reading literals" do
+    _, port = with_server_socket do |sock|
+      sock.gets # => NOOP
+      sock.print("RUBY0001 OK done\r\n")
+      sock.gets # => NOOP
+      sock.print("* 1 FETCH (BODY[] {12345}\r\n" + "a" * 12_345 + ")\r\n")
+      sock.print("RUBY0002 OK done\r\n")
+      "RUBY0003"
+    end
+    Timeout.timeout(5) do
+      imap = Net::IMAP.new("localhost", port: port, max_response_size: 640 << 20)
+      assert_equal 640 << 20, imap.max_response_size
+      imap.max_response_size = 12_345 + 30
+      assert_equal 12_345 + 30, imap.max_response_size
+      imap.noop # to reset the get_response limit
+      imap.noop # to send the FETCH
+      assert_equal "a" * 12_345, imap.responses["FETCH"].first.attr["BODY[]"]
+    ensure
+      imap.logout rescue nil
+      imap.disconnect rescue nil
+    end
+  end
+
+  test "#max_response_size closes connection for too long line" do
+    _, port = with_server_socket do |sock|
+      sock.gets or next # => never called
+      fail "client disconnects first"
+    end
+    assert_raise_with_message(
+      Net::IMAP::ResponseTooLargeError, /exceeds max_response_size .*\b10B\b/
+    ) do
+      Net::IMAP.new("localhost", port: port, max_response_size: 10)
+      fail "should not get here (greeting longer than max_response_size)"
+    end
+  end
+
+  test "#max_response_size closes connection for too long literal" do
+    _, port = with_server_socket(ignore_io_error: true) do |sock|
+      sock.gets # => NOOP
+      sock.print "* 1 FETCH (BODY[] {1000}\r\n" + "a" * 1000 + ")\r\n"
+      sock.print("RUBY0001 OK done\r\n")
+    end
+    client = Net::IMAP.new("localhost", port: port, max_response_size: 1000)
+    assert_equal 1000, client.max_response_size
+    client.max_response_size = 50
+    assert_equal 50, client.max_response_size
+    assert_raise_with_message(
+      Net::IMAP::ResponseTooLargeError,
+      /\d+B read \+ 1000B literal.* exceeds max_response_size .*\b50B\b/
+    ) do
+      client.noop
+      fail "should not get here (FETCH literal longer than max_response_size)"
+    end
+  end
+
+  def with_server_socket(ignore_io_error: false)
+    server = create_tcp_server
+    port   = server.addr[1]
+    start_server do
+      Timeout.timeout(5) do
+        sock = server.accept
+        sock.print("* OK connection established\r\n")
+        logout_tag = yield sock if block_given?
+        sock.gets # => LOGOUT
+        sock.print("* BYE terminating connection\r\n")
+        sock.print("#{logout_tag} OK LOGOUT completed\r\n") if logout_tag
+      rescue IOError, EOFError, Errno::ECONNABORTED, Errno::ECONNRESET,
+        Errno::EPIPE, Errno::ETIMEDOUT
+        ignore_io_error or raise
+      ensure
+        sock.close rescue nil
+        server.close rescue nil
+      end
+    end
+    return server, port
+  end
+
+  def start_server
+    th = Thread.new do
+      yield
+    end
+    @threads << th
+    sleep 0.1 until th.stop?
+  end
+
+  def create_tcp_server
+    return TCPServer.new(server_addr, 0)
+  end
+
+  def server_addr
+    Addrinfo.tcp("localhost", 0).ip_address
+  end
+end

test/net/imap/test_response_reader.rb

@@ -6,6 +6,7 @@
 
 class ResponseReaderTest < Test::Unit::TestCase
   class FakeClient
+    attr_accessor :max_response_size
   end
 
   def literal(str) "{#{str.bytesize}}\r\n#{str}" end
@@ -44,4 +45,19 @@ def literal(str) "{#{str.bytesize}}\r\n#{str}" end
     assert_equal "",           rcvr.read_response_buffer.to_str
   end
 
+  test "#read_response_buffer with max_response_size" do
+    client = FakeClient.new
+    client.max_response_size = 10
+    under = "+ 3456\r\n"
+    exact = "+ 345678\r\n"
+    over  = "+ 3456789\r\n"
+    io = StringIO.new([under, exact, over].join)
+    rcvr = Net::IMAP::ResponseReader.new(client, io)
+    assert_equal under, rcvr.read_response_buffer.to_str
+    assert_equal exact, rcvr.read_response_buffer.to_str
+    assert_raise Net::IMAP::ResponseTooLargeError do
+      rcvr.read_response_buffer
+    end
+  end
+
 end