Ticket #18972: 0001-Fixed-18972-Refactored-bundled-wsgi-server-s-chunkin.patch

File 0001-Fixed-18972-Refactored-bundled-wsgi-server-s-chunkin.patch, 6.0 KB (added by Simon Charette, 12 years ago)
  • django/core/servers/basehttp.py

    From 8c31b4d6447d7615b56128e9bf4cc484cb14dc16 Mon Sep 17 00:00:00 2001
    From: Matthew Wood <woodm1979@gmail.com>
    Date: Mon, 18 Mar 2013 16:06:24 -0600
    Subject: [PATCH] Fixed #18972 -- Refactored bundled wsgi server's chunking
     algorithm.
    
    Thanks to amosonn at yahoo.com for the report, @doda for the initial patch and
    @datagrok for the revamped logic and test case.
    ---
     django/core/servers/basehttp.py |   26 +++++++---------
     tests/builtin_server/tests.py   |   63 ++++++++++++++++++++++++++++++++++-----
     2 files changed, 66 insertions(+), 23 deletions(-)
    
    diff --git a/django/core/servers/basehttp.py b/django/core/servers/basehttp.py
    index 89ec2b3..080529a 100644
    a b been reviewed for security issues. DON'T USE IT FOR PRODUCTION USE!  
    99
    1010from __future__ import unicode_literals
    1111
    12 import os
     12from io import BytesIO
    1313import socket
    1414import sys
    1515import traceback
    from django.core.management.color import color_style  
    2525from django.core.wsgi import get_wsgi_application
    2626from django.utils.module_loading import import_by_path
    2727
    28 __all__ = ['WSGIServer', 'WSGIRequestHandler']
     28__all__ = ('WSGIServer', 'WSGIRequestHandler', 'MAX_SOCKET_CHUNK_SIZE')
     29
     30
     31# If data is too large, socket will choke, so write chunks no larger than 32MB
     32# at a time. The rationale behind the 32MB can be found on Django's Trac:
     33# https://code.djangoproject.com/ticket/5596#comment:4
     34MAX_SOCKET_CHUNK_SIZE = 32 * 1024 * 1024  # 32 MB
    2935
    3036
    3137def get_internal_wsgi_application():
    class ServerHandler(simple_server.ServerHandler, object):  
    7783            self.bytes_sent += len(data)
    7884
    7985        # XXX check Content-Length and truncate if too many bytes written?
    80 
    81         # If data is too large, socket will choke, so write chunks no larger
    82         # than 32MB at a time.
    83         length = len(data)
    84         if length > 33554432:
    85             offset = 0
    86             while offset < length:
    87                 chunk_size = min(33554432, length)
    88                 self._write(data[offset:offset+chunk_size])
    89                 self._flush()
    90                 offset += chunk_size
    91         else:
    92             self._write(data)
     86        data = BytesIO(data)
     87        for chunk in iter(lambda: data.read(MAX_SOCKET_CHUNK_SIZE), b''):
     88            self._write(chunk)
    9389            self._flush()
    9490
    9591    def error_output(self, environ, start_response):
  • tests/builtin_server/tests.py

    diff --git a/tests/builtin_server/tests.py b/tests/builtin_server/tests.py
    index 041bb3c..b769a8a 100644
    a b from __future__ import unicode_literals  
    22
    33from io import BytesIO
    44
    5 from django.core.servers.basehttp import ServerHandler
     5from django.core.servers.basehttp import ServerHandler, MAX_SOCKET_CHUNK_SIZE
    66from django.utils.unittest import TestCase
    77
    8 #
    9 # Tests for #9659: wsgi.file_wrapper in the builtin server.
    10 # We need to mock a couple of handlers and keep track of what
    11 # gets called when using a couple kinds of WSGI apps.
    12 #
    138
    149class DummyHandler(object):
    15     def log_request(*args, **kwargs):
     10    def log_request(self, *args, **kwargs):
    1611        pass
    1712
     13
    1814class FileWrapperHandler(ServerHandler):
    1915    def __init__(self, *args, **kwargs):
    20         ServerHandler.__init__(self, *args, **kwargs)
     16        super(FileWrapperHandler, self).__init__(*args, **kwargs)
    2117        self.request_handler = DummyHandler()
    2218        self._used_sendfile = False
    2319
    class FileWrapperHandler(ServerHandler):  
    2521        self._used_sendfile = True
    2622        return True
    2723
     24
    2825def wsgi_app(environ, start_response):
    2926    start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
    3027    return [b'Hello World!']
    3128
     29
    3230def wsgi_app_file_wrapper(environ, start_response):
    3331    start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
    3432    return environ['wsgi.file_wrapper'](BytesIO(b'foo'))
    3533
     34
    3635class WSGIFileWrapperTests(TestCase):
    3736    """
    3837    Test that the wsgi.file_wrapper works for the builting server.
     38
     39    Tests for #9659: wsgi.file_wrapper in the builtin server.
     40    We need to mock a couple of handlers and keep track of what
     41    gets called when using a couple kinds of WSGI apps.
    3942    """
    4043
    4144    def test_file_wrapper_uses_sendfile(self):
    class WSGIFileWrapperTests(TestCase):  
    5356        self.assertFalse(handler._used_sendfile)
    5457        self.assertEqual(handler.stdout.getvalue().splitlines()[-1], b'Hello World!')
    5558        self.assertEqual(handler.stderr.getvalue(), b'')
     59
     60
     61class WriteChunkCounterHandler(ServerHandler):
     62    """
     63    Server handler that counts the number of chunks written after headers we're
     64    sent. Used to make sure large response body chunking works properly.
     65    """
     66
     67    def __init__(self, *args, **kwargs):
     68        super(WriteChunkCounterHandler, self).__init__(*args, **kwargs)
     69        self.request_handler = DummyHandler()
     70        self.headers_written = False
     71        self.write_chunk_counter = 0
     72
     73    def send_headers(self):
     74        super(WriteChunkCounterHandler, self).send_headers()
     75        self.headers_written = True
     76
     77    def _write(self, data):
     78        if self.headers_written:
     79            self.write_chunk_counter += 1
     80        self.stdout.write(data)
     81
     82
     83def send_big_data_app(environ, start_response):
     84    start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
     85    # Return a blob of data that is 1.5 times the maximum chunk size.
     86    return [b'x' * (MAX_SOCKET_CHUNK_SIZE + MAX_SOCKET_CHUNK_SIZE // 2)]
     87
     88
     89class ServerHandlerChunksProperly(TestCase):
     90    """
     91    Test that the ServerHandler chunks data properly.
     92
     93    Tests for #18972: The logic that performs the math to break data into
     94    32MB (MAX_SOCKET_CHUNK_SIZE) chunks was flawed, BUT it didn't actually
     95    cause any problems.
     96    """
     97
     98    def test_chunked_data(self):
     99        env = {'SERVER_PROTOCOL': 'HTTP/1.0'}
     100        handler = WriteChunkCounterHandler(None, BytesIO(), BytesIO(), env)
     101        handler.run(send_big_data_app)
     102        self.assertEqual(handler.write_chunk_counter, 2)
Back to Top