Ticket #2070: modpyton-ok-needs-fcgi-testing.3.diff

django/http/__init__.py

@@ -1 +1 @@
 import os
+import cgi
 from Cookie import SimpleCookie
 from pprint import pformat
 from urllib import urlencode, quote
@@ -42 +43 @@
     def is_secure(self):
         return os.environ.get("HTTPS") == "on"
 
-def parse_file_upload(header_dict, post_data):
+def default_parse_file_upload(req):
     "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
     import email, email.Message
     from cgi import parse_header
-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
-    raw_message += '\r\n\r\n' + post_data
+    raw_message = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()])
+    raw_message += '\r\n\r\n' + req.raw_post_data
     msg = email.message_from_string(raw_message)
     POST = MultiValueDict()
     FILES = MultiValueDict()
@@ -73 +74 @@
                 POST.appendlist(name_dict['name'], submessage.get_payload())
     return POST, FILES
 
+def parse_file_upload(req):
+    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
+
+    if hasattr(req, 'parse_file_upload'):
+        return req.parse_file_upload(req)
+
+    return default_parse_file_upload(req)
+
 class QueryDict(MultiValueDict):
     """A specialized MultiValueDict that takes a query string when initialized.
     This is immutable unless you create a copy of it."""

django/oldforms/__init__.py

@@ -661 +661 @@
         self.validator_list = [self.isNonEmptyFile] + validator_list
 
     def isNonEmptyFile(self, field_data, all_data):
+        if not isinstance(field_data, dict):
+            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
         try:
+            content = field_data.tell()>0
+        except: 
             content = field_data['content']
-        except TypeError:
-            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
         if not content:
             raise validators.CriticalValidationError, gettext("The submitted file is empty.")
 

django/db/models/base.py

@@ -322 +322 @@
     def _get_FIELD_size(self, field):
         return os.path.getsize(self._get_FIELD_filename(field))
 
-    def _save_FIELD_file(self, field, filename, raw_contents):
+    def _save_FIELD_file(self, field, filename, raw_field):
         directory = field.get_directory_name()
         try: # Create the date-based directory if it doesn't exist.
             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
@@ -344 +344 @@
         setattr(self, field.attname, filename)
 
         full_filename = self._get_FIELD_filename(field)
-        fp = open(full_filename, 'wb')
-        fp.write(raw_contents)
-        fp.close()
+        if not raw_field.get('payload', False):
+            fp = open(full_filename, 'wb')
+            fp.write(raw_field['content'])
+            fp.close()
+        else:
+            os.rename(raw_field['payload']['name'], full_filename)
 
         # Save the width and/or height, if applicable.
         if isinstance(field, ImageField) and (field.width_field or field.height_field):

django/db/models/fields/__init__.py

@@ -648 +648 @@
         if new_data.get(upload_field_name, False):
             func = getattr(new_object, 'save_%s_file' % self.name)
             if rel:
-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"])
+                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0])
             else:
-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"])
+                func(new_data[upload_field_name]["filename"], new_data[upload_field_name])
 
     def get_directory_name(self):
         return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))

django/core/handlers/wsgi.py

@@ -109 +109 @@
         # Populates self._post and self._files
         if self.method == 'POST':
             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
+                self._post, self._files = http.parse_file_upload(self)
             else:
                 self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
         else:
@@ -166 +164 @@
             self._raw_post_data = buf.getvalue()
             buf.close()
             return self._raw_post_data
+    
+    def _get_raw_request(self):
+        return self.environ['wsgi.input']
 
+    def _get_header_dict(self):
+        header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
+        header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
+        return header_dict
+
     GET = property(_get_get, _set_get)
     POST = property(_get_post, _set_post)
     COOKIES = property(_get_cookies, _set_cookies)
     FILES = property(_get_files)
     REQUEST = property(_get_request)
     raw_post_data = property(_get_raw_post_data)
+    raw_request = property(_get_raw_request)
+    header_dict = property(_get_header_dict)
 
 class WSGIHandler(BaseHandler):
     def __call__(self, environ, start_response):

django/core/handlers/modpython.py

@@ -47 +47 @@
     def _load_post_and_files(self):
         "Populates self._post and self._files"
         if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
+            self._post, self._files = http.parse_file_upload(self)
         else:
             self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
 
@@ -122 +122 @@
     def _get_method(self):
         return self.META['REQUEST_METHOD'].upper()
 
+    def _get_raw_request(self):
+        return self._req
+
+    def _get_header_dict(self):
+        return self._req.headers_in
+
     GET = property(_get_get, _set_get)
     POST = property(_get_post, _set_post)
     COOKIES = property(_get_cookies, _set_cookies)
@@ -130 +136 @@
     REQUEST = property(_get_request)
     raw_post_data = property(_get_raw_post_data)
     method = property(_get_method)
+    raw_request = property(_get_raw_request)
+    header_dict = property(_get_header_dict)    
 
 class ModPythonHandler(BaseHandler):
     def __call__(self, req):

django/contrib/admin/media/js/UploadProgress.js

@@ +1 @@
+
+function getxy(){
+    var x,y;
+    if (self.innerHeight) // all except Explorer
+        {
+        x = self.innerWidth;
+        y = self.innerHeight;
+        }
+    else if (document.documentElement && document.documentElement.clientHeight)
+        // Explorer 6 Strict Mode
+        {
+        x = document.documentElement.clientWidth;
+        y = document.documentElement.clientHeight;
+        }
+    else if (document.body) // other Explorers
+        {
+        x = document.body.clientWidth;
+        y = document.body.clientHeight;
+        }
+    return {'x':x,'y':y}
+    }
+
+var humanvalue = ['B','KB','MB','GB']
+function humanize(bytes) {
+    curbytes = bytes
+    iterations = 0
+    while (curbytes>1024) {
+        iterations++
+        curbytes=curbytes/1024
+        }
+    return curbytes.toFixed(1) + ' ' + humanvalue[iterations]
+    }
+
+interval = null;
+function fetch(uuid) {
+    req = xmlhttp
+    req.open("GET", "/progress/", 1);
+    req.setRequestHeader("X-Progress-Id", uuid);
+    req.onreadystatechange = function () {
+    if (req.readyState == 4) {
+        if (req.status == 200) {
+
+            var upload = eval( '(' + req.responseText + ')' );
+
+            if (upload.state == 'done' || upload.state == 'uploading') {
+                bar = document.getElementById('progress_bar');
+                bar_txt = document.getElementById('progress_text')
+                bar_txt.innerHTML = ((upload.received / upload.size) * 100).toFixed(1) + '% - ' +
+                    humanize(upload.received) + ' of ' + humanize(upload.size) 
+                w = 400 * upload.received / upload.size;
+                bar.style.width = w + 'px';
+
+                }
+                if (upload.state == 'done') {
+                    window.clearTimeout(interval);
+                    }
+                }
+            }
+        }
+    req.send(null); 
+
+    }
+
+function openprogress(e) {
+
+    uuid = "";
+    for (i = 0; i < 32; i++) {
+        uuid += Math.floor(Math.random() * 16).toString(16);
+        }
+    frm = e.target||e.srcElement
+
+    frm.action=frm.action+"?" + uuid; 
+
+    pos = getxy()
+    posx = parseInt((pos.x/2)-(420/2), 10)
+    posy = parseInt((pos.y/2)-(50/2), 10)
+
+    progress_wrap = quickElement('div', document.body, '', 'style', 
+        'position: absolute; top: '+posy+'px; left: '+posx+'px; height: 50px; ' +
+        'padding: 10px; width: 420px; background: #ffffff; ' +
+        'border: solid 1px #dddddd;', 'id', 'progress_wrap')
+
+    progress_label = quickElement('h1', progress_wrap, 'Upload progress')
+
+    progress = quickElement('div', progress_wrap, '', 'style', 
+        'top: 0; left: 0; width: 0px; ', 'id', 'progress_bar', 'class', 'submit-row')
+
+    progress_text = quickElement('div', progress_wrap, '0%', 'style',
+        'color: #000000; ', 'id', 'progress_text')
+ 
+    interval = window.setInterval(
+        function () {
+            fetch(uuid);
+            },
+        1000
+        );
+    }
+
+addEvent(window, 'load', function() {
+        frm = document.getElementsByTagName('form')[0]
+        addEvent(frm, 'submit',  openprogress)    
+        }
+    )

django/middleware/upload.py

@@ +1 @@
+"streaming upload middleware"
+import cgi
+import os
+import tempfile
+import re
+import email, email.Message, email.FeedParser
+from email import Errors
+from email import Message
+from email.FeedParser import NLCRE, NLCRE_bol, NLCRE_eol, NLCRE_crack,headerRE, EMPTYSTRING, NL, NeedMoreData
+
+from django.conf import settings
+from django.utils.datastructures import MultiValueDict
+from django.utils import simplejson
+
+from django.core import signals
+from django.dispatch import dispatcher
+
+def delete_tempfile(sender):
+    sender.delete()
+
+class TempFileDict(dict):
+    "Keeps uploaded file as a file-like object and reads its content on demand"
+    
+    def __init__(self):
+        self['name'] = tempfile.mktemp()
+        self['file'] = open(self['name'], "w+b")
+        dispatcher.connect(delete_tempfile, sender=self, signal=signals.request_finished) 
+
+    def write(self, data):
+        self['file'].write(data)
+
+    def close(self):
+        self['file'].close()
+
+    def delete(self):
+        if os.path.exsists(self['name']):
+            os.remove(self['name'])
+    
+    def tell(self):
+        return os.path.getsize(self['name']) 
+
+    def __repr__(self):
+        return '<TempFileDict>'
+
+class FileDict(dict):
+
+    def __init__(self, filename, contenttype, payload):
+        self['filename'] , self['content-type'], self['payload'] = filename, contenttype, payload 
+
+    def __getitem__(self, name):
+        if name=='content' and not 'content' in self:
+            size = self.tell()
+            f = open(self['payload']['name'], 'r')
+            self['content'] = f.read(size)
+            f.close()
+        return dict.__getitem__(self, name)
+
+    def tell(self):
+        return self['payload'].tell()
+
+    def __repr__(self):
+        return '<FileDict>'
+
+
+class StreamingFileFeedParser(email.FeedParser.FeedParser):
+
+    def _parsegen(self):
+        # Create a new message and start by parsing headers.
+        self._new_message()
+        headers = []
+        # Collect the headers, searching for a line that doesn't match the RFC
+        # 2822 header or continuation pattern (including an empty line).
+        for line in self._input:
+            if line is NeedMoreData:
+                yield NeedMoreData
+                continue
+            if not headerRE.match(line):
+                # If we saw the RFC defined header/body separator
+                # (i.e. newline), just throw it away. Otherwise the line is
+                # part of the body so push it back.
+                if not NLCRE.match(line):
+                    self._input.unreadline(line)
+                break
+            headers.append(line)
+        # Done with the headers, so parse them and figure out what we're
+        # supposed to see in the body of the message.
+        self._parse_headers(headers)
+        # Headers-only parsing is a backwards compatibility hack, which was
+        # necessary in the older parser, which could throw errors.  All
+        # remaining lines in the input are thrown into the message body.
+        if self._headersonly:
+            lines = []
+            while True:
+                line = self._input.readline()
+                if line is NeedMoreData:
+                    yield NeedMoreData
+                    continue
+                if line == '':
+                    break
+                lines.append(line)
+            self._cur.set_payload(EMPTYSTRING.join(lines))
+            return
+        if self._cur.get_content_type() == 'message/delivery-status':
+            # message/delivery-status contains blocks of headers separated by
+            # a blank line.  We'll represent each header block as a separate
+            # nested message object, but the processing is a bit different
+            # than standard message/* types because there is no body for the
+            # nested messages.  A blank line separates the subparts.
+            while True:
+                self._input.push_eof_matcher(NLCRE.match)
+                for retval in self._parsegen():
+                    if retval is NeedMoreData:
+                        yield NeedMoreData
+                        continue
+                    break
+                msg = self._pop_message()
+                # We need to pop the EOF matcher in order to tell if we're at
+                # the end of the current file, not the end of the last block
+                # of message headers.
+                self._input.pop_eof_matcher()
+                # The input stream must be sitting at the newline or at the
+                # EOF.  We want to see if we're at the end of this subpart, so
+                # first consume the blank line, then test the next line to see
+                # if we're at this subpart's EOF.
+                while True:
+                    line = self._input.readline()
+                    if line is NeedMoreData:
+                        yield NeedMoreData
+                        continue
+                    break
+                while True:
+                    line = self._input.readline()
+                    if line is NeedMoreData:
+                        yield NeedMoreData
+                        continue
+                    break
+                if line == '':
+                    break
+                # Not at EOF so this is a line we're going to need.
+                self._input.unreadline(line)
+            return
+        if self._cur.get_content_maintype() == 'message':
+            # The message claims to be a message/* type, then what follows is
+            # another RFC 2822 message.
+            for retval in self._parsegen():
+                if retval is NeedMoreData:
+                    yield NeedMoreData
+                    continue
+                break
+            self._pop_message()
+            return
+        if self._cur.get_content_maintype() == 'multipart':
+            boundary = self._cur.get_boundary()
+            if boundary is None:
+                # The message /claims/ to be a multipart but it has not
+                # defined a boundary.  That's a problem which we'll handle by
+                # reading everything until the EOF and marking the message as
+                # defective.
+                self._cur.defects.append(Errors.NoBoundaryInMultipartDefect())
+                lines = []
+                for line in self._input:
+                    if line is NeedMoreData:
+                        yield NeedMoreData
+                        continue
+                    lines.append(line)
+                self._cur.set_payload(EMPTYSTRING.join(lines))
+                return
+            # Create a line match predicate which matches the inter-part
+            # boundary as well as the end-of-multipart boundary.  Don't push
+            # this onto the input stream until we've scanned past the
+            # preamble.
+            separator = '--' + boundary
+            boundaryre = re.compile(
+                '(?P<sep>' + re.escape(separator) +
+                r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
+            capturing_preamble = True
+            preamble = []
+            linesep = False
+            while True:
+                line = self._input.readline()
+                if line is NeedMoreData:
+                    yield NeedMoreData
+                    continue
+                if line == '':
+                    break
+                mo = boundaryre.match(line)
+                if mo:
+                    # If we're looking at the end boundary, we're done with
+                    # this multipart.  If there was a newline at the end of
+                    # the closing boundary, then we need to initialize the
+                    # epilogue with the empty string (see below).
+                    if mo.group('end'):
+                        linesep = mo.group('linesep')
+                        break
+                    # We saw an inter-part boundary.  Were we in the preamble?
+                    if capturing_preamble:
+                        if preamble:
+                            # According to RFC 2046, the last newline belongs
+                            # to the boundary.
+                            lastline = preamble[-1]
+                            eolmo = NLCRE_eol.search(lastline)
+                            if eolmo:
+                                preamble[-1] = lastline[:-len(eolmo.group(0))]
+                            self._cur.preamble = EMPTYSTRING.join(preamble)
+                        capturing_preamble = False
+                        self._input.unreadline(line)
+                        continue
+                    # We saw a boundary separating two parts.  Consume any
+                    # multiple boundary lines that may be following.  Our
+                    # interpretation of RFC 2046 BNF grammar does not produce
+                    # body parts within such double boundaries.
+                    while True:
+                        line = self._input.readline()
+                        if line is NeedMoreData:
+                            yield NeedMoreData
+                            continue
+                        mo = boundaryre.match(line)
+                        if not mo:
+                            self._input.unreadline(line)
+                            break
+                    # Recurse to parse this subpart; the input stream points
+                    # at the subpart's first line.
+                    self._input.push_eof_matcher(boundaryre.match)
+                    for retval in self._parsegen():
+                        if retval is NeedMoreData:
+                            yield NeedMoreData
+                            continue
+                        break
+                    # Because of RFC 2046, the newline preceding the boundary
+                    # separator actually belongs to the boundary, not the
+                    # previous subpart's payload (or epilogue if the previous
+                    # part is a multipart).
+                    if self._last.get_content_maintype() == 'multipart':
+                        epilogue = self._last.epilogue
+                        if epilogue == '':
+                            self._last.epilogue = None
+                        elif epilogue is not None:
+                            mo = NLCRE_eol.search(epilogue)
+                            if mo:
+                                end = len(mo.group(0))
+                                self._last.epilogue = epilogue[:-end]
+                    else:
+                        payload = self._last.get_payload()
+                        if isinstance(payload, basestring):
+                            mo = NLCRE_eol.search(payload)
+                            if mo:
+                                payload = payload[:-len(mo.group(0))]
+                                self._last.set_payload(payload)
+                    self._input.pop_eof_matcher()
+                    self._pop_message()
+                    # Set the multipart up for newline cleansing, which will
+                    # happen if we're in a nested multipart.
+                    self._last = self._cur
+                else:
+                    # I think we must be in the preamble
+                    assert capturing_preamble
+                    preamble.append(line)
+            # We've seen either the EOF or the end boundary.  If we're still
+            # capturing the preamble, we never saw the start boundary.  Note
+            # that as a defect and store the captured text as the payload.
+            # Everything from here to the EOF is epilogue.
+            if capturing_preamble:
+                self._cur.defects.append(Errors.StartBoundaryNotFoundDefect())
+                self._cur.set_payload(EMPTYSTRING.join(preamble))
+                epilogue = []
+                for line in self._input:
+                    if line is NeedMoreData:
+                        yield NeedMoreData
+                        continue
+                self._cur.epilogue = EMPTYSTRING.join(epilogue)
+                return
+            # If the end boundary ended in a newline, we'll need to make sure
+            # the epilogue isn't None
+            if linesep:
+                epilogue = ['']
+            else:
+                epilogue = []
+            for line in self._input:
+                if line is NeedMoreData:
+                    yield NeedMoreData
+                    continue
+                epilogue.append(line)
+            # Any CRLF at the front of the epilogue is not technically part of
+            # the epilogue.  Also, watch out for an empty string epilogue,
+            # which means a single newline.
+            if epilogue:
+                firstline = epilogue[0]
+                bolmo = NLCRE_bol.match(firstline)
+                if bolmo:
+                    epilogue[0] = firstline[len(bolmo.group(0)):]
+            self._cur.epilogue = EMPTYSTRING.join(epilogue)
+            return
+        # Otherwise, it's some non-multipart type, so the entire rest of the
+        # file contents becomes the payload.
+        name_dict = cgi.parse_header(self._cur['Content-Disposition'])[1]
+        if name_dict.has_key('filename'):
+            tmpfile = TempFileDict()
+            for line in self._input:
+                if line is NeedMoreData:
+                    yield NeedMoreData
+                    continue
+                tmpfile.write(line)
+            tmpfile.close()
+            self._cur.set_payload(tmpfile)
+        else:
+            lines = []
+            for line in self._input:
+                if line is NeedMoreData:
+                    yield NeedMoreData
+                    continue
+                lines.append(line)
+            self._cur.set_payload(EMPTYSTRING.join(lines))
+
+def parse_streaming_file_upload(req):
+    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
+
+    try:
+        BUFFER_SIZE=settings.UPLOAD_BUFFER_SIZE
+    except:
+        BUFFER_SIZE=200000
+
+    if hasattr(req, 'upload_state'): 
+        upload_state = req.upload_state(req) 
+    else: 
+        upload_state = None 
+
+    raw_headers = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()])
+    raw_headers += '\r\n\r\n' 
+    POST = MultiValueDict()
+    FILES = MultiValueDict()
+    parser = StreamingFileFeedParser()
+    parser.feed(raw_headers)
+    while 1:
+        # make this a non-blocing read
+        line=req.raw_request.read(BUFFER_SIZE)
+        if upload_state:
+            upload_state.addlen(len(line))
+        parser.feed(line)
+        if line == '':
+            break 
+    msg=parser.close()
+    POST = MultiValueDict()
+    FILES = MultiValueDict()
+    for submessage in msg.get_payload():
+        if isinstance(submessage, email.Message.Message):
+            name_dict = cgi.parse_header(submessage['Content-Disposition'])[1]
+            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
+            # or {'name': 'blah'} for POST fields
+            # We assume all uploaded files have a 'filename' set.
+            if name_dict.has_key('filename'):
+                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
+                if not name_dict['filename'].strip():
+                    continue
+                # IE submits the full path, so trim everything but the basename.
+                # (We can't use os.path.basename because it expects Linux paths.)
+                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
+                FILES.appendlist(name_dict['name'], FileDict(
+                    filename,
+                    (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
+                    submessage.get_payload()
+                ))
+            else:
+                POST.appendlist(name_dict['name'], submessage.get_payload())
+    return POST, FILES
+
+class StreamingUploadMiddleware:
+
+    def process_request(self, request):
+        request.parse_file_upload = parse_streaming_file_upload
+
+def get_temp_file(identifier):
+    return os.path.join(tempfile.gettempdir(),identifier)
+
+class UploadState:
+
+    def __init__(self, req):
+        self.identifier = req.META['QUERY_STRING']
+        self.state = {'size': int(req.header_dict.get('content-length')),
+             'state': 'starting', 'received': 0}
+        self.save()
+
+    def addlen(self, toadd):
+        self.state['received'] = self.state['received'] + toadd
+        if self.state['size']-1 <= self.state['received']:
+            self.state['state'] = 'done'
+        else:
+             self.state['state'] = 'uploading'
+        self.save()
+
+    def save(self): 
+        simplejson.dump(self.state,open(get_temp_file(self.identifier), 'w')) 
+
+class UploadStateMiddleware:
+
+    def process_request(self, request):
+
+        try:
+            progress_url=settings.PROGRESS_URL
+        except:
+            progress_url='/progress/'
+
+        if request.META['QUERY_STRING']:
+            request.upload_state = UploadState
+
+        if request.path == progress_url: 
+            for header in request.header_dict.items():
+                if header[0].upper().replace('-', '_').endswith('X_PROGRESS_ID'):
+                    progress_id = header[1]
+            try:
+                content = open(get_temp_file(progress_id), 'r').read()
+            except:
+                content="{}"
+            if not content:
+                content="{}"
+
+            from django.http import HttpResponse
+            return HttpResponse(content=content, mimetype='text/plain')