Ticket #2070: 3581-streaming_uploads_and_uploadprogress_middleware_x_progress_id.diff

django/http/__init__.py

@@ -1 +1 @@
 import os
+import cgi
 from Cookie import SimpleCookie
 from pprint import pformat
 from urllib import urlencode, quote
@@ -42 +43 @@
     def is_secure(self):
         return os.environ.get("HTTPS") == "on"
 
-def parse_file_upload(header_dict, post_data):
+def default_parse_file_upload(req):
     "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
     import email, email.Message
     from cgi import parse_header
-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
-    raw_message += '\r\n\r\n' + post_data
+    raw_message = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()])
+    raw_message += '\r\n\r\n' + req.raw_post_data
     msg = email.message_from_string(raw_message)
     POST = MultiValueDict()
     FILES = MultiValueDict()
@@ -73 +74 @@
                 POST.appendlist(name_dict['name'], submessage.get_payload())
     return POST, FILES
 
+def parse_file_upload(req):
+    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
+
+    if hasattr(req, 'parse_file_upload'):
+        return req.parse_file_upload(req)
+
+    return default_parse_file_upload(req)
+
 class QueryDict(MultiValueDict):
     """A specialized MultiValueDict that takes a query string when initialized.
     This is immutable unless you create a copy of it."""

django/db/models/base.py

@@ -322 +322 @@
     def _get_FIELD_size(self, field):
         return os.path.getsize(self._get_FIELD_filename(field))
 
-    def _save_FIELD_file(self, field, filename, raw_contents):
+    def _save_FIELD_file(self, field, filename, raw_field):
         directory = field.get_directory_name()
         try: # Create the date-based directory if it doesn't exist.
             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
@@ -344 +344 @@
         setattr(self, field.attname, filename)
 
         full_filename = self._get_FIELD_filename(field)
-        fp = open(full_filename, 'wb')
-        fp.write(raw_contents)
-        fp.close()
 
+        if not hasattr(raw_field, 'get_size'):
+            fp = open(full_filename, 'wb')
+            fp.write(raw_field['content'])
+            fp.close()
+        else:
+            os.rename(raw_field['tmp_name'], full_filename)
+
         # Save the width and/or height, if applicable.
         if isinstance(field, ImageField) and (field.width_field or field.height_field):
             from django.utils.images import get_image_dimensions

django/db/models/fields/__init__.py

@@ -600 +600 @@
         if new_data.get(upload_field_name, False):
             func = getattr(new_object, 'save_%s_file' % self.name)
             if rel:
-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"])
+                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0])
             else:
-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"])
+                func(new_data[upload_field_name]["filename"], new_data[upload_field_name])
 
     def get_directory_name(self):
         return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))

django/forms/__init__.py

@@ -655 +655 @@
         self.validator_list = [self.isNonEmptyFile] + validator_list
 
     def isNonEmptyFile(self, field_data, all_data):
+
         try:
-            content = field_data['content']
-        except TypeError:
-            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
+            content = field_data.get_size()
+
+        except:
+            try:
+                content = field_data['content']
+            except TypeError:
+                raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
+
         if not content:
             raise validators.CriticalValidationError, gettext("The submitted file is empty.")
 

django/core/handlers/wsgi.py

@@ -72 +72 @@
         # Populates self._post and self._files
         if self.method == 'POST':
             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
+                self._post, self._files = http.parse_file_upload(self)
             else:
                 self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
         else:
@@ -121 +119 @@
         except AttributeError:
             self._raw_post_data = self.environ['wsgi.input'].read(int(self.environ["CONTENT_LENGTH"]))
             return self._raw_post_data
+    
+    def _get_raw_request(self):
+        return self.environ['wsgi.input']
 
+    def _get_header_dict(self):
+        header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
+        header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
+        return header_dict
+
     GET = property(_get_get, _set_get)
     POST = property(_get_post, _set_post)
     COOKIES = property(_get_cookies, _set_cookies)
     FILES = property(_get_files)
     REQUEST = property(_get_request)
     raw_post_data = property(_get_raw_post_data)
+    raw_request = property(_get_raw_request)
+    header_dict = property(_get_header_dict)
 
 class WSGIHandler(BaseHandler):
     def __call__(self, environ, start_response):

django/core/handlers/modpython.py

@@ -29 +29 @@
     def _load_post_and_files(self):
         "Populates self._post and self._files"
         if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
+            self._post, self._files = http.parse_file_upload(self)
         else:
             self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
 
@@ -104 +104 @@
     def _get_method(self):
         return self.META['REQUEST_METHOD'].upper()
 
+    def _get_raw_request(self):
+        return self._req
+
+    def _get_header_dict(self):
+        return self._req.headers_in
+
     GET = property(_get_get, _set_get)
     POST = property(_get_post, _set_post)
     COOKIES = property(_get_cookies, _set_cookies)
@@ -112 +118 @@
     REQUEST = property(_get_request)
     raw_post_data = property(_get_raw_post_data)
     method = property(_get_method)
+    raw_request = property(_get_raw_request)
+    header_dict = property(_get_header_dict)    
 
 class ModPythonHandler(BaseHandler):
     def __call__(self, req):

django/contrib/admin/media/js/UploadProgress.js

@@ +1 @@
+
+function getxy(){
+    var x,y;
+    if (self.innerHeight) // all except Explorer
+        {
+        x = self.innerWidth;
+        y = self.innerHeight;
+        }
+    else if (document.documentElement && document.documentElement.clientHeight)
+        // Explorer 6 Strict Mode
+        {
+        x = document.documentElement.clientWidth;
+        y = document.documentElement.clientHeight;
+        }
+    else if (document.body) // other Explorers
+        {
+        x = document.body.clientWidth;
+        y = document.body.clientHeight;
+        }
+    return {'x':x,'y':y}
+    }
+
+var humanvalue = ['B','KB','MB','GB']
+function humanize(bytes) {
+    curbytes = bytes
+    iterations = 0
+    while (curbytes>1024) {
+        iterations++
+        curbytes=curbytes/1024
+        }
+    return curbytes.toFixed(1) + ' ' + humanvalue[iterations]
+    }
+
+interval = null;
+function fetch(uuid) {
+    req = xmlhttp
+    req.open("GET", "/progress/", 1);
+    req.setRequestHeader("X-Progress-Id", uuid);
+    req.onreadystatechange = function () {
+    if (req.readyState == 4) {
+        if (req.status == 200) {
+
+            var upload = eval( '(' + req.responseText + ')' );
+
+            if (upload.state == 'done' || upload.state == 'uploading') {
+                bar = document.getElementById('progress_bar');
+                bar_txt = document.getElementById('progress_text')
+                bar_txt.innerHTML = ((upload.received / upload.size) * 100).toFixed(1) + '% - ' +
+                    humanize(upload.received) + ' of ' + humanize(upload.size) 
+                w = 400 * upload.received / upload.size;
+                bar.style.width = w + 'px';
+
+                }
+                if (upload.state == 'done') {
+                    window.clearTimeout(interval);
+                    }
+                }
+            }
+        }
+    req.send(null); 
+
+    }
+
+function openprogress(e) {
+
+    uuid = "";
+    for (i = 0; i < 32; i++) {
+        uuid += Math.floor(Math.random() * 16).toString(16);
+        }
+    frm = e.target||e.srcElement
+
+    frm.action=frm.action+"?" + uuid; 
+
+    pos = getxy()
+    posx = parseInt((pos.x/2)-(420/2), 10)
+    posy = parseInt((pos.y/2)-(50/2), 10)
+
+    progress_wrap = quickElement('div', document.body, '', 'style', 
+        'position: absolute; top: '+posy+'px; left: '+posx+'px; height: 50px; ' +
+        'padding: 10px; width: 420px; background: #ffffff; ' +
+        'border: solid 1px #dddddd;', 'id', 'progress_wrap')
+
+    progress_label = quickElement('h1', progress_wrap, 'Upload progress')
+
+    progress = quickElement('div', progress_wrap, '', 'style', 
+        'top: 0; left: 0; width: 0px; ', 'id', 'progress_bar', 'class', 'submit-row')
+
+    progress_text = quickElement('div', progress_wrap, '0%', 'style',
+        'color: #000000; ', 'id', 'progress_text')
+ 
+    interval = window.setInterval(
+        function () {
+            fetch(uuid);
+            },
+        1000
+        );
+    }
+
+addEvent(window, 'load', function() {
+        frm = document.getElementsByTagName('form')[0]
+        addEvent(frm, 'submit',  openprogress)    
+        }
+    )

django/middleware/upload.py

@@ +1 @@
+"streaming upload middleware"
+import cgi
+import os
+import tempfile
+from django.conf import settings
+from django.utils.datastructures import MultiValueDict
+from django.utils import simplejson
+
+try:
+    UPLOAD_BUFFER_SIZE = settings.UPLOAD_BUFFER_SIZE
+except:
+    UPLOAD_BUFFER_SIZE = 64000
+
+class FileDict(dict):
+    "Keeps uploaded file as a file-like object and reads its content on demand"
+    def __getitem__(self, name):
+        if name=='content' and not 'content' in self:
+            self['file'].seek(0, 2)
+            size = self['file'].tell()
+            self['file'].seek(0, 0)
+            self['content']=self['file'].read(size)
+        return dict.__getitem__(self, name)
+        
+    def get_size(self):
+        self['file'].seek(0, 2)    
+        size = self['file'].tell()
+        return size
+
+    def __repr__(self):
+        return '<FileDict>'
+
+class PostStream:
+
+    def __init__(self, fp, upload_state=None):
+        self.fp = fp
+        self.upload_state = upload_state
+        self.bufsize = UPLOAD_BUFFER_SIZE
+ 
+    def readline(self):
+        data = self.fp.readline(self.bufsize)
+        if self.upload_state:
+            self.upload_state.addlen(len(data)) 
+        return data
+
+    def read(self):
+        data = self.fp.read(self.bufsize)
+        if self.upload_state:
+            self.upload_state.addlen(len(data)) 
+        return data 
+
+class FieldStorageM(cgi.FieldStorage, object):
+    pass
+ 
+class FieldStorage(FieldStorageM):
+
+    upload_state = None
+    bufsize = UPLOAD_BUFFER_SIZE
+     
+    "cgi.FieldStorage with ability to store files on disk"
+
+    def __init__(self, fp=None, headers=None, outerboundary="",
+                 environ=os.environ, keep_blank_values=0, strict_parsing=0, upload_state=None):
+
+        if not isinstance(fp, PostStream):
+            stream = PostStream(fp, upload_state=upload_state)
+        else: 
+            stream = fp
+
+        super(FieldStorage, self).__init__(fp=stream, headers=headers,
+            outerboundary=outerboundary, environ=environ,
+            keep_blank_values=keep_blank_values, strict_parsing=strict_parsing)
+
+    def make_file(self, binary=None):
+     
+        tmpfile = tempfile.NamedTemporaryFile("w+b")
+        self.tmp_name = tmpfile.name
+        return tmpfile
+
+
+def parse_streaming_file_upload(req):
+    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
+    if hasattr(req, 'upload_state'):
+        upload_state = req.upload_state(req)
+    else:
+        upload_state = None
+
+    fs = FieldStorage(req.raw_request, environ=req.META, headers=req.header_dict, upload_state=upload_state )
+    POST = MultiValueDict()
+    FILES = MultiValueDict()
+    for key in fs.keys():
+        # We can't use FieldStorage.getlist to get contents of a 
+        # field as a list because for file fields it returns only filenames
+        if type(fs[key]) == type([]):
+            field_list = fs[key]
+        else:
+            field_list = [fs[key]]
+        for field in field_list:
+            if hasattr(field, 'filename') and field.filename is not None:
+                if not field.filename.strip():
+                    continue
+                # IE submits the full path, so trim everything but the basename.
+                # (We can't use os.path.basename because it expects Linux paths.)
+                filename = field.filename[field.filename.rfind("\\") + 1:]
+                FILES.appendlist(key, FileDict({
+                    'filename': filename,
+                    'content-type': field.type,
+                    'file': field.file,
+                    'tmp_name': field.tmp_name
+                }))
+            else:
+                POST.appendlist(key, field.value)
+    return POST, FILES
+
+class StreamingUploadMiddleware:
+
+    def process_request(self, request):
+        request.parse_file_upload = parse_streaming_file_upload
+
+def get_temp_file(identifier):
+    return os.path.join(tempfile.gettempdir(),identifier)
+
+class UploadState:
+
+    def __init__(self, req):
+        self.identifier = req.META['QUERY_STRING']
+        self.state = {'size': int(req.header_dict.get('content-length')),
+             'state': 'starting', 'received': 0}
+        self.save()
+
+    def addlen(self, toadd):
+        self.state['received'] = self.state['received'] + toadd
+        if self.state['size']-1 <= self.state['received']:
+            self.state['state'] = 'done'
+        else:
+             self.state['state'] = 'uploading'
+        self.save()
+
+    def save(self): 
+        simplejson.dump(self.state,open(get_temp_file(self.identifier), 'w')) 
+
+class UploadStateMiddleware:
+    def process_request(self, request):
+        if request.META['QUERY_STRING']:
+            request.upload_state = UploadState
+        if request.path == '/progress/':
+            for header in request.header_dict.items():
+                if header[0].upper().replace('-', '_').endswith('X_PROGRESS_ID'):
+                    progress_id = header[1]
+            try:
+                content = open(get_temp_file(progress_id), 'r').read()
+            except:
+                content="{}"
+            if not content:
+                content="{}"
+
+            from django.http import HttpResponse
+            return HttpResponse(content=content, mimetype='text/plain')