Ticket #1484: 3013-streaming-file.diff

django/http/__init__.py

@@ -2 +2 @@
 from pprint import pformat
 from urllib import urlencode
 from django.utils.datastructures import MultiValueDict
+import cgi
+from StringIO import StringIO
 
 try:
     # The mod_python version is more efficient, so try importing it first.
@@ -35 +37 @@
     def get_full_path(self):
         return ''
 
-def parse_file_upload(header_dict, post_data):
+class FileDict(dict):
+    "Keeps uploaded file as a file-like object and reads its content on demand"
+    def __getitem__(self, name):
+        if name=='content' and not 'content' in self:
+            self['file'].seek(0, 2)
+            size = self['file'].tell()
+            self['file'].seek(0, 0)
+            self['content']=self['file'].read(size)
+        return dict.__getitem__(self, name)
+
+    def __deepcopy__(self, memo={}):
+        self['content'] # make sure file content is loaded
+        import copy
+        result = self.__class__()
+        memo[id(self)] = result
+        for key, value in dict.items(self):
+            dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
+        return result
+
+class FieldStorage(cgi.FieldStorage):
+    "cgi.FieldStorage with ability to store files on disk or in memory"
+    def make_file(self, binary=None):
+        return cgi.FieldStorage.make_file(self, binary)    
+
+    def read_lines_to_eof(self):
+        """Internal: read lines until EOF."""
+        while 1:
+            line = self.fp.readline(64000)
+            if not line:
+                self.done = -1
+                break
+            self.__write(line)
+
+    def read_lines_to_outerboundary(self):
+        """Internal: read lines until outerboundary."""
+        next = "--" + self.outerboundary
+        last = next + "--"
+        delim = ""
+        while 1:
+            line = self.fp.readline(64000)
+            if not line:
+                self.done = -1
+                break
+            if line[:2] == "--":
+                strippedline = line.strip()
+                if strippedline == next:
+                    break
+                if strippedline == last:
+                    self.done = 1
+                    break
+            odelim = delim
+            if line[-2:] == "\r\n":
+                delim = "\r\n"
+                line = line[:-2]
+            elif line[-1] == "\n":
+                delim = "\n"
+                line = line[:-1]
+            else:
+                delim = ""
+            self.__write(odelim + line)
+
+
+
+def parse_file_upload(post_stream, environ):
     "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
-    import email, email.Message
-    from cgi import parse_header
-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
-    raw_message += '\r\n\r\n' + post_data
-    msg = email.message_from_string(raw_message)
+    fs = FieldStorage(post_stream, environ=environ)
     POST = MultiValueDict()
     FILES = MultiValueDict()
-    for submessage in msg.get_payload():
-        if isinstance(submessage, email.Message.Message):
-            name_dict = parse_header(submessage['Content-Disposition'])[1]
-            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
-            # or {'name': 'blah'} for POST fields
-            # We assume all uploaded files have a 'filename' set.
-            if name_dict.has_key('filename'):
-                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
-                if not name_dict['filename'].strip():
+    for key in fs.keys():
+        # We can't use FieldStorage.getlist to get contents of a 
+        # field as a list because for file fields it returns only filenames
+        if type(fs[key]) == type([]):
+            field_list = fs[key]
+        else:
+            field_list = [fs[key]]
+        for field in field_list:
+            if hasattr(field, 'filename') and field.filename is not None:
+                if not field.filename.strip():
                     continue
                 # IE submits the full path, so trim everything but the basename.
                 # (We can't use os.path.basename because it expects Linux paths.)
-                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
-                FILES.appendlist(name_dict['name'], {
+                filename = field.filename[field.filename.rfind("\\") + 1:]
+                FILES.appendlist(key, FileDict({
                     'filename': filename,
-                    'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
-                    'content': submessage.get_payload(),
-                })
+                    'content-type': field.type,
+                    'file': field.file,
+                }))
             else:
-                POST.appendlist(name_dict['name'], submessage.get_payload())
+                POST.appendlist(key, field.value)
     return POST, FILES
 
 class QueryDict(MultiValueDict):

django/conf/global_settings.py

@@ -211 +211 @@
 # Hint: you really don't!
 TRANSACTIONS_MANAGED = False
 
+# Whether to store uploaded files in temp files rather than in memory.
+# Storing files on disk may be necessary for accepting large files.
+STORE_UPLOAD_ON_DISK = False
+
 ##############
 # MIDDLEWARE #
 ##############

django/core/handlers/wsgi.py

@@ -69 +69 @@
         # Populates self._post and self._files
         if self.environ['REQUEST_METHOD'] == 'POST':
             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
+                self._post, self._files = http.parse_file_upload(self.environ['wsgi.input'], self.environ)
             else:
                 self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
         else:

django/core/handlers/modpython.py

@@ -26 +26 @@
     def _load_post_and_files(self):
         "Populates self._post and self._files"
         if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
+            environ = dict(self.META)
+            environ['CONTENT_LENGTH'] = environ['HTTP_CONTENT_LENGTH']
+            environ['CONTENT_TYPE'] = environ['HTTP_CONTENT_TYPE']
+            self._post, self._files = http.parse_file_upload(self._req, environ)
         else:
             self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()