Ticket #2070: 5722.2.diff

File 5722.2.diff, 22.2 KB (added by simonbun <simonbun@…>, 17 years ago)

Sorry, that last one patch also included #4165. This is the correct one.

  • django/http/__init__.py

     
    44from urllib import urlencode
    55from django.utils.datastructures import MultiValueDict
    66from django.utils.encoding import smart_str, iri_to_uri, force_unicode
     7from django.http.multipartparser import MultiPartParser, MultiPartParserError
     8import re
    79
     10upload_id_re = re.compile(r'^[a-fA-F0-9]{32}$') # file progress id Regular expression
     11
    812RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
    913
    1014try:
    … …  
    6468
    6569    encoding = property(_get_encoding, _set_encoding)
    6670
    67 def parse_file_upload(header_dict, post_data):
    68     "Returns a tuple of (POST QueryDict, FILES MultiValueDict)"
    69     import email, email.Message
    70     from cgi import parse_header
    71     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    72     raw_message += '\r\n\r\n' + post_data
    73     msg = email.message_from_string(raw_message)
    74     POST = QueryDict('', mutable=True)
    75     FILES = MultiValueDict()
    76     for submessage in msg.get_payload():
    77         if submessage and isinstance(submessage, email.Message.Message):
    78             name_dict = parse_header(submessage['Content-Disposition'])[1]
    79             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    80             # or {'name': 'blah'} for POST fields
    81             # We assume all uploaded files have a 'filename' set.
    82             if 'filename' in name_dict:
    83                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    84                 if not name_dict['filename'].strip():
    85                     continue
    86                 # IE submits the full path, so trim everything but the basename.
    87                 # (We can't use os.path.basename because it expects Linux paths.)
    88                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    89                 FILES.appendlist(name_dict['name'], {
    90                     'filename': filename,
    91                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    92                     'content': submessage.get_payload(),
    93                 })
    94             else:
    95                 POST.appendlist(name_dict['name'], submessage.get_payload())
    96     return POST, FILES
     71    def _get_file_progress(self):
     72        return {}
    9773
     74    def _set_file_progress(self,value):
     75        pass
     76
     77    def _del_file_progress(self):
     78        pass
     79
     80    file_progress = property(_get_file_progress,
     81                             _set_file_progress,
     82                             _del_file_progress)
     83
     84    def _get_file_progress_from_args(self, headers, get, querystring):
     85        """
     86        This parses the request for a file progress_id value.
     87        Note that there are two distinct ways of getting the progress
     88        ID -- header and GET. One is used primarily to attach via JavaScript
     89        to the end of an HTML form action while the other is used for AJAX
     90        communication.
     91       
     92        All progress IDs must be valid 32-digit hexadecimal numbers.
     93        """
     94        if 'X-Upload-ID' in headers:
     95            progress_id = headers['X-Upload-ID']
     96        elif 'progress_id' in get:
     97            progress_id = get['progress_id']
     98        else:
     99            return None
     100       
     101        if not upload_id_re.match(progress_id):
     102            return None
     103
     104        return progress_id
     105
     106def parse_file_upload(headers, input, request):
     107    from django.conf import settings
     108
     109    # Only stream files to disk if FILE_STREAMING_DIR is set
     110    file_upload_dir = settings.FILE_UPLOAD_DIR
     111    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE
     112   
     113    try:
     114        parser = MultiPartParser(headers, input, request, file_upload_dir, streaming_min_post_size)
     115        return parser.parse()
     116    except MultiPartParserError, e:
     117        return MultiValueDict({ '_file_upload_error': [e.message] }), {}
     118
     119
    98120class QueryDict(MultiValueDict):
    99121    """
    100122    A specialized MultiValueDict that takes a query string when initialized.

  • django/oldforms/__init__.py

     
    676676        self.validator_list = [self.isNonEmptyFile] + validator_list
    677677
    678678    def isNonEmptyFile(self, field_data, all_data):
    679         try:
    680             content = field_data['content']
    681         except TypeError:
     679        if field_data.has_key('_file_upload_error'):
     680            raise validators.CriticalValidationError, field_data['_file_upload_error']
     681        if not field_data.has_key('filename'):
    682682            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    683         if not content:
     683        if not field_data['content-length']:
    684684            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    685685
    686686    def render(self, data):
    687687        return u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    688688            (self.get_id(), self.__class__.__name__, self.field_name)
     689   
     690    def prepare(self, new_data):
     691        if new_data.has_key('_file_upload_error'):
     692            # pretend we got something in the field to raise a validation error later
     693            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
    689694
    690695    def html2python(data):
    691696        if data is None:

  • django/db/models/base.py

     
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
    1515from django.utils.encoding import smart_str, force_unicode
     16from django.utils.file import file_move_safe
    1617from django.conf import settings
    1718from itertools import izip
    1819import types
    … …  
    365366    def _get_FIELD_size(self, field):
    366367        return os.path.getsize(self._get_FIELD_filename(field))
    367368
    368     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     369    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    369370        directory = field.get_directory_name()
    370371        try: # Create the date-based directory if it doesn't exist.
    371372            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    372373        except OSError: # Directory probably already exists.
    373374            pass
     375
     376        if filename is None:
     377            filename = raw_field['filename']
     378
    374379        filename = field.get_filename(filename)
    375380
    376381        # If the filename already exists, keep adding an underscore to the name of
    … …  
    387392        setattr(self, field.attname, filename)
    388393
    389394        full_filename = self._get_FIELD_filename(field)
    390         fp = open(full_filename, 'wb')
    391         fp.write(raw_contents)
    392         fp.close()
     395        if raw_field.has_key('tmpfilename'):
     396            raw_field['tmpfile'].close()
     397            file_move_safe(raw_field['tmpfilename'], full_filename)
     398        else:
     399            from django.utils import file_locks
     400            fp = open(full_filename, 'wb')
     401            # exclusive lock
     402            file_locks.lock(fp, file_locks.LOCK_EX)
     403            fp.write(raw_field['content'])
     404            fp.close()
    393405
    394406        # Save the width and/or height, if applicable.
    395407        if isinstance(field, ImageField) and (field.width_field or field.height_field):

  • django/db/models/fields/__init__.py

     
    707707        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    708708        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    709709        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    710         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     710        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
     711        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))       
    711712        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    712713
    713714    def delete_file(self, instance):
    … …  
    730731        if new_data.get(upload_field_name, False):
    731732            func = getattr(new_object, 'save_%s_file' % self.name)
    732733            if rel:
    733                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     734                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
    734735            else:
    735                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     736                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
    736737
    737738    def get_directory_name(self):
    738739        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))

  • django/conf/global_settings.py

     
    247247from django import get_version
    248248URL_VALIDATOR_USER_AGENT = "Django/%s (http://www.djangoproject.com)" % get_version()
    249249
     250# The directory to place streamed file uploads. The web server needs write
     251# permissions on this directory.
     252# If this is None, streaming uploads are disabled.
     253FILE_UPLOAD_DIR = None
     254
     255# The minimum size of a POST before file uploads are streamed to disk.
     256# Any less than this number, and the file is uploaded to memory.
     257# Size is in bytes.
     258STREAMING_MIN_POST_SIZE = 512 * (2**10)
     259
    250260##############
    251261# MIDDLEWARE #
    252262##############

  • django/core/handlers/wsgi.py

     
    7676        self.environ = environ
    7777        self.path = force_unicode(environ['PATH_INFO'])
    7878        self.META = environ
     79        self.META['UPLOAD_PROGRESS_ID'] = self._get_file_progress_id()
    7980        self.method = environ['REQUEST_METHOD'].upper()
    8081
    8182    def __repr__(self):
    … …  
    112113            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    113114                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    114115                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    115                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     116                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
     117                header_dict['X-Progress-ID'] = self.environ.get('HTTP_X_PROGRESS_ID', '')
     118                try:
     119                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self)
     120                except:
     121                    self._post, self._files = {}, {} # make sure we dont read the input stream again
     122                    raise
     123                self._raw_post_data = None # raw data is not available for streamed multipart messages
    116124            else:
    117125                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    118126        else:
    … …  
    168176            buf.close()
    169177            return self._raw_post_data
    170178
     179    def _get_file_progress_id(self):
     180        """
     181        Returns the Progress ID of the request,
     182        usually provided if there is a file upload
     183        going on.
     184        Returns ``None`` if no progress ID is specified.
     185        """
     186        return self._get_file_progress_from_args(self.environ,
     187                                                 self.GET,
     188                                                 self.environ.get('QUERY_STRING', ''))
     189
    171190    GET = property(_get_get, _set_get)
    172191    POST = property(_get_post, _set_post)
    173192    COOKIES = property(_get_cookies, _set_cookies)

  • django/core/handlers/modpython.py

     
    4848    def _load_post_and_files(self):
    4949        "Populates self._post and self._files"
    5050        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    51             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     51            self._raw_post_data = None # raw data is not available for streamed multipart messages
     52            try:
     53                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self)
     54            except:
     55                self._post, self._files = {}, {} # make sure we dont read the input stream again
     56                raise
    5257        else:
    5358            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5459
    … …  
    9398                'AUTH_TYPE':         self._req.ap_auth_type,
    9499                'CONTENT_LENGTH':    self._req.clength, # This may be wrong
    95100                'CONTENT_TYPE':      self._req.content_type, # This may be wrong
    96                 'GATEWAY_INTERFACE': 'CGI/1.1',
    97                 'PATH_INFO':         self._req.path_info,
    98                 'PATH_TRANSLATED':   None, # Not supported
    99                 'QUERY_STRING':      self._req.args,
    100                 'REMOTE_ADDR':       self._req.connection.remote_ip,
    101                 'REMOTE_HOST':       None, # DNS lookups not supported
    102                 'REMOTE_IDENT':      self._req.connection.remote_logname,
    103                 'REMOTE_USER':       self._req.user,
    104                 'REQUEST_METHOD':    self._req.method,
    105                 'SCRIPT_NAME':       None, # Not supported
    106                 'SERVER_NAME':       self._req.server.server_hostname,
    107                 'SERVER_PORT':       self._req.server.port,
    108                 'SERVER_PROTOCOL':   self._req.protocol,
    109                 'SERVER_SOFTWARE':   'mod_python'
     101                'GATEWAY_INTERFACE':  'CGI/1.1',
     102                'PATH_INFO':          self._req.path_info,
     103                'PATH_TRANSLATED':    None, # Not supported
     104                'QUERY_STRING':       self._req.args,
     105                'REMOTE_ADDR':        self._req.connection.remote_ip,
     106                'REMOTE_HOST':        None, # DNS lookups not supported
     107                'REMOTE_IDENT':       self._req.connection.remote_logname,
     108                'REMOTE_USER':        self._req.user,
     109                'REQUEST_METHOD':     self._req.method,
     110                'SCRIPT_NAME':        None, # Not supported
     111                'SERVER_NAME':        self._req.server.server_hostname,
     112                'SERVER_PORT':        self._req.server.port,
     113                'SERVER_PROTOCOL':    self._req.protocol,
     114                'UPLOAD_PROGRESS_ID': self._get_file_progress_id(),
     115                'SERVER_SOFTWARE':    'mod_python'
    110116            }
    111117            for key, value in self._req.headers_in.items():
    112118                key = 'HTTP_' + key.upper().replace('-', '_')
    … …  
    123129    def _get_method(self):
    124130        return self.META['REQUEST_METHOD'].upper()
    125131
     132    def _get_file_progress_id(self):
     133        """
     134        Returns the Progress ID of the request,
     135        usually provided if there is a file upload
     136        going on.
     137        Returns ``None`` if no progress ID is specified.
     138        """
     139        return self._get_file_progress_from_args(self._req.headers_in,
     140                                                 self.GET,
     141                                                 self._req.args)
     142
    126143    GET = property(_get_get, _set_get)
    127144    POST = property(_get_post, _set_post)
    128145    COOKIES = property(_get_cookies, _set_cookies)

  • tests/modeltests/test_client/views.py

     
    4646
    4747    return HttpResponse(t.render(c))
    4848
     49def post_file_view(request):
     50    "A view that expects a multipart post and returns a file in the context"
     51    t = Template('File {{ file.filename }} received', name='POST Template')
     52    c = Context({'file': request.FILES['file_file']})
     53    return HttpResponse(t.render(c))
     54
    4955def redirect_view(request):
    5056    "A view that redirects all requests to the GET view"
    5157    return HttpResponseRedirect('/test_client/get_view/')

  • tests/modeltests/test_client/models.py

     
    44
    55The test client is a class that can act like a simple
    66browser for testing purposes.
    7  
     7
    88It allows the user to compose GET and POST requests, and
    99obtain the response that the server gave to those requests.
    1010The server Response objects are annotated with the details
    … …  
    8080        self.assertEqual(response.template.name, "Book template")
    8181        self.assertEqual(response.content, "Blink - Malcolm Gladwell")
    8282
     83    def test_post_file_view(self):
     84        "POST this python file to a view"
     85        import os, tempfile
     86        from django.conf import settings
     87        file = __file__.replace('.pyc', '.py')
     88        for upload_dir in [None, tempfile.gettempdir()]:
     89            settings.FILE_UPLOAD_DIR = upload_dir
     90            post_data = { 'name': file, 'file': open(file) }
     91            response = self.client.post('/test_client/post_file_view/', post_data)
     92            self.failUnless('models.py' in response.context['file']['filename'])
     93            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
     94            if upload_dir:
     95                self.failUnless(response.context['file']['tmpfilename'])
     96
    8397    def test_redirect(self):
    8498        "GET a URL that redirects elsewhere"
    8599        response = self.client.get('/test_client/redirect_view/')

  • tests/modeltests/test_client/urls.py

     
    55urlpatterns = patterns('',
    66    (r'^get_view/$', views.get_view),
    77    (r'^post_view/$', views.post_view),
     8    (r'^post_file_view/$', views.post_file_view),
    89    (r'^raw_post_view/$', views.raw_post_view),
    910    (r'^redirect_view/$', views.redirect_view),
    1011    (r'^permanent_redirect_view/$', redirect_to, { 'url': '/test_client/get_view/' }),

  • docs/request_response.txt

     
    7272``FILES``
    7373    A dictionary-like object containing all uploaded files. Each key in
    7474    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    75     value in ``FILES`` is a standard Python dictionary with the following three
     75    value in ``FILES`` is a standard Python dictionary with the following four
    7676    keys:
    7777
    7878        * ``filename`` -- The name of the uploaded file, as a Python string.
    7979        * ``content-type`` -- The content type of the uploaded file.
    8080        * ``content`` -- The raw content of the uploaded file.
     81        * ``content-length`` -- The length of the content in bytes.
    8182
     83    If streaming file uploads are enabled two additional keys
     84    describing the uploaded file will be present:
     85
     86        * ``tmpfilename`` -- The filename for the temporary file.
     87        * ``tmpfile`` -- An open file object for the temporary file.
     88
     89    The temporary file will be removed when the request finishes.
     90
     91    Note that accessing ``content`` when streaming uploads are enabled
     92    will read the whole file into memory which may not be what you want.
     93
    8294    Note that ``FILES`` will only contain data if the request method was POST
    8395    and the ``<form>`` that posted to the request had
    8496    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank

  • docs/settings.txt

     
    472472
    473473.. _Testing Django Applications: ../testing/
    474474
     475FILE_UPLOAD_DIR
     476---------------
     477
     478Default: ``None``
     479
     480Path to a directory where temporary files should be written during
     481file uploads. Leaving this as ``None`` will disable streaming file uploads,
     482and cause all uploaded files to be stored (temporarily) in memory.
     483
    475484IGNORABLE_404_ENDS
    476485------------------
    477486
    … …  
    788797
    789798.. _site framework docs: ../sites/
    790799
     800STREAMING_MIN_POST_SIZE
     801-----------------------
     802
     803Default: 524288 (``512*1024``)
     804
     805An integer specifying the minimum number of bytes that has to be
     806received (in a POST) for file upload streaming to take place. Any
     807request smaller than this will be handled in memory.
     808Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
     809
    791810TEMPLATE_CONTEXT_PROCESSORS
    792811---------------------------
    793812

  • docs/forms.txt

     
    475475   new_data = request.POST.copy()
    476476   new_data.update(request.FILES)
    477477
     478Streaming file uploads.
     479-----------------------
     480
     481File uploads will be read into memory by default. This works fine for
     482small to medium sized uploads (from 1MB to 100MB depending on your
     483setup and usage). If you want to support larger uploads you can enable
     484upload streaming where only a small part of the file will be in memory
     485at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
     486setting (see the settings_ document for more details).
     487
     488See `request object`_ for more details about ``request.FILES`` objects
     489with streaming file uploads enabled.
     490
    478491Validators
    479492==========
    480493
    … …  
    698711.. _`generic views`: ../generic_views/
    699712.. _`models API`: ../model-api/
    700713.. _settings: ../settings/
     714.. _request object: ../request_response/#httprequest-objects
Back to Top