Ticket #2070: 5820_streaming_file_upload.diff

File 5820_streaming_file_upload.diff, 37.8 KB (added by Brian Rosner <brosner@…>, 17 years ago)

updated to r5820

  • django/http/multipartparser.py

     
     1"""
     2MultiPart parsing for file uploads.
     3If both a progress id is sent (either through ``X-Progress-ID``
     4header or ``progress_id`` GET) and ``FILE_UPLOAD_DIR`` is set
     5in the settings, then the file progress will be tracked using
     6``request.file_progress``.
     7
     8To use this feature, consider creating a middleware with an appropriate
     9``process_request``::
     10
     11    class FileProgressTrack(object):
     12        def __get__(self, request, HttpRequest):
     13            progress_id = request.META['UPLOAD_PROGRESS_ID']
     14            status = # get progress from progress_id here
     15
     16            return status
     17
     18        def __set__(self, request, new_value):
     19            progress_id = request.META['UPLOAD_PROGRESS_ID']
     20
     21            # set the progress using progress_id here.
     22
     23    # example middleware
     24    class FileProgressExample(object):
     25        def process_request(self, request):
     26            request.__class__.file_progress = FileProgressTrack()
     27
     28
     29
     30"""
     31
     32__all__ = ['MultiPartParserError','MultiPartParser']
     33
     34
     35from django.utils.datastructures import MultiValueDict
     36import os
     37
     38try:
     39    from cStringIO import StringIO
     40except ImportError:
     41    from StringIO import StringIO
     42
     43
     44class MultiPartParserError(Exception):
     45    def __init__(self, message):
     46        self.message = message
     47    def __str__(self):
     48        return repr(self.message)
     49
     50class MultiPartParser(object):
     51    """
     52    A rfc2388 multipart/form-data parser.
     53   
     54    parse() reads the input stream in chunk_size chunks and returns a
     55    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     56    file_upload_dir is defined files will be streamed to temporary
     57    files in the specified directory.
     58
     59    The FILES dictionary will have 'filename', 'content-type',
     60    'content' and 'content-length' entries. For streamed files it will
     61    also have 'tmpfilename' and 'tmpfile'. The 'content' entry will
     62    only be read from disk when referenced for streamed files.
     63
     64    If the X-Progress-ID is sent (in one of many formats), then
     65    object.file_progress will be given a dictionary of the progress.
     66    """
     67    def __init__(self, headers, input, request, file_upload_dir=None, streaming_min_post_size=None, chunk_size=1024*64):
     68        try:
     69            content_length = int(headers['Content-Length'])
     70        except:
     71            raise MultiPartParserError('Invalid Content-Length: %s' % headers.get('Content-Length'))
     72
     73        content_type = headers.get('Content-Type')
     74
     75        if not content_type or not content_type.startswith('multipart/'):
     76            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)
     77           
     78        ctype, opts = self.parse_header(content_type)
     79        boundary = opts.get('boundary')
     80        from cgi import valid_boundary
     81        if not boundary or not valid_boundary(boundary):
     82            raise MultiPartParserError('Invalid boundary in multipart form: %s' % boundary)
     83
     84        progress_id = request.META['UPLOAD_PROGRESS_ID']
     85
     86        self._track_progress = file_upload_dir and progress_id # whether or not to track progress
     87        self._boundary = '--' + boundary
     88        self._input = input
     89        self._size = content_length
     90        self._received = 0
     91        self._file_upload_dir = file_upload_dir
     92        self._chunk_size = chunk_size
     93        self._state = 'PREAMBLE'
     94        self._partial = ''
     95        self._post = MultiValueDict()
     96        self._files = MultiValueDict()
     97        self._request = request
     98
     99        if streaming_min_post_size is not None and content_length < streaming_min_post_size:
     100            self._file_upload_dir = None # disable file streaming for small request
     101        elif self._track_progress:
     102            request.file_progress = {'state': 'starting'}
     103
     104        try:
     105            # Use mx fast string search if available.
     106            from mx.TextTools import FS
     107            self._fs = FS(self._boundary)
     108        except ImportError:
     109            self._fs = None
     110
     111    def parse(self):
     112        try:
     113            self._parse()
     114        finally:
     115            if self._track_progress:
     116                self._request.file_progress = {'state': 'done'}
     117        return self._post, self._files
     118
     119    def _parse(self):
     120        size = self._size
     121
     122        try:
     123            while size > 0:
     124                n = self._read(self._input, min(self._chunk_size, size))
     125                if not n:
     126                    break
     127                size -= n
     128        except:
     129            # consume any remaining data so we dont generate a "Connection Reset" error
     130            size = self._size - self._received
     131            while size > 0:
     132                data = self._input.read(min(self._chunk_size, size))
     133                size -= len(data)
     134            raise
     135
     136    def _find_boundary(self, data, start, stop):
     137        """
     138        Find the next boundary and return the end of current part
     139        and start of next part.
     140        """
     141        if self._fs:
     142            boundary = self._fs.find(data, start, stop)
     143        else:
     144            boundary = data.find(self._boundary, start, stop)
     145        if boundary >= 0:
     146            end = boundary
     147            next = boundary + len(self._boundary)
     148
     149            # backup over CRLF
     150            if end > 0 and data[end-1] == '\n': end -= 1
     151            if end > 0 and data[end-1] == '\r': end -= 1
     152            # skip over --CRLF
     153            if next < stop and data[next] == '-': next += 1
     154            if next < stop and data[next] == '-': next += 1
     155            if next < stop and data[next] == '\r': next += 1
     156            if next < stop and data[next] == '\n': next += 1
     157
     158            return True, end, next
     159        else:
     160            return False, stop, stop
     161
     162    class TemporaryFile(object):
     163        "A temporary file that tries to delete itself when garbage collected."
     164        def __init__(self, dir):
     165            import tempfile
     166            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     167            self.file = os.fdopen(fd, 'w+b')
     168            self.name = name
     169
     170        def __getattr__(self, name):
     171            a = getattr(self.__dict__['file'], name)
     172            if type(a) != type(0):
     173                setattr(self, name, a)
     174            return a
     175
     176        def __del__(self):
     177            try:
     178                os.unlink(self.name)
     179            except OSError:
     180                pass
     181
     182    class LazyContent(dict):
     183        """
     184        A lazy FILES dictionary entry that reads the contents from
     185        tmpfile only when referenced.
     186        """
     187        def __init__(self, data):
     188            dict.__init__(self, data)
     189
     190        def __getitem__(self, key):
     191            if key == 'content' and not self.has_key(key):
     192                self['tmpfile'].seek(0)
     193                self['content'] = self['tmpfile'].read()
     194            return dict.__getitem__(self, key)
     195
     196    def _read(self, input, size):
     197        data = input.read(size)
     198
     199        if not data:
     200            return 0
     201
     202        read_size = len(data)
     203        self._received += read_size
     204
     205        if self._partial:
     206            data = self._partial + data
     207
     208        start = 0
     209        stop = len(data)
     210
     211        while start < stop:
     212            boundary, end, next = self._find_boundary(data, start, stop)
     213
     214            if not boundary and read_size:
     215                # make sure we dont treat a partial boundary (and its separators) as data
     216                stop -= len(self._boundary) + 16
     217                end = next = stop
     218                if end <= start:
     219                    break # need more data
     220
     221            if self._state == 'PREAMBLE':
     222                # Preamble, just ignore it
     223                self._state = 'HEADER'
     224
     225            elif self._state == 'HEADER':
     226                # Beginning of header, look for end of header and parse it if found.
     227
     228                header_end = data.find('\r\n\r\n', start, stop)
     229                if header_end == -1:
     230                    break # need more data
     231
     232                header = data[start:header_end]
     233
     234                self._fieldname = None
     235                self._filename = None
     236                self._content_type = None
     237
     238                for line in header.split('\r\n'):
     239                    ctype, opts = self.parse_header(line)
     240                    if ctype == 'content-disposition: form-data':
     241                        self._fieldname = opts.get('name')
     242                        self._filename = opts.get('filename')
     243                    elif ctype.startswith('content-type: '):
     244                        self._content_type = ctype[14:]
     245
     246                if self._filename is not None:
     247                    # cleanup filename from IE full paths:
     248                    self._filename = self._filename[self._filename.rfind("\\")+1:].strip()
     249
     250                    if self._filename: # ignore files without filenames
     251                        if self._file_upload_dir:
     252                            try:
     253                                self._file = self.TemporaryFile(dir=self._file_upload_dir)
     254                            except (OSError, IOError), e:
     255                                raise MultiPartParserError("Failed to create temporary file. Error was %s" % e)
     256                        else:
     257                            self._file = StringIO()
     258                    else:
     259                        self._file = None
     260                    self._filesize = 0
     261                    self._state = 'FILE'
     262                else:
     263                    self._field = StringIO()
     264                    self._state = 'FIELD'
     265                next = header_end + 4
     266
     267            elif self._state == 'FIELD':
     268                # In a field, collect data until a boundary is found.
     269
     270                self._field.write(data[start:end])
     271                if boundary:
     272                    if self._fieldname:
     273                        self._post.appendlist(self._fieldname, self._field.getvalue())
     274                    self._field.close()
     275                    self._state = 'HEADER'
     276
     277            elif self._state == 'FILE':
     278                # In a file, collect data until a boundary is found.
     279
     280                if self._file:
     281                    try:
     282                        self._file.write(data[start:end])
     283                    except IOError, e:
     284                        raise MultiPartParserError("Failed to write to temporary file.")
     285                    self._filesize += end-start
     286
     287                    if self._track_progress:
     288                        self._request.file_progress = {'received': self._received,
     289                                                       'size':     self._size,
     290                                                       'state':    'uploading'}
     291
     292                if boundary:
     293                    if self._file:
     294                        if self._file_upload_dir:
     295                            self._file.seek(0)
     296                            file = self.LazyContent({
     297                                'filename': self._filename,
     298                                'content-type':  self._content_type,
     299                                # 'content': is read on demand
     300                                'content-length': self._filesize,
     301                                'tmpfilename': self._file.name,
     302                                'tmpfile': self._file
     303                            })
     304                        else:
     305                            file = {
     306                                'filename': self._filename,
     307                                'content-type':  self._content_type,
     308                                'content': self._file.getvalue(),
     309                                'content-length': self._filesize
     310                            }
     311                            self._file.close()
     312
     313                        self._files.appendlist(self._fieldname, file)
     314
     315                    self._state = 'HEADER'
     316
     317            start = next
     318
     319        self._partial = data[start:]
     320
     321        return read_size
     322
     323    def parse_header(self, line):
     324        from cgi import parse_header
     325        return parse_header(line)

  • django/http/__init__.py

     
    11import os
     2import re
    23from Cookie import SimpleCookie
    34from pprint import pformat
    45from urllib import urlencode
    56from django.utils.datastructures import MultiValueDict
     7from django.http.multipartparser import MultiPartParser, MultiPartParserError
    68from django.utils.encoding import smart_str, iri_to_uri, force_unicode
    79
     10upload_id_re = re.compile(r'^[a-fA-F0-9]{32}$') # file progress id Regular expression
     11
    812RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
    913
    1014try:
    … …  
    4650
    4751    def is_secure(self):
    4852        return os.environ.get("HTTPS") == "on"
    49 
     53       
    5054    def _set_encoding(self, val):
    5155        """
    5256        Sets the encoding used for GET/POST accesses. If the GET or POST
    … …  
    6468
    6569    encoding = property(_get_encoding, _set_encoding)
    6670
    67 def parse_file_upload(header_dict, post_data):
    68     "Returns a tuple of (POST QueryDict, FILES MultiValueDict)"
    69     import email, email.Message
    70     from cgi import parse_header
    71     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    72     raw_message += '\r\n\r\n' + post_data
    73     msg = email.message_from_string(raw_message)
    74     POST = QueryDict('', mutable=True)
    75     FILES = MultiValueDict()
    76     for submessage in msg.get_payload():
    77         if submessage and isinstance(submessage, email.Message.Message):
    78             name_dict = parse_header(submessage['Content-Disposition'])[1]
    79             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    80             # or {'name': 'blah'} for POST fields
    81             # We assume all uploaded files have a 'filename' set.
    82             if 'filename' in name_dict:
    83                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    84                 if not name_dict['filename'].strip():
    85                     continue
    86                 # IE submits the full path, so trim everything but the basename.
    87                 # (We can't use os.path.basename because it expects Linux paths.)
    88                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    89                 FILES.appendlist(name_dict['name'], {
    90                     'filename': filename,
    91                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    92                     'content': submessage.get_payload(),
    93                 })
    94             else:
    95                 POST.appendlist(name_dict['name'], submessage.get_payload())
    96     return POST, FILES
     71    def _get_file_progress(self):
     72        return {}
     73   
     74    def _set_file_progress(self,value):
     75        pass
    9776
     77    def _del_file_progress(self):
     78        pass
     79
     80    file_progress = property(_get_file_progress,
     81                             _set_file_progress,
     82                             _del_file_progress)
     83
     84    def _get_file_progress_from_args(self, headers, get, querystring):
     85        """
     86        This parses the request for a file progress_id value.
     87        Note that there are two distinct ways of getting the progress
     88        ID -- header and GET. One is used primarily to attach via JavaScript
     89        to the end of an HTML form action while the other is used for AJAX
     90        communication.
     91
     92        All progress IDs must be valid 32-digit hexadecimal numbers.
     93        """
     94        if 'X-Upload-ID' in headers:
     95            progress_id = headers['X-Upload-ID']
     96        elif 'progress_id' in get:
     97            progress_id = get['progress_id']
     98        else:
     99            return None
     100
     101        if not upload_id_re.match(progress_id):
     102            return None
     103
     104        return progress_id
     105
     106def parse_file_upload(headers, input, request):
     107    from django.conf import settings
     108
     109    # Only stream files to disk if FILE_STREAMING_DIR is set
     110    file_upload_dir = settings.FILE_UPLOAD_DIR
     111    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE
     112
     113    try:
     114        parser = MultiPartParser(headers, input, request, file_upload_dir, streaming_min_post_size)
     115        return parser.parse()
     116    except MultiPartParserError, e:
     117        return MultiValueDict({ '_file_upload_error': [e.message] }), {}
     118
     119
    98120class QueryDict(MultiValueDict):
    99121    """
    100122    A specialized MultiValueDict that takes a query string when initialized.

  • django/conf/global_settings.py

     
    247247from django import get_version
    248248URL_VALIDATOR_USER_AGENT = "Django/%s (http://www.djangoproject.com)" % get_version()
    249249
     250# The directory to place streamed file uploads. The web server needs write
     251# permissions on this directory.
     252# If this is None, streaming uploads are disabled.
     253FILE_UPLOAD_DIR = None
     254
     255# The minimum size of a POST before file uploads are streamed to disk.
     256# Any less than this number, and the file is uploaded to memory.
     257# Size is in bytes.
     258STREAMING_MIN_POST_SIZE = 512 * (2**10)
     259
    250260##############
    251261# MIDDLEWARE #
    252262##############

  • django/db/models/base.py

     
    1212from django.dispatch import dispatcher
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
     15from django.utils.file import file_move_safe
    1516from django.utils.encoding import smart_str, force_unicode
    1617from django.conf import settings
    1718from itertools import izip
    … …  
    365366    def _get_FIELD_size(self, field):
    366367        return os.path.getsize(self._get_FIELD_filename(field))
    367368
    368     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     369    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    369370        directory = field.get_directory_name()
    370371        try: # Create the date-based directory if it doesn't exist.
    371372            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    372373        except OSError: # Directory probably already exists.
    373374            pass
     375
     376        if filename is None:
     377            filename = raw_field['filename']
     378
    374379        filename = field.get_filename(filename)
    375380
    376381        # If the filename already exists, keep adding an underscore to the name of
    … …  
    387392        setattr(self, field.attname, filename)
    388393
    389394        full_filename = self._get_FIELD_filename(field)
    390         fp = open(full_filename, 'wb')
    391         fp.write(raw_contents)
    392         fp.close()
     395        if raw_field.has_key('tmpfilename'):
     396            raw_field['tmpfile'].close()
     397            file_move_safe(raw_field['tmpfilename'], full_filename)
     398        else:
     399            from django.utils import file_locks
     400            fp = open(full_filename, 'wb')
     401            # exclusive lock
     402            file_locks.lock(fp, file_locks.LOCK_EX)
     403            fp.write(raw_field['content'])
     404            fp.close()
    393405
    394406        # Save the width and/or height, if applicable.
    395407        if isinstance(field, ImageField) and (field.width_field or field.height_field):

  • django/db/models/fields/__init__.py

     
    747747        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    748748        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    749749        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    750         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     750        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
     751        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))       
    751752        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    752753
    753754    def delete_file(self, instance):
    … …  
    770771        if new_data.get(upload_field_name, False):
    771772            func = getattr(new_object, 'save_%s_file' % self.name)
    772773            if rel:
    773                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     774                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
    774775            else:
    775                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     776                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
    776777
    777778    def get_directory_name(self):
    778779        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))

  • django/oldforms/__init__.py

     
    680680        self.validator_list = [self.isNonEmptyFile] + validator_list
    681681
    682682    def isNonEmptyFile(self, field_data, all_data):
    683         try:
    684             content = field_data['content']
    685         except TypeError:
     683        if field_data.has_key('_file_upload_error'):
     684            raise validators.CriticalValidationError, field_data['_file_upload_error']
     685        if not field_data.has_key('filename'):
    686686            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    687         if not content:
     687        if not field_data['content-length']:
    688688            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    689689
    690690    def render(self, data):
    691691        return u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    692692            (self.get_id(), self.__class__.__name__, self.field_name)
    693693
     694    def prepare(self, new_data):
     695        if new_data.has_key('_file_upload_error'):
     696            # pretend we got something in the field to raise a validation error later
     697            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
     698
    694699    def html2python(data):
    695700        if data is None:
    696701            raise EmptyValue

  • django/core/handlers/wsgi.py

     
    7676        self.environ = environ
    7777        self.path = force_unicode(environ['PATH_INFO'])
    7878        self.META = environ
     79        self.META['UPLOAD_PROGRESS_ID'] = self._get_file_progress_id()
    7980        self.method = environ['REQUEST_METHOD'].upper()
    8081
    8182    def __repr__(self):
    … …  
    112113            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    113114                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    114115                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    115                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     116                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
     117                header_dict['X-Progress-ID'] = self.environ.get('HTTP_X_PROGRESS_ID', '')
     118                try:
     119                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self)
     120                except:
     121                    self._post, self._files = {}, {} # make sure we dont read the input stream again
     122                    raise
     123                self._raw_post_data = None # raw data is not available for streamed multipart messages
    116124            else:
    117125                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    118126        else:
    … …  
    168176            buf.close()
    169177            return self._raw_post_data
    170178
     179    def _get_file_progress_id(self):
     180        """
     181        Returns the Progress ID of the request,
     182        usually provided if there is a file upload
     183        going on.
     184        Returns ``None`` if no progress ID is specified.
     185        """
     186        return self._get_file_progress_from_args(self.environ,
     187                                                 self.GET,
     188                                                 self.environ.get('QUERY_STRING', ''))
     189
    171190    GET = property(_get_get, _set_get)
    172191    POST = property(_get_post, _set_post)
    173192    COOKIES = property(_get_cookies, _set_cookies)

  • django/core/handlers/modpython.py

     
    4848    def _load_post_and_files(self):
    4949        "Populates self._post and self._files"
    5050        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    51             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     51            self._raw_post_data = None # raw data is not available for streamed multipart messages
     52            try:
     53                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self)
     54            except:
     55                self._post, self._files = {}, {} # make sure we dont read the input stream again
     56                raise
    5257        else:
    5358            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5459
    … …  
    9398                'AUTH_TYPE':         self._req.ap_auth_type,
    9499                'CONTENT_LENGTH':    self._req.clength, # This may be wrong
    95100                'CONTENT_TYPE':      self._req.content_type, # This may be wrong
    96                 'GATEWAY_INTERFACE': 'CGI/1.1',
    97                 'PATH_INFO':         self._req.path_info,
    98                 'PATH_TRANSLATED':   None, # Not supported
    99                 'QUERY_STRING':      self._req.args,
    100                 'REMOTE_ADDR':       self._req.connection.remote_ip,
    101                 'REMOTE_HOST':       None, # DNS lookups not supported
    102                 'REMOTE_IDENT':      self._req.connection.remote_logname,
    103                 'REMOTE_USER':       self._req.user,
    104                 'REQUEST_METHOD':    self._req.method,
    105                 'SCRIPT_NAME':       None, # Not supported
    106                 'SERVER_NAME':       self._req.server.server_hostname,
    107                 'SERVER_PORT':       self._req.server.port,
    108                 'SERVER_PROTOCOL':   self._req.protocol,
    109                 'SERVER_SOFTWARE':   'mod_python'
     101                'GATEWAY_INTERFACE':  'CGI/1.1',
     102                'PATH_INFO':          self._req.path_info,
     103                'PATH_TRANSLATED':    None, # Not supported
     104                'QUERY_STRING':       self._req.args,
     105                'REMOTE_ADDR':        self._req.connection.remote_ip,
     106                'REMOTE_HOST':        None, # DNS lookups not supported
     107                'REMOTE_IDENT':       self._req.connection.remote_logname,
     108                'REMOTE_USER':        self._req.user,
     109                'REQUEST_METHOD':     self._req.method,
     110                'SCRIPT_NAME':        None, # Not supported
     111                'SERVER_NAME':        self._req.server.server_hostname,
     112                'SERVER_PORT':        self._req.server.port,
     113                'SERVER_PROTOCOL':    self._req.protocol,
     114                'UPLOAD_PROGRESS_ID': self._get_file_progress_id(),
     115                'SERVER_SOFTWARE':    'mod_python'
    110116            }
    111117            for key, value in self._req.headers_in.items():
    112118                key = 'HTTP_' + key.upper().replace('-', '_')
    … …  
    123129    def _get_method(self):
    124130        return self.META['REQUEST_METHOD'].upper()
    125131
     132    def _get_file_progress_id(self):
     133        """
     134        Returns the Progress ID of the request,
     135        usually provided if there is a file upload
     136        going on.
     137        Returns ``None`` if no progress ID is specified.
     138        """
     139        return self._get_file_progress_from_args(self._req.headers_in,
     140                                                 self.GET,
     141                                                 self._req.args)
     142
    126143    GET = property(_get_get, _set_get)
    127144    POST = property(_get_post, _set_post)
    128145    COOKIES = property(_get_cookies, _set_cookies)

  • django/utils/file_locks.py

     
     1"""
     2Locking portability by Jonathan Feignberg <jdf@pobox.com> in python cookbook
     3
     4Example Usage::
     5
     6    from django.utils import file_locks
     7
     8    f = open('./file', 'wb')
     9
     10    file_locks.lock(f, file_locks.LOCK_EX)
     11    f.write('Django')
     12    f.close()
     13"""
     14
     15
     16import os
     17
     18__all__ = ['LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock']
     19
     20if os.name == 'nt':
     21        import win32con
     22        import win32file
     23        import pywintypes
     24        LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     25        LOCK_SH = 0
     26        LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     27        __overlapped = pywintypes.OVERLAPPED()
     28elif os.name == 'posix':
     29        import fcntl
     30        LOCK_EX = fcntl.LOCK_EX
     31        LOCK_SH = fcntl.LOCK_SH
     32        LOCK_NB = fcntl.LOCK_NB
     33else:
     34        raise RuntimeError("Locking only defined for nt and posix platforms")
     35
     36if os.name == 'nt':
     37        def lock(file, flags):
     38                hfile = win32file._get_osfhandle(file.fileno())
     39                win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     40
     41        def unlock(file):
     42                hfile = win32file._get_osfhandle(file.fileno())
     43                win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     44
     45elif os.name =='posix':
     46        def lock(file, flags):
     47                fcntl.flock(file.fileno(), flags)
     48
     49        def unlock(file):
     50                fcntl.flock(file.fileno(), fcntl.LOCK_UN)

  • django/utils/file.py

     
     1import os
     2
     3__all__ = ['file_move_safe']
     4
     5try:
     6    import shutil
     7    file_move = shutil.move
     8except ImportError:
     9    file_move = os.rename
     10
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     12    """
     13    Moves a file from one location to another in the safest way possible.
     14   
     15    First, it tries using shutils.move, which is OS-dependent but doesn't
     16    break with change of filesystems. Then it tries os.rename, which will
     17    break if it encounters a change in filesystems. Lastly, it streams
     18    it manually from one file to another in python.
     19
     20    Without ``allow_overwrite``, if the destination file exists, the
     21    file will raise an IOError.
     22    """
     23
     24    from django.utils import file_locks
     25
     26    if old_file_name == new_file_name:
     27        # No file moving takes place.
     28        return
     29
     30    if not allow_overwrite and os.path.exists(new_file_name):
     31        raise IOError, "Django does not allow overwriting files."
     32
     33    try:
     34        file_move(old_file_name, new_file_name)
     35        return
     36    except OSError: # moving to another filesystem
     37        pass
     38
     39    new_file = open(new_file_name, 'wb')
     40    # exclusive lock
     41    file_locks.lock(new_file, file_locks.LOCK_EX)
     42    old_file = open(old_file_name, 'rb')
     43    current_chunk = None
     44
     45    while current_chunk != '':
     46        current_chunk = old_file.read(chunk_size)
     47        new_file.write(current_chunk)
     48
     49    new_file.close()
     50    old_file.close()
     51
     52    os.remove(old_file_name)
     53

  • tests/modeltests/test_client/views.py

     
    4646
    4747    return HttpResponse(t.render(c))
    4848
     49def post_file_view(request):
     50    "A view that expects a multipart post and returns a file in the context"
     51    t = Template('File {{ file.filename }} received', name='POST Template')
     52    c = Context({'file': request.FILES['file_file']})
     53    return HttpResponse(t.render(c))
     54
    4955def redirect_view(request):
    5056    "A view that redirects all requests to the GET view"
    5157    return HttpResponseRedirect('/test_client/get_view/')

  • tests/modeltests/test_client/models.py

     
    44
    55The test client is a class that can act like a simple
    66browser for testing purposes.
    7  
     7
    88It allows the user to compose GET and POST requests, and
    99obtain the response that the server gave to those requests.
    1010The server Response objects are annotated with the details
    … …  
    8080        self.assertEqual(response.template.name, "Book template")
    8181        self.assertEqual(response.content, "Blink - Malcolm Gladwell")
    8282
     83    def test_post_file_view(self):
     84        "POST this python file to a view"
     85        import os, tempfile
     86        from django.conf import settings
     87        file = __file__.replace('.pyc', '.py')
     88        for upload_dir in [None, tempfile.gettempdir()]:
     89            settings.FILE_UPLOAD_DIR = upload_dir
     90            post_data = { 'name': file, 'file': open(file) }
     91            response = self.client.post('/test_client/post_file_view/', post_data)
     92            self.failUnless('models.py' in response.context['file']['filename'])
     93            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
     94            if upload_dir:
     95                self.failUnless(response.context['file']['tmpfilename'])
     96
    8397    def test_redirect(self):
    8498        "GET a URL that redirects elsewhere"
    8599        response = self.client.get('/test_client/redirect_view/')

  • tests/modeltests/test_client/urls.py

     
    55urlpatterns = patterns('',
    66    (r'^get_view/$', views.get_view),
    77    (r'^post_view/$', views.post_view),
     8    (r'^post_file_view/$', views.post_file_view),
    89    (r'^raw_post_view/$', views.raw_post_view),
    910    (r'^redirect_view/$', views.redirect_view),
    1011    (r'^permanent_redirect_view/$', redirect_to, { 'url': '/test_client/get_view/' }),

  • docs/request_response.txt

     
    7272``FILES``
    7373    A dictionary-like object containing all uploaded files. Each key in
    7474    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    75     value in ``FILES`` is a standard Python dictionary with the following three
     75    value in ``FILES`` is a standard Python dictionary with the following four
    7676    keys:
    7777
    7878        * ``filename`` -- The name of the uploaded file, as a Python string.
    7979        * ``content-type`` -- The content type of the uploaded file.
    8080        * ``content`` -- The raw content of the uploaded file.
     81        * ``content-length`` -- The length of the content in bytes.
    8182
     83    If streaming file uploads are enabled two additional keys
     84    describing the uploaded file will be present:
     85
     86        * ``tmpfilename`` -- The filename for the temporary file.
     87        * ``tmpfile`` -- An open file object for the temporary file.
     88
     89    The temporary file will be removed when the request finishes.
     90
     91    Note that accessing ``content`` when streaming uploads are enabled
     92    will read the whole file into memory which may not be what you want.
     93
    8294    Note that ``FILES`` will only contain data if the request method was POST
    8395    and the ``<form>`` that posted to the request had
    8496    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank

  • docs/settings.txt

     
    472472
    473473.. _Testing Django Applications: ../testing/
    474474
     475FILE_UPLOAD_DIR
     476---------------
     477
     478Default: ``None``
     479
     480Path to a directory where temporary files should be written during
     481file uploads. Leaving this as ``None`` will disable streaming file uploads,
     482and cause all uploaded files to be stored (temporarily) in memory.
     483
    475484IGNORABLE_404_ENDS
    476485------------------
    477486
    … …  
    788797
    789798.. _site framework docs: ../sites/
    790799
     800STREAMING_MIN_POST_SIZE
     801-----------------------
     802
     803Default: 524288 (``512*1024``)
     804
     805An integer specifying the minimum number of bytes that has to be
     806received (in a POST) for file upload streaming to take place. Any
     807request smaller than this will be handled in memory.
     808Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
     809
    791810TEMPLATE_CONTEXT_PROCESSORS
    792811---------------------------
    793812

  • docs/forms.txt

     
    475475   new_data = request.POST.copy()
    476476   new_data.update(request.FILES)
    477477
     478Streaming file uploads.
     479-----------------------
     480
     481File uploads will be read into memory by default. This works fine for
     482small to medium sized uploads (from 1MB to 100MB depending on your
     483setup and usage). If you want to support larger uploads you can enable
     484upload streaming where only a small part of the file will be in memory
     485at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
     486setting (see the settings_ document for more details).
     487
     488See `request object`_ for more details about ``request.FILES`` objects
     489with streaming file uploads enabled.
     490
    478491Validators
    479492==========
    480493
    … …  
    698711.. _`generic views`: ../generic_views/
    699712.. _`models API`: ../model-api/
    700713.. _settings: ../settings/
     714.. _request object: ../request_response/#httprequest-objects
Back to Top