Ticket #2070: 6654_fixed_tests_and_file_clean.diff

File 6654_fixed_tests_and_file_clean.diff, 45.0 KB (added by Øyvind Saltvik <oyvind@…>, 17 years ago)

diff did not apply cleanly, fixed

  • django/http/multipartparser.py

     
     1"""
     2MultiPart parsing for file uploads.
     3If both a progress id is sent (either through ``X-Progress-ID``
     4header or ``progress_id`` GET) and ``FILE_UPLOAD_DIR`` is set
     5in the settings, then the file progress will be tracked using
     6``request.file_progress``.
     7
     8To use this feature, consider creating a middleware with an appropriate
     9``process_request``::
     10
     11    class FileProgressTrack(object):
     12        def __get__(self, request, HttpRequest):
     13            progress_id = request.META['UPLOAD_PROGRESS_ID']
     14            status = # get progress from progress_id here
     15
     16            return status
     17
     18        def __set__(self, request, new_value):
     19            progress_id = request.META['UPLOAD_PROGRESS_ID']
     20
     21            # set the progress using progress_id here.
     22
     23    # example middleware
     24    class FileProgressExample(object):
     25        def process_request(self, request):
     26            request.__class__.file_progress = FileProgressTrack()
     27
     28
     29
     30"""
     31
     32__all__ = ['MultiPartParserError','MultiPartParser']
     33
     34
     35from django.utils.datastructures import MultiValueDict
     36from django.http.utils import str_to_unicode
     37from django.conf import settings
     38import os
     39
     40try:
     41    from cStringIO import StringIO
     42except ImportError:
     43    from StringIO import StringIO
     44
     45
     46class MultiPartParserError(Exception):
     47    def __init__(self, message):
     48        self.message = message
     49    def __str__(self):
     50        return repr(self.message)
     51
     52class MultiPartParser(object):
     53    """
     54    A rfc2388 multipart/form-data parser.
     55   
     56    parse() reads the input stream in chunk_size chunks and returns a
     57    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     58    file_upload_dir is defined files will be streamed to temporary
     59    files in the specified directory.
     60
     61    The FILES dictionary will have 'filename', 'content-type',
     62    'content' and 'content-length' entries. For streamed files it will
     63    also have 'tmpfilename' and 'tmpfile'. The 'content' entry will
     64    only be read from disk when referenced for streamed files.
     65
     66    If the X-Progress-ID is sent (in one of many formats), then
     67    object.file_progress will be given a dictionary of the progress.
     68    """
     69    def __init__(self, headers, input, request, file_upload_dir=None, streaming_min_post_size=None, chunk_size=1024*64):
     70        try:
     71            content_length = int(headers['Content-Length'])
     72        except:
     73            raise MultiPartParserError('Invalid Content-Length: %s' % headers.get('Content-Length'))
     74
     75        content_type = headers.get('Content-Type')
     76
     77        if not content_type or not content_type.startswith('multipart/'):
     78            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)
     79           
     80        ctype, opts = self.parse_header(content_type)
     81        boundary = opts.get('boundary')
     82        from cgi import valid_boundary
     83        if not boundary or not valid_boundary(boundary):
     84            raise MultiPartParserError('Invalid boundary in multipart form: %s' % boundary)
     85
     86        progress_id = request.META['UPLOAD_PROGRESS_ID']
     87
     88        self._track_progress = file_upload_dir and progress_id # whether or not to track progress
     89        self._boundary = '--' + boundary
     90        self._input = input
     91        self._size = content_length
     92        self._received = 0
     93        self._file_upload_dir = file_upload_dir
     94        self._chunk_size = chunk_size
     95        self._state = 'PREAMBLE'
     96        self._partial = ''
     97        self._post = MultiValueDict()
     98        self._files = MultiValueDict()
     99        self._request = request
     100        self._encoding = request.encoding or settings.DEFAULT_CHARSET
     101
     102        if streaming_min_post_size is not None and content_length < streaming_min_post_size:
     103            self._file_upload_dir = None # disable file streaming for small request
     104        elif self._track_progress:
     105            request.file_progress = {'state': 'starting'}
     106
     107        try:
     108            # Use mx fast string search if available.
     109            from mx.TextTools import FS
     110            self._fs = FS(self._boundary)
     111        except ImportError:
     112            self._fs = None
     113
     114    def parse(self):
     115        try:
     116            self._parse()
     117        finally:
     118            if self._track_progress:
     119                self._request.file_progress = {'state': 'done'}
     120        return self._post, self._files
     121
     122    def _parse(self):
     123        size = self._size
     124
     125        try:
     126            while size > 0:
     127                n = self._read(self._input, min(self._chunk_size, size))
     128                if not n:
     129                    break
     130                size -= n
     131        except:
     132            # consume any remaining data so we dont generate a "Connection Reset" error
     133            size = self._size - self._received
     134            while size > 0:
     135                data = self._input.read(min(self._chunk_size, size))
     136                size -= len(data)
     137            raise
     138
     139    def _find_boundary(self, data, start, stop):
     140        """
     141        Find the next boundary and return the end of current part
     142        and start of next part.
     143        """
     144        if self._fs:
     145            boundary = self._fs.find(data, start, stop)
     146        else:
     147            boundary = data.find(self._boundary, start, stop)
     148        if boundary >= 0:
     149            end = boundary
     150            next = boundary + len(self._boundary)
     151
     152            # backup over CRLF
     153            if end > 0 and data[end-1] == '\n': end -= 1
     154            if end > 0 and data[end-1] == '\r': end -= 1
     155            # skip over --CRLF
     156            if next < stop and data[next] == '-': next += 1
     157            if next < stop and data[next] == '-': next += 1
     158            if next < stop and data[next] == '\r': next += 1
     159            if next < stop and data[next] == '\n': next += 1
     160
     161            return True, end, next
     162        else:
     163            return False, stop, stop
     164
     165    class TemporaryFile(object):
     166        "A temporary file that tries to delete itself when garbage collected."
     167        def __init__(self, dir):
     168            import tempfile
     169            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     170            self.file = os.fdopen(fd, 'w+b')
     171            self.name = name
     172
     173        def __getattr__(self, name):
     174            a = getattr(self.__dict__['file'], name)
     175            if type(a) != type(0):
     176                setattr(self, name, a)
     177            return a
     178
     179        def __del__(self):
     180            try:
     181                os.unlink(self.name)
     182            except OSError:
     183                pass
     184
     185    class LazyContent(dict):
     186        """
     187        A lazy FILES dictionary entry that reads the contents from
     188        tmpfile only when referenced.
     189        """
     190        def __init__(self, data):
     191            dict.__init__(self, data)
     192
     193        def __getitem__(self, key):
     194            if key == 'content' and not self.has_key(key):
     195                self['tmpfile'].seek(0)
     196                self['content'] = self['tmpfile'].read()
     197            return dict.__getitem__(self, key)
     198
     199    def _read(self, input, size):
     200        data = input.read(size)
     201
     202        if not data:
     203            return 0
     204
     205        read_size = len(data)
     206        self._received += read_size
     207
     208        if self._partial:
     209            data = self._partial + data
     210
     211        start = 0
     212        stop = len(data)
     213
     214        while start < stop:
     215            boundary, end, next = self._find_boundary(data, start, stop)
     216
     217            if not boundary and read_size:
     218                # make sure we dont treat a partial boundary (and its separators) as data
     219                stop -= len(self._boundary) + 16
     220                end = next = stop
     221                if end <= start:
     222                    break # need more data
     223
     224            if self._state == 'PREAMBLE':
     225                # Preamble, just ignore it
     226                self._state = 'HEADER'
     227
     228            elif self._state == 'HEADER':
     229                # Beginning of header, look for end of header and parse it if found.
     230
     231                header_end = data.find('\r\n\r\n', start, stop)
     232                if header_end == -1:
     233                    break # need more data
     234
     235                header = data[start:header_end]
     236
     237                self._fieldname = None
     238                self._filename = None
     239                self._content_type = None
     240
     241                for line in header.split('\r\n'):
     242                    ctype, opts = self.parse_header(line)
     243                    if ctype == 'content-disposition: form-data':
     244                        self._fieldname = opts.get('name')
     245                        self._filename = opts.get('filename')
     246                    elif ctype.startswith('content-type: '):
     247                        self._content_type = ctype[14:]
     248
     249                if self._filename is not None:
     250                    # cleanup filename from IE full paths:
     251                    self._filename = self._filename[self._filename.rfind("\\")+1:].strip()
     252
     253                    if self._filename: # ignore files without filenames
     254                        if self._file_upload_dir:
     255                            try:
     256                                self._file = self.TemporaryFile(dir=self._file_upload_dir)
     257                            except (OSError, IOError), e:
     258                                raise MultiPartParserError("Failed to create temporary file. Error was %s" % e)
     259                        else:
     260                            self._file = StringIO()
     261                    else:
     262                        self._file = None
     263                    self._filesize = 0
     264                    self._state = 'FILE'
     265                else:
     266                    self._field = StringIO()
     267                    self._state = 'FIELD'
     268                next = header_end + 4
     269
     270            elif self._state == 'FIELD':
     271                # In a field, collect data until a boundary is found.
     272
     273                self._field.write(data[start:end])
     274                if boundary:
     275                    if self._fieldname:
     276                        self._post.appendlist(self._fieldname, str_to_unicode(self._field.getvalue(), self._encoding))
     277                    self._field.close()
     278                    self._state = 'HEADER'
     279
     280            elif self._state == 'FILE':
     281                # In a file, collect data until a boundary is found.
     282
     283                if self._file:
     284                    try:
     285                        self._file.write(data[start:end])
     286                    except IOError, e:
     287                        raise MultiPartParserError("Failed to write to temporary file.")
     288                    self._filesize += end-start
     289
     290                    if self._track_progress:
     291                        self._request.file_progress = {'received': self._received,
     292                                                       'size':     self._size,
     293                                                       'state':    'uploading'}
     294
     295                if boundary:
     296                    if self._file:
     297                        if self._file_upload_dir:
     298                            self._file.seek(0)
     299                            file = self.LazyContent({
     300                                'filename': str_to_unicode(self._filename, self._encoding),
     301                                'content-type':  self._content_type,
     302                                # 'content': is read on demand
     303                                'content-length': self._filesize,
     304                                'tmpfilename': self._file.name,
     305                                'tmpfile': self._file
     306                            })
     307                        else:
     308                            file = {
     309                                'filename': str_to_unicode(self._filename, self._encoding),
     310                                'content-type':  self._content_type,
     311                                'content': self._file.getvalue(),
     312                                'content-length': self._filesize
     313                            }
     314                            self._file.close()
     315
     316                        self._files.appendlist(self._fieldname, file)
     317
     318                    self._state = 'HEADER'
     319
     320            start = next
     321
     322        self._partial = data[start:]
     323
     324        return read_size
     325
     326    def parse_header(self, line):
     327        from cgi import parse_header
     328        return parse_header(line)

  • django/http/__init__.py

     
    11import os
     2import re
    23from Cookie import SimpleCookie
    34from pprint import pformat
    45from urllib import urlencode
    56from urlparse import urljoin
     7from django.http.utils import str_to_unicode
     8from django.http.multipartparser import MultiPartParser, MultiPartParserError
    69from django.utils.datastructures import MultiValueDict, FileDict
    710from django.utils.encoding import smart_str, iri_to_uri, force_unicode
    811
     12upload_id_re = re.compile(r'^[a-fA-F0-9]{32}$') # file progress id Regular expression
     13
    914RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
    1015
    1116try:
    … …  
    7883
    7984    def is_secure(self):
    8085        return os.environ.get("HTTPS") == "on"
    81 
     86       
    8287    def _set_encoding(self, val):
    8388        """
    8489        Sets the encoding used for GET/POST accesses. If the GET or POST
    … …  
    96101
    97102    encoding = property(_get_encoding, _set_encoding)
    98103
    99 def parse_file_upload(header_dict, post_data):
    100     "Returns a tuple of (POST QueryDict, FILES MultiValueDict)"
    101     import email, email.Message
    102     from cgi import parse_header
    103     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    104     raw_message += '\r\n\r\n' + post_data
    105     msg = email.message_from_string(raw_message)
    106     POST = QueryDict('', mutable=True)
    107     FILES = MultiValueDict()
    108     for submessage in msg.get_payload():
    109         if submessage and isinstance(submessage, email.Message.Message):
    110             name_dict = parse_header(submessage['Content-Disposition'])[1]
    111             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    112             # or {'name': 'blah'} for POST fields
    113             # We assume all uploaded files have a 'filename' set.
    114             if 'filename' in name_dict:
    115                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    116                 if not name_dict['filename'].strip():
    117                     continue
    118                 # IE submits the full path, so trim everything but the basename.
    119                 # (We can't use os.path.basename because that uses the server's
    120                 # directory separator, which may not be the same as the
    121                 # client's one.)
    122                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    123                 FILES.appendlist(name_dict['name'], FileDict({
    124                     'filename': filename,
    125                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    126                     'content': submessage.get_payload(),
    127                 }))
    128             else:
    129                 POST.appendlist(name_dict['name'], submessage.get_payload())
    130     return POST, FILES
     104    def _get_file_progress(self):
     105        return {}
     106   
     107    def _set_file_progress(self,value):
     108        pass
    131109
     110    def _del_file_progress(self):
     111        pass
     112
     113    file_progress = property(_get_file_progress,
     114                             _set_file_progress,
     115                             _del_file_progress)
     116
     117    def _get_file_progress_from_args(self, headers, get, querystring):
     118        """
     119        This parses the request for a file progress_id value.
     120        Note that there are two distinct ways of getting the progress
     121        ID -- header and GET. One is used primarily to attach via JavaScript
     122        to the end of an HTML form action while the other is used for AJAX
     123        communication.
     124
     125        All progress IDs must be valid 32-digit hexadecimal numbers.
     126        """
     127        if 'X-Upload-ID' in headers:
     128            progress_id = headers['X-Upload-ID']
     129        elif 'progress_id' in get:
     130            progress_id = get['progress_id']
     131        else:
     132            return None
     133
     134        if not upload_id_re.match(progress_id):
     135            return None
     136
     137        return progress_id
     138
     139def parse_file_upload(headers, input, request):
     140    from django.conf import settings
     141
     142    # Only stream files to disk if FILE_STREAMING_DIR is set
     143    file_upload_dir = settings.FILE_UPLOAD_DIR
     144    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE
     145
     146    try:
     147        parser = MultiPartParser(headers, input, request, file_upload_dir, streaming_min_post_size)
     148        return parser.parse()
     149    except MultiPartParserError, e:
     150        return MultiValueDict({ '_file_upload_error': [e.message] }), {}
     151
     152
    132153class QueryDict(MultiValueDict):
    133154    """
    134155    A specialized MultiValueDict that takes a query string when initialized.
    … …  
    399420# A backwards compatible alias for HttpRequest.get_host.
    400421def get_host(request):
    401422    return request.get_host()
    402 
    403 # It's neither necessary nor appropriate to use
    404 # django.utils.encoding.smart_unicode for parsing URLs and form inputs. Thus,
    405 # this slightly more restricted function.
    406 def str_to_unicode(s, encoding):
    407     """
    408     Convert basestring objects to unicode, using the given encoding. Illegaly
    409     encoded input characters are replaced with Unicode "unknown" codepoint
    410     (\ufffd).
    411 
    412     Returns any non-basestring objects without change.
    413     """
    414     if isinstance(s, str):
    415         return unicode(s, encoding, 'replace')
    416     else:
    417         return s
    418 

  • django/http/utils.py

     
     1# It's neither necessary nor appropriate to use
     2# django.utils.encoding.smart_unicode for parsing URLs and form inputs. Thus,
     3# this slightly more restricted function.
     4def str_to_unicode(s, encoding):
     5    """
     6    Convert basestring objects to unicode, using the given encoding. Illegaly
     7    encoded input characters are replaced with Unicode "unknown" codepoint
     8    (\ufffd).
     9
     10    Returns any non-basestring objects without change.
     11    """
     12    if isinstance(s, str):
     13        return unicode(s, encoding, 'replace')
     14    else:
     15        return s
     16

  • django/oldforms/__init__.py

     
    680680        self.validator_list = [self.isNonEmptyFile] + validator_list
    681681
    682682    def isNonEmptyFile(self, field_data, all_data):
    683         try:
    684             content = field_data['content']
    685         except TypeError:
     683        if field_data.has_key('_file_upload_error'):
     684            raise validators.CriticalValidationError, field_data['_file_upload_error']
     685        if not field_data.has_key('filename'):
    686686            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    687         if not content:
     687        if not field_data['content-length']:
    688688            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    689689
    690690    def render(self, data):
    691691        return u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    692692            (self.get_id(), self.__class__.__name__, self.field_name)
    693693
     694    def prepare(self, new_data):
     695        if new_data.has_key('_file_upload_error'):
     696            # pretend we got something in the field to raise a validation error later
     697            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
     698
    694699    def html2python(data):
    695700        if data is None:
    696701            raise EmptyValue

  • django/db/models/base.py

     
    1212from django.dispatch import dispatcher
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
     15from django.utils.file import file_move_safe
    1516from django.utils.encoding import smart_str, force_unicode, smart_unicode
    1617from django.conf import settings
    1718from itertools import izip
    … …  
    379380    def _get_FIELD_size(self, field):
    380381        return os.path.getsize(self._get_FIELD_filename(field))
    381382
    382     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     383    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    383384        directory = field.get_directory_name()
    384385        try: # Create the date-based directory if it doesn't exist.
    385386            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    386387        except OSError: # Directory probably already exists.
    387388            pass
     389
     390        if filename is None:
     391            filename = raw_field['filename']
     392
    388393        filename = field.get_filename(filename)
    389394
    390395        # If the filename already exists, keep adding an underscore to the name of
    … …  
    401406        setattr(self, field.attname, filename)
    402407
    403408        full_filename = self._get_FIELD_filename(field)
    404         fp = open(full_filename, 'wb')
    405         fp.write(raw_contents)
    406         fp.close()
     409        if raw_field.has_key('tmpfilename'):
     410            raw_field['tmpfile'].close()
     411            file_move_safe(raw_field['tmpfilename'], full_filename)
     412        else:
     413            from django.utils import file_locks
     414            fp = open(full_filename, 'wb')
     415            # exclusive lock
     416            file_locks.lock(fp, file_locks.LOCK_EX)
     417            fp.write(raw_field['content'])
     418            fp.close()
    407419
    408420        # Save the width and/or height, if applicable.
    409421        if isinstance(field, ImageField) and (field.width_field or field.height_field):

  • django/db/models/fields/__init__.py

     
    761761        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    762762        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    763763        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    764         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     764        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
     765        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))       
    765766        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    766767
    767768    def delete_file(self, instance):
    … …  
    784785        if new_data.get(upload_field_name, False):
    785786            func = getattr(new_object, 'save_%s_file' % self.name)
    786787            if rel:
    787                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     788                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
    788789            else:
    789                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     790                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
    790791
    791792    def get_directory_name(self):
    792793        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
    … …  
    798799
    799800    def save_form_data(self, instance, data):
    800801        if data:
    801             getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
     802            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
    802803
    803804    def formfield(self, **kwargs):
    804805        defaults = {'form_class': forms.FileField}

  • django/conf/global_settings.py

     
    251251from django import get_version
    252252URL_VALIDATOR_USER_AGENT = "Django/%s (http://www.djangoproject.com)" % get_version()
    253253
     254# The directory to place streamed file uploads. The web server needs write
     255# permissions on this directory.
     256# If this is None, streaming uploads are disabled.
     257FILE_UPLOAD_DIR = None
     258
     259# The minimum size of a POST before file uploads are streamed to disk.
     260# Any less than this number, and the file is uploaded to memory.
     261# Size is in bytes.
     262STREAMING_MIN_POST_SIZE = 512 * (2**10)
     263
    254264##############
    255265# MIDDLEWARE #
    256266##############

  • django/core/validators.py

     
    177177    from PIL import Image
    178178    from cStringIO import StringIO
    179179    try:
    180         content = field_data['content']
     180        filename = field_data['filename']
    181181    except TypeError:
    182182        raise ValidationError, _("No file was submitted. Check the encoding type on the form.")
    183183    try:
    184184        # load() is the only method that can spot a truncated JPEG,
    185185        #  but it cannot be called sanely after verify()
    186         trial_image = Image.open(StringIO(content))
     186        trial_image = Image.open(field_data.get('tmpfilename') or StringIO(field_data.get('content','')))
    187187        trial_image.load()
    188188        # verify() is the only method that can spot a corrupt PNG,
    189189        #  but it must be called immediately after the constructor
    190         trial_image = Image.open(StringIO(content))
     190        trial_image = Image.open(field_data.get('tmpfilename') or StringIO(field_data.get('content','')))
    191191        trial_image.verify()
    192192    except Exception: # Python Imaging Library doesn't recognize it as an image
    193193        raise ValidationError, _("Upload a valid image. The file you uploaded was either not an image or a corrupted image.")

  • django/core/handlers/wsgi.py

     
    7777        self.environ = environ
    7878        self.path = force_unicode(environ['PATH_INFO'])
    7979        self.META = environ
     80        self.META['UPLOAD_PROGRESS_ID'] = self._get_file_progress_id()
    8081        self.method = environ['REQUEST_METHOD'].upper()
    8182
    8283    def __repr__(self):
    … …  
    114115            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    115116                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    116117                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    117                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     118                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
     119                header_dict['X-Progress-ID'] = self.environ.get('HTTP_X_PROGRESS_ID', '')
     120                try:
     121                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self)
     122                except:
     123                    self._post, self._files = {}, {} # make sure we dont read the input stream again
     124                    raise
     125                self._raw_post_data = None # raw data is not available for streamed multipart messages
    118126            else:
    119127                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    120128        else:
    … …  
    172180            buf.close()
    173181            return self._raw_post_data
    174182
     183    def _get_file_progress_id(self):
     184        """
     185        Returns the Progress ID of the request,
     186        usually provided if there is a file upload
     187        going on.
     188        Returns ``None`` if no progress ID is specified.
     189        """
     190        return self._get_file_progress_from_args(self.environ,
     191                                                 self.GET,
     192                                                 self.environ.get('QUERY_STRING', ''))
     193
    175194    GET = property(_get_get, _set_get)
    176195    POST = property(_get_post, _set_post)
    177196    COOKIES = property(_get_cookies, _set_cookies)

  • django/core/handlers/modpython.py

     
    5151    def _load_post_and_files(self):
    5252        "Populates self._post and self._files"
    5353        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    54             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     54            self._raw_post_data = None # raw data is not available for streamed multipart messages
     55            try:
     56                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self)
     57            except:
     58                self._post, self._files = {}, {} # make sure we dont read the input stream again
     59                raise
    5560        else:
    5661            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5762
    … …  
    96101                'AUTH_TYPE':         self._req.ap_auth_type,
    97102                'CONTENT_LENGTH':    self._req.clength, # This may be wrong
    98103                'CONTENT_TYPE':      self._req.content_type, # This may be wrong
    99                 'GATEWAY_INTERFACE': 'CGI/1.1',
    100                 'PATH_INFO':         self._req.path_info,
    101                 'PATH_TRANSLATED':   None, # Not supported
    102                 'QUERY_STRING':      self._req.args,
    103                 'REMOTE_ADDR':       self._req.connection.remote_ip,
    104                 'REMOTE_HOST':       None, # DNS lookups not supported
    105                 'REMOTE_IDENT':      self._req.connection.remote_logname,
    106                 'REMOTE_USER':       self._req.user,
    107                 'REQUEST_METHOD':    self._req.method,
    108                 'SCRIPT_NAME':       None, # Not supported
    109                 'SERVER_NAME':       self._req.server.server_hostname,
    110                 'SERVER_PORT':       self._req.server.port,
    111                 'SERVER_PROTOCOL':   self._req.protocol,
    112                 'SERVER_SOFTWARE':   'mod_python'
     104                'GATEWAY_INTERFACE':  'CGI/1.1',
     105                'PATH_INFO':          self._req.path_info,
     106                'PATH_TRANSLATED':    None, # Not supported
     107                'QUERY_STRING':       self._req.args,
     108                'REMOTE_ADDR':        self._req.connection.remote_ip,
     109                'REMOTE_HOST':        None, # DNS lookups not supported
     110                'REMOTE_IDENT':       self._req.connection.remote_logname,
     111                'REMOTE_USER':        self._req.user,
     112                'REQUEST_METHOD':     self._req.method,
     113                'SCRIPT_NAME':        None, # Not supported
     114                'SERVER_NAME':        self._req.server.server_hostname,
     115                'SERVER_PORT':        self._req.server.port,
     116                'SERVER_PROTOCOL':    self._req.protocol,
     117                'UPLOAD_PROGRESS_ID': self._get_file_progress_id(),
     118                'SERVER_SOFTWARE':    'mod_python'
    113119            }
    114120            for key, value in self._req.headers_in.items():
    115121                key = 'HTTP_' + key.upper().replace('-', '_')
    … …  
    126132    def _get_method(self):
    127133        return self.META['REQUEST_METHOD'].upper()
    128134
     135    def _get_file_progress_id(self):
     136        """
     137        Returns the Progress ID of the request,
     138        usually provided if there is a file upload
     139        going on.
     140        Returns ``None`` if no progress ID is specified.
     141        """
     142        return self._get_file_progress_from_args(self._req.headers_in,
     143                                                 self.GET,
     144                                                 self._req.args)
     145
    129146    GET = property(_get_get, _set_get)
    130147    POST = property(_get_post, _set_post)
    131148    COOKIES = property(_get_cookies, _set_cookies)

  • django/newforms/fields.py

     
    421421
    422422class UploadedFile(StrAndUnicode):
    423423    "A wrapper for files uploaded in a FileField"
    424     def __init__(self, filename, content):
     424    def __init__(self, filename, data):
    425425        self.filename = filename
    426         self.content = content
     426        self.data = data
    427427
    428428    def __unicode__(self):
    429429        """
    … …  
    448448        if not self.required and data in EMPTY_VALUES:
    449449            return None
    450450        try:
    451             f = UploadedFile(data['filename'], data['content'])
     451            f = UploadedFile(data['filename'], data)
    452452        except TypeError:
    453453            raise ValidationError(self.error_messages['invalid'])
    454454        except KeyError:
    455455            raise ValidationError(self.error_messages['missing'])
    456         if not f.content:
     456        if not f.data.get('content-length'):
    457457            raise ValidationError(self.error_messages['empty'])
    458458        return f
    459459
    … …  
    475475        try:
    476476            # load() is the only method that can spot a truncated JPEG,
    477477            #  but it cannot be called sanely after verify()
    478             trial_image = Image.open(StringIO(f.content))
     478            trial_image = Image.open(f.data.get('tmpfilename') or StringIO(f.data['content']))
    479479            trial_image.load()
    480480            # verify() is the only method that can spot a corrupt PNG,
    481481            #  but it must be called immediately after the constructor
    482             trial_image = Image.open(StringIO(f.content))
     482            trial_image = Image.open(f.data.get('tmpfilename') or StringIO(f.data['content']))
    483483            trial_image.verify()
    484484        except Exception: # Python Imaging Library doesn't recognize it as an image
    485485            raise ValidationError(self.error_messages['invalid_image'])

  • django/utils/file_locks.py

     
     1"""
     2Locking portability by Jonathan Feignberg <jdf@pobox.com> in python cookbook
     3
     4Example Usage::
     5
     6    from django.utils import file_locks
     7
     8    f = open('./file', 'wb')
     9
     10    file_locks.lock(f, file_locks.LOCK_EX)
     11    f.write('Django')
     12    f.close()
     13"""
     14
     15
     16import os
     17
     18__all__ = ['LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock']
     19
     20if os.name == 'nt':
     21        import win32con
     22        import win32file
     23        import pywintypes
     24        LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     25        LOCK_SH = 0
     26        LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     27        __overlapped = pywintypes.OVERLAPPED()
     28elif os.name == 'posix':
     29        import fcntl
     30        LOCK_EX = fcntl.LOCK_EX
     31        LOCK_SH = fcntl.LOCK_SH
     32        LOCK_NB = fcntl.LOCK_NB
     33else:
     34        raise RuntimeError("Locking only defined for nt and posix platforms")
     35
     36if os.name == 'nt':
     37        def lock(file, flags):
     38                hfile = win32file._get_osfhandle(file.fileno())
     39                win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     40
     41        def unlock(file):
     42                hfile = win32file._get_osfhandle(file.fileno())
     43                win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     44
     45elif os.name =='posix':
     46        def lock(file, flags):
     47                fcntl.flock(file.fileno(), flags)
     48
     49        def unlock(file):
     50                fcntl.flock(file.fileno(), fcntl.LOCK_UN)

  • django/utils/file.py

     
     1import os
     2
     3__all__ = ['file_move_safe']
     4
     5try:
     6    import shutil
     7    file_move = shutil.move
     8except ImportError:
     9    file_move = os.rename
     10
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     12    """
     13    Moves a file from one location to another in the safest way possible.
     14   
     15    First, it tries using shutils.move, which is OS-dependent but doesn't
     16    break with change of filesystems. Then it tries os.rename, which will
     17    break if it encounters a change in filesystems. Lastly, it streams
     18    it manually from one file to another in python.
     19
     20    Without ``allow_overwrite``, if the destination file exists, the
     21    file will raise an IOError.
     22    """
     23
     24    from django.utils import file_locks
     25
     26    if old_file_name == new_file_name:
     27        # No file moving takes place.
     28        return
     29
     30    if not allow_overwrite and os.path.exists(new_file_name):
     31        raise IOError, "Django does not allow overwriting files."
     32
     33    try:
     34        file_move(old_file_name, new_file_name)
     35        return
     36    except OSError: # moving to another filesystem
     37        pass
     38
     39    new_file = open(new_file_name, 'wb')
     40    # exclusive lock
     41    file_locks.lock(new_file, file_locks.LOCK_EX)
     42    old_file = open(old_file_name, 'rb')
     43    current_chunk = None
     44
     45    while current_chunk != '':
     46        current_chunk = old_file.read(chunk_size)
     47        new_file.write(current_chunk)
     48
     49    new_file.close()
     50    old_file.close()
     51
     52    os.remove(old_file_name)
     53

  • tests/modeltests/test_client/views.py

     
    4747
    4848    return HttpResponse(t.render(c))
    4949
     50def post_file_view(request):
     51    "A view that expects a multipart post and returns a file in the context"
     52    t = Template('File {{ file.filename }} received', name='POST Template')
     53    c = Context({'file': request.FILES['file_file']})
     54    return HttpResponse(t.render(c))
     55
    5056def redirect_view(request):
    5157    "A view that redirects all requests to the GET view"
    5258    if request.GET:

  • tests/modeltests/test_client/models.py

     
    44
    55The test client is a class that can act like a simple
    66browser for testing purposes.
    7  
     7
    88It allows the user to compose GET and POST requests, and
    99obtain the response that the server gave to those requests.
    1010The server Response objects are annotated with the details
    … …  
    8080        self.assertEqual(response.template.name, "Book template")
    8181        self.assertEqual(response.content, "Blink - Malcolm Gladwell")
    8282
     83    def test_post_file_view(self):
     84        "POST this python file to a view"
     85        import os, tempfile
     86        from django.conf import settings
     87        file = __file__.replace('.pyc', '.py')
     88        for upload_dir, streaming_size in [(None,512*1000), (tempfile.gettempdir(), 1)]:
     89            settings.FILE_UPLOAD_DIR = upload_dir
     90            settings.STREAMING_MIN_POST_SIZE = streaming_size
     91            post_data = { 'name': file, 'file_file': open(file) }
     92            response = self.client.post('/test_client/post_file_view/', post_data)
     93            self.failUnless('models.py' in response.context['file']['filename'])
     94            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
     95            if upload_dir:
     96                self.failUnless(response.context['file']['tmpfilename'])
     97
    8398    def test_redirect(self):
    8499        "GET a URL that redirects elsewhere"
    85100        response = self.client.get('/test_client/redirect_view/')

  • tests/modeltests/test_client/urls.py

     
    55urlpatterns = patterns('',
    66    (r'^get_view/$', views.get_view),
    77    (r'^post_view/$', views.post_view),
     8    (r'^post_file_view/$', views.post_file_view),
    89    (r'^raw_post_view/$', views.raw_post_view),
    910    (r'^redirect_view/$', views.redirect_view),
    1011    (r'^permanent_redirect_view/$', redirect_to, { 'url': '/test_client/get_view/' }),

  • tests/regressiontests/forms/fields.py

     
    760760...
    761761ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    762762
    763 >>> f.clean({'filename': 'name', 'content':None})
     763>>> f.clean({'filename': 'name', 'content':None, 'content-length': 0})
    764764Traceback (most recent call last):
    765765...
    766766ValidationError: [u'The submitted file is empty.']
    … …  
    770770...
    771771ValidationError: [u'The submitted file is empty.']
    772772
    773 >>> type(f.clean({'filename': 'name', 'content':'Some File Content'}))
     773>>> type(f.clean({'filename': 'name', 'content':'Some File Content', 'content-length': len('Some File Content')}))
    774774<class 'django.newforms.fields.UploadedFile'>
    775775
    776776# URLField ##################################################################

  • tests/regressiontests/forms/forms.py

     
    14101410>>> print f
    14111411<tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
    14121412
    1413 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
     1413>>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content', 'content-length': len('some content')}}, auto_id=False)
    14141414>>> print f
    14151415<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
    14161416>>> f.is_valid()

  • docs/request_response.txt

     
    8282``FILES``
    8383    A dictionary-like object containing all uploaded files. Each key in
    8484    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    85     value in ``FILES`` is a standard Python dictionary with the following three
     85    value in ``FILES`` is a standard Python dictionary with the following four
    8686    keys:
    8787
    8888        * ``filename`` -- The name of the uploaded file, as a Python string.
    8989        * ``content-type`` -- The content type of the uploaded file.
    9090        * ``content`` -- The raw content of the uploaded file.
     91        * ``content-length`` -- The length of the content in bytes.
    9192
     93    If streaming file uploads are enabled two additional keys
     94    describing the uploaded file will be present:
     95
     96        * ``tmpfilename`` -- The filename for the temporary file.
     97        * ``tmpfile`` -- An open file object for the temporary file.
     98
     99    The temporary file will be removed when the request finishes.
     100
     101    Note that accessing ``content`` when streaming uploads are enabled
     102    will read the whole file into memory which may not be what you want.
     103
    92104    Note that ``FILES`` will only contain data if the request method was POST
    93105    and the ``<form>`` that posted to the request had
    94106    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank

  • docs/settings.txt

     
    480480
    481481.. _Testing Django Applications: ../testing/
    482482
     483FILE_UPLOAD_DIR
     484---------------
     485
     486Default: ``None``
     487
     488Path to a directory where temporary files should be written during
     489file uploads. Leaving this as ``None`` will disable streaming file uploads,
     490and cause all uploaded files to be stored (temporarily) in memory.
     491
    483492IGNORABLE_404_ENDS
    484493------------------
    485494
    … …  
    845854
    846855.. _site framework docs: ../sites/
    847856
     857STREAMING_MIN_POST_SIZE
     858-----------------------
     859
     860Default: 524288 (``512*1024``)
     861
     862An integer specifying the minimum number of bytes that has to be
     863received (in a POST) for file upload streaming to take place. Any
     864request smaller than this will be handled in memory.
     865Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
     866
    848867TEMPLATE_CONTEXT_PROCESSORS
    849868---------------------------
    850869

  • docs/forms.txt

     
    475475   new_data = request.POST.copy()
    476476   new_data.update(request.FILES)
    477477
     478Streaming file uploads.
     479-----------------------
     480
     481File uploads will be read into memory by default. This works fine for
     482small to medium sized uploads (from 1MB to 100MB depending on your
     483setup and usage). If you want to support larger uploads you can enable
     484upload streaming where only a small part of the file will be in memory
     485at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
     486setting (see the settings_ document for more details).
     487
     488See `request object`_ for more details about ``request.FILES`` objects
     489with streaming file uploads enabled.
     490
    478491Validators
    479492==========
    480493
    … …  
    698711.. _`generic views`: ../generic_views/
    699712.. _`models API`: ../model-api/
    700713.. _settings: ../settings/
     714.. _request object: ../request_response/#httprequest-objects
Back to Top