Ticket #5361: filestorage.diff

django/core/filestorage/__init__.py

@@ +1 @@
+from StringIO import StringIO
+
+class Backend(object):
+    def get_available_filename(self, filename):
+        # If the filename already exists, keep adding an underscore to the name
+        # of the file until the filename doesn't exist.
+        while self.file_exists(filename):
+            try:
+                dot_index = filename.rindex('.')
+            except ValueError: # filename has no dot
+                filename += '_'
+            else:
+                filename = filename[:dot_index] + '_' + filename[dot_index:]
+        return filename
+
+    def get_filename(self, filename):
+        return filename
+
+class RemoteFile(StringIO):
+    """Sends files to a remote backend automatically, when necessary."""
+
+    def __init__(self, data, mode, writer):
+        self._mode = mode
+        self._write_to_backend = writer
+        self._is_dirty = False
+        StringIO.__init__(self, data)
+
+    def write(self, data):
+        if 'w' not in self._mode:
+            raise AttributeError, "File was opened for read-only access."
+        StringIO.write(self, data)
+        self._is_dirty = True
+
+    def close(self):
+        if self._is_dirty:
+            self._write_to_backend(self.getvalue())
+        StringIO.close(self)
+

django/core/filestorage/filesystem.py

@@ +1 @@
+import datetime
+import os
+
+from django.conf import settings
+from django.utils.encoding import force_unicode, smart_str
+
+from django.core.filestorage import Backend
+
+class FileSystemBackend(Backend):
+    """Standard filesystem storage"""
+
+    def __init__(self, location='', media_root=None, media_url=None):
+        self.location = location
+        if media_root != None and media_url != None:
+            # Both were provided, so use them
+            pass
+        elif media_root is None and media_url is None:
+            # Neither were provided, so use global settings
+            from django.conf import settings
+            try:
+                media_root = settings.MEDIA_ROOT
+                media_url = settings.MEDIA_URL
+            except AttributeError:
+                raise ImproperlyConfigured, "Media settings not defined."
+        else:
+            # One or the other were provided, but not both
+            raise ImproperlyConfigured, "Both media_root and media_url must be provided."
+        self.media_root = media_root
+        self.media_url = media_url
+
+    def _get_directory_name(self):
+        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.location))))
+
+    def _get_absolute_path(self, filename):
+        return os.path.normpath(os.path.join(self.media_root, filename))
+
+    # The following methods define the Backend API
+
+    def get_available_filename(self, filename):
+        from django.utils.text import get_valid_filename
+        f = os.path.join(self._get_directory_name(), get_valid_filename(os.path.basename(filename)))
+        return Backend.get_available_filename(self, os.path.normpath(f))
+
+    def get_filesize(self, filename):
+        return os.path.getsize(self._get_absolute_path(filename))
+
+    def get_absolute_url(self, filename):
+        import urlparse
+        return urlparse.urljoin(self.media_url, filename).replace('\\', '/')
+
+    def open(self, filename, mode='rb'):
+        return open(self._get_absolute_path(filename), mode)
+
+    def file_exists(self, filename):
+        return os.path.exists(self._get_absolute_path(filename))
+
+    def save_file(self, filename, raw_contents):
+        directory = self._get_directory_name()
+        try: # Create the date-based directory if it doesn't exist.
+            os.makedirs(os.path.join(self.media_root, directory))
+        except OSError: # Directory probably already exists.
+            pass
+        filename = self.get_available_filename(filename)
+
+        # Write the file to disk.
+        fp = open(self._get_absolute_path(filename), 'wb')
+        fp.write(raw_contents)
+        fp.close()
+
+        return filename
+
+    def delete_file(self, filename):
+        file_name = self._get_absolute_path(filename)
+        # If the file exists, delete it from the filesystem.
+        if os.path.exists(file_name):
+            os.remove(file_name)

django/core/filestorage/__init__.py

@@ +1 @@
+from StringIO import StringIO
+
+class Backend(object):
+    def get_available_filename(self, filename):
+        # If the filename already exists, keep adding an underscore to the name
+        # of the file until the filename doesn't exist.
+        while self.file_exists(filename):
+            try:
+                dot_index = filename.rindex('.')
+            except ValueError: # filename has no dot
+                filename += '_'
+            else:
+                filename = filename[:dot_index] + '_' + filename[dot_index:]
+        return filename
+
+    def get_filename(self, filename):
+        return filename
+
+class RemoteFile(StringIO):
+    """Sends files to a remote backend automatically, when necessary."""
+
+    def __init__(self, data, mode, writer):
+        self._mode = mode
+        self._write_to_backend = writer
+        self._is_dirty = False
+        StringIO.__init__(self, data)
+
+    def write(self, data):
+        if 'w' not in self._mode:
+            raise AttributeError, "File was opened for read-only access."
+        StringIO.write(self, data)
+        self._is_dirty = True
+
+    def close(self):
+        if self._is_dirty:
+            self._write_to_backend(self.getvalue())
+        StringIO.close(self)
+

django/core/filestorage/filesystem.py

@@ +1 @@
+import datetime
+import os
+
+from django.conf import settings
+from django.utils.encoding import force_unicode, smart_str
+
+from django.core.filestorage import Backend
+
+class FileSystemBackend(Backend):
+    """Standard filesystem storage"""
+
+    def __init__(self, location='', media_root=None, media_url=None):
+        self.location = location
+        if media_root != None and media_url != None:
+            # Both were provided, so use them
+            pass
+        elif media_root is None and media_url is None:
+            # Neither were provided, so use global settings
+            from django.conf import settings
+            try:
+                media_root = settings.MEDIA_ROOT
+                media_url = settings.MEDIA_URL
+            except AttributeError:
+                raise ImproperlyConfigured, "Media settings not defined."
+        else:
+            # One or the other were provided, but not both
+            raise ImproperlyConfigured, "Both media_root and media_url must be provided."
+        self.media_root = media_root
+        self.media_url = media_url
+
+    def _get_directory_name(self):
+        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.location))))
+
+    def _get_absolute_path(self, filename):
+        return os.path.normpath(os.path.join(self.media_root, filename))
+
+    # The following methods define the Backend API
+
+    def get_available_filename(self, filename):
+        from django.utils.text import get_valid_filename
+        f = os.path.join(self._get_directory_name(), get_valid_filename(os.path.basename(filename)))
+        return Backend.get_available_filename(self, os.path.normpath(f))
+
+    def get_filesize(self, filename):
+        return os.path.getsize(self._get_absolute_path(filename))
+
+    def get_absolute_url(self, filename):
+        import urlparse
+        return urlparse.urljoin(self.media_url, filename).replace('\\', '/')
+
+    def open(self, filename, mode='rb'):
+        return open(self._get_absolute_path(filename), mode)
+
+    def file_exists(self, filename):
+        return os.path.exists(self._get_absolute_path(filename))
+
+    def save_file(self, filename, raw_contents):
+        directory = self._get_directory_name()
+        try: # Create the date-based directory if it doesn't exist.
+            os.makedirs(os.path.join(self.media_root, directory))
+        except OSError: # Directory probably already exists.
+            pass
+        filename = self.get_available_filename(filename)
+
+        # Write the file to disk.
+        fp = open(self._get_absolute_path(filename), 'wb')
+        fp.write(raw_contents)
+        fp.close()
+
+        return filename
+
+    def delete_file(self, filename):
+        file_name = self._get_absolute_path(filename)
+        # If the file exists, delete it from the filesystem.
+        if os.path.exists(file_name):
+            os.remove(file_name)

django/db/models/base.py

@@ -18 +18 @@
 import types
 import sys
 import os
+from warnings import warn
 
 class ModelBase(type):
     "Metaclass for all models"
@@ -357 +358 @@
         return getattr(self, cachename)
 
     def _get_FIELD_filename(self, field):
-        if getattr(self, field.attname): # value is not blank
-            return os.path.join(settings.MEDIA_ROOT, getattr(self, field.attname))
-        return ''
+        warn("Use instance.%s.open() if you need access to the file." % field.attname, DeprecationWarning)
+        return field.backend._get_absolute_path(self.__dict__[field.attname])
 
     def _get_FIELD_url(self, field):
-        if getattr(self, field.attname): # value is not blank
-            import urlparse
-            return urlparse.urljoin(settings.MEDIA_URL, getattr(self, field.attname)).replace('\\', '/')
-        return ''
+        warn("Use instance.%s.get_absolute_url()." % field.attname, DeprecationWarning)
+        return getattr(self, field.attname).get_absolute_url()
 
     def _get_FIELD_size(self, field):
-        return os.path.getsize(self._get_FIELD_filename(field))
+        warn("Use instance.%s.get_filesize()." % field.attname, DeprecationWarning)
+        return getattr(self, field.attname).get_filesize()
 
     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
-        directory = field.get_directory_name()
-        try: # Create the date-based directory if it doesn't exist.
-            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
-        except OSError: # Directory probably already exists.
-            pass
-        filename = field.get_filename(filename)
+        warn("Use instance.%s.save_file()." % field.attname, DeprecationWarning)
+        return getattr(self, field.attname).save_file(filename, raw_contents, save)
 
-        # If the filename already exists, keep adding an underscore to the name of
-        # the file until the filename doesn't exist.
-        while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
-            try:
-                dot_index = filename.rindex('.')
-            except ValueError: # filename has no dot
-                filename += '_'
-            else:
-                filename = filename[:dot_index] + '_' + filename[dot_index:]
-
-        # Write the file to disk.
-        setattr(self, field.attname, filename)
-
-        full_filename = self._get_FIELD_filename(field)
-        fp = open(full_filename, 'wb')
-        fp.write(raw_contents)
-        fp.close()
-
-        # Save the width and/or height, if applicable.
-        if isinstance(field, ImageField) and (field.width_field or field.height_field):
-            from django.utils.images import get_image_dimensions
-            width, height = get_image_dimensions(full_filename)
-            if field.width_field:
-                setattr(self, field.width_field, width)
-            if field.height_field:
-                setattr(self, field.height_field, height)
-
-        # Save the object because it has changed unless save is False
-        if save:
-            self.save()
-
-    _save_FIELD_file.alters_data = True
-
     def _get_FIELD_width(self, field):
-        return self._get_image_dimensions(field)[0]
+        warn("Use instance.%s.get_width()." % field.attname, DeprecationWarning)
+        return getattr(self, field.attname).get_width()
 
     def _get_FIELD_height(self, field):
-        return self._get_image_dimensions(field)[1]
+        warn("Use instance.%s.get_height()." % field.attname, DeprecationWarning)
+        return getattr(self, field.attname).get_height()
 
-    def _get_image_dimensions(self, field):
-        cachename = "__%s_dimensions_cache" % field.name
-        if not hasattr(self, cachename):
-            from django.utils.images import get_image_dimensions
-            filename = self._get_FIELD_filename(field)
-            setattr(self, cachename, get_image_dimensions(filename))
-        return getattr(self, cachename)
-
 ############################################
 # HELPER FUNCTIONS (CURRIED MODEL METHODS) #
 ############################################

django/db/models/fields/__init__.py

@@ -694 +694 @@
         defaults.update(kwargs)
         return super(EmailField, self).formfield(**defaults)
 
+class File(object):
+    def __init__(self, obj, field, filename):
+        self.obj = obj
+        self.field = field
+        self.backend = field.backend
+        self.filename = filename
+
+    def __str__(self):
+        return self.backend.get_filename(self.filename)
+
+    def get_absolute_url(self):
+        return self.backend.get_absolute_url(self.filename)
+
+    def get_filesize(self):
+        if not hasattr(self, '_filesize'):
+            self._filesize = self.backend.get_filesize(self.filename))
+        return self._filesize
+
+    def open(self, mode='rb'):
+        return self.backend.open(self.filename, mode)
+
+    def save_file(self, filename, raw_contents, save=True):
+        self.filename = self.backend.save_file(filename, raw_contents)
+
+        # Save the object because it has changed, unless save is False
+        if save:
+            self.obj.save()
+
+class FileProxy(object):
+    def __init__(self, field):
+        self.field = field
+        self.cache_name = self.field.get_cache_name()
+
+    def __get__(self, instance=None, owner=None):
+        if instance is None:
+            raise AttributeError, "%s can only be accessed from %s instances." % (self.field.attname, self.owner.__name__)
+        return getattr(instance, self.cache_name)
+
+    def __set__(self, instance, value):
+        if hasattr(instance, self.cache_name):
+            raise AttributeError, "%s can not be set in this manner." % self.field.attname
+        instance.__dict__[self.field.attname] = value
+        attr = self.field.attr_class(instance, self.field, value)
+        setattr(instance, self.cache_name, attr)
+
 class FileField(Field):
-    def __init__(self, verbose_name=None, name=None, upload_to='', **kwargs):
-        self.upload_to = upload_to
+    attr_class = File
+
+    def __init__(self, verbose_name=None, name=None, upload_to='', backend=None, **kwargs):
+        if backend is None:
+            from django.db.models.fields.backends.filesystem import FileSystemBackend
+            backend = FileSystemBackend(location=upload_to)
+        self.backend = self.upload_to = backend
         Field.__init__(self, verbose_name, name, **kwargs)
 
     def get_db_prep_save(self, value):
@@ -704 +754 @@
         # Need to convert UploadedFile objects provided via a form to unicode for database insertion
         if value is None:
             return None
-        return unicode(value)
+        return unicode(value.filename)
 
     def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True):
         field_list = Field.get_manipulator_fields(self, opts, manipulator, change, name_prefix, rel, follow)
@@ -744 +794 @@
 
     def contribute_to_class(self, cls, name):
         super(FileField, self).contribute_to_class(cls, name)
+        setattr(cls, self.attname, FileProxy(self))
         setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
         setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
         setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
         dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
 
-    def delete_file(self, instance):
-        if getattr(instance, self.attname):
-            file_name = getattr(instance, 'get_%s_filename' % self.name)()
-            # If the file exists and no other object of this type references it,
-            # delete it from the filesystem.
-            if os.path.exists(file_name) and \
-                not instance.__class__._default_manager.filter(**{'%s__exact' % self.name: getattr(instance, self.attname)}):
-                os.remove(file_name)
+    def delete_file(self, instance, sender):
+        filename = getattr(instance, self.attname).filename
+        # If no other object of this type references the file,
+        # delete it from the backend.
+        if not sender._default_manager.filter(**{self.name: filename}):
+            self.backend.delete_file(filename)
 
     def get_manipulator_field_objs(self):
         return [oldforms.FileUploadField, oldforms.HiddenField]
@@ -768 +817 @@
     def save_file(self, new_data, new_object, original_object, change, rel, save=True):
         upload_field_name = self.get_manipulator_field_names('')[0]
         if new_data.get(upload_field_name, False):
-            func = getattr(new_object, 'save_%s_file' % self.name)
             if rel:
-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
+                field = new_data[upload_field_name][0]
             else:
-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
+                field = new_data[upload_field_name]
+            getattr(new_object, self.attname).save_file(field["filename"], field["content"], save)
 
-    def get_directory_name(self):
-        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
-
-    def get_filename(self, filename):
-        from django.utils.text import get_valid_filename
-        f = os.path.join(self.get_directory_name(), get_valid_filename(os.path.basename(filename)))
-        return os.path.normpath(f)
-
     def save_form_data(self, instance, data):
         if data:
-            getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
-        
+            getattr(instance, self.attnamename).save_file(data.filename, data.content, save=False)
+
     def formfield(self, **kwargs):
         defaults = {'form_class': forms.FileField}
         # If a file has been provided previously, then the form doesn't require 
@@ -814 +855 @@
         defaults.update(kwargs)
         return super(FloatField, self).formfield(**defaults)
 
+class ImageFile(File):
+    def get_width(self):
+        return self._get_image_dimensions()[0]
+
+    def get_height(self):
+        return self._get_image_dimensions()[1]
+
+    def _get_image_dimensions(self):
+        if not hasattr(self, '_dimensions_cache'):
+            from django.utils.images import get_image_dimensions
+            self._dimensions_cache = get_image_dimensions(self.open())
+        return self._dimensions_cache
+
 class ImageField(FileField):
+    attr_class = ImageFile
+
     def __init__(self, verbose_name=None, name=None, width_field=None, height_field=None, **kwargs):
         self.width_field, self.height_field = width_field, height_field
         FileField.__init__(self, verbose_name, name, **kwargs)
@@ -832 +888 @@
             setattr(cls, 'get_%s_height' % self.name, curry(cls._get_FIELD_height, field=self))
 
     def save_file(self, new_data, new_object, original_object, change, rel, save=True):
-        FileField.save_file(self, new_data, new_object, original_object, change, rel, save)
         # If the image has height and/or width field(s) and they haven't
         # changed, set the width and/or height field(s) back to their original
         # values.
-        if change and (self.width_field or self.height_field) and save:
-            if self.width_field:
-                setattr(new_object, self.width_field, getattr(original_object, self.width_field))
-            if self.height_field:
-                setattr(new_object, self.height_field, getattr(original_object, self.height_field))
-            new_object.save()
+        if self.width_field or self.height_field:
+            if original_object and not change:
+                if self.width_field:
+                    setattr(new_object, self.width_field, getattr(original_object, self.width_field))
+                if self.height_field:
+                    setattr(new_object, self.height_field, getattr(original_object, self.height_field))
+            else:
+                from cStringIO import StringIO
+                from django.utils.images import get_image_dimensions
 
+                upload_field_name = self.get_manipulator_field_names('')[0]
+                if rel:
+                    field = new_data[upload_field_name][0]
+                else:
+                    field = new_data[upload_field_name]
+
+                # Get the width and height from the raw content to avoid extra
+                # unnecessary trips to the file backend.
+                width, height = get_image_dimensions(StringIO(field["content"]))
+
+                if self.width_field:
+                    setattr(new_object, self.width_field, width)
+                if self.height_field:
+                    setattr(new_object, self.height_field, height)
+        FileField.save_file(self, new_data, new_object, original_object, change, rel, save)
+
     def formfield(self, **kwargs):
         defaults = {'form_class': forms.ImageField}
         return super(ImageField, self).formfield(**defaults)

django/utils/images.py

@@ -6 +6 @@
 
 import ImageFile
 
-def get_image_dimensions(path):
-    """Returns the (width, height) of an image at a given path."""
+def get_image_dimensions(file_or_path):
+    """Returns the (width, height) of an image, given an open file or a path."""
     p = ImageFile.Parser()
-    fp = open(path, 'rb')
+    if hasattr(file_or_path, 'read'):
+        fp = file_or_path
+    else:
+        fp = open(file_or_path, 'rb')
     while 1:
         data = fp.read(1024)
         if not data:
@@ -19 +22 @@
             return p.image.size
             break
     fp.close()
-    return None
+    return None
+ No newline at end of file

docs/db-api.txt

@@ -1848 +1848 @@
 get_FOO_filename()
 ------------------
 
+**Deprecated in Django development version. See `managing files` for the new,
+preferred method for dealing with files.**
+
 For every ``FileField``, the object will have a ``get_FOO_filename()`` method,
 where ``FOO`` is the name of the field. This returns the full filesystem path
 to the file, according to your ``MEDIA_ROOT`` setting.
@@ -1858 +1861 @@
 get_FOO_url()
 -------------
 
+**Deprecated in Django development version. See `managing files` for the new,
+preferred method for dealing with files.**
+
 For every ``FileField``, the object will have a ``get_FOO_url()`` method,
 where ``FOO`` is the name of the field. This returns the full URL to the file,
 according to your ``MEDIA_URL`` setting. If the value is blank, this method
@@ -1866 +1872 @@
 get_FOO_size()
 --------------
 
+**Deprecated in Django development version. See `managing files` for the new,
+preferred method for dealing with files.**
+
 For every ``FileField``, the object will have a ``get_FOO_size()`` method,
 where ``FOO`` is the name of the field. This returns the size of the file, in
 bytes. (Behind the scenes, it uses ``os.path.getsize``.)
@@ -1873 +1882 @@
 save_FOO_file(filename, raw_contents)
 -------------------------------------
 
+**Deprecated in Django development version. See `managing files` for the new,
+preferred method for dealing with files.**
+
 For every ``FileField``, the object will have a ``save_FOO_file()`` method,
 where ``FOO`` is the name of the field. This saves the given file to the
 filesystem, using the given filename. If a file with the given filename already
@@ -1882 +1894 @@
 get_FOO_height() and get_FOO_width()
 ------------------------------------
 
+**Deprecated in Django development version. See `managing files` for the new,
+preferred method for dealing with files.**
+
 For every ``ImageField``, the object will have ``get_FOO_height()`` and
 ``get_FOO_width()`` methods, where ``FOO`` is the name of the field. This
 returns the height (or width) of the image, as an integer, in pixels.

docs/model-api.txt

@@ -227 +227 @@
 ``FileField``
 ~~~~~~~~~~~~~
 
-A file-upload field. Has one **required** argument:
+A file-upload field. **Requires** exactly one of the following two arguments:
 
     ======================  ===================================================
     Argument                Description
     ======================  ===================================================
     ``upload_to``           A local filesystem path that will be appended to
                             your ``MEDIA_ROOT`` setting to determine the
-                            output of the ``get_<fieldname>_url()`` helper
-                            function.
+                            final storage destination. If this argument is
+                            supplied, the storage backend will default to
+                            ``FileSystemBackend``.
+    ``backend``             **New in Django development version**
+
+                            A storage backend object, which handles the storage
+                            and retrieval of your files. See `managing files`_
+                            for details on how to provide this object.
     ======================  ===================================================
 
-This path may contain `strftime formatting`_, which will be replaced by the
-date/time of the file upload (so that uploaded files don't fill up the given
-directory).
+.. _managing files: ../files/
 
+The ``upload_to`` path may contain `strftime formatting`_, which will be
+replaced by the date/time of the file upload (so that uploaded files don't fill
+up the given directory).
+
 The admin represents this field as an ``<input type="file">`` (a file-upload
 widget).
 
-Using a ``FileField`` or an ``ImageField`` (see below) in a model takes a few
-steps:
+Using a ``FileField`` or an ``ImageField`` (see below) in a model without a
+specified backend takes a few steps:
 
     1. In your settings file, you'll need to define ``MEDIA_ROOT`` as the
        full path to a directory where you'd like Django to store uploaded