Ordered Many-to-Many Relation Fields for Django

Version: 1.0
Date: Tue Mar 31 21:47:05 JDT 2009

For more information, see http://www.aryehleib.com/MutableLists.html

    Todo
    ----

    1. List-interface API to ordered ForeignKey relations
    2. Doubly-ordered many-to-many relation fields
    3. Symmetric, ordered self-referential many-to-many relations
    4. Allow multiple occurences, i.e. (to, from, order) is unique,
       not just (to, from)

Index: django/db/models/fields/ordered.py
===================================================================
--- django/db/models/fields/ordered.py	(revision 0)
+++ django/db/models/fields/ordered.py	(revision 0)
@@ -0,0 +1,292 @@
+from django.db import connection, transaction
+from django.db.models.fields import related as relfields
+from django.utils.functional import curry
+from django.utils.mutable_list import ListMixin
+from django.core.exceptions import ObjectDoesNotExist
+"""
+Provides Singly Ordered Many To Many Relation Field
+"""
+
+class OrderedManyIndexError(ObjectDoesNotExist):
+    "Out of range index on ordered many-to-many relation list"
+
+def create_ordered_many_related_manager(manager_base, through=None):
+    """
+    Creates a manager that subclasses manager_base
+    to provide a set-style interface for the non-ordered side
+    for partially ordered many-to-many related objects
+    """
+    ManyRelatedManager = relfields.create_many_related_manager(manager_base, through)
+
+    class OrderedManyRelatedManager(ManyRelatedManager):
+        def __init__(self, model, *args, **kwargs):
+            self.order_col_name = kwargs.pop('order_col_name')
+            super(OrderedManyRelatedManager, self).__init__(model, *args, **kwargs)
+
+        def _add_one_item(self, target_pk, cursor=None):
+            if cursor is None:
+                cursor = connection.cursor()
+
+            # get the ordering for object to add and append this rel
+            cursor.execute("SELECT MAX(%s) FROM %s WHERE %s = %%s" % \
+                            (self.order_col_name, self.join_table, self.target_col_name),
+                            [ target_pk ])
+            result  = cursor.fetchone()[0]
+            if result is None:
+                next_order = 0
+            else:
+                next_order = result + 1
+
+            # Add the ones that aren't there already
+            cursor.execute("INSERT INTO %s (%s, %s, %s) VALUES (%%s, %%s, %%s)" % \
+                (self.join_table, self.source_col_name, self.target_col_name, 
+                    self.order_col_name),
+                [self._pk_val, target_pk, next_order])
+
+        def _add_items(self, source_col_name, target_col_name, *objs):
+            # join_table: name of the m2m link table
+            # source_col_name: the PK colname in join_table for the source object
+            # target_col_name: the PK colname in join_table for the target object
+            # *objs - objects to add. Either object instances, or primary keys of object instances.
+            assert source_col_name == self.source_col_name, 'source is "%s", should be "%s"' % \
+                    (source_col_name, self.source_col_name)
+            assert target_col_name == self.target_col_name, 'target is "%s", should be "%s"' % \
+                    (target_col_name, self.target_col_name)
+
+            # If there aren't any objects, there is nothing to do.
+            if objs:
+                # Check that all the objects are of the right type
+                new_ids = set()
+                for obj in objs:
+                    if isinstance(obj, self.model):
+                        new_ids.add(obj._get_pk_val())
+                    else:
+                        new_ids.add(obj)
+                # Add the newly created or already existing objects to the join table.
+                # First find out which items are already added, to avoid adding them twice
+                cursor = connection.cursor()
+                cursor.execute("SELECT %s FROM %s WHERE %s = %%s AND %s IN (%s)" % \
+                    (target_col_name, self.join_table, source_col_name,
+                    target_col_name, ",".join(['%s'] * len(new_ids))),
+                    [self._pk_val] + list(new_ids))
+                existing_ids = set([row[0] for row in cursor.fetchall()])
+
+                # Add the ones that aren't there already
+                for obj_id in (new_ids - existing_ids):
+                    self._add_one_item(obj_id, cursor)
+                transaction.commit_unless_managed()
+
+    return OrderedManyRelatedManager
+
+def create_reverse_ordered_many_related_manager(manager_base, through=None):
+    """
+    Creates a manager that subclasses manager_base
+    as well as ListMixin to provide list-style interface
+    to ordered many-to-many related objects
+    """
+    ManyRelatedManager = relfields.create_many_related_manager(manager_base, through)
+
+    class ReverseOrderedManyRelatedManager(ManyRelatedManager, ListMixin):
+        _IndexError = OrderedManyIndexError
+
+        def __init__(self, model, *args, **kwargs):
+            self.order_col_name = kwargs.pop('order_col_name')
+            super(ReverseOrderedManyRelatedManager, self).__init__(model, *args, **kwargs)
+            # TODO: dynamically get the pk type for related field, 
+            # instead of assuming it's an int/long
+            self._allowed = (model, int, long)
+
+        def get_query_set(self):
+            extras = { 'order_by'    : ['%s.%s' % (self.join_table, self.order_col_name)], }
+            return manager_base.get_query_set(self)._next_is_sticky().filter(**(self.core_filters)).extra(**extras)
+
+        def __len__(self):
+            return self.count()
+
+        def __getitem__(self, index):
+            self._begin()
+            result = super(ReverseOrderedManyRelatedManager, self).__getitem__(index)
+            self._end(commit=False)
+            return result
+
+        def __setitem__(self, index, val):
+            self._begin()
+            super(ReverseOrderedManyRelatedManager, self).__setitem__(index, val)
+            self._end(commit=True)
+        __setitem__.alters_data = True
+
+        def __delitem__(self, index):
+            self._begin()
+            super(ReverseOrderedManyRelatedManager, self).__delitem__(index)
+            self._end(commit=True)
+        __delitem__.alters_data = True
+
+        def _begin(self):
+            query   = 'SELECT "id", %s FROM %s WHERE %s = %%s ORDER BY %s' % \
+                                (self.source_col_name, self.join_table,
+                                    self.target_col_name, self.order_col_name)
+            self._cursor  = connection.cursor()
+            self._cursor.execute(query, [self._pk_val])
+            self._cache = self._cursor.fetchall()
+
+        def _end(self, commit=False):
+            del self._cursor
+            del self._cache
+            if commit:
+                transaction.commit_unless_managed()
+
+        def add(self, *objs):
+            self.extend(objs)
+        add.alters_data = True
+
+        def remove(self, *objs):
+            for o in objs:
+                ListMixin.remove(self, o)
+        remove.alters_data = True
+
+        def clear(self):
+            self[:] = []
+        clear.alters_data = True
+
+        def _get_single_external(self, i):
+            target_pk  = self._cache[i][1]
+            return super(ReverseOrderedManyRelatedManager, self).get(pk=target_pk)
+
+        def _set_single(self, i, obj):
+            join_id = self._cache[i][0]
+            query   = 'UPDATE %s SET %s = %%s WHERE "id" = %%s' % \
+                        (self.join_table, self.source_col_name)
+            self._cursor.execute(query, [obj.pk, join_id])
+
+        # need to use int as pk of related object for serialization
+        # so we use a special class to distinguish indices into the cache
+        class _PK(object):
+            def __init__(self, pk):
+                self.pk = pk
+
+        def _get_single_internal(self, i):
+            return self._PK(i)
+
+        def _set_list(self, length, items):
+            # Optimize this by re-using existing records?
+            query   = 'DELETE FROM %s WHERE %s = %%s' % \
+                        (self.join_table, self.target_col_name)
+            self._cursor.execute(query, [self._pk_val])
+            
+            vals_list   = []
+            my_pk       = self._pk_val
+            for i, item in enumerate(items):
+                if isinstance(item, self._PK):
+                    vals_list.append((my_pk, self._cache[item.pk][1], i))
+                elif isinstance(item, (int, long)):
+                    vals_list.append((my_pk, item, i))
+                else:
+                    vals_list.append((my_pk, item.pk, i))
+
+            query = 'INSERT INTO %s (%s, %s, %s) VALUES (%%s, %%s, %%s)' % \
+                    (self.join_table, self.target_col_name,
+                        self.source_col_name, self.order_col_name)
+            self._cursor.executemany(query, vals_list)
+
+    return ReverseOrderedManyRelatedManager
+
+class OrderedManyRelatedObjectsDescriptor(relfields.ManyRelatedObjectsDescriptor):
+    def __get__(self, instance, instance_type=None):
+        if instance is None:
+            return self
+
+        # Dynamically create a class that subclasses the related
+        # model's default manager.
+        rel_model = self.related.model
+        superclass = rel_model._default_manager.__class__
+        OrderedRelatedManager = create_ordered_many_related_manager(superclass, self.related.field.rel.through)
+
+        qn = connection.ops.quote_name
+        manager = OrderedRelatedManager(
+            model=rel_model,
+            core_filters={'%s__pk' % self.related.field.name: instance._get_pk_val()},
+            instance=instance,
+            symmetrical=False,
+            join_table=qn(self.related.field.m2m_db_table()),
+            source_col_name=qn(self.related.field.m2m_reverse_name()),
+            target_col_name=qn(self.related.field.m2m_column_name()),
+            order_col_name=qn(self.related.field.m2m_order_name()),
+        )
+
+        return manager
+
+class ReverseOrderedManyRelatedObjectsDescriptor(relfields.ReverseManyRelatedObjectsDescriptor):
+    def __get__(self, instance, instance_type=None):
+        # probably should not allow this
+        if instance is None:
+            return self
+
+        # Dynamically create a class that subclasses the related
+        # model's default manager.
+        rel_model = self.field.rel.to
+        superclass = rel_model._default_manager.__class__
+        ReverseOrderedRelatedManager = create_reverse_ordered_many_related_manager(superclass, self.field.rel.through)
+
+        qn = connection.ops.quote_name
+        manager = ReverseOrderedRelatedManager(
+            model=rel_model,
+            core_filters={'%s__pk' % self.field.related_query_name(): instance._get_pk_val()},
+            instance=instance,
+            symmetrical=False,
+            join_table=qn(self.field.m2m_db_table()),
+            source_col_name=qn(self.field.m2m_reverse_name()),
+            target_col_name=qn(self.field.m2m_column_name()),
+            order_col_name=qn(self.field.m2m_order_name()),
+        )
+
+        return manager
+
+    def __set__(self, instance, val):
+        if instance is None:
+            raise AttributeError, "Manager must be accessed via instance"
+        manager = self.__get__(instance)
+        manager[:] = list(val)
+
+
+class OrderedManyToManyField(relfields.ManyToManyField):
+    ordered = 1
+    def __init__(self, *args, **kwargs):
+        kwargs['symmetrical'] = False
+        if 'through' in kwargs:
+            raise ValueError, "Use of 'through' model not allowed for OrderedManyToManyField"
+    
+        super(OrderedManyToManyField, self).__init__(*args, **kwargs)
+
+    def contribute_to_class(self, cls, name):
+        super(OrderedManyToManyField, self).contribute_to_class(cls, name)
+        # Add the descriptor for the m2m relation
+        setattr(cls, self.name, ReverseOrderedManyRelatedObjectsDescriptor(self))
+
+        # Set up the accessor for the m2m table name for the relation
+        self.m2m_db_table = curry(self._get_m2m_db_table, cls._meta)
+
+        if isinstance(self.rel.to, basestring):
+            target = self.rel.to
+        else:
+            target = self.rel.to._meta.db_table
+        cls._meta.duplicate_targets[self.column] = (target, "m2m")
+
+    def contribute_to_related_class(self, cls, related):
+        # Set up the accessors for the column names on the m2m table
+        self.m2m_column_name = curry(self._get_m2m_column_name, related)
+        self.m2m_reverse_name = curry(self._get_m2m_reverse_name, related)
+        self.m2m_order_name = curry(self._get_m2m_order_name, related)
+
+        setattr(cls, related.get_accessor_name(), OrderedManyRelatedObjectsDescriptor(related))
+
+    def _get_m2m_order_name(self, related):
+        "Function that can be curried to provide the related column name for the m2m table"
+        try:
+            return self._m2m_order_name_cache
+        except:
+            self._m2m_order_name_cache = related.parent_model._meta.object_name.lower() + '_order'
+            return self._m2m_order_name_cache
+

Property changes on: django/db/models/fields/ordered.py
___________________________________________________________________
Name: svn:executable
   + *

Index: django/db/models/__init__.py
===================================================================
--- django/db/models/__init__.py	(revision 10131)
+++ django/db/models/__init__.py	(working copy)
@@ -11,6 +11,7 @@
 from django.db.models.fields.subclassing import SubfieldBase
 from django.db.models.fields.files import FileField, ImageField
 from django.db.models.fields.related import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel
+from django.db.models.fields.ordered import OrderedManyToManyField
 from django.db.models import signals
 
 # Admin stages.
Index: django/db/backends/creation.py
===================================================================
--- django/db/backends/creation.py	(revision 10131)
+++ django/db/backends/creation.py	(working copy)
@@ -223,6 +223,13 @@
                 style.SQL_FIELD(qn(field.rel.to._meta.pk.column)),
                 self.connection.ops.deferrable_sql())
         ]
+        if hasattr(field, 'ordered'):
+            table_output.append(
+            '    %s %s %s,' %
+                (style.SQL_FIELD(qn(field.m2m_order_name())),
+                style.SQL_COLTYPE(models.IntegerField().db_type()),
+                style.SQL_KEYWORD('NOT NULL'),)
+                )
         deferred = []
 
         return table_output, deferred
Index: django/utils/mutable_list.py
===================================================================
--- django/utils/mutable_list.py	(revision 0)
+++ django/utils/mutable_list.py	(revision 0)
@@ -0,0 +1,309 @@
+# Copyright (c) 2008-2009 Aryeh Leib Taurog, all rights reserved.
+# Released under the New BSD license.
+"""
+This module contains a base type which provides list-style mutations
+without specific data storage methods.
+
+See also http://www.aryehleib.com/MutableLists.html
+
+Author: Aryeh Leib Taurog.
+"""
+class ListMixin(object):
+    """
+    A base class which provides complete list interface.
+    Derived classes must call ListMixin's __init__() function
+    and implement the following:
+
+    function _get_single_external(self, i):
+        Return single item with index i for general use.
+        The index i will always satisfy 0 <= i < len(self).
+
+    function _get_single_internal(self, i):
+        Same as above, but for use within the class [Optional]
+        Note that if _get_single_internal and _get_single_internal return
+        different types of objects, _set_list must distinguish
+        between the two and handle each appropriately.
+
+    function _set_list(self, length, items):
+        Recreate the entire object.
+
+        NOTE: items may be a generator which calls _get_single_internal.
+        Therefore, it is necessary to cache the values in a temporary:
+            temp = list(items)
+        before clobbering the original storage.
+
+    function _set_single(self, i, value):
+        Set the single item at index i to value [Optional]
+        If left undefined, all mutations will result in rebuilding
+        the object using _set_list.
+
+    function __len__(self):
+        Return the length
+
+    int _minlength:
+        The minimum legal length [Optional]
+
+    int _maxlength:
+        The maximum legal length [Optional]
+
+    type or tuple _allowed:
+        A type or tuple of allowed item types [Optional]
+
+    class _IndexError:
+        The type of exception to be raise on invalid index [Optional]
+    """
+
+    _minlength = 0
+    _maxlength = None
+    _IndexError = IndexError
+
+    ### Python initialization and special list interface methods ###
+
+    def __init__(self, *args, **kwargs):
+        if not hasattr(self, '_get_single_internal'):
+            self._get_single_internal = self._get_single_external
+
+        if not hasattr(self, '_set_single'):
+            self._set_single = self._set_single_rebuild
+            self._assign_extended_slice = self._assign_extended_slice_rebuild
+
+        super(ListMixin, self).__init__(*args, **kwargs)
+
+    def __getitem__(self, index):
+        "Get the item(s) at the specified index/slice."
+        if isinstance(index, slice):
+            return [self._get_single_external(i) for i in xrange(*index.indices(len(self)))]
+        else:
+            index = self._checkindex(index)
+            return self._get_single_external(index)
+
+    def __delitem__(self, index):
+        "Delete the item(s) at the specified index/slice."
+        if not isinstance(index, (int, long, slice)):
+            raise TypeError("%s is not a legal index" % index)
+
+        # calculate new length and dimensions
+        origLen     = len(self)
+        if isinstance(index, (int, long)):
+            index = self._checkindex(index)
+            indexRange  = [index]
+        else:
+            indexRange  = range(*index.indices(origLen))
+
+        newLen      = origLen - len(indexRange)
+        newItems    = ( self._get_single_internal(i)
+                        for i in xrange(origLen)
+                        if i not in indexRange )
+
+        self._rebuild(newLen, newItems)
+
+    def __setitem__(self, index, val):
+        "Set the item(s) at the specified index/slice."
+        if isinstance(index, slice):
+            self._set_slice(index, val)
+        else:
+            index = self._checkindex(index)
+            self._check_allowed((val,))
+            self._set_single(index, val)
+
+    def __iter__(self):
+        "Iterate over the items in the list"
+        for i in xrange(len(self)):
+            yield self[i]
+
+    ### Special methods for arithmetic operations ###
+    def __add__(self, other):
+        'add another list-like object'
+        return self.__class__(list(self) + list(other))
+
+    def __radd__(self, other):
+        'add to another list-like object'
+        return other.__class__(list(other) + list(self))
+
+    def __iadd__(self, other):
+        'add another list-like object to self'
+        self.extend(list(other))
+        return self
+
+    def __mul__(self, n):
+        'multiply'
+        return self.__class__(list(self) * n)
+
+    def __rmul__(self, n):
+        'multiply'
+        return self.__class__(list(self) * n)
+
+    def __imul__(self, n):
+        'multiply'
+        if n <= 0:
+            del self[:]
+        else:
+            cache = list(self)
+            for i in range(n-1):
+                self.extend(cache)
+        return self
+
+    def __cmp__(self, other):
+        'cmp'
+        slen = len(self)
+        for i in range(slen):
+            try:
+                c = cmp(self[i], other[i])
+            except IndexError:
+                # must be other is shorter
+                return 1
+            else:
+                # elements not equal
+                if c: return c
+
+        return cmp(slen, len(other))
+
+    ### Public list interface Methods ###
+    ## Non-mutating ##
+    def count(self, val):
+        "Standard list count method"
+        count = 0
+        for i in self:
+            if val == i: count += 1
+        return count
+
+    def index(self, val):
+        "Standard list index method"
+        for i in xrange(0, len(self)):
+            if self[i] == val: return i
+        raise ValueError('%s not found in object' % str(val))
+
+    ## Mutating ##
+    def append(self, val):
+        "Standard list append method"
+        self[len(self):] = [val]
+
+    def extend(self, vals):
+        "Standard list extend method"
+        self[len(self):] = vals
+
+    def insert(self, index, val):
+        "Standard list insert method"
+        if not isinstance(index, (int, long)):
+            raise TypeError("%s is not a legal index" % index)
+        self[index:index] = [val]
+
+    def pop(self, index=-1):
+        "Standard list pop method"
+        result = self[index]
+        del self[index]
+        return result
+
+    def remove(self, val):
+        "Standard list remove method"
+        del self[self.index(val)]
+
+    def reverse(self):
+        "Standard list reverse method"
+        self[:] = self[-1::-1]
+
+    def sort(self, cmp=cmp, key=None, reverse=False):
+        "Standard list sort method"
+        if key:
+            temp = [(key(v),v) for v in self]
+            temp.sort(cmp=cmp, key=lambda x: x[0], reverse=reverse)
+            self[:] = [v[1] for v in temp]
+        else:
+            temp = list(self)
+            temp.sort(cmp=cmp, reverse=reverse)
+            self[:] = temp
+
+    ### Private routines ###
+    def _rebuild(self, newLen, newItems):
+        if newLen < self._minlength:
+            raise ValueError('Must have at least %d items' % self._minlength)
+        if self._maxlength is not None and newLen > self._maxlength:
+            raise ValueError('Cannot have more than %d items' % self._maxlength)
+
+        self._set_list(newLen, newItems)
+
+    def _set_single_rebuild(self, index, value):
+        self._set_slice(slice(index, index + 1, 1), [value])
+
+    def _checkindex(self, index, correct=True):
+        length = len(self)
+        if 0 <= index < length:
+            return index
+        if correct and -length <= index < 0:
+            return index + length
+        raise self._IndexError('invalid index: %s' % str(index))
+
+    def _check_allowed(self, items):
+        if hasattr(self, '_allowed'):
+            if False in [isinstance(val, self._allowed) for val in items]:
+                raise TypeError('Invalid type encountered in the arguments.')
+
+    def _set_slice(self, index, values):
+        "Assign values to a slice of the object"
+        try:
+            iter(values)
+        except TypeError:
+            raise TypeError('can only assign an iterable to a slice')
+
+        self._check_allowed(values)
+
+        origLen     = len(self)
+        valueList   = list(values)
+        start, stop, step = index.indices(origLen)
+
+        # CAREFUL: index.step and step are not the same!
+        # step will never be None
+        if index.step is None:
+            self._assign_simple_slice(start, stop, valueList)
+        else:
+            self._assign_extended_slice(start, stop, step, valueList)
+
+    def _assign_extended_slice_rebuild(self, start, stop, step, valueList):
+        'Assign an extended slice by rebuilding entire list'
+        indexList   = range(start, stop, step)
+        # extended slice, only allow assigning slice of same size
+        if len(valueList) != len(indexList):
+            raise ValueError('attempt to assign sequence of size %d '
+                             'to extended slice of size %d'
+                             % (len(valueList), len(indexList)))
+
+        # we're not changing the length of the sequence
+        newLen  = len(self)
+        newVals = dict(zip(indexList, valueList))
+        def newItems():
+            for i in xrange(newLen):
+                if i in newVals:
+                    yield newVals[i]
+                else:
+                    yield self._get_single_internal(i)
+
+        self._rebuild(newLen, newItems())
+
+    def _assign_extended_slice(self, start, stop, step, valueList):
+        'Assign an extended slice by re-assigning individual items'
+        indexList   = range(start, stop, step)
+        # extended slice, only allow assigning slice of same size
+        if len(valueList) != len(indexList):
+            raise ValueError('attempt to assign sequence of size %d '
+                             'to extended slice of size %d'
+                             % (len(valueList), len(indexList)))
+
+        for i, val in zip(indexList, valueList):
+            self._set_single(i, val)
+
+    def _assign_simple_slice(self, start, stop, valueList):
+        'Assign a simple slice; Can assign slice of any length'
+        origLen = len(self)
+        stop = max(start, stop)
+        newLen  = origLen - stop + start + len(valueList)
+        def newItems():
+            for i in xrange(origLen + 1):
+                if i == start:
+                    for val in valueList:
+                        yield val
+
+                if i < origLen:
+                    if i < start or i >= stop:
+                        yield self._get_single_internal(i)
+
+        self._rebuild(newLen, newItems())

Property changes on: django/utils/mutable_list.py
___________________________________________________________________
Name: svn:executable
   + *

