Ticket #7210: 7210-F-support.2.patch

django/db/models/sql/where.py

@@ -4 +4 @@
 import datetime
 
 from django.utils import tree
+from django.utils.functional import curry
 from django.db import connection
 from django.db.models.fields import Field
 from django.db.models.query_utils import QueryWrapper
+from django.db.models.sql.expressions import Expression, Literal
 from datastructures import EmptyResultSet, FullResultSet
 
 # Connection types
@@ -26 +28 @@
     """
     default = AND
 
-    def as_sql(self, node=None, qn=None):
+    def as_sql(self, opts, node=None, qn=None):
         """
         Returns the SQL version of the where clause and the value to be
         substituted in. Returns None, None if this node is empty.
@@ -47 +49 @@
         for child in node.children:
             try:
                 if hasattr(child, 'as_sql'):
-                    sql, params = child.as_sql(qn=qn)
+                    sql, params = child.as_sql(opts, qn=qn)
                     format = '(%s)'
                 elif isinstance(child, tree.Node):
-                    sql, params = self.as_sql(child, qn)
+                    sql, params = self.as_sql(opts, child, qn)
                     if child.negated:
                         format = 'NOT (%s)'
                     elif len(child.children) == 1:
@@ -58 +60 @@
                     else:
                         format = '(%s)'
                 else:
-                    sql, params = self.make_atom(child, qn)
+                    sql, params = self.make_atom(opts, child, qn)
                     format = '%s'
             except EmptyResultSet:
                 if node.connector == AND and not node.negated:
@@ -86 +88 @@
         conn = ' %s ' % node.connector
         return conn.join(result), result_params
 
-    def make_atom(self, child, qn):
+    def make_atom(self, opts, child, qn):
         """
         Turn a tuple (table_alias, field_name, field_class, lookup_type, value)
         into valid SQL.
@@ -99 +101 @@
             lhs = '%s.%s' % (qn(table_alias), qn(name))
         else:
             lhs = qn(name)
+        if not field:
+            field = Field()
         db_type = field and field.db_type() or None
         field_sql = connection.ops.field_cast_sql(db_type) % lhs
+        prep_func = curry(field.get_db_prep_lookup, lookup_type)
 
-        if isinstance(value, datetime.datetime):
-            cast_sql = connection.ops.datetime_cast_sql()
-        else:
-            cast_sql = '%s'
+        if lookup_type in connection.operators:
+            if isinstance(value, Expression):
+                sql, params = value.as_sql(opts, field, prep_func, qn)
+            else:
+                sql, params = Literal(value).as_sql(opts, field, prep_func, qn)
 
-        if field:
-            params = field.get_db_prep_lookup(lookup_type, value)
-        else:
-            params = Field().get_db_prep_lookup(lookup_type, value)
+            format = '%s %s' % (
+                connection.ops.lookup_cast(lookup_type),
+                connection.operators[lookup_type])
+            return format % (field_sql, sql), params
+
+        if isinstance(value, Expression):
+            raise TypeError('Invalid lookup_type for use with %s object: %r' % (
+                value.__class__.__name__, lookup_type))
+
+        params = prep_func(value)
         if isinstance(params, QueryWrapper):
             extra, params = params.data
         else:
             extra = ''
 
-        if lookup_type in connection.operators:
-            format = "%s %%s %s" % (connection.ops.lookup_cast(lookup_type),
-                    extra)
-            return (format % (field_sql,
-                    connection.operators[lookup_type] % cast_sql), params)
-
         if lookup_type == 'in':
             if not value:
                 raise EmptyResultSet
             if extra:
                 return ('%s IN %s' % (field_sql, extra), params)
-            return ('%s IN (%s)' % (field_sql, ', '.join(['%s'] * len(value))),
-                    params)
+            return ('%s IN (%s)' % (field_sql, ', '.join(['%s'] * len(value))), params)
         elif lookup_type in ('range', 'year'):
             return ('%s BETWEEN %%s and %%s' % field_sql, params)
         elif lookup_type in ('month', 'day'):
@@ -164 +169 @@
     """
     A node that matches everything.
     """
-    def as_sql(self, qn=None):
+    def as_sql(self, opts, qn=None):
         raise FullResultSet
 
     def relabel_aliases(self, change_map, node=None):

django/db/models/sql/expressions.py

@@ +1 @@
+from copy import deepcopy
+from datetime import datetime
+
+from django.utils import tree
+from django.core.exceptions import FieldError
+from django.db import connection
+from django.db.models.fields import Field, FieldDoesNotExist
+from django.db.models.query_utils import QueryWrapper
+
+class Expression(object):
+    """
+    Base class for all sql expressions, expected by QuerySet.update.
+    """
+    # Arithmetic connection types
+    ADD = '+'
+    SUB = '-'
+    MUL = '*'
+    DIV = '/'
+    MOD = '%%'
+
+    # Bitwise connection types
+    AND = '&'
+    OR = '|'
+
+    def _combine(self, other, conn, reversed, node=None):
+        if reversed:
+            obj = ExpressionNode([Literal(other)], conn)
+            obj.add(node or self, conn)
+        else:
+            obj = node or ExpressionNode([self], conn)
+            if isinstance(other, Expression):
+                obj.add(other, conn)
+            else:
+                obj.add(Literal(other), conn)
+        return obj
+
+    def __add__(self, other):
+        return self._combine(other, self.ADD, False)
+
+    def __sub__(self, other):
+        return self._combine(other, self.SUB, False)
+
+    def __mul__(self, other):
+        return self._combine(other, self.MUL, False)
+
+    def __div__(self, other):
+        return self._combine(other, self.DIV, False)
+
+    def __mod__(self, other):
+        return self._combine(other, self.MOD, False)
+
+    def __and__(self, other):
+        return self._combine(other, self.AND, False)
+
+    def __or__(self, other):
+        return self._combine(other, self.OR, False)
+
+    def __radd__(self, other):
+        return self._combine(other, self.ADD, True)
+
+    def __rsub__(self, other):
+        return self._combine(other, self.SUB, True)
+
+    def __rmul__(self, other):
+        return self._combine(other, self.MUL, True)
+
+    def __rdiv__(self, other):
+        return self._combine(other, self.DIV, True)
+
+    def __rmod__(self, other):
+        return self._combine(other, self.MOD, True)
+
+    def __rand__(self, other):
+        return self._combine(other, self.AND, True)
+
+    def __ror__(self, other):
+        return self._combine(other, self.OR, True)
+
+    def as_sql(self, opts, field, prep_func=None, qn=None):
+        raise NotImplementedError
+
+class ExpressionNode(Expression, tree.Node):
+    default = None
+
+    def __init__(self, children=None, connector=None, negated=False):
+        if children is not None and len(children) > 1 and connector is None:
+            raise TypeError('You have to specify a connector.')
+        super(ExpressionNode, self).__init__(children, connector, negated)
+
+    def _combine(self, *args, **kwargs):
+        return super(ExpressionNode, self)._combine(node=deepcopy(self), *args, **kwargs)
+
+    def as_sql(self, opts, field, prep_func=None, qn=None, node=None):
+        if not qn:
+            qn = connection.ops.quote_name
+        if node is None:
+            node = self
+
+        result = []
+        result_params = []
+        for child in node.children:
+            if hasattr(child, 'as_sql'):
+                sql, params = child.as_sql(opts, field, prep_func, qn)
+                format = '%s'
+            else:
+                sql, params = self.as_sql(opts, field, prep_func, qn, child)
+                if len(child.children) > 1:
+                    format = '(%s)'
+                else:
+                    format = '%s'
+            if sql:
+                result.append(format % sql)
+                result_params.extend(params)
+        conn = ' %s ' % node.connector
+        return conn.join(result), result_params
+
+class Literal(Expression):
+    """
+    An expression representing the given value.
+    """
+    def __init__(self, value):
+        self.value = value
+
+    def as_sql(self, opts, field, prep_func=None, qn=None):
+        if self.value is None:
+            return 'NULL', ()
+
+        if isinstance(self.value, datetime):
+            sql = connection.ops.datetime_cast_sql()
+        else:
+            sql = '%s'
+        params = prep_func and prep_func(self.value) or (self.value,)
+
+        if isinstance(params, QueryWrapper):
+            return params.data
+        return sql, params
+
+class F(Expression):
+    """
+    An expression representing the value of the given field.
+    """
+    def __init__(self, name):
+        self.name = name
+
+    def as_sql(self, opts, field, prep_func=None, qn=None):
+        if not qn:
+            qn = connection.ops.quote_name
+
+        try:
+            src_field = opts.get_field(self.name)
+        except FieldDoesNotExist:
+            names = opts.get_all_field_names()
+            raise FieldError('Cannot resolve keyword %r into field. '
+                    'Choices are: %s' % (self.name, ', '.join(names)))
+
+        field_sql = connection.ops.field_cast_sql(src_field.db_type())
+        lhs = '%s.%s' % (qn(opts.db_table), qn(src_field.column))
+        return field_sql % lhs, ()

django/db/models/sql/query.py

@@ -250 +250 @@
         # get_from_clause() for details.
         from_, f_params = self.get_from_clause()
 
-        where, w_params = self.where.as_sql(qn=self.quote_name_unless_alias)
+        where, w_params = self.where.as_sql(self.get_meta(), qn=self.quote_name_unless_alias)
         params = list(self.extra_select_params)
 
         result = ['SELECT']

django/db/models/sql/subqueries.py

@@ -8 +8 @@
 from django.db.models.sql.datastructures import RawValue, Date
 from django.db.models.sql.query import Query
 from django.db.models.sql.where import AND
+from django.db.models.sql.expressions import Expression, Literal
 
 __all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'DateQuery',
         'CountQuery']
@@ -25 +26 @@
         assert len(self.tables) == 1, \
                 "Can only delete from one table at a time."
         result = ['DELETE FROM %s' % self.quote_name_unless_alias(self.tables[0])]
-        where, params = self.where.as_sql()
+        where, params = self.where.as_sql(self.get_meta())
         result.append('WHERE %s' % where)
         return ' '.join(result), tuple(params)
 
@@ -126 +127 @@
         result = ['UPDATE %s' % qn(table)]
         result.append('SET')
         values, update_params = [], []
-        for name, val, placeholder in self.values:
-            if val is not None:
-                values.append('%s = %s' % (qn(name), placeholder))
-                update_params.append(val)
-            else:
-                values.append('%s = NULL' % qn(name))
+        for name, sql, params in self.values:
+            values.append('%s = %s' % (qn(name), sql))
+            update_params.extend(params)
         result.append(', '.join(values))
-        where, params = self.where.as_sql()
+        where, params = self.where.as_sql(self.get_meta())
         if where:
             result.append('WHERE %s' % where)
         return ' '.join(result), tuple(update_params + params)
@@ -207 +205 @@
             self.where.add((None, f.column, f, 'in',
                     pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]),
                     AND)
-            self.values = [(related_field.column, None, '%s')]
+            self.values = [(related_field.column, 'NULL', ())]
             self.execute_sql(None)
 
     def add_update_values(self, values):
@@ -232 +230 @@
         """
         from django.db.models.base import Model
         for field, model, val in values_seq:
-            # FIXME: Some sort of db_prep_* is probably more appropriate here.
-            if field.rel and isinstance(val, Model):
-                val = val.pk
-
-            # Getting the placeholder for the field.
-            if hasattr(field, 'get_placeholder'):
-                placeholder = field.get_placeholder(val)
+            if isinstance(val, Expression):
+                expr = val
             else:
-                placeholder = '%s'
+                expr = Literal(val)
 
+            sql, params = expr.as_sql(
+                self.get_meta(),
+                field,
+                lambda x: field.get_db_prep_lookup('exact', field.get_db_prep_save(x)),
+                self.connection.ops.quote_name)
+
             if model:
-                self.add_related_update(model, field.column, val, placeholder)
+                self.add_related_update(model, field.column, sql, params)
             else:
-                self.values.append((field.column, val, placeholder))
+                self.values.append((field.column, sql, params))
 
-    def add_related_update(self, model, column, value, placeholder):
+    def add_related_update(self, model, column, sql, params):
         """
         Adds (name, value) to an update query for an ancestor model.
 
         Updates are coalesced so that we only run one update query per ancestor.
         """
         try:
-            self.related_updates[model].append((column, value, placeholder))
+            self.related_updates[model].append((column, sql, params))
         except KeyError:
-            self.related_updates[model] = [(column, value, placeholder)]
+            self.related_updates[model] = [(column, sql, params)]
 
     def get_related_updates(self):
         """
@@ -312 +311 @@
         parameters. This provides a way to insert NULL and DEFAULT keywords
         into the query, for example.
         """
-        placeholders, values = [], []
+        values = []
         for field, val in insert_values:
-            if hasattr(field, 'get_placeholder'):
-                # Some fields (e.g. geo fields) need special munging before
-                # they can be inserted.
-                placeholders.append(field.get_placeholder(val))
-            else:
-                placeholders.append('%s')
-
             self.columns.append(field.column)
             values.append(val)
         if raw_values:
             self.values.extend(values)
         else:
             self.params += tuple(values)
-            self.values.extend(placeholders)
+            self.values.extend(['%s'] * len(values))
 
 class DateQuery(Query):
     """

django/db/models/__init__.py

@@ -4 +4 @@
 from django.db import connection
 from django.db.models.loading import get_apps, get_app, get_models, get_model, register_models
 from django.db.models.query import Q
+from django.db.models.sql.expressions import F
 from django.db.models.manager import Manager
 from django.db.models.base import Model, AdminOptions
 from django.db.models.fields import *

tests/modeltests/expressions/models.py

@@ +1 @@
+"""
+Tests for the update() queryset method that allows in-place, multi-object
+updates.
+"""
+
+from django.db import models
+
+#
+# Model for testing arithmetic expressions.
+#
+
+class Number(models.Model):
+    integer = models.IntegerField()
+    float = models.FloatField(null=True)
+
+    def __unicode__(self):
+        return u'%i, %.3f' % (self.integer, self.float)
+
+#
+# A more ordinary use case.
+#
+
+class Employee(models.Model):
+    firstname = models.CharField(max_length=50)
+    lastname = models.CharField(max_length=50)
+
+    def __unicode__(self):
+        return u'%s %s' % (self.firstname, self.lastname)
+
+class Company(models.Model):
+    name = models.CharField(max_length=100)
+    num_employees = models.PositiveIntegerField()
+    num_chairs = models.PositiveIntegerField()
+    ceo = models.ForeignKey(
+        Employee,
+        related_name='company_ceo_set')
+    point_of_contact = models.ForeignKey(
+        Employee,
+        related_name='company_point_of_contact_set',
+        null=True)
+
+    def __unicode__(self):
+        return self.name
+
+
+__test__ = {'API_TESTS': """
+>>> from django.db.models import F
+
+>>> Number(integer=-1).save()
+>>> Number(integer=42).save()
+>>> Number(integer=1337).save()
+
+We can fill a value in all objects with an other value of the same object.
+
+>>> Number.objects.update(float=F('integer'))
+>>> Number.objects.all()
+[<Number: -1, -1.000>, <Number: 42, 42.000>, <Number: 1337, 1337.000>]
+
+We can increment a value of all objects in a query set.
+
+>>> Number.objects.filter(integer__gt=0).update(integer=F('integer') + 1)
+>>> Number.objects.all()
+[<Number: -1, -1.000>, <Number: 43, 42.000>, <Number: 1338, 1337.000>]
+
+We can filter for objects, where a value is not equals the value of an other field.
+
+>>> Number.objects.exclude(float=F('integer'))
+[<Number: 43, 42.000>, <Number: 1338, 1337.000>]
+
+Complex expressions of different connection types are possible.
+
+>>> n = Number.objects.create(integer=10, float=123.45)
+>>> Number.objects.filter(pk=n.pk).update(float=F('integer') + F('float') * 2)
+>>> Number.objects.get(pk=n.pk)
+<Number: 10, 256.900>
+
+All supported operators, work as expected in native and reverse order.
+
+>>> from operator import add, sub, mul, div, mod, and_, or_
+>>> for op in (add, sub, mul, div, mod, and_, or_):
+...     n = Number.objects.create(integer=42, float=15.)
+...     Number.objects.filter(pk=n.pk).update(
+...         integer=op(F('integer'), 15), float=op(42., F('float')))
+...     Number.objects.get(pk=n.pk)
+<Number: 57, 57.000>
+<Number: 27, 27.000>
+<Number: 630, 630.000>
+<Number: 3, 2.800>
+<Number: 12, 12.000>
+<Number: 10, 10.000>
+<Number: 47, 47.000>
+
+
+>>> Company(name='Example Inc.', num_employees=2300, num_chairs=5,
+...     ceo=Employee.objects.create(firstname='Joe', lastname='Smith')).save()
+>>> Company(name='Foobar Ltd.', num_employees=3, num_chairs=3,
+...     ceo=Employee.objects.create(firstname='Frank', lastname='Meyer')).save()
+>>> Company(name='Test GmbH', num_employees=32, num_chairs=1,
+...     ceo=Employee.objects.create(firstname='Max', lastname='Mustermann')).save()
+
+We can filter for companies where the number of employees is greater than the
+number of chairs.
+
+>>> Company.objects.filter(num_employees__gt=F('num_chairs'))
+[<Company: Example Inc.>, <Company: Test GmbH>]
+
+The relation of a foreign key can become copied over to an other foreign key.
+
+>>> Company.objects.update(point_of_contact=F('ceo'))
+>>> [c.point_of_contact for c in Company.objects.all()]
+[<Employee: Joe Smith>, <Employee: Frank Meyer>, <Employee: Max Mustermann>]
+
+"""}

tests/modeltests/update/models.py

@@ -4 +4 @@
 """
 
 from django.db import models
+from django.conf import settings
 
-class DataPoint(models.Model):
+class Product(models.Model):
     name = models.CharField(max_length=20)
-    value = models.CharField(max_length=20)
-    another_value = models.CharField(max_length=20, blank=True)
+    description = models.CharField(max_length=20)
+    expires = models.DateTimeField(null=True)
 
     def __unicode__(self):
         return unicode(self.name)
 
-class RelatedPoint(models.Model):
+class RelatedProduct(models.Model):
     name = models.CharField(max_length=20)
-    data = models.ForeignKey(DataPoint)
+    data = models.ForeignKey(Product)
 
     def __unicode__(self):
         return unicode(self.name)
 
 
 __test__ = {'API_TESTS': """
->>> DataPoint(name="d0", value="apple").save()
->>> DataPoint(name="d2", value="banana").save()
->>> d3 = DataPoint(name="d3", value="banana")
->>> d3.save()
->>> RelatedPoint(name="r1", data=d3).save()
+>>> from datetime import datetime
 
+>>> Product(name="p0", description="apple").save()
+>>> Product(name="p2", description="banana").save()
+>>> p3 = Product(name="p3", description="banana")
+>>> p3.save()
+>>> RelatedProduct(name="r1", data=p3).save()
+
 Objects are updated by first filtering the candidates into a queryset and then
 calling the update() method. It executes immediately and returns nothing.
 
->>> DataPoint.objects.filter(value="apple").update(name="d1")
->>> DataPoint.objects.filter(value="apple")
-[<DataPoint: d1>]
+>>> Product.objects.filter(description="apple").update(name="p1")
+>>> Product.objects.filter(description="apple")
+[<Product: p1>]
 
 We can update multiple objects at once.
 
->>> DataPoint.objects.filter(value="banana").update(value="pineapple")
->>> DataPoint.objects.get(name="d2").value
+>>> Product.objects.filter(description="banana").update(description="pineapple")
+>>> Product.objects.get(name="p2").description
 u'pineapple'
 
 Foreign key fields can also be updated, although you can only update the object
 referred to, not anything inside the related object.
 
->>> d = DataPoint.objects.get(name="d1")
->>> RelatedPoint.objects.filter(name="r1").update(data=d)
->>> RelatedPoint.objects.filter(data__name="d1")
-[<RelatedPoint: r1>]
+>>> p = Product.objects.get(name="p1")
+>>> RelatedProduct.objects.filter(name="r1").update(data=p)
+>>> RelatedProduct.objects.filter(data__name="p1")
+[<RelatedProduct: r1>]
 
-Multiple fields can be updated at once
+Multiple fields can be updated at once. If DATABASE_ENGINE is mysql microseconds
+must be truncated.
 
->>> DataPoint.objects.filter(value="pineapple").update(value="fruit", another_value="peaches")
->>> d = DataPoint.objects.get(name="d2")
->>> d.value, d.another_value
-(u'fruit', u'peaches')
+>>> Product.objects.filter(description="pineapple").update(
+...     description="fruit",
+...     expires=datetime(2010, 1, 1, 12, 0, 0, 123456))
+>>> p = Product.objects.get(name="p2")
+>>> p.description, p.expires
+"""}
 
+if settings.DATABASE_ENGINE == 'mysql':
+    __test__['API_TESTS'] += "(u'fruit', datetime.datetime(2010, 1, 1, 12, 0))"
+else:
+    __test__['API_TESTS'] += "(u'fruit', datetime.datetime(2010, 1, 1, 12, 0, 0, 123456))"
+
+__test__['API_TESTS'] += """
+
 In the rare case you want to update every instance of a model, update() is also
-a manager method.
+a manager method and update with None works as well.
 
->>> DataPoint.objects.update(value='thing')
->>> DataPoint.objects.values('value').distinct()
-[{'value': u'thing'}]
+>>> Product.objects.update(expires=None)
+>>> Product.objects.values('expires').distinct()
+[{'expires': None}]
 
 We do not support update on already sliced query sets.
 
@@ -71 +84 @@
 AssertionError: Cannot update a query once a slice has been taken.
 
 """
-}