Index: tests/regressiontests/nested_foreign_keys/__init__.py
===================================================================
Index: tests/regressiontests/nested_foreign_keys/tests.py
===================================================================
--- tests/regressiontests/nested_foreign_keys/tests.py	(revision 0)
+++ tests/regressiontests/nested_foreign_keys/tests.py	(revision 0)
@@ -0,0 +1,165 @@
+from django.test import TestCase
+
+from models import Person, Movie, Event, Screening, ScreeningNullFK, Package, PackageNullFK
+
+
+# These are tests for #16715. The basic scheme is always the same: 3 models with
+# 2 relations. The first relation may be null, while the second is non-nullable.
+# In some cases, Django would pick the wrong join type for the second relation,
+# resulting in missing objects in the queryset.
+#
+#   Model A
+#   | (Relation A/B : nullable)
+#   Model B
+#   | (Relation B/C : non-nullable)
+#   Model C
+#
+# Because of the possibility of NULL rows resulting from the LEFT OUTER JOIN
+# between Model A and Model B (i.e. instances of A without reference to B),
+# the second join must also be LEFT OUTER JOIN, so that we do not ignore
+# instances of A that do not reference B.
+#
+# Relation A/B can either be an explicit foreign key or an implicit reverse
+# relation such as introduced by one-to-one relations (through multi-table
+# inheritance).
+class NestedForeignKeysTests(TestCase):
+    def setUp(self):
+        self.director = Person.objects.create(name=u'Terry Gilliam / Terry Jones')
+        self.movie = Movie.objects.create(title=u'Monty Python and the Holy Grail', director=self.director)
+
+
+    # This test failed in #16715 because in some cases INNER JOIN was selected
+    # for the second foreign key relation instead of LEFT OUTER JOIN.
+    def testInheritance(self):
+        some_event = Event.objects.create()
+        screening = Screening.objects.create(movie=self.movie)
+
+        self.assertEqual(len(Event.objects.all()), 2)
+        self.assertEqual(len(Event.objects.select_related('screening')), 2)
+        # This failed.
+        self.assertEqual(len(Event.objects.select_related('screening__movie')), 2)
+
+        self.assertEqual(len(Event.objects.values()), 2)
+        self.assertEqual(len(Event.objects.values('screening__pk')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__pk')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__title')), 2)
+        # This failed.
+        self.assertEqual(len(Event.objects.values('screening__movie__pk', 'screening__movie__title')), 2)
+
+        # Simple filter/exclude queries for good measure.
+        self.assertEqual(Event.objects.filter(screening__movie=self.movie).count(), 1)
+        self.assertEqual(Event.objects.exclude(screening__movie=self.movie).count(), 1)
+
+
+    # These all work because the second foreign key in the chain has null=True.
+    def testInheritanceNullFK(self):
+        some_event = Event.objects.create()
+        screening = ScreeningNullFK.objects.create(movie=None)
+        screening_with_movie = ScreeningNullFK.objects.create(movie=self.movie)
+
+        self.assertEqual(len(Event.objects.all()), 3)
+        self.assertEqual(len(Event.objects.select_related('screeningnullfk')), 3)
+        self.assertEqual(len(Event.objects.select_related('screeningnullfk__movie')), 3)
+
+        self.assertEqual(len(Event.objects.values()), 3)
+        self.assertEqual(len(Event.objects.values('screeningnullfk__pk')), 3)
+        self.assertEqual(len(Event.objects.values('screeningnullfk__movie__pk')), 3)
+        self.assertEqual(len(Event.objects.values('screeningnullfk__movie__title')), 3)
+        self.assertEqual(len(Event.objects.values('screeningnullfk__movie__pk', 'screeningnullfk__movie__title')), 3)
+
+        self.assertEqual(Event.objects.filter(screeningnullfk__movie=self.movie).count(), 1)
+        self.assertEqual(Event.objects.exclude(screeningnullfk__movie=self.movie).count(), 2)
+
+
+    # This test failed in #16715 because in some cases INNER JOIN was selected
+    # for the second foreign key relation instead of LEFT OUTER JOIN.
+    def testExplicitForeignKey(self):
+        package = Package.objects.create()
+        screening = Screening.objects.create(movie=self.movie)
+        package_with_screening = Package.objects.create(screening=screening)
+
+        self.assertEqual(len(Package.objects.all()), 2)
+        self.assertEqual(len(Package.objects.select_related('screening')), 2)
+        self.assertEqual(len(Package.objects.select_related('screening__movie')), 2)
+
+        self.assertEqual(len(Package.objects.values()), 2)
+        self.assertEqual(len(Package.objects.values('screening__pk')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__pk')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__title')), 2)
+        # This failed.
+        self.assertEqual(len(Package.objects.values('screening__movie__pk', 'screening__movie__title')), 2)
+
+        self.assertEqual(Package.objects.filter(screening__movie=self.movie).count(), 1)
+        self.assertEqual(Package.objects.exclude(screening__movie=self.movie).count(), 1)
+
+
+    # These all work because the second foreign key in the chain has null=True.
+    def testExplicitForeignKeyNullFK(self):
+        package = PackageNullFK.objects.create()
+        screening = ScreeningNullFK.objects.create(movie=None)
+        screening_with_movie = ScreeningNullFK.objects.create(movie=self.movie)
+        package_with_screening = PackageNullFK.objects.create(screening=screening)
+        package_with_screening_with_movie = PackageNullFK.objects.create(screening=screening_with_movie)
+
+        self.assertEqual(len(PackageNullFK.objects.all()), 3)
+        self.assertEqual(len(PackageNullFK.objects.select_related('screening')), 3)
+        self.assertEqual(len(PackageNullFK.objects.select_related('screening__movie')), 3)
+
+        self.assertEqual(len(PackageNullFK.objects.values()), 3)
+        self.assertEqual(len(PackageNullFK.objects.values('screening__pk')), 3)
+        self.assertEqual(len(PackageNullFK.objects.values('screening__movie__pk')), 3)
+        self.assertEqual(len(PackageNullFK.objects.values('screening__movie__title')), 3)
+        self.assertEqual(len(PackageNullFK.objects.values('screening__movie__pk', 'screening__movie__title')), 3)
+
+        self.assertEqual(PackageNullFK.objects.filter(screening__movie=self.movie).count(), 1)
+        self.assertEqual(PackageNullFK.objects.exclude(screening__movie=self.movie).count(), 2)
+
+
+# Some additional tests for #16715. The only difference is the depth of the
+# nesting as we now use 4 models instead of 3 (and thus 3 relations). This
+# checks if promotion of join types works for deeper nesting too.
+class DeeplyNestedForeignKeysTests(TestCase):
+    def setUp(self):
+        self.director = Person.objects.create(name=u'Terry Gilliam / Terry Jones')
+        self.movie = Movie.objects.create(title=u'Monty Python and the Holy Grail', director=self.director)
+
+
+    def testInheritance(self):
+        some_event = Event.objects.create()
+        screening = Screening.objects.create(movie=self.movie)
+
+        self.assertEqual(len(Event.objects.all()), 2)
+        self.assertEqual(len(Event.objects.select_related('screening__movie__director')), 2)
+
+        self.assertEqual(len(Event.objects.values()), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__director__pk')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__director__name')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__director__pk', 'screening__movie__director__name')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__pk', 'screening__movie__director__pk')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__pk', 'screening__movie__director__name')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__title', 'screening__movie__director__pk')), 2)
+        self.assertEqual(len(Event.objects.values('screening__movie__title', 'screening__movie__director__name')), 2)
+
+        self.assertEqual(Event.objects.filter(screening__movie__director=self.director).count(), 1)
+        self.assertEqual(Event.objects.exclude(screening__movie__director=self.director).count(), 1)
+
+
+    def testExplicitForeignKey(self):
+        package = Package.objects.create()
+        screening = Screening.objects.create(movie=self.movie)
+        package_with_screening = Package.objects.create(screening=screening)
+
+        self.assertEqual(len(Package.objects.all()), 2)
+        self.assertEqual(len(Package.objects.select_related('screening__movie__director')), 2)
+
+        self.assertEqual(len(Package.objects.values()), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__director__pk')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__director__name')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__director__pk', 'screening__movie__director__name')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__pk', 'screening__movie__director__pk')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__pk', 'screening__movie__director__name')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__title', 'screening__movie__director__pk')), 2)
+        self.assertEqual(len(Package.objects.values('screening__movie__title', 'screening__movie__director__name')), 2)
+
+        self.assertEqual(Package.objects.filter(screening__movie__director=self.director).count(), 1)
+        self.assertEqual(Package.objects.exclude(screening__movie__director=self.director).count(), 1)
Index: tests/regressiontests/nested_foreign_keys/models.py
===================================================================
--- tests/regressiontests/nested_foreign_keys/models.py	(revision 0)
+++ tests/regressiontests/nested_foreign_keys/models.py	(revision 0)
@@ -0,0 +1,28 @@
+from django.db import models
+
+
+class Person(models.Model):
+    name = models.CharField(max_length=200)
+
+
+class Movie(models.Model):
+    title = models.CharField(max_length=200)
+    director = models.ForeignKey(Person)
+
+
+class Event(models.Model):
+    pass
+
+
+class Screening(Event):
+    movie = models.ForeignKey(Movie)
+
+class ScreeningNullFK(Event):
+    movie = models.ForeignKey(Movie, null=True)
+
+
+class Package(models.Model):
+    screening = models.ForeignKey(Screening, null=True)
+
+class PackageNullFK(models.Model):
+    screening = models.ForeignKey(ScreeningNullFK, null=True)
Index: django/db/models/sql/compiler.py
===================================================================
--- django/db/models/sql/compiler.py	(revision 16928)
+++ django/db/models/sql/compiler.py	(working copy)
@@ -397,8 +397,7 @@
             self.query.ref_alias(alias)
 
         # Must use left outer joins for nullable fields and their relations.
-        self.query.promote_alias_chain(joins,
-            self.query.alias_map[joins[0]][JOIN_TYPE] == self.query.LOUTER)
+        self.query.promote_joins(joins)
 
         # If we get to this point and the field is a relation to another model,
         # append the default ordering for that model.
@@ -579,8 +578,7 @@
                     alias_chain.append(alias)
                     for (dupe_opts, dupe_col) in dupe_set:
                         self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias)
-                if self.query.alias_map[root_alias][JOIN_TYPE] == self.query.LOUTER:
-                    self.query.promote_alias_chain(alias_chain, True)
+                self.query.promote_joins(alias_chain)
             else:
                 alias = root_alias
 
@@ -597,8 +595,7 @@
             columns, aliases = self.get_default_columns(start_alias=alias,
                     opts=f.rel.to._meta, as_pairs=True)
             self.query.related_select_cols.extend(columns)
-            if self.query.alias_map[alias][JOIN_TYPE] == self.query.LOUTER:
-                self.query.promote_alias_chain(aliases, True)
+            self.query.promote_joins(aliases)
             self.query.related_select_fields.extend(f.rel.to._meta.fields)
             if restricted:
                 next = requested.get(f.name, {})
@@ -671,7 +668,9 @@
                 self.query.related_select_fields.extend(model._meta.fields)
 
                 next = requested.get(f.related_query_name(), {})
-                new_nullable = f.null or None
+                # Use True here because we are looking at the _reverse_ side of
+                # the relation, which is always nullable.
+                new_nullable = True
 
                 self.fill_related_selections(model._meta, table, cur_depth+1,
                     used, next, restricted, new_nullable)
Index: django/db/models/sql/query.py
===================================================================
--- django/db/models/sql/query.py	(revision 16928)
+++ django/db/models/sql/query.py	(working copy)
@@ -497,7 +497,7 @@
                 # Again, some of the tables won't have aliases due to
                 # the trimming of unnecessary tables.
                 if self.alias_refcount.get(alias) or rhs.alias_refcount.get(alias):
-                    self.promote_alias(alias, True)
+                    self.promote_joins([alias], True)
 
         # Now relabel a copy of the rhs where-clause and add it to the current
         # one.
@@ -677,33 +677,29 @@
         """ Decreases the reference count for this alias. """
         self.alias_refcount[alias] -= 1
 
-    def promote_alias(self, alias, unconditional=False):
+    def promote_joins(self, aliases, unconditional=False):
         """
-        Promotes the join type of an alias to an outer join if it's possible
-        for the join to contain NULL values on the left. If 'unconditional' is
-        False, the join is only promoted if it is nullable, otherwise it is
-        always promoted.
-
-        Returns True if the join was promoted by this call.
+        Promotes recursively the join type of given aliases and its children to
+        an outer join if it's possible for the join to contain NULL values on
+        the left.  If 'unconditional' is False, the join is only promoted if it
+        is nullable or the parent join is an outer join, otherwise it is always
+        promoted.
         """
-        if ((unconditional or self.alias_map[alias][NULLABLE]) and
-                self.alias_map[alias][JOIN_TYPE] != self.LOUTER):
-            data = list(self.alias_map[alias])
-            data[JOIN_TYPE] = self.LOUTER
-            self.alias_map[alias] = tuple(data)
-            return True
-        return False
+        aliases = list(aliases)
+        while aliases:
+            alias = aliases.pop(0)
+            parent_alias = self.alias_map[alias][LHS_ALIAS]
+            parent_louter = parent_alias and self.alias_map[parent_alias][JOIN_TYPE] == self.LOUTER or False
+            if ((unconditional or self.alias_map[alias][NULLABLE] or parent_louter) and
+                    self.alias_map[alias][JOIN_TYPE] != self.LOUTER):
+                data = list(self.alias_map[alias])
+                data[JOIN_TYPE] = self.LOUTER
+                self.alias_map[alias] = tuple(data)
+                # Join type of 'alias' changed, so re-examine all aliases that
+                # refer to this one (and, in turn, their children).
+                aliases.extend(child for child in self.alias_map.iterkeys()
+                    if self.alias_map[child][LHS_ALIAS] == alias and child not in aliases)
 
-    def promote_alias_chain(self, chain, must_promote=False):
-        """
-        Walks along a chain of aliases, promoting the first nullable join and
-        any joins following that. If 'must_promote' is True, all the aliases in
-        the chain are promoted.
-        """
-        for alias in chain:
-            if self.promote_alias(alias, must_promote):
-                must_promote = True
-
     def promote_unused_aliases(self, initial_refcounts, used_aliases):
         """
         Given a "before" copy of the alias_refcounts dictionary (as
@@ -712,19 +708,10 @@
         then and which ones haven't been used and promotes all of those
         aliases, plus any children of theirs in the alias tree, to outer joins.
         """
-        # FIXME: There's some (a lot of!) overlap with the similar OR promotion
-        # in add_filter(). It's not quite identical, but is very similar. So
-        # pulling out the common bits is something for later.
-        considered = {}
         for alias in self.tables:
-            if alias not in used_aliases:
-                continue
-            if (alias not in initial_refcounts or
+            if alias in used_aliases and (alias not in initial_refcounts or
                     self.alias_refcount[alias] == initial_refcounts[alias]):
-                parent = self.alias_map[alias][LHS_ALIAS]
-                must_promote = considered.get(parent, False)
-                promoted = self.promote_alias(alias, must_promote)
-                considered[alias] = must_promote or promoted
+                self.promote_joins([alias])
 
     def change_aliases(self, change_map):
         """
@@ -891,7 +878,7 @@
                         continue
                     self.ref_alias(alias)
                     if promote:
-                        self.promote_alias(alias)
+                        self.promote_joins([alias])
                     return alias
 
         # No reuse is possible, so we need a new alias.
@@ -1000,8 +987,7 @@
             # If the aggregate references a model or field that requires a join,
             # those joins must be LEFT OUTER - empty join rows must be returned
             # in order for zeros to be returned for those aggregates.
-            for column_alias in join_list:
-                self.promote_alias(column_alias, unconditional=True)
+            self.promote_joins(join_list, True)
 
             col = (join_list[-1], col)
         else:
@@ -1100,7 +1086,7 @@
             # If the comparison is against NULL, we may need to use some left
             # outer joins when creating the join chain. This is only done when
             # needed, as it's less efficient at the database level.
-            self.promote_alias_chain(join_list)
+            self.promote_joins(join_list)
             join_promote = True
 
         # Process the join list to see if we can remove any inner joins from
@@ -1131,16 +1117,16 @@
                     # This means that we are dealing with two different query
                     # subtrees, so we don't need to do any join promotion.
                     continue
-                join_promote = join_promote or self.promote_alias(join, unconditional)
+                join_promote = join_promote or self.promote_joins([join], unconditional)
                 if table != join:
-                    table_promote = self.promote_alias(table)
+                    table_promote = self.promote_joins([table])
                 # We only get here if we have found a table that exists
                 # in the join list, but isn't on the original tables list.
                 # This means we've reached the point where we only have
                 # new tables, so we can break out of this promotion loop.
                 break
-            self.promote_alias_chain(join_it, join_promote)
-            self.promote_alias_chain(table_it, table_promote or join_promote)
+            self.promote_joins(join_it, join_promote)
+            self.promote_joins(table_it, table_promote or join_promote)
 
         if having_clause or force_having:
             if (alias, col) not in self.group_by:
@@ -1152,7 +1138,7 @@
                 connector)
 
         if negate:
-            self.promote_alias_chain(join_list)
+            self.promote_joins(join_list)
             if lookup_type != 'isnull':
                 if len(join_list) > 1:
                     for alias in join_list:
@@ -1617,7 +1603,7 @@
                         final_alias = join[LHS_ALIAS]
                         col = join[LHS_JOIN_COL]
                         joins = joins[:-1]
-                self.promote_alias_chain(joins[1:])
+                self.promote_joins(joins[1:])
                 self.select.append((final_alias, col))
                 self.select_fields.append(field)
         except MultiJoin:
