Ticket #3214: sql-statement-iterator.diff

core/management.py

@@ -6 +6 @@
 import os, re, shutil, sys, textwrap
 from optparse import OptionParser
 from django.utils import termcolors
+from django.utils.sqltools import sql_statement_iterator
 
 # For Python 2.3
 if not hasattr(__builtins__, 'set'):
@@ -348 +349 @@
     for sql_file in sql_files:
         if os.path.exists(sql_file):
             fp = open(sql_file, 'U')
-            for statement in statements.split(fp.read()):
+            for statement in sql_statement_iterator( fp.read(), double_percent_signs = True ):
                 # Remove any comments from the file
                 statement = re.sub(r"--.*[\n\Z]", "", statement)
                 if statement.strip():

utils/sqltools.py

@@ +1 @@
+r"""
+Contains tools to massage sql text.
+
+#>>> re.findall( r'(?<!\%)(\%)(?!\%)', ' %a%% ' )
+
+
+"""
+import re
+
+_single_or_stmtend = r"(?P<semi>\;)|(?P<comment>\-\-)|(?P<ccomment>\/\*)|(?P<endccomment>\*\/)|(?P<lineend>\n|\r$)|(?P<firstquo>^\')|(?P<escquo>\'\')|(?:(?P<midquo>\')(?!\'))"
+_dollarq = r"(?P<dollarquo>\$\w*\$)"
+_percent = r"(?<!\%)\%(?!\%)"
+
+_single_id_re = re.compile( _single_or_stmtend )
+_id_re = re.compile( '|'.join( ( _single_or_stmtend, _dollarq ) ) )
+_percent_re = re.compile( _percent )
+
+def sql_statement_iterator(
+        script, dollar_quotes = True, double_percent_signs = False
+        ):
+    r"""
+    Iterate through the statements in an sql script,
+    while respecting the boundaries of strings and comments.
+
+    Supports normal single quotes, dollar_quotes (unless turned off).
+    Supports normal sql comments, and c-style comments
+    If 'double_percent_signs' is specified, standalone percent signs
+    will be doubled. This is useful when no parameters are intended,
+    as otherwise python db api treats them as substitution points.
+
+
+    Test various quirks of quotes:
+
+    >>> def show( s, **kw ):
+    ...     for line in sql_statement_iterator( s, **kw ):
+    ...         print line
+
+    >>> show( 'this is a \'simple;\'; test' )
+    this is a 'simple;';
+     test
+    >>> show( ';' )
+    ;
+    >>> show( 'let\'s test a $DOLLAR$ quoted; $DOLLAR$ string' )
+    let's test a $DOLLAR$ quoted; $DOLLAR$ string
+
+    >>> show( 'testing $$ nested; $A$ quotes; \'with;\' $A$;$$ many; fake; semicolons;'  )
+    testing $$ nested; $A$ quotes; 'with;' $A$;$$ many;
+     fake;
+     semicolons;
+
+    >>> show( '''nested misformed $$ quote's $$; dont; cause; problems;''' )
+    nested misformed $$ quote's $$;
+     dont;
+     cause;
+     problems;
+
+    A real function:
+    
+    >>> show( '''CREATE OR REPLACE FUNCTION serviceIsCurrent( srow client_service )
+    ... RETURNS bool LANGUAGE PLPGSQL AS $BODY$
+    ... BEGIN
+    ...     RETURN srow.start <= current_date AND (
+    ...         srow.end IS NULL OR srow.end >= current_date ) AND 
+    ...         srow."_superceededBy_id" IS NULL;
+    ... END;
+    ... $BODY$;
+    ...  ''' )
+    CREATE OR REPLACE FUNCTION serviceIsCurrent( srow client_service )
+    RETURNS bool LANGUAGE PLPGSQL AS $BODY$
+    BEGIN
+        RETURN srow.start <= current_date AND (
+            srow.end IS NULL OR srow.end >= current_date ) AND 
+            srow."_superceededBy_id" IS NULL;
+    END;
+    $BODY$;
+    <BLANKLINE>
+    <BLANKLINE>
+    
+    Test ''
+    
+#    >>> show( "that''s it ';' ." )
+    that''s it ';' .
+    >>> show( "a''''b" )
+    a''''b
+
+    Now test percent doubling
+    
+    >>> show( "hello there %1 %2 %%; %" )
+    hello there %%1 %%2 %%;
+     %%
+
+    Test w/o dollar quote:
+    
+    >>> show( "$$ d;d $$", dollar_quotes = False )
+    $$ d;
+    d $$
+
+    Test comments:
+
+    >>> show( "this is a line; -- this; doesn't; break\n'now; quote was in comment" )
+    this is a line;
+     -- this; doesn't; break
+    'now; quote was in comment
+
+    >>> show( "/* ; a\n -- ; */;z b\n;c" )
+    /* ; a
+     -- ; */;
+    z b
+    ;
+    c
+
+    """
+    outer_quote = None
+    is_comment = False
+    is_c_comment = False
+    if dollar_quotes:
+        re = _id_re
+    else:
+        re = _single_id_re
+
+    lastPos = 0
+    for match in re.finditer( script ):
+        if match.group( 'semi' ):
+            if not ( outer_quote or is_comment or is_c_comment ):
+                newPos = match.end( 1 )
+                
+                statement = script[ lastPos : newPos ]
+                yield _percent_re.sub( '%%', statement )
+                lastPos = newPos
+        elif match.group( 'comment' ) and not is_c_comment:
+            if not outer_quote:
+                is_comment = True
+        elif match.group( 'ccomment' ) and not is_comment:
+            if not outer_quote:
+                is_c_comment = True
+        elif match.group( 'lineend' ):
+            is_comment = False
+        elif match.group( 'endccomment' ):
+            is_c_comment = False
+        elif not ( is_comment or is_c_comment ):
+            quote = filter( lambda g: g is not None, match.groups() )[ 0 ]
+            if quote == "''":
+                # double-quote is escape for quote
+                continue
+            elif outer_quote and quote == outer_quote:
+                # strings embedded in other strings needn't be
+                # well-formed -- throw away nesting if outer quote
+                # is found
+                outer_quote = None
+            elif not outer_quote:
+                outer_quote = quote
+                
+
+    if lastPos < len( script ):
+        yield _percent_re.sub( '%%', script[ lastPos : ] )
+
+
+    
+    
+    
+def _test():
+    import doctest
+    doctest.testmod()
+
+if __name__ == "__main__":
+    _test()