Merge branch '5.0/rt-importer-split-batch-for-large-query-2' into 5.0-trunk

Author: sunnavy

lib/RT/Migrate/Importer.pm

@@ -842,11 +842,33 @@ sub _BatchCreate {
             $values_paren = $1;
         }
 
-        # DBs have placeholder limitations(64k for Pg), here we replace
-        # placeholders to support bigger batch sizes. The performance is similar.
-        my $batch_sql
-            = $RT::Handle->FillIn( $sql . ( ", $values_paren" x ( $count - 1 ) ), [ map @$_, @{ $query{$sql} } ] );
-        $self->RunSQL($batch_sql);
+        # Enforce a 256MB limit on query strings to get around database limitation.
+        my @query_bind_vals;
+        my ( $vals_size, $bind_vals_batch ) = ( 0, [] );
+        foreach my $bind_vals ( @{ $query{$sql} } ) {
+            my $length;
+            $length += length $_ for grep { defined $_ } @$bind_vals;
+
+            # check if over 250 MB to leave some extra room for $sql, commas, and quotes
+            if ( $vals_size + $length > 250 * 1024**2 ) {
+                push @query_bind_vals, $bind_vals_batch;
+                $vals_size       = 0;
+                $bind_vals_batch = [];
+            }
+            push @$bind_vals_batch, $bind_vals;
+            $vals_size += $length;
+        }
+        push @query_bind_vals, $bind_vals_batch if @$bind_vals_batch;
+
+        foreach my $bind_vals (@query_bind_vals) {
+            $count = @$bind_vals;
+
+            # DBs have placeholder limitations(64k for Pg), here we replace
+            # placeholders to support bigger batch sizes. The performance is similar.
+            my $batch_sql
+                = $RT::Handle->FillIn( $sql . ( ", $values_paren" x ( $count - 1 ) ), [ map @$_, @$bind_vals ] );
+            $self->RunSQL($batch_sql);
+        }
     }
 
     # Clone doesn't need to return anything