[10.x] Add scout:queue command to optimize large imports (#929)

* Extract searchable query

* Add scout:queue command to optimize reindexing

* Dispatch on appropriate queue and connection

* Add tests

* CS fixes

* Remove strange assertion edge case in lower laravel versions

* Fix test php 8.0

* Filter searchable models

* Return early if searchable models is empty

* Use collection isEmpty method for consistency across jobs

* Add ability to enter custom range via --min and --max options

* CS fixes

* Fix comparison assertions

* formatting

---------

Co-authored-by: Taylor Otwell <taylor@laravel.com>

Author: stevebauman

src/Console/QueueImportCommand.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace Laravel\Scout\Console;
+
+use Illuminate\Console\Command;
+use Laravel\Scout\Jobs\MakeRangeSearchable;
+use Symfony\Component\Console\Attribute\AsCommand;
+
+#[AsCommand(name: 'scout:queue-import')]
+class QueueImportCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'scout:queue-import
+            {model : Class name of model to bulk queue}
+            {--min= : The minimum ID to start queuing from}
+            {--max= : The maximum ID to queue up to}
+            {--c|chunk= : The number of records to queue in a single job (Defaults to configuration value: `scout.chunk.searchable`)}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Import the given model into the search index via chunked, queued jobs';
+
+    /**
+     * Execute the console command.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $class = $this->argument('model');
+
+        $model = new $class;
+
+        $query = $model::makeAllSearchableQuery();
+
+        $min = $this->option('min') ?? $query->min($model->getScoutKeyName());
+        $max = $this->option('max') ?? $query->max($model->getScoutKeyName());
+
+        $chunk = max(1, (int) ($this->option('chunk') ?? config('scout.chunk.searchable', 500)));
+
+        if (! $min || ! $max) {
+            $this->info('No records found for ['.$class.'].');
+
+            return;
+        }
+
+        if (! is_numeric($min) || ! is_numeric($max)) {
+            $this->error('The primary key for ['.$class.'] is not numeric.');
+
+            return;
+        }
+
+        for ($start = $min; $start <= $max; $start += $chunk) {
+            $end = min($start + $chunk - 1, $max);
+
+            dispatch(new MakeRangeSearchable($model, $start, $end))
+                ->onQueue($model->syncWithSearchUsingQueue())
+                ->onConnection($model->syncWithSearchUsing());
+
+            $this->line('<comment>Queued ['.$class.'] models up to ID:</comment> '.$end);
+        }
+
+        $this->info('All ['.$class.'] records have been queued for importing.');
+    }
+}

src/Jobs/MakeRangeSearchable.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace Laravel\Scout\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Queue\SerializesModels;
+use Laravel\Scout\Scout;
+
+class MakeRangeSearchable implements ShouldQueue
+{
+    use Queueable, SerializesModels;
+
+    /**
+     * The model to be made searchable.
+     *
+     * @var \Illuminate\Database\Eloquent\Model
+     */
+    public $model;
+
+    /**
+     * The start id to be made searchable.
+     *
+     * @var int
+     */
+    public $start;
+
+    /**
+     * The end id to be made searchable.
+     *
+     * @var int
+     */
+    public $end;
+
+    /**
+     * Create a new job instance.
+     *
+     * @param  \Illuminate\Database\Eloquent\Model  $model
+     * @param  int  $start
+     * @param  int  $end
+     * @return void
+     */
+    public function __construct($model, $start, $end)
+    {
+        $this->model = $model;
+        $this->start = $start;
+        $this->end = $end;
+    }
+
+    /**
+     * Handle the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $models = $this->model::makeAllSearchableQuery()
+            ->whereBetween($this->model->getScoutKeyName(), [$this->start, $this->end])
+            ->get()
+            ->filter
+            ->shouldBeSearchable();
+
+        if ($models->isEmpty()) {
+            return;
+        }
+
+        dispatch(new Scout::$makeSearchableJob($models))
+            ->onQueue($this->model->syncWithSearchUsingQueue())
+            ->onConnection($this->model->syncWithSearchUsing());
+    }
+}

src/Jobs/MakeSearchable.php

@@ -35,7 +35,7 @@ public function __construct($models)
      */
     public function handle()
     {
-        if (count($this->models) === 0) {
+        if ($this->models->isEmpty()) {
             return;
         }
 

src/ScoutServiceProvider.php

@@ -8,6 +8,7 @@
 use Laravel\Scout\Console\FlushCommand;
 use Laravel\Scout\Console\ImportCommand;
 use Laravel\Scout\Console\IndexCommand;
+use Laravel\Scout\Console\QueueImportCommand;
 use Laravel\Scout\Console\SyncIndexSettingsCommand;
 use Meilisearch\Client as Meilisearch;
 
@@ -48,6 +49,7 @@ public function boot()
     {
         if ($this->app->runningInConsole()) {
             $this->commands([
+                QueueImportCommand::class,
                 FlushCommand::class,
                 ImportCommand::class,
                 IndexCommand::class,

src/Searchable.php

@@ -171,12 +171,22 @@ public static function search($query = '', $callback = null)
      * @return void
      */
     public static function makeAllSearchable($chunk = null)
+    {
+        static::makeAllSearchableQuery()->searchable($chunk);
+    }
+
+    /**
+     * Get a query builder for making all instances of the model searchable.
+     *
+     * @return \Illuminate\Database\Eloquent\Builder
+     */
+    public static function makeAllSearchableQuery()
     {
         $self = new static;
 
         $softDelete = static::usesSoftDelete() && config('scout.soft_delete', false);
 
-        $self->newQuery()
+        return $self->newQuery()
             ->when(true, function ($query) use ($self) {
                 $self->makeAllSearchableUsing($query);
             })
@@ -185,8 +195,7 @@ public static function makeAllSearchable($chunk = null)
             })
             ->orderBy(
                 $self->qualifyColumn($self->getScoutKeyName())
-            )
-            ->searchable($chunk);
+            );
     }
 
     /**