FOUR-33159 POC Performance issue when open the columns configuration in saved search

ProcessMaker/Http/Controllers/Api/V1_1/ProcessVariableController.php

@@ -4,17 +4,18 @@
 
 namespace ProcessMaker\Http\Controllers\Api\V1_1;
 
+use Illuminate\Database\QueryException;
 use Illuminate\Http\Request;
 use Illuminate\Pagination\LengthAwarePaginator;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Schema;
 use ProcessMaker\Http\Controllers\Controller;
-use ProcessMaker\Managers\ExportManager;
 use ProcessMaker\Models\Process;
-use ProcessMaker\Models\Screen;
 use ProcessMaker\Package\SavedSearch\Models\SavedSearch;
 use ProcessMaker\Package\VariableFinder\Models\ProcessVariable;
+use ProcessMaker\Services\ProcessScreenVariableService;
 
 class ProcessVariableController extends Controller
 {
@@ -119,15 +120,22 @@ public function index(Request $request)
         $processIds = !empty($validated['processIds'])
             ? array_map('intval', explode(',', $validated['processIds']))
             : [];
-        $perPage = $validated['per_page'] ?? 20;
-        $page = $validated['page'] ?? 1;
+        $perPage = (int) ($validated['per_page'] ?? 20);
+        $page = (int) ($validated['page'] ?? 1);
         $excludeSavedSearch = $validated['savedSearchId'] ?? 0;
 
-        // Generate mock data
-        if (static::$mockData) {
+        // Available columns and process variables have independent pagination.
+        if ($request->has('onlyAvailable')) {
+            $paginator = $this->getAvailableColumnsPaginator(
+                $excludeSavedSearch,
+                $page,
+                $perPage,
+                $request
+            );
+        } elseif (static::$mockData) {
             $paginator = $this->getProcessesVariablesFromMock($processIds, $excludeSavedSearch, $page, $perPage, $request);
         } else {
-            $paginator = $this->getProcessesVariables($processIds, $excludeSavedSearch, $page, $perPage, $request);
+            $paginator = $this->getProcessesVariables($processIds, $excludeSavedSearch, $page, $perPage);
         }
 
         return response()->json([
@@ -203,10 +211,9 @@ private function getProcessesVariablesFromMock(array $processIds, $excludeSavedS
      * @param bool $excludeSavedSearch Flag to exclude saved searches.
      * @param int $page The page number for pagination.
      * @param int $perPage The number of items per page for pagination.
-     * @param Request $request The HTTP request instance.
-     * @return \Illuminate\Http\JsonResponse JSON response containing the process variables.
+     * @return LengthAwarePaginator
      */
-    public function getProcessesVariables(array $processIds, $excludeSavedSearch, $page, $perPage, $request)
+    public function getProcessesVariables(array $processIds, $excludeSavedSearch, $page, $perPage)
     {
         // Determine which columns to exclude based on the saved search
         $activeColumns = [];
@@ -224,12 +231,11 @@ public function getProcessesVariables(array $processIds, $excludeSavedSearch, $p
             || !Schema::hasTable('process_variables')
             || !self::$useVarFinder
         ) {
-            $paginator = $this->getProcessesVariablesFrom($processIds);
-            if ($request->has('onlyAvailable')) {
-                return $this->mergeOnlyAvailableColumns($paginator, $savedSearch, $activeColumns);
-            }
+            return $this->getProcessesVariablesFrom($processIds, $activeColumns);
+        }
 
-            return $paginator;
+        if ($processIds === []) {
+            return new LengthAwarePaginator([], 0, $perPage, $page);
         }
 
         // Build a single query that joins asset_variables, and var_finder_variables
@@ -249,33 +255,57 @@ public function getProcessesVariables(array $processIds, $excludeSavedSearch, $p
                 DB::raw('NULL AS `default`'),
             ]);
 
-        // Return the paginated result
-        $paginator = $query->paginate($perPage, ['*'], 'page', $page);
+        $activeDataColumns = collect($activeColumns)
+            ->filter(fn ($column) => str_starts_with($column, 'data.'))
+            ->map(fn ($column) => substr($column, 5))
+            ->values()
+            ->all();
+        if ($activeDataColumns !== []) {
+            $query->whereNotIn('vfv.field', $activeDataColumns);
+        }
 
-        if ($request->has('onlyAvailable')) {
-            return $this->mergeOnlyAvailableColumns($paginator, $savedSearch, $activeColumns);
+        try {
+            $paginator = $query->paginate($perPage, ['*'], 'page', $page);
+        } catch (QueryException $exception) {
+            if ((int) ($exception->errorInfo[1] ?? 0) !== 1038) {
+                throw $exception;
+            }
+
+            Log::warning('Variable Finder exceeded MySQL sort memory; using screen variables', [
+                'process_count' => count($processIds),
+                'page' => $page,
+                'per_page' => $perPage,
+            ]);
+            $paginator = $this->getProcessesVariablesFrom($processIds, $activeColumns);
         }
 
-        return $query->paginate($perPage, ['*'], 'page', $page);
+        return $paginator;
     }
 
     /**
-     * Merge only available columns with collection items
-     *
-     * @param LengthAwarePaginator $paginator
-     * @param SavedSearch|null $savedSearch
-     * @param array $activeColumns
+     * Paginate the saved search columns separately from process variables.
      *
      * @return LengthAwarePaginator
      */
-    private function mergeOnlyAvailableColumns($paginator, $savedSearch, $activeColumns)
+    private function getAvailableColumnsPaginator($savedSearchId, int $page, int $perPage, Request $request)
     {
+        $savedSearch = $savedSearchId ? SavedSearch::find($savedSearchId) : null;
+        $activeColumns = $savedSearch?->current_columns?->pluck('field')->toArray() ?? [];
         $availableColumns = $this->mergeAvailableColumns($savedSearch);
-        $availableColumns = $availableColumns->merge($paginator->items());
-        $availableColumns = $this->filterActiveColumns($availableColumns, $activeColumns);
-        $paginator->setCollection($availableColumns);
+        $availableColumns = $this->filterActiveColumns($availableColumns, $activeColumns)
+            ->unique('field')
+            ->values();
 
-        return $paginator;
+        return new LengthAwarePaginator(
+            $availableColumns->forPage($page, $perPage)->values(),
+            $availableColumns->count(),
+            $perPage,
+            $page,
+            [
+                'path' => $request->url(),
+                'query' => $request->query(),
+            ]
+        );
     }
 
     /**
@@ -285,15 +315,13 @@ private function mergeOnlyAvailableColumns($paginator, $savedSearch, $activeColu
      */
     private function mergeAvailableColumns(?SavedSearch $savedSearch = null)
     {
-        $availableColumns = collect();
-
-        if ($savedSearch?->available_columns) {
-            $availableColumns = $savedSearch->available_columns->merge(
-                $savedSearch->getDataColumnsAttribute() ?? collect()
-            );
+        if (!$savedSearch) {
+            return collect();
         }
 
-        return $availableColumns;
+        $availableColumns = $savedSearch->available_columns;
+
+        return $availableColumns->merge($savedSearch->getDataColumnsAttribute() ?? collect());
     }
 
     /**
@@ -333,41 +361,23 @@ public static function useVarFinder(bool $value = true)
      * Retrieve process variables from its screens.
      *
      * @param array $processIds
+     * @param array $activeColumns
      *
      * @return LengthAwarePaginator
      */
-    private function getProcessesVariablesFrom(array $processIds)
+    private function getProcessesVariablesFrom(array $processIds, array $activeColumns = [])
     {
         $perPage = request()->get('per_page', 20);
         // Validate processIds input is required
         if (empty($processIds)) {
             return new LengthAwarePaginator([], 0, $perPage, 1);
         }
 
-        // Get screens used in the processes
         $processes = Process::whereIn('id', $processIds)->get();
-        $ids = collect([]);
-        foreach ($processes as $process) {
-            $manager = app(ExportManager::class);
-            try {
-                $ids = $ids->merge($manager->getDependenciesOfType(Screen::class, $process));
-            } catch (\Exception $e) {
-                $ids = collect([]);
-            }
-        }
-
-        // Get columns from screens
-        $columns = collect([]);
-        $screens = Screen::whereIn('id', $ids->unique())->where('type', '!=', 'DISPLAY')->get();
-        foreach ($screens as $screen) {
-            $screenColumns = $screen->fields->map(function ($item) {
-                $item->field = "data.{$item->field}";
-
-                return $item;
-            });
-
-            $columns = $columns->merge($screenColumns);
-        }
+        $columns = app(ProcessScreenVariableService::class)->forProcesses($processes);
+        $columns = $this->filterActiveColumns($columns, $activeColumns)
+            ->unique('field')
+            ->values();
 
         // Paginate the result
         $page = request()->get('page', 1);

ProcessMaker/Services/ProcessScreenVariableService.php

@@ -0,0 +1,148 @@
+<?php
+
+namespace ProcessMaker\Services;
+
+use Illuminate\Contracts\Cache\LockTimeoutException;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Log;
+use ProcessMaker\Assets\ScreensInProcess;
+use ProcessMaker\Models\Column;
+use ProcessMaker\Models\Process;
+use ProcessMaker\Models\Screen;
+use Throwable;
+
+class ProcessScreenVariableService
+{
+    private const CACHE_FRESH_SECONDS = 300;
+
+    private const CACHE_STALE_SECONDS = 3600;
+
+    private const CACHE_LOCK_SECONDS = 120;
+
+    /**
+     * Resolve screen variables without traversing the full export dependency graph.
+     */
+    public function forProcesses(iterable $processes): Collection
+    {
+        $processes = collect($processes)
+            ->filter(fn ($process) => $process instanceof Process)
+            ->unique('id')
+            ->sortBy('id')
+            ->values();
+
+        if ($processes->isEmpty()) {
+            return collect();
+        }
+
+        $processSignature = $processes->map(function (Process $process) {
+            return implode(':', [
+                $process->id,
+                (string) $process->getRawOriginal('updated_at'),
+                sha1((string) $process->getRawOriginal('bpmn')),
+            ]);
+        })->implode('|');
+        $cacheKey = 'process-screen-variables:v1:' . sha1($processSignature);
+        $lastKnownKey = $cacheKey . ':last-known-good';
+        $payload = Cache::flexible(
+            $cacheKey,
+            [self::CACHE_FRESH_SECONDS, self::CACHE_STALE_SECONDS],
+            fn () => $this->refresh($processes, $cacheKey, $lastKnownKey)
+        );
+
+        return collect(is_array($payload) ? $payload : [])
+            ->map(fn (array $column) => new Column($column));
+    }
+
+    private function refresh(Collection $processes, string $cacheKey, string $lastKnownKey): array
+    {
+        $lastKnown = Cache::get($lastKnownKey);
+        $lock = Cache::lock($cacheKey . ':lock', self::CACHE_LOCK_SECONDS);
+
+        if (!$lock->get()) {
+            try {
+                return $lock->block(
+                    5,
+                    fn () => Cache::get($lastKnownKey, is_array($lastKnown) ? $lastKnown : [])
+                );
+            } catch (LockTimeoutException) {
+                return is_array($lastKnown) ? $lastKnown : [];
+            }
+        }
+
+        $startedAt = microtime(true);
+        $failed = false;
+
+        try {
+            $resolver = new ScreensInProcess();
+            $screenIds = collect();
+
+            foreach ($processes as $process) {
+                try {
+                    foreach ($resolver->referencesToExport($process) as [$class, $id]) {
+                        if ($class === Screen::class) {
+                            $screenIds->push($id);
+                        }
+                    }
+                } catch (Throwable $exception) {
+                    $failed = true;
+                    Log::warning('Unable to resolve process screens for variable discovery', [
+                        'process_id' => $process->id,
+                        'message' => $exception->getMessage(),
+                    ]);
+                }
+            }
+
+            $columns = collect();
+            $screens = Screen::whereIn('id', $screenIds->unique())
+                ->where('type', '!=', 'DISPLAY')
+                ->get();
+
+            foreach ($screens as $screen) {
+                try {
+                    $columns = $columns->merge($screen->fields->map(function ($column) {
+                        $column->field = 'data.' . $column->field;
+
+                        return $column;
+                    }));
+                } catch (Throwable $exception) {
+                    $failed = true;
+                    Log::warning('Unable to resolve screen fields for variable discovery', [
+                        'screen_id' => $screen->id,
+                        'message' => $exception->getMessage(),
+                    ]);
+                }
+            }
+
+            $payload = $columns
+                ->unique('field')
+                ->values()
+                ->map(fn ($column) => get_object_vars($column))
+                ->all();
+
+            if ($failed && is_array($lastKnown)) {
+                return $lastKnown;
+            }
+
+            if (!$failed) {
+                Cache::put($lastKnownKey, $payload, self::CACHE_STALE_SECONDS);
+            }
+
+            return $payload;
+        } catch (Throwable $exception) {
+            Log::warning('Screen variable discovery failed', [
+                'process_count' => $processes->count(),
+                'message' => $exception->getMessage(),
+            ]);
+
+            return is_array($lastKnown) ? $lastKnown : [];
+        } finally {
+            $durationMs = round((microtime(true) - $startedAt) * 1000, 2);
+            Log::debug('Process screen variable discovery timing', [
+                'process_count' => $processes->count(),
+                'duration_ms' => $durationMs,
+            ]);
+            $lock->release();
+        }
+    }
+}