From f9734bbd907b2a9baf488d543aee49a04f42171b Mon Sep 17 00:00:00 2001 From: jwilson232 Date: Tue, 27 Oct 2026 15:01:38 +0000 Subject: [PATCH 1/2] api: cache dashboard overview until next pipeline run `cache.set()` was being called without an explicit timeout so was using the default 300 seconds. Updated to have no timeout as the cache is cleared on a pipeline run by `rewrite_quality_data`. Added a step to warm the cache once it has been cleared. --- datastore/api/dashboard/api.py | 3 ++- tools/data_run.sh | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/datastore/api/dashboard/api.py b/datastore/api/dashboard/api.py index cee489be..4823c64d 100644 --- a/datastore/api/dashboard/api.py +++ b/datastore/api/dashboard/api.py @@ -88,6 +88,7 @@ def get(self, *args, **kwargs): ret = quality_data.generate_stats(mode, source_file_set) - cache.set(full_request_uri, ret) + # As cache is cleared on a pipeline run, set without a timeout + cache.set(full_request_uri, ret, timeout=None) return JsonResponse(ret, safe=False) diff --git a/tools/data_run.sh b/tools/data_run.sh index fce83c8f..a05350f6 100755 --- a/tools/data_run.sh +++ b/tools/data_run.sh @@ -35,6 +35,7 @@ else MAX_TOTAL_RUNS_IN_DB=31 # Based on running this script each day and Keep a few extra for safety MAX_PACKAGE_AGE_DAYS=`expr $MAX_TOTAL_RUNS_IN_DB + 2` + DASHBOARD_HOST=localhost:8000 ### End CONFIG ### PERFORMANCE_CSV=~/data_run_profile.csv fi @@ -100,6 +101,12 @@ profile_block_stop "REWRITE_QUALITY_DATA" ./datastore/manage.py set_status --what datastore --status IDLE +## +## Warm the dashboard endpoints cache for the DQD +## +curl -s -o /dev/null "https://$DASHBOARD_HOST/api/dashboard/overview?mode=publishers" || true +curl -s -o /dev/null "https://$DASHBOARD_HOST/api/dashboard/overview?mode=grants" || true + ## ## Clean up before creating the GrantNav data package. ## From 9fc5c336c976c82eab4abb495e3253aee806625e Mon Sep 17 00:00:00 2001 From: jwilson232 Date: Tue, 27 Oct 2026 15:12:07 +0000 Subject: [PATCH 2/2] chore: ignore .ve files in black check --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6f3a66fd..0bd7726f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,4 +17,4 @@ jobs: - run: flake8 # To local srcs, we must specifically exclude src - - run: black --check --exclude "/(\.eggs|\.git|_cache|src)/" ./ + - run: black --check --exclude "/(\.eggs|\.git|_cache|src|.ve)/" ./