diff --git a/.github/workflows/jsonschema.yaml b/.github/workflows/jsonschema.yaml index fd5ee39e..e04b3859 100644 --- a/.github/workflows/jsonschema.yaml +++ b/.github/workflows/jsonschema.yaml @@ -20,7 +20,7 @@ jobs: persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index d1685801..3a0ebf11 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ pattern-vault.init vault.init super-linter.log common/pattern-vault.init +tests/ci/.results/ # Super-linter outputs super-linter-output diff --git a/Makefile-common b/Makefile-common index b632653c..29e34341 100644 --- a/Makefile-common +++ b/Makefile-common @@ -18,7 +18,7 @@ operator-deploy operator-upgrade: ## Installs/updates the pattern on a cluster ( @$(ANSIBLE_RUN) rhvp.cluster_utils.operator_deploy .PHONY: install -#install: pattern-install ## Installs the pattern onto a cluster (Loads secrets as well if configured) +#install: pattern-install ## Installs the pattern onto a cluster (Loads secrets as well if configured) install: operator-deploy load-secrets .PHONY: uninstall ## Prints a notice that patterns cannot currently be uninstalled @@ -53,3 +53,8 @@ validate-schema: ## validates values files against schema in common/clustergroup .PHONY: argo-healthcheck argo-healthcheck: ## Checks if all argo applications are synced @$(ANSIBLE_RUN) rhvp.cluster_utils.argo_healthcheck + +##@ Testing (CI) Tasks +.PHONY: run-ci-tests +run-ci-tests: argo-healthcheck ## To run ci-tests, set any needed env vars + @$(ANSIBLE_RUN) rhvp.cluster_utils.run_ci_tests diff --git a/overrides/values-ci.yaml b/overrides/values-ci.yaml new file mode 100644 index 00000000..fe92cb45 --- /dev/null +++ b/overrides/values-ci.yaml @@ -0,0 +1,172 @@ +# These overrides are needed in CI system to test everything +# The CI will pass these overrides with +# EXTRA_HELM_OPTS='--set clusterGroup.sharedValueFiles[3]=/overrides/values-ci.yaml' +global: + registry: + enabled: true + domain: quay-registry-quay-quay-enterprise.apps.{{ .Values.global.clusterDomain }} + repository: ztvp/qtodo + user: quay-admin + vaultPath: "secret/data/hub/infra/quay/quay-users" + passwordVaultKey: "quay-admin-password" + +clusterGroup: + namespaces: + openshift-storage: + operatorGroup: true + targetNamespace: openshift-storage + annotations: + openshift.io/cluster-monitoring: "true" + argocd.argoproj.io/sync-wave: "26" # Propagated to OperatorGroup by framework + quay-enterprise: + annotations: + argocd.argoproj.io/sync-wave: "32" # Create before NooBaa and all Quay components + labels: + openshift.io/cluster-monitoring: "true" + trusted-artifact-signer: + annotations: + argocd.argoproj.io/sync-wave: "32" # Auto-created by RHTAS operator + labels: + openshift.io/cluster-monitoring: "true" + rhtpa-operator: + operatorGroup: true + targetNamespace: rhtpa-operator + annotations: + argocd.argoproj.io/sync-wave: "26" # Create before operator subscription + trusted-profile-analyzer: + annotations: + argocd.argoproj.io/sync-wave: "32" # Create before RHTPA components + labels: + openshift.io/cluster-monitoring: "true" + openshift-pipelines: + subscriptions: + openshift-pipelines: + name: openshift-pipelines-operator-rh + namespace: openshift-operators + channel: pipelines-1.23 + odf: + name: odf-operator + namespace: openshift-storage + annotations: + argocd.argoproj.io/sync-wave: "27" # Install after OperatorGroup (26) + quay-operator: + name: quay-operator + namespace: openshift-operators + annotations: + argocd.argoproj.io/sync-wave: "28" # Install after ODF operator + rhtas-operator: + name: rhtas-operator + namespace: openshift-operators + channel: stable-v1.4 + annotations: + argocd.argoproj.io/sync-wave: "29" # Install after Quay operator, before applications + catalogSource: redhat-operators + rhtpa-operator: + name: rhtpa-operator + namespace: rhtpa-operator # MUST use dedicated namespace (not openshift-operators) + catalogSource: redhat-operators + annotations: + argocd.argoproj.io/sync-wave: "27" # Install after OperatorGroup (26), before applications + applications: + noobaa-mcg: + name: noobaa-mcg + namespace: openshift-storage + argoProject: hub + path: charts/noobaa-mcg + annotations: + argocd.argoproj.io/sync-wave: "36" # Deploy after core services + quay-registry: + name: quay-registry + namespace: quay-enterprise + argoProject: hub + chart: quay + chartVersion: 0.2.* + annotations: + argocd.argoproj.io/sync-wave: "41" # Deploy after NooBaa storage backend + extraValueFiles: + - /overrides/values-quay.yaml + trusted-artifact-signer: + name: trusted-artifact-signer + namespace: trusted-artifact-signer + argoProject: hub + path: charts/rhtas-operator + annotations: + argocd.argoproj.io/sync-wave: "46" # Deploy after dependencies + overrides: + - name: rhtas.zeroTrust.spire.enabled + value: "true" + - name: rhtas.zeroTrust.spire.trustDomain + value: "apps.{{ $.Values.global.clusterDomain }}" + - name: rhtas.zeroTrust.spire.issuer + value: "https://spire-spiffe-oidc-discovery-provider.apps.{{ $.Values.global.clusterDomain }}" + - name: rhtas.zeroTrust.email.enabled + value: "true" + - name: rhtas.zeroTrust.email.issuer + value: https://keycloak.apps.{{ $.Values.global.clusterDomain }}/realms/ztvp + trusted-profile-analyzer: + name: trusted-profile-analyzer + namespace: trusted-profile-analyzer + argoProject: hub + path: charts/rhtpa-operator + annotations: + argocd.argoproj.io/sync-wave: "41" # Create chart resources (OBC, DB, etc.) + ignoreDifferences: + - group: batch + kind: Job + jsonPointers: + - /status + qtodo: + overrides: + - name: app.oidc.enabled + value: "true" + - name: app.spire.enabled + value: "true" + - name: app.vault.url + value: https://vault.vault.svc.cluster.local:8200 + - name: app.vault.role + value: qtodo + - name: app.vault.secretPath + value: secret/data/apps/qtodo/qtodo-db + extraValueFiles: + - '/overrides/values-ci.yaml' + tekton-chains: + name: tekton-chains + project: hub + path: charts/tekton-chains + annotations: + argocd.argoproj.io/sync-wave: "47" + supply-chain: + name: supply-chain + argoProject: hub + path: charts/supply-chain + annotations: + argocd.argoproj.io/sync-wave: "48" + ignoreDifferences: + - kind: ServiceAccount + jqPathExpressions: + - .imagePullSecrets[]|select(.name | contains("-dockercfg-")) + - group: tekton.dev + kind: Task + jsonPointers: + - /spec/steps + - group: tekton.dev + kind: Pipeline + jsonPointers: + - /spec/tasks + overrides: + - name: registry.tlsVerify + value: "false" + - name: rhtas.enabled + value: "true" + - name: rhtpa.enabled + value: "true" + - name: pipelinerun.enabled + value: "true" + extraValueFiles: + - '/overrides/values-ci.yaml' + acs-central: + overrides: + - name: integration.quay.enabled + value: "true" + - name: integration.quay.url + value: "quay-quay-enterprise.apps.{{ .Values.global.domain }}" diff --git a/pattern-metadata.yaml b/pattern-metadata.yaml index 75e47906..a4846a5c 100644 --- a/pattern-metadata.yaml +++ b/pattern-metadata.yaml @@ -25,7 +25,7 @@ requirements: type: Standard_D8s_v3 aws: replicas: 3 - type: m5.2xlarge + type: m5.4xlarge controlPlane: platform: gcp: @@ -36,7 +36,7 @@ requirements: type: Standard_D4s_v3 aws: replicas: 3 - type: m5.xlarge + type: m5.2xlarge spoke: # optional - represents the clusters imported into ACM compute: platform: diff --git a/pattern.sh b/pattern.sh index 9da10890..8ed19b32 100755 --- a/pattern.sh +++ b/pattern.sh @@ -15,6 +15,19 @@ function version { echo "$1" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }' } +# We need this check mostly for CI testing, we do not want to run container in container +function is_container() { + [ -n "${KUBERNETES_SERVICE_HOST:-}" ] && return 0 + [ -f /.dockerenv ] && return 0 + [ -f /run/.containerenv ] && return 0 + return 1 +} + +if is_container; then + echo "Already running in a container" + exec "$@" +fi + if [ -z "${PATTERN_UTILITY_CONTAINER:-}" ]; then PATTERN_UTILITY_CONTAINER="quay.io/validatedpatterns/utility-container" fi @@ -120,6 +133,7 @@ podman run -it --rm --pull=newer \ -e TOKEN_SECRET \ -e UUID_FILE \ -e VALUES_SECRET \ + -e 'VP_*' \ ${PKI_HOST_MOUNT_ARGS[@]+"${PKI_HOST_MOUNT_ARGS[@]}"} \ -v "$(pwd -P)":"$(pwd -P)" \ -v "${HOME}":"${HOME}" \ diff --git a/tests/ci/README.md b/tests/ci/README.md new file mode 100644 index 00000000..5743becf --- /dev/null +++ b/tests/ci/README.md @@ -0,0 +1,39 @@ +# The requirements.txt file is just a placeholder to show the installed packages in the utility container + + + +## The ci will run pytest based on file pattern + +pytest -lv test_\.py --junit-xml .results/test_\.xml + +## To run upstream tests locally + +Set the env variables pointing to the clusters needed to run the tests on: +`HUBCONFIG` (`SPOKECONFIG` if applicable) pointing to the kubconfig of hub (and spoke) clusters +`TARGET_CLUSTERGROUP` if its different from the default (values-global.yaml main.clusterGroupName) +Test logs and junit-xml will be saved in ci/.results/ +Run from repository root + +```bash +./pattern.sh make run-ci-tests +``` + +## Writing additonal tests + +The openshift_dyn_client fixture will return a DynamicClient which can be used inside test functions. +It requires only an env variable param to use it as a kubeconfig. + +```python +@pytest.mark.parametrize( + "openshift_dyn_client", + ["HUBCONFIG"], + indirect=True, +) +``` + +If your tests requires additional python packages, you might want to either run them fully locally in a venv or similar. +Or if you want to use the pattern framework (make/Ansible wrappers) you need to use your own util container + +```bash +PATTERN_UTILITY_CONTAINER="your_utility container" ./pattern.sh make run-ci-tests +``` diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py new file mode 100644 index 00000000..3dc1f76b --- /dev/null +++ b/tests/ci/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/tests/ci/conftest.py b/tests/ci/conftest.py new file mode 100644 index 00000000..633b39ee --- /dev/null +++ b/tests/ci/conftest.py @@ -0,0 +1 @@ +from validatedpatterns_tests.interop.conftest_openshift import * # noqa: F401, F403 diff --git a/tests/ci/requirements.txt b/tests/ci/requirements.txt new file mode 100644 index 00000000..42aecfdf --- /dev/null +++ b/tests/ci/requirements.txt @@ -0,0 +1,15 @@ +ansible-core==2.18.* +ansible-runner +awxkit +kubernetes +openshift +boto3>=1.21 +botocore>=1.24 +awscli>=1.22 +azure-cli>=2.34 +gcloud +humanize +pytz +pytest +junitparser +vp-qe-test-common @ git+https://github.com/darkdoc/vp-qe-test-common.git@v1 \ No newline at end of file diff --git a/tests/ci/test_hub.py b/tests/ci/test_hub.py new file mode 100644 index 00000000..c89cab7e --- /dev/null +++ b/tests/ci/test_hub.py @@ -0,0 +1,96 @@ +import pytest +from validatedpatterns_tests.interop import application, components, subscription + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_subscription_status_hub(openshift_dyn_client): + expected_subs = { # subscription: [namespace] + "advanced-cluster-management": ["open-cluster-management"], + "openshift-gitops-operator": ["openshift-gitops-operator"], + "openshift-external-secrets-operator": ["external-secrets-operator"], + "openshift-cert-manager-operator": ["cert-manager-operator"], + "rhbk-operator": ["keycloak-system"], + "compliance-operator": ["openshift-compliance"], + "rhacs-operator": ["openshift-operators"], + "openshift-zero-trust-workload-identity-manager": [ + "zero-trust-workload-identity-manager" + ], + "openshift-pipelines-operator-rh": ["openshift-operators"], + "odf-operator": ["openshift-storage"], + "quay-operator": ["openshift-operators"], + "rhtas-operator": ["openshift-operators"], + "rhtpa-operator": ["rhtpa-operator"], + } + + subscription.assert_subscription_status(openshift_dyn_client, expected_subs) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_site_reachable(openshift_dyn_client): + + components.assert_site_reachable(openshift_dyn_client) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_pod_status_hub(openshift_dyn_client): + projects = [ + "external-secrets-operator", # application.argoproj.io/openshift-external-secrets + "external-secrets", # application.argoproj.io/openshift-external-secrets + "keycloak-system", # application.argoproj.io/rh-keycloak + "openshift-compliance", # application.argoproj.io/compliance-scanning + "openshift-operators", + "openshift-gitops", + "openshift-gitops-operator", + "openshift-storage", # application.argoproj.io/noobaa-mcg + "open-cluster-management", # application.argoproj.io/acm + "vault", # application.argoproj.io/vault + "zero-trust-workload-identity-manager", # application.argoproj.io/zero-trust-workload-identity-manager + "stackrox", # application.argoproj.io/acs-central-services + # application.argoproj.io/acs-policies + # application.argoproj.io/acs-secured-cluster + "cert-manager", # application.argoproj.io/rh-cert-manager + "cert-manager-operator", # application.argoproj.io/rh-cert-manager + "qtodo", # application.argoproj.io/qtodo + "qtodo-db", # application.argoproj.io/qtodo-db + "quay-enterprise", # application.argoproj.io/quay-registry + "vp-gitops", + "patterns-operator", + "trusted-artifact-signer", # application.argoproj.io/trusted-artifact-signer + "trusted-profile-analyzer", # application.argoproj.io/trusted-profile-analyzer + "openshift-config", # application.argoproj.io/ztvp-certificates + ] + + skip_check = ["create-auth-provider"] + components.assert_pod_status(openshift_dyn_client, projects, skip_check=skip_check) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_argocd_reachable(openshift_dyn_client): + components.assert_argocd_reachable(openshift_dyn_client) + + +@pytest.mark.parametrize( + "openshift_dyn_client", + ["VP_HUBCONFIG"], + indirect=True, +) +def test_argocd_applications_health(openshift_dyn_client): + projects = ["vp-gitops", "layered-zero-trust-hub"] + + application.assert_argocd_applications(openshift_dyn_client, projects)