Add provider compat tests against v3-X-test branch tips #54454

.github/workflows/test-providers.yml

@@ -47,7 +47,8 @@ on:  # yamllint disable-line rule:truthy
       providers-compatibility-tests-matrix:
         description: >
           JSON-formatted array of providers compatibility tests in the form of array of dicts
-          (airflow-version, python-versions, remove-providers, run-unit-tests)
+          (airflow-version, python-versions, remove-providers, run-unit-tests,
+          airflow-constraints-reference)
         required: true
         type: string
       providers-test-types-list-as-strings-in-json:
@@ -231,20 +232,21 @@ jobs:
       - name: "Download airflow package: wheel"
         run: |
           pip download "apache-airflow==${{ matrix.compat.airflow-version }}" -d dist --no-deps
+        if: "!contains(matrix.compat.airflow-version, ':')"
       - name: >
           Install and verify all provider distributions and airflow on
           Airflow ${{ matrix.compat.airflow-version }}:Python ${{ matrix.compat.python-version }}
         # We do not need to run import check if we run tests, the tests should cover all the import checks
         # automatically
         if: matrix.compat.run-unit-tests != 'true'
         env:
-          AIRFLOW_VERSION: "${{ matrix.compat.airflow-version }}"
+          CONSTRAINTS_REF: "${{ matrix.compat.airflow-constraints-reference }}"
         run: >
           breeze release-management verify-provider-distributions
           --use-distributions-from-dist
           --distribution-format wheel
           --use-airflow-version wheel
-          --airflow-constraints-reference constraints-${AIRFLOW_VERSION}
+          --airflow-constraints-reference "${CONSTRAINTS_REF}"
           --providers-skip-constraints
           --install-airflow-with-constraints
       - name: Check amount of disk space available
@@ -258,13 +260,14 @@ jobs:
           PROVIDERS_TEST_TYPES: "${{ matrix.test-types.test_types }}"
           AIRFLOW_VERSION: "${{ matrix.compat.airflow-version }}"
           REMOVE_PROVIDERS: "${{ matrix.compat.remove-providers }}"
+          CONSTRAINTS_REF: "${{ matrix.compat.airflow-constraints-reference }}"
         run: >
           breeze testing providers-tests --run-in-parallel
           --parallel-test-types "${PROVIDERS_TEST_TYPES}"
           --use-distributions-from-dist
           --distribution-format wheel
           --use-airflow-version "${AIRFLOW_VERSION}"
-          --airflow-constraints-reference constraints-${AIRFLOW_VERSION}
+          --airflow-constraints-reference "${CONSTRAINTS_REF}"
           --install-airflow-with-constraints
           --providers-skip-constraints
           --skip-providers "${REMOVE_PROVIDERS}"

dev/breeze/src/airflow_breeze/global_constants.py

@@ -789,24 +789,35 @@ def get_airflow_extras():
     {
         "python-version": "3.10",
         "airflow-version": "2.11.1",
+        "airflow-constraints-reference": "constraints-2.11.1",
         "remove-providers": "common.messaging edge3 fab git keycloak informatica common.ai opensearch",
         "run-unit-tests": "true",
     },
     {
         "python-version": "3.10",
         "airflow-version": "3.0.6",
+        "airflow-constraints-reference": "constraints-3.0.6",
         "remove-providers": "",
         "run-unit-tests": "true",
     },
     {
         "python-version": "3.10",
         "airflow-version": "3.1.8",
+        "airflow-constraints-reference": "constraints-3.1.8",
         "remove-providers": "",
         "run-unit-tests": "true",
     },
     {
         "python-version": "3.10",
         "airflow-version": "3.2.1",
+        "airflow-constraints-reference": "constraints-3.2.1",
+        "remove-providers": "",
+        "run-unit-tests": "true",
+    },
+    {
+        "python-version": "3.10",
+        "airflow-version": "apache/airflow:v3-2-test",
+        "airflow-constraints-reference": "constraints-3-2",
         "remove-providers": "",
         "run-unit-tests": "true",
     },

scripts/in_container/install_airflow_and_providers.py

@@ -473,7 +473,7 @@ def find_installation_spec(
             github_repository=github_repository,
             python_version=python_version,
         )
-        compile_ui_assets = True
+        compile_ui_assets = not install_airflow_with_constraints
         console.print(f"\nInstalling airflow task-sdk from GitHub {resolved_version}\n")
         airflow_task_sdk_distribution = f"apache-airflow-task-sdk @ {vcs_url}#subdirectory=task-sdk"
         airflow_constraints_location = get_airflow_constraints_location(
@@ -695,10 +695,8 @@ def compile_ui_assets(
         console.print(
             f"[bright_blue]Copying UI source from: {extracted_ui_directory} to: {source_ui_directory}"
         )
-        if source_ui_directory.exists():
-            shutil.rmtree(source_ui_directory)
         source_ui_directory.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copytree(extracted_ui_directory, source_ui_directory)
+        shutil.copytree(extracted_ui_directory, source_ui_directory, dirs_exist_ok=True)
     else:
         console.print(f"[yellow]Main UI directory not found at: {extracted_ui_directory}")
 
@@ -1024,7 +1022,9 @@ def install_airflow_and_providers(
         if installation_spec.airflow_ctl_distribution:
             _install_airflow_ctl_with_constraints(installation_spec, github_actions)
     if installation_spec.provider_distributions or not install_airflow_with_constraints:
-        _install_airflow_and_optionally_providers_together(installation_spec, github_actions)
+        _install_airflow_and_optionally_providers_together(
+            installation_spec, github_actions, airflow_already_installed=install_airflow_with_constraints
+        )
     if mount_sources == "providers-and-tests":
         if (
             use_airflow_version
@@ -1118,7 +1118,9 @@ def install_airflow_and_providers(
 
 
 def _install_airflow_and_optionally_providers_together(
-    installation_spec: InstallationSpec, github_actions: bool
+    installation_spec: InstallationSpec,
+    github_actions: bool,
+    airflow_already_installed: bool = False,
 ):
     console.print("[bright_blue]Installing airflow and optionally providers together")
     base_install_cmd = [
@@ -1129,26 +1131,36 @@ def _install_airflow_and_optionally_providers_together(
     if installation_spec.pre_release:
         console.print("[bright_blue]Allowing pre-release versions of airflow and providers")
         base_install_cmd.extend(["--pre"])
-    if installation_spec.airflow_distribution:
-        console.print(
-            f"\n[bright_blue]Adding airflow distribution to installation: {installation_spec.airflow_distribution} "
-        )
-        base_install_cmd.append(installation_spec.airflow_distribution)
-    if installation_spec.airflow_core_distribution:
-        console.print(
-            f"\n[bright_blue]Adding airflow core distribution to installation: {installation_spec.airflow_core_distribution}"
-        )
-        base_install_cmd.append(installation_spec.airflow_core_distribution)
-    if installation_spec.airflow_task_sdk_distribution:
-        console.print(
-            f"\n[bright_blue]Adding task-sdk distribution to installation: {installation_spec.airflow_task_sdk_distribution}"
-        )
-        base_install_cmd.append(installation_spec.airflow_task_sdk_distribution)
-    if installation_spec.airflow_ctl_distribution:
+    if airflow_already_installed:
         console.print(
-            f"\n[bright_blue]Adding airflow ctl distribution to installation: {installation_spec.airflow_ctl_distribution}"
+            "\n[bright_blue]Airflow already installed with constraints - "
+            "only installing provider distributions"
         )
-        base_install_cmd.append(installation_spec.airflow_ctl_distribution)
+    else:
+        if installation_spec.airflow_distribution:
+            console.print(
+                f"\n[bright_blue]Adding airflow distribution to installation: "
+                f"{installation_spec.airflow_distribution} "
+            )
+            base_install_cmd.append(installation_spec.airflow_distribution)
+        if installation_spec.airflow_core_distribution:
+            console.print(
+                f"\n[bright_blue]Adding airflow core distribution to installation: "
+                f"{installation_spec.airflow_core_distribution}"
+            )
+            base_install_cmd.append(installation_spec.airflow_core_distribution)
+        if installation_spec.airflow_task_sdk_distribution:
+            console.print(
+                f"\n[bright_blue]Adding task-sdk distribution to installation: "
+                f"{installation_spec.airflow_task_sdk_distribution}"
+            )
+            base_install_cmd.append(installation_spec.airflow_task_sdk_distribution)
+        if installation_spec.airflow_ctl_distribution:
+            console.print(
+                f"\n[bright_blue]Adding airflow ctl distribution to installation: "
+                f"{installation_spec.airflow_ctl_distribution}"
+            )
+            base_install_cmd.append(installation_spec.airflow_ctl_distribution)
     console.print("\n[bright_blue]Adding provider distributions to installation:")
     for provider_package in sorted(installation_spec.provider_distributions):
         console.print(f"  {provider_package}")