diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index 51beb7b4dc..26d047a288 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -28,7 +28,7 @@ jobs: needs: get-version strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 552c6d2a6c..81e1982e88 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -27,9 +27,9 @@ jobs: - name: Set Python versions for run run: | if [[ ${{ github.event_name }} == "schedule" ]]; then - echo "python_versions=[\"3.10\",\"3.11\",\"3.12\"]" >> $GITHUB_ENV + echo "python_versions=[\"3.10\",\"3.11\",\"3.12\",\"3.13\",\"3.14\"]" >> $GITHUB_ENV else - echo "python_versions=[\"3.12\"]" >> $GITHUB_ENV + echo "python_versions=[\"3.14\"]" >> $GITHUB_ENV fi build: @@ -110,9 +110,6 @@ jobs: uv pip uninstall --system pandas pyarrow uv pip freeze - name: Run extras unit tests with coverage - # Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003 - # and https://github.com/pytorch/pytorch/issues/110436 - if: ${{ matrix.python-version != '3.12' }} run: | make unit_test_extras_codecov - name: Codecov @@ -139,6 +136,11 @@ jobs: pandas: "pandas>=2.0.0" - pandas: "pandas<2.0.0" python-version: "3.12" + # pandas 1.x has no wheels for python 3.13+ + - pandas: "pandas<2.0.0" + python-version: "3.13" + - pandas: "pandas<2.0.0" + python-version: "3.14" steps: - uses: actions/checkout@v4 @@ -389,6 +391,56 @@ jobs: # Segmentation fault on python 3.12: https://github.com/flyteorg/flyte/issues/5020 - python-version: 3.12 plugin-names: "flytekit-kf-pytorch" + # Python 3.13: the exclusions below mirror the python 3.12 ones. They have not been verified on 3.13 + # (several of the underlying issues are closed by now); revisit both lists together. + - python-version: 3.13 + plugin-names: "flytekit-airflow" + - python-version: 3.13 + plugin-names: "flytekit-kf-pytorch" + - python-version: 3.13 + plugin-names: "flytekit-modin" + - python-version: 3.13 + plugin-names: "flytekit-onnx-pytorch" + - python-version: 3.13 + plugin-names: "flytekit-ray" + - python-version: 3.13 + plugin-names: "flytekit-vaex" + # flytekitplugins-duckdb declares python_requires<3.13 (duckdb<=1.0.0 pin) + - python-version: 3.13 + plugin-names: "flytekit-duckdb" + # Python 3.14: same as python 3.13. Not verified either. + - python-version: 3.14 + plugin-names: "flytekit-airflow" + - python-version: 3.14 + plugin-names: "flytekit-kf-pytorch" + - python-version: 3.14 + plugin-names: "flytekit-modin" + - python-version: 3.14 + plugin-names: "flytekit-onnx-pytorch" + - python-version: 3.14 + plugin-names: "flytekit-ray" + - python-version: 3.14 + plugin-names: "flytekit-vaex" + - python-version: 3.14 + plugin-names: "flytekit-duckdb" + # Tensorflow, a dependency of the flytekit-mlflow tests, has no python 3.14 wheels yet + - python-version: 3.14 + plugin-names: "flytekit-mlflow" + # great-expectations<1.0 and whylogs require numpy<2, which has no python 3.13+ wheels + - python-version: 3.13 + plugin-names: "flytekit-greatexpectations" + - python-version: 3.14 + plugin-names: "flytekit-greatexpectations" + - python-version: 3.13 + plugin-names: "flytekit-whylogs" + - python-version: 3.14 + plugin-names: "flytekit-whylogs" + # ydata-profiling declares python_requires<3.14 + - python-version: 3.14 + plugin-names: "flytekit-deck-standard" + # dbt-core<1.8 (pinned by the plugin) fails to validate its config on python 3.14 + - python-version: 3.14 + plugin-names: "flytekit-dbt" steps: - uses: actions/checkout@v4 - name: "Clear action cache" @@ -416,7 +468,8 @@ jobs: echo "Workspace: $GITHUB_WORKSPACE" uv pip install --system -U $GITHUB_WORKSPACE "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl" # TODO: remove this when numpy v2 in onnx has been resolved - if [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" || ${{ matrix.plugin-names }} == "flytekit-pandera" ]]; then + # Not on python 3.13+: the pandas/pyarrow releases with 3.13 wheels require numpy>=2. + if [[ ${{ matrix.python-version }} != "3.13" && ${{ matrix.python-version }} != "3.14" ]] && [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" || ${{ matrix.plugin-names }} == "flytekit-pandera" ]]; then uv pip install --system "numpy<2.0.0" fi # The -U step above re-resolves aiohttp (a flytekit dependency) back to the latest, diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index f5cd80c6ce..feb13b466b 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -124,6 +124,8 @@ jobs: - "3.10" - "3.11" - "3.12" + - "3.13" + - "3.14" steps: - uses: actions/checkout@v4 with: diff --git a/Dockerfile.dev b/Dockerfile.dev index aaac1739d3..fbb7511005 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -44,10 +44,11 @@ RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_FLYTEKIT=$PSEUDO_VERSION \ pandas \ pillow \ plotly \ + pydantic \ pyarrow \ pygments \ scikit-learn \ - ydata-profiling \ + "ydata-profiling; python_version<'3.14'" \ && apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ diff --git a/dev-requirements.in b/dev-requirements.in index 16dec73e85..909c5ba2b7 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -21,14 +21,15 @@ keyrings.alt setuptools_scm pytest-icdiff -# Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003 tensorflow<=2.15.1; python_version<'3.12' +# tensorflow>=2.16 ships python 3.12 wheels and >=2.20 ships python 3.13 wheels (linux x86_64/aarch64, macos arm64, +# windows). There are no python 3.14 wheels yet, the tensorflow tests are skipped there. +tensorflow; python_version>='3.12' and python_version<'3.14' # Newer versions of torch bring in nvidia dependencies that are not present in windows, so # we put this constraint while we do not have per-environment requirements files torch<=1.12.1; python_version<'3.11' -# pytorch 2 supports python 3.11 -# pytorch 2 does not support 3.12 yet: https://github.com/pytorch/pytorch/issues/110436 -torch; python_version<'3.12' +# pytorch>=2.2 ships python 3.12 wheels, >=2.6 python 3.13 wheels and >=2.9 python 3.14 wheels +torch; python_version>='3.11' pydantic # TODO: Currently, the python-magic library causes build errors on Windows due to its dependency on DLLs for libmagic. diff --git a/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/type_information.py b/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/type_information.py index b6a7b247e6..a17ea369ce 100644 --- a/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/type_information.py +++ b/plugins/flytekit-omegaconf/flytekitplugins/omegaconf/type_information.py @@ -1,4 +1,5 @@ import dataclasses +import inspect import typing from collections import ChainMap @@ -33,7 +34,9 @@ def all_annotations(cls: typing.Type) -> ChainMap: Returns a dictionary-like ChainMap that includes annotations for all attributes defined in cls or inherited from superclasses. """ - return ChainMap(*(c.__annotations__ for c in cls.__mro__ if "__annotations__" in c.__dict__)) + # inspect.get_annotations also covers python 3.14, where class annotations are evaluated lazily and are + # no longer stored in the class __dict__. + return ChainMap(*(inspect.get_annotations(c) for c in cls.__mro__)) def extract_node_type( diff --git a/pyproject.toml b/pyproject.toml index 684ceda4f0..528aab63ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,8 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", "Topic :: Software Development", diff --git a/tests/flytekit/unit/configuration/test_image_config.py b/tests/flytekit/unit/configuration/test_image_config.py index ca841fa497..8e2bf2904f 100644 --- a/tests/flytekit/unit/configuration/test_image_config.py +++ b/tests/flytekit/unit/configuration/test_image_config.py @@ -16,6 +16,7 @@ [ (PythonVersion.PYTHON_3_9, "cr.flyte.org/flyteorg/flytekit:py3.9-latest"), (PythonVersion.PYTHON_3_10, "cr.flyte.org/flyteorg/flytekit:py3.10-latest"), + (PythonVersion.PYTHON_3_13, "cr.flyte.org/flyteorg/flytekit:py3.13-latest"), (PythonVersion.PYTHON_3_14, "cr.flyte.org/flyteorg/flytekit:py3.14-latest"), ], ) @@ -27,6 +28,7 @@ def test_defaults(python_version_enum, expected_image_string): "python_version_enum, flytekit_version, expected_image_string", [ (PythonVersion.PYTHON_3_9, "v0.32.0", "cr.flyte.org/flyteorg/flytekit:py3.9-0.32.0"), + (PythonVersion.PYTHON_3_13, "v1.17.0", "cr.flyte.org/flyteorg/flytekit:py3.13-1.17.0"), ], ) def test_set_both(python_version_enum, flytekit_version, expected_image_string): diff --git a/tests/flytekit/unit/extras/tensorflow/conftest.py b/tests/flytekit/unit/extras/tensorflow/conftest.py new file mode 100644 index 0000000000..dd12e9f7ce --- /dev/null +++ b/tests/flytekit/unit/extras/tensorflow/conftest.py @@ -0,0 +1,5 @@ +import sys + +# tensorflow does not ship python 3.14 wheels yet, see dev-requirements.in +if sys.version_info >= (3, 14): + collect_ignore_glob = ["*"]