Skip to content
DC' Blog
Go back

01-polars代码结构

Tip

所有的代码分析基于 polars 1.43.0 版本

分析整体代码结构,及调用流程

polars目录结构

polars整体目录结构如下

目录描述
cratesRust 核心代码,包含 29 个 crate,polars 几乎所有核心逻辑都在这里
py-polarsPython 绑定 polars 包,提供面向用户的 Python API,内含 src/(Python 源码)和 tests/
pyo3-polarsRust→Python 桥接工具层,提供 polars 类型与 PyO3 之间的转换,供第三方 Rust 插件作者使用
tools开发辅助脚本(Cargo 编译失败警告、环境变量更新等)
docs用户文档站点(基于 mkdocs),含 source/ 下的 Markdown 文档和 assets/ 静态资源
examples示例代码/数据集

核心结构上,py-polars是入口,调用crates中的rust代码;pyo3-polars可以用来写第三方插件

调用流程

找一个最简单的代码将整个调用流程穿起来,比如

pl.Series("ints", [1, 2, 3, 4, 5])
  1. 调用Series.__init__
py-polars/src/polars/series/series.pyGitHub
@expr_dispatch
class Series:
    """
    A Series represents a single column in a Polars DataFrame.
展开折叠代码 (178-264 行,共 87 行)
    Parameters
    ----------
    name : str, default None
        Name of the Series. Will be used as a column name when used in a DataFrame.
        When not specified, name is set to an empty string.
    values : ArrayLike, default None
        One-dimensional data in various forms. Supported are: Sequence, Series,
        pyarrow Array, and numpy ndarray.
    dtype : DataType, default None
        Data type of the resulting Series. If set to `None` (default), the data type is
        inferred from the `values` input. The strategy for data type inference depends
        on the `strict` parameter:

        - If `strict` is set to True (default), the inferred data type is equal to the
          first non-null value, or `Null` if all values are null.
        - If `strict` is set to False, the inferred data type is the supertype of the
          values, or :class:`Object` if no supertype can be found. **WARNING**: A full
          pass over the values is required to determine the supertype.
        - If no values were passed, the resulting data type is :class:`Null`.

    strict : bool, default True
        Throw an error if any value does not exactly match the given or inferred data
        type. If set to `False`, values that do not match the data type are cast to
        that data type or, if casting is not possible, set to null instead.
    nan_to_null : bool, default False
        In case a numpy array is used to create this Series, indicate how to deal
        with np.nan values. (This parameter is a no-op on non-numpy data).

    Examples
    --------
    Constructing a Series by specifying name and values positionally:

    >>> s = pl.Series("a", [1, 2, 3])
    >>> s
    shape: (3,)
    Series: 'a' [i64]
    [
        1
        2
        3
    ]

    Notice that the dtype is automatically inferred as a polars Int64:

    >>> s.dtype
    Int64

    Constructing a Series with a specific dtype:

    >>> s2 = pl.Series("a", [1, 2, 3], dtype=pl.Float32)
    >>> s2
    shape: (3,)
    Series: 'a' [f32]
    [
        1.0
        2.0
        3.0
    ]

    It is possible to construct a Series with values as the first positional argument.
    This syntax considered an anti-pattern, but it can be useful in certain
    scenarios. You must specify any other arguments through keywords.

    >>> s3 = pl.Series([1, 2, 3])
    >>> s3
    shape: (3,)
    Series: '' [i64]
    [
        1
        2
        3
    ]
    """

    # NOTE: This `= None` is needed to generate the docs with sphinx_accessor.
    _s: PySeries = None  # type: ignore[assignment]
    _accessors: ClassVar[set_[str_]] = {
        "arr",
        "bin",
        "cat",
        "dt",
        "ext",
        "list",
        "plot",
        "str",
        "struct",
    }

    def __init__(
        self,
        name: str_ | ArrayLike | None = None,
        values: ArrayLike | None = None,
        dtype: PolarsDataType | None = None,
        *,
        strict: bool = True,
        nan_to_null: bool = False,
    ) -> None:
        # If 'Unknown' treat as None to trigger type inference
展开折叠代码 (276-293 行,共 18 行)
        if dtype == Unknown:
            dtype = None
        elif dtype is not None and not is_polars_dtype(dtype):
            dtype = parse_into_dtype(dtype)

        # Handle case where values are passed as the first argument
        original_name: str | None = None
        if name is None:
            name = ""
        elif isinstance(name, str):
            original_name = name
        else:
            if values is None:
                values = name
                name = ""
            else:
                msg = "Series name must be a string"
                raise TypeError(msg)

        if isinstance(values, Sequence):
            self._s = sequence_to_pyseries(
                name,
                values,
                dtype=dtype,
                strict=strict,
                nan_to_null=nan_to_null,
            )
展开折叠代码 (304-370 行,共 67 行)
        elif values is None:
            self._s = sequence_to_pyseries(name, [], dtype=dtype)

        elif _check_for_numpy(values) and isinstance(values, np.ndarray):
            self._s = numpy_to_pyseries(
                name, values, strict=strict, nan_to_null=nan_to_null
            )
            if values.dtype.type in [np.datetime64, np.timedelta64]:
                # cast to appropriate dtype, handling NaT values
                input_dtype = _resolve_temporal_dtype(None, values.dtype)
                dtype = _resolve_temporal_dtype(dtype, values.dtype)
                if dtype is not None:
                    self._s = (
                        # `values.dtype` has already been validated in
                        # `numpy_to_pyseries`, so `input_dtype` can't be `None`
                        self.cast(input_dtype, strict=False)  # type: ignore[arg-type]
                        .cast(dtype)
                        .scatter(np.argwhere(np.isnat(values)).flatten(), None)
                        ._s
                    )
                    return

            if dtype is not None:
                self._s = self.cast(dtype, strict=strict)._s

        elif _check_for_torch(values) and isinstance(values, torch.Tensor):
            self._s = numpy_to_pyseries(
                name, values.numpy(force=False), strict=strict, nan_to_null=nan_to_null
            )
            if dtype is not None:
                self._s = self.cast(dtype, strict=strict)._s

        elif _check_for_pyarrow(values) and isinstance(
            values, (pa.Array, pa.ChunkedArray)
        ):
            self._s = arrow_to_pyseries(name, values, dtype=dtype, strict=strict)

        elif _check_for_pandas(values) and isinstance(
            values, (pd.Series, pd.Index, pd.DatetimeIndex)
        ):
            self._s = pandas_to_pyseries(name, values, dtype=dtype, strict=strict)

        elif not hasattr(values, "__arrow_c_stream__") and _is_generator(values):
            self._s = iterable_to_pyseries(name, values, dtype=dtype, strict=strict)

        elif isinstance(values, Series):
            self._s = series_to_pyseries(
                original_name, values, dtype=dtype, strict=strict
            )

        elif isinstance(values, pl.DataFrame):
            self._s = dataframe_to_pyseries(
                original_name, values, dtype=dtype, strict=strict
            )

        elif hasattr(values, "__arrow_c_array__"):
            self._s = PySeries.from_arrow_c_array(values)

        elif hasattr(values, "__arrow_c_stream__"):
            self._s = PySeries.from_arrow_c_stream(values)

        else:
            msg = (
                f"Series constructor called with unsupported type {type(values).__name__!r}"
                " for the `values` parameter"
            )
            raise TypeError(msg)
  1. 这里面是在判断values的类型,然后调用不同的_to_pyseries方法,比如这个就是调用sequence_to_pyseries,看看定义
py-polars/src/polars/_utils/construction/series.pyGitHub
def sequence_to_pyseries(
    name: str,
    values: Sequence[Any],
    dtype: PolarsDataType | None = None,
    *,
    strict: bool = True,
    nan_to_null: bool = False,
) -> PySeries:
    """Construct a PySeries from a sequence."""
展开折叠代码 (85-323 行,共 239 行)
    python_dtype: type | None = None

    if isinstance(dtype, BaseExtension):
        storage = dtype.ext_storage()
        pys = sequence_to_pyseries(
            name, values, storage, strict=strict, nan_to_null=nan_to_null
        )
        return pys.ext_to(dtype)

    if isinstance(values, range):
        return range_to_series(name, values, dtype=dtype)._s

    # empty sequence
    if len(values) == 0 and dtype is None:
        # if dtype for empty sequence could be guessed
        # (e.g comparisons between self and other), default to Null
        dtype = Null

    # lists defer to subsequent handling; identify nested type
    elif dtype in (List, Array):
        python_dtype = list

    # infer temporal type handling
    py_temporal_types = {date, datetime, timedelta, time}
    pl_temporal_types = {Date, Datetime, Duration, Time}

    value = get_first_non_none(values)
    if value is not None:
        if (
            dataclasses.is_dataclass(value)
            or is_pydantic_model(value)
            or is_namedtuple(value.__class__)
            or is_sqlalchemy_row(value)
        ) and dtype != Object:
            return pl.DataFrame(values).to_struct(name)._s
        elif (
            not isinstance(value, dict) and isinstance(value, Mapping)
        ) and dtype != Object:
            return _sequence_of_dict_to_pydf(
                value,
                data=values,
                strict=strict,
                schema_overrides=None,
                infer_schema_length=None,
                schema=None,
            ).to_struct(name, [])
        elif isinstance(value, range) and dtype is None:
            values = [range_to_series("", v) for v in values]
        else:
            # for temporal dtypes:
            # * if the values are integer, we take the physical branch.
            # * if the values are python types, take the temporal branch.
            # * if the values are ISO-8601 strings, init then convert via strptime.
            # * if the values are floats/other dtypes, this is an error.
            if dtype in py_temporal_types and isinstance(value, int):
                dtype = parse_into_dtype(dtype)  # construct from integer
            elif (
                dtype in pl_temporal_types or type(dtype) in pl_temporal_types
            ) and not isinstance(value, int):
                python_dtype = dtype_to_py_type(dtype)  # type: ignore[arg-type]

    # if values are enums, infer and load the appropriate dtype/values
    if issubclass(type(value), PyEnum):
        if dtype is None and python_dtype is None:
            with contextlib.suppress(TypeError):
                dtype = Enum(type(value))
        if not isinstance(value, (str, int)):
            values = [v.value for v in values]

    # physical branch
    # flat data
    if (
        dtype is not None
        and is_polars_dtype(dtype)
        and not dtype.is_nested()
        and dtype != Unknown
        and (python_dtype is None)
    ):
        constructor = polars_type_to_constructor(dtype)
        pyseries = _construct_series_with_fallbacks(
            constructor, name, values, dtype, strict=strict
        )
        if dtype in (
            Date,
            Datetime,
            Duration,
            Time,
            Boolean,
            Categorical,
            Enum,
        ) or isinstance(dtype, (Categorical, Decimal)):
            if pyseries.dtype() != dtype:
                pyseries = pyseries.cast(dtype, strict=strict, wrap_numerical=False)

        # Uninstanced Decimal is a bit special and has various inference paths
        if dtype == Decimal:
            if pyseries.dtype() == String:
                pyseries = pyseries.str_to_decimal_infer(inference_length=0)
            elif pyseries.dtype().is_float():
                # Go through string so we infer an appropriate scale.
                pyseries = pyseries.cast(
                    String, strict=strict, wrap_numerical=False
                ).str_to_decimal_infer(inference_length=0)
            elif pyseries.dtype().is_integer() or pyseries.dtype() == Null:
                pyseries = pyseries.cast(
                    Decimal(scale=0), strict=strict, wrap_numerical=False
                )
            elif not isinstance(pyseries.dtype(), Decimal):
                msg = f"can't convert {pyseries.dtype()} to Decimal"
                raise TypeError(msg)

        return pyseries

    elif dtype == Struct:
        # This is very bad. Goes via rows? And needs to do outer nullability separate.
        # It also has two data passes.
        # TODO: eventually go into struct builder
        struct_schema = dtype.to_schema() if isinstance(dtype, Struct) else None
        empty = {}  # type: ignore[var-annotated]

        data = []
        invalid = []
        for i, v in enumerate(values):
            if v is None:
                invalid.append(i)
                data.append(empty)
            else:
                data.append(v)

        return plc.sequence_to_pydf(
            data=data,
            schema=struct_schema,
            orient="row",
        ).to_struct(name, invalid)

    if python_dtype is None:
        if value is None:
            constructor = polars_type_to_constructor(Null)
            return constructor(name, values, strict)

        # generic default dtype
        python_dtype = type(value)

    # temporal branch
    if issubclass(python_dtype, tuple(py_temporal_types)):
        if dtype is None:
            dtype = parse_into_dtype(python_dtype)  # construct from integer
        elif dtype in py_temporal_types:
            dtype = parse_into_dtype(dtype)

        values_dtype = None if value is None else try_parse_into_dtype(type(value))
        if values_dtype is not None and values_dtype.is_float():
            msg = f"'float' object cannot be interpreted as a {python_dtype.__name__!r}"
            raise TypeError(
                # we do not accept float values as temporal; if this is
                # required, the caller should explicitly cast to int first.
                msg
            )

        # We use the AnyValue builder to create the datetime array
        # We store the values internally as UTC and set the timezone
        py_series = PySeries.new_from_any_values(name, values, strict)

        time_unit = getattr(dtype, "time_unit", None)
        time_zone = getattr(dtype, "time_zone", None)

        if dtype.is_temporal() and values_dtype == String and dtype != Duration:
            s = wrap_s(py_series).str.strptime(dtype, strict=strict)  # type: ignore[arg-type]
        elif time_unit is not None and values_dtype != Date:
            s = wrap_s(py_series).dt.cast_time_unit(time_unit)
        else:
            s = wrap_s(py_series)

        if (values_dtype == Date) & (dtype == Datetime):
            s = s.cast(Datetime(time_unit or "us"))

        if dtype == Datetime and time_zone is not None:
            return s.dt.convert_time_zone(time_zone)._s
        return s._s

    elif (
        _check_for_numpy(value)
        and isinstance(value, np.ndarray)
        and len(value.shape) == 1
    ):
        n_elems = len(value)
        if all(len(v) == n_elems for v in values):
            # can take (much) faster path if all lists are the same length
            return numpy_to_pyseries(
                name,
                np.vstack(values),
                strict=strict,
                nan_to_null=nan_to_null,
            )
        else:
            return PySeries.new_series_list(
                name,
                [
                    numpy_to_pyseries("", v, strict=strict, nan_to_null=nan_to_null)
                    for v in values
                ],
                strict,
            )

    elif python_dtype in (list, tuple):
        if dtype is None:
            return PySeries.new_from_any_values(name, values, strict=strict)
        elif dtype == Object:
            return PySeries.new_object(name, values, strict)
        else:
            if (inner_dtype := getattr(dtype, "inner", None)) is not None:
                pyseries_list = [
                    None
                    if value is None
                    else sequence_to_pyseries(
                        "",
                        value,
                        inner_dtype,
                        strict=strict,
                        nan_to_null=nan_to_null,
                    )
                    for value in values
                ]
                pyseries = PySeries.new_series_list(name, pyseries_list, strict)
            else:
                pyseries = PySeries.new_from_any_values_and_dtype(
                    name, values, dtype, strict=strict
                )
            if dtype != pyseries.dtype():
                pyseries = pyseries.cast(dtype, strict=False, wrap_numerical=False)
            return pyseries

    elif python_dtype == pl.Series:
        return PySeries.new_series_list(
            name, [v._s if v is not None else None for v in values], strict
        )

    elif python_dtype == PySeries:
        return PySeries.new_series_list(name, values, strict)
    else:
        constructor = py_type_to_constructor(python_dtype)
        if constructor == PySeries.new_object:
            try:
                srs = PySeries.new_from_any_values(name, values, strict)
                if _check_for_numpy(python_dtype, check_type=False) and isinstance(
                    np.bool_(True), np.generic
                ):
                    dtype = numpy_char_code_to_dtype(np.dtype(python_dtype).char)
                    return srs.cast(dtype, strict=strict, wrap_numerical=False)
                else:
                    return srs

            except RuntimeError:
                return PySeries.new_from_any_values(name, values, strict=strict)

        return _construct_series_with_fallbacks(
            constructor, name, values, dtype, strict=strict
        )

这里面核心判断数据类型,去py_type_to_constructor获取constructor,并且用该constructor方法去调用_construct_series_with_fallbacks构建series

  1. 因为只是为了了解调用流程,只看py_type_to_constructor就行,_construct_series_with_fallbacks同理
py-polars/src/polars/datatypes/constructor.pyGitHub
    _PY_TYPE_TO_CONSTRUCTOR: dict[
        Any, Callable[[str, Sequence[Any], bool], PySeries]
    ] = {
        float: PySeries.new_opt_f64,
        bool: PySeries.new_opt_bool,
        int: PySeries.new_opt_i64,
        str: PySeries.new_str,
        bytes: PySeries.new_binary,
        PyDecimal: PySeries.new_decimal,
    }
展开折叠代码 (60-159 行,共 100 行)


def polars_type_to_constructor(
    dtype: PolarsDataType,
) -> Callable[[str, Sequence[Any], bool], PySeries]:
    """Get the right PySeries constructor for the given Polars dtype."""
    # Special case for Array as it needs to pass the dtype argument on construction
    if isinstance(dtype, dt.Array):
        return functools.partial(PySeries.new_array, dtype=dtype)

    try:
        base_type = dtype.base_type()
        return _POLARS_TYPE_TO_CONSTRUCTOR[base_type]
    except KeyError:  # pragma: no cover
        msg = f"cannot construct PySeries for type {dtype!r}"
        raise ValueError(msg) from None


_NUMPY_TYPE_TO_CONSTRUCTOR = None


def _set_numpy_to_constructor() -> None:
    global _NUMPY_TYPE_TO_CONSTRUCTOR
    _NUMPY_TYPE_TO_CONSTRUCTOR = {
        np.float16: PySeries.new_f16,
        np.float32: PySeries.new_f32,
        np.float64: PySeries.new_f64,
        np.int8: PySeries.new_i8,
        np.int16: PySeries.new_i16,
        np.int32: PySeries.new_i32,
        np.int64: PySeries.new_i64,
        np.uint8: PySeries.new_u8,
        np.uint16: PySeries.new_u16,
        np.uint32: PySeries.new_u32,
        np.uint64: PySeries.new_u64,
        np.str_: PySeries.new_str,
        np.bytes_: PySeries.new_binary,
        np.bool_: PySeries.new_bool,
        np.datetime64: PySeries.new_i64,
        np.timedelta64: PySeries.new_i64,
    }


@functools.lru_cache(maxsize=32)
def _normalise_numpy_dtype(dtype: Any) -> tuple[Any, Any]:
    normalised_dtype = (
        np.dtype(dtype.base.name) if dtype.kind in ("i", "u", "f") else dtype
    ).type
    if normalised_dtype in (np.datetime64, np.timedelta64):
        time_unit = np.datetime_data(dtype)[0]
        if time_unit in dt.DTYPE_TEMPORAL_UNITS or (
            time_unit == "D" and normalised_dtype == np.datetime64
        ):
            return normalised_dtype, np.int64
        else:
            msg = (
                "incorrect NumPy datetime resolution"
                "\n\n'D' (datetime only), 'ms', 'us', and 'ns' resolutions are supported when converting from numpy.{datetime64,timedelta64}."
                " Please cast to the closest supported unit before converting."
            )
            raise ValueError(msg)
    return normalised_dtype, None


def numpy_values_and_dtype(
    values: np.ndarray[Any, Any],
) -> tuple[np.ndarray[Any, Any], type]:
    """Return numpy values and their associated dtype, adjusting if required."""
    # Create new dtype object from dtype base name so architecture specific
    # dtypes (np.longlong np.ulonglong np.intc np.uintc np.longdouble, ...)
    # get converted to their normalized dtype (np.int*, np.uint*, np.float*).
    dtype, cast_as = _normalise_numpy_dtype(values.dtype)
    if cast_as:
        values = values.astype(cast_as)
    return values, dtype


def numpy_type_to_constructor(
    values: np.ndarray[Any, Any], dtype: type[np.dtype[Any]]
) -> Callable[..., PySeries]:
    """Get the right PySeries constructor for the given Polars dtype."""
    if _NUMPY_TYPE_TO_CONSTRUCTOR is None:
        _set_numpy_to_constructor()
    try:
        return _NUMPY_TYPE_TO_CONSTRUCTOR[dtype]  # type:ignore[index]
    except KeyError:
        if len(values) > 0:
            first_non_nan = next(
                (v for v in values if isinstance(v, np.ndarray) or v == v), None
            )
            if isinstance(first_non_nan, str):
                return PySeries.new_str
            if isinstance(first_non_nan, bytes):
                return PySeries.new_binary
        return PySeries.new_object
    except NameError:  # pragma: no cover
        msg = f"'numpy' is required to convert numpy dtype {dtype!r}"
        raise ModuleNotFoundError(msg) from None
def py_type_to_constructor(py_type: type[Any]) -> Callable[..., PySeries]:
    """Get the right PySeries constructor for the given Python dtype."""
    py_type = (
        next((tp for tp in _PY_TYPE_TO_CONSTRUCTOR if issubclass(py_type, tp)), py_type)
        if py_type not in _PY_TYPE_TO_CONSTRUCTOR
        else py_type
    )
    return _PY_TYPE_TO_CONSTRUCTOR.get(py_type, PySeries.new_object)
  1. 可以看到int拿到的是PySeries.new_opt_i64,该方法定义为
py-polars/src/polars/_plr.pyiGitHub
    @staticmethod
    def new_opt_i64(name: str, obj: Any, strict: bool) -> PySeries: ...
  1. 该方法在_plr.pyi文件里,只有定义没有实现,他是以库的方式通过_plr.py加载进来
py-polars/src/polars/_plr.pyGitHub
def rt_32() -> None:
    from _polars_runtime_32 import BUILD_FEATURE_FLAGS

    check_cpu_flags(BUILD_FEATURE_FLAGS)

    import _polars_runtime_32._polars_runtime as plr

    sys.modules[__name__] = plr
  1. _polars_runtime_32._polars_runtime这个模块是通过maturin编译的
py-polars/runtime/polars-runtime-32/pyproject.tomlGitHub
[build-system]
requires = ["maturin>=1.3.2"]
build-backend = "maturin"
展开折叠代码 (5-39 行,共 35 行)
[project]
name = "polars-runtime-32"
description = "Blazingly fast DataFrame library"
readme = "README.md"
authors = [
  { name = "Ritchie Vink", email = "[email protected]" },
]
license = { file = "LICENSE" }
requires-python = ">=3.10"

keywords = ["dataframe", "arrow", "out-of-core"]
classifiers = [
  "Development Status :: 5 - Production/Stable",
  "Environment :: Console",
  "Intended Audience :: Science/Research",
  "License :: OSI Approved :: MIT License",
  "Operating System :: OS Independent",
  "Programming Language :: Python",
  "Programming Language :: Python :: 3",
  "Programming Language :: Python :: 3 :: Only",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
  "Programming Language :: Python :: 3.13",
  "Programming Language :: Rust",
  "Topic :: Scientific/Engineering",
  "Typing :: Typed",
]
dynamic = ["version"]

[project.urls]
Homepage = "https://www.pola.rs/"
Documentation = "https://docs.pola.rs/api/python/stable/reference/index.html"
Repository = "https://github.com/pola-rs/polars"
Changelog = "https://github.com/pola-rs/polars/releases"

[tool.maturin]
include = [
  { path = "rust-toolchain.toml", format = "sdist" },
  { path = "_polars_runtime_32/build_feature_flags.py", format = ["sdist", "wheel"] },
]
module-name = "_polars_runtime_32._polars_runtime"
  1. 而实际接口是从polars_python这个crate导入的
py-polars/runtime/polars-runtime-32/src/lib.rsGitHub
pub use polars_python::c_api::*;
  1. c_api,用pyo3导出了PySeries
crates/polars-python/src/c_api/mod.rsGitHub
#[pymodule(gil_used = false)] // gil_used = false will be default in PyO3 0.28.
pub fn _polars_runtime(py: Python, m: &Bound<PyModule>) -> PyResult<()> {
    // Classes
    m.add_class::<PySeries>().unwrap();
    m.add_class::<PyDataFrame>().unwrap();
    m.add_class::<PyLazyFrame>().unwrap();
展开折叠代码 (119-482 行,共 364 行)
    m.add_class::<PyOptFlags>().unwrap();
    #[cfg(not(target_arch = "wasm32"))]
    m.add_class::<PyInProcessQuery>().unwrap();
    m.add_class::<PyLazyGroupBy>().unwrap();
    m.add_class::<PyExpr>().unwrap();
    m.add_class::<PyDataTypeExpr>().unwrap();
    m.add_class::<PySelector>().unwrap();
    #[cfg(feature = "sql")]
    m.add_class::<PySQLContext>().unwrap();
    m.add_class::<PyCategories>().unwrap();
    m.add_class::<PyArrowCStreamReader>().unwrap();

    // Submodules
    // LogicalPlan objects
    m.add_wrapped(wrap_pymodule!(_ir_nodes))?;
    // Expr objects
    m.add_wrapped(wrap_pymodule!(_expr_nodes))?;

    // Functions - eager
    m.add_wrapped(wrap_pyfunction!(functions::concat_df))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_series))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_df_diagonal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_df_horizontal))
        .unwrap();

    // Functions - range
    m.add_wrapped(wrap_pyfunction!(functions::int_range))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::eager_int_range))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::int_ranges))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::linear_space))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::linear_spaces))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::date_range))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::date_ranges))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::datetime_range))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::datetime_ranges))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::time_range))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::time_ranges))
        .unwrap();

    // Functions - business
    m.add_wrapped(wrap_pyfunction!(functions::business_day_count))
        .unwrap();

    // Functions - aggregation
    m.add_wrapped(wrap_pyfunction!(functions::all_horizontal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::any_horizontal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::max_horizontal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::min_horizontal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::sum_horizontal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::mean_horizontal))
        .unwrap();

    // Functions - lazy
    m.add_wrapped(wrap_pyfunction!(functions::arg_sort_by))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::arg_where))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::as_struct))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::coalesce))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::field)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::col)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::collect_all))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::collect_all_lazy))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::element)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::explain_all))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::collect_all_with_callback))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_lf))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_arr))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_list))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::as_list)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_str))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::len)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::cov)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::cum_fold))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::cum_reduce))
        .unwrap();
    #[cfg(feature = "trigonometry")]
    m.add_wrapped(wrap_pyfunction!(functions::arctan2)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::datetime))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_expr))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_lf_diagonal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::concat_lf_horizontal))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::duration))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::fold)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::lit)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::map_expr))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::pearson_corr))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::rolling_corr))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::rolling_cov))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::reduce)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::repeat)).unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::spearman_rank_corr))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::when)).unwrap();

    // Functions: schema
    m.add_wrapped(wrap_pyfunction!(
        crate::interop::arrow::init_polars_schema_from_arrow_c_schema
    ))
    .unwrap();
    m.add_wrapped(wrap_pyfunction!(
        crate::interop::arrow::polars_schema_field_from_arrow_c_schema
    ))
    .unwrap();
    m.add_wrapped(wrap_pyfunction!(
        crate::interop::arrow::to_py::polars_schema_to_pycapsule
    ))
    .unwrap();

    // Functions: other
    m.add_wrapped(wrap_pyfunction!(functions::check_length))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::py_get_engine_affinity))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::config_reload_env_vars))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::config_reload_env_var))
        .unwrap();

    #[cfg(feature = "sql")]
    m.add_wrapped(wrap_pyfunction!(functions::sql_expr))
        .unwrap();

    // Functions - I/O
    #[cfg(feature = "ipc")]
    m.add_wrapped(wrap_pyfunction!(functions::read_ipc_schema))
        .unwrap();
    #[cfg(feature = "parquet")]
    m.add_wrapped(wrap_pyfunction!(functions::read_parquet_metadata))
        .unwrap();
    #[cfg(all(feature = "parquet", feature = "json"))]
    m.add_wrapped(wrap_pyfunction!(
        functions::_bench_parquet_metadata_bincode_size
    ))
    .unwrap();
    #[cfg(all(feature = "parquet", feature = "json"))]
    m.add_wrapped(wrap_pyfunction!(functions::_parquet_metadata_pruned_json))
        .unwrap();
    #[cfg(feature = "clipboard")]
    m.add_wrapped(wrap_pyfunction!(functions::read_clipboard_string))
        .unwrap();
    #[cfg(feature = "clipboard")]
    m.add_wrapped(wrap_pyfunction!(functions::write_clipboard_string))
        .unwrap();
    #[cfg(feature = "catalog")]
    m.add_class::<PyCatalogClient>().unwrap();

    // Functions - meta
    m.add_wrapped(wrap_pyfunction!(functions::get_index_type))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::thread_pool_size))
        .unwrap();

    // Numeric formatting
    m.add_wrapped(wrap_pyfunction!(functions::get_thousands_separator))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::set_thousands_separator))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::get_float_fmt))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::get_float_precision))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::get_decimal_separator))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::get_trim_decimal_zeros))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::set_float_fmt))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::set_float_precision))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::set_decimal_separator))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::set_trim_decimal_zeros))
        .unwrap();

    // Functions - misc
    m.add_wrapped(wrap_pyfunction!(functions::dtype_str_repr))
        .unwrap();
    #[cfg(feature = "object")]
    m.add_wrapped(wrap_pyfunction!(functions::__register_startup_deps))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(functions::gen_uuid_v7))
        .unwrap();

    // Functions - random
    m.add_wrapped(wrap_pyfunction!(functions::set_random_seed))
        .unwrap();

    // Functions - escape_regex
    m.add_wrapped(wrap_pyfunction!(functions::escape_regex))
        .unwrap();

    // Dtype helpers
    m.add_wrapped(wrap_pyfunction!(datatypes::_get_dtype_max))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(datatypes::_get_dtype_min))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(datatypes::_known_timezones))
        .unwrap();

    // Extension type registry.
    m.add_wrapped(wrap_pyfunction!(extension::_register_extension_type))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(extension::_unregister_extension_type))
        .unwrap();

    // Testing
    m.add_wrapped(wrap_pyfunction!(testing::assert_series_equal_py))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(testing::assert_dataframe_equal_py))
        .unwrap();
    m.add_wrapped(wrap_pyfunction!(testing::assert_schema_equal_py))
        .unwrap();

    // Exceptions - Errors
    m.add("PolarsError", py.get_type::<exceptions::PolarsError>())
        .unwrap();
    m.add(
        "ColumnNotFoundError",
        py.get_type::<exceptions::ColumnNotFoundError>(),
    )
    .unwrap();
    m.add("ComputeError", py.get_type::<exceptions::ComputeError>())
        .unwrap();
    m.add(
        "DuplicateError",
        py.get_type::<exceptions::DuplicateError>(),
    )
    .unwrap();
    m.add(
        "InvalidOperationError",
        py.get_type::<exceptions::InvalidOperationError>(),
    )
    .unwrap();
    m.add("NoDataError", py.get_type::<exceptions::NoDataError>())
        .unwrap();
    m.add(
        "OutOfBoundsError",
        py.get_type::<exceptions::OutOfBoundsError>(),
    )
    .unwrap();
    m.add(
        "SQLInterfaceError",
        py.get_type::<exceptions::SQLInterfaceError>(),
    )
    .unwrap();
    m.add(
        "SQLSyntaxError",
        py.get_type::<exceptions::SQLSyntaxError>(),
    )
    .unwrap();
    m.add("SchemaError", py.get_type::<exceptions::SchemaError>())
        .unwrap();
    m.add(
        "SchemaFieldNotFoundError",
        py.get_type::<exceptions::SchemaFieldNotFoundError>(),
    )
    .unwrap();
    m.add("ShapeError", py.get_type::<exceptions::ShapeError>())
        .unwrap();
    m.add(
        "StringCacheMismatchError",
        py.get_type::<exceptions::StringCacheMismatchError>(),
    )
    .unwrap();
    m.add(
        "StructFieldNotFoundError",
        py.get_type::<exceptions::StructFieldNotFoundError>(),
    )
    .unwrap();

    // Exceptions - Warnings
    m.add("PolarsWarning", py.get_type::<exceptions::PolarsWarning>())
        .unwrap();
    m.add(
        "PerformanceWarning",
        py.get_type::<exceptions::PerformanceWarning>(),
    )
    .unwrap();
    m.add(
        "CategoricalRemappingWarning",
        py.get_type::<exceptions::CategoricalRemappingWarning>(),
    )
    .unwrap();
    m.add(
        "MapWithoutReturnDtypeWarning",
        py.get_type::<exceptions::MapWithoutReturnDtypeWarning>(),
    )
    .unwrap();

    // Exceptions - Panic
    m.add(
        "PanicException",
        py.get_type::<pyo3::panic::PanicException>(),
    )
    .unwrap();

    // Cloud
    #[cfg(feature = "polars_cloud_client")]
    m.add_wrapped(wrap_pyfunction!(cloud_client::prepare_cloud_plan))
        .unwrap();
    #[cfg(feature = "polars_cloud_server")]
    m.add_wrapped(wrap_pyfunction!(cloud_server::_execute_ir_plan_with_gpu))
        .unwrap();

    // Build info
    m.add("__version__", PYPOLARS_VERSION)?;
    m.add("RUNTIME_REPR", RUNTIME_REPR)?;

    // Plugins
    #[cfg(feature = "ffi_plugin")]
    m.add_wrapped(wrap_pyfunction!(functions::register_plugin_function))
        .unwrap();

    // Capsules
    #[cfg(feature = "allocator")]
    {
        m.add("_allocator", allocator::create_allocator_capsule(py)?)?;
        m.add_wrapped(wrap_pyfunction!(allocator::_estimate_memory_usage))
            .unwrap();
    }

    m.add("_debug", cfg!(debug_assertions))?;

    Ok(())
}
  1. 而在construction里面导出了new_opt_i64
crates/polars-python/src/series/construction.rsGitHub
// Init with lists that can contain Nones
macro_rules! init_method_opt {
    ($name:ident, $type:ty, $native: ty) => {
        #[pymethods]
        impl PySeries {
            #[staticmethod]
            fn $name(name: &str, obj: &Bound<PyAny>, strict: bool) -> PyResult<Self> {
                new_primitive::<$type, _>(name, obj, strict, |v| v.extract::<$native>())
            }
        }
    };
}

init_method_opt!(new_opt_u8, UInt8Type, u8);
init_method_opt!(new_opt_u16, UInt16Type, u16);
init_method_opt!(new_opt_u32, UInt32Type, u32);
init_method_opt!(new_opt_u64, UInt64Type, u64);
init_method_opt!(new_opt_u128, UInt128Type, u128);
init_method_opt!(new_opt_i8, Int8Type, i8);
init_method_opt!(new_opt_i16, Int16Type, i16);
init_method_opt!(new_opt_i32, Int32Type, i32);
init_method_opt!(new_opt_i64, Int64Type, i64);
init_method_opt!(new_opt_i128, Int128Type, i128);
init_method_opt!(new_opt_f32, Float32Type, f32);
init_method_opt!(new_opt_f64, Float64Type, f64);

这个宏展开相当于

#[pymethods]
impl PySeries {
    #[staticmethod]
    fn new_opt_i64(name: &str, obj: &Bound<PyAny>, strict: bool) -> PyResult<Self> {
        new_primitive::<Int64Type, _>(name, obj, strict, |v| v.extract::<i64>())
    }
}
  1. 继续看new_primitive,其中核心是用了PrimitiveChunkedBuilder
crates/polars-python/src/series/construction.rsGitHub
fn new_primitive<'py, T, F>(
    name: &str,
    values: &Bound<'py, PyAny>,
    _strict: bool,
    extract: F,
) -> PyResult<PySeries>
where
    T: PolarsNumericType,
    F: Fn(Bound<'py, PyAny>) -> PyResult<T::Native>,
{
    let len = values.len()?;
    let mut builder = PrimitiveChunkedBuilder::<T>::new(name.into(), len);

    for res in values.try_iter()? {
        let value = res?;
        if value.is_none() {
            builder.append_null()
        } else {
            let v = extract(value)?;
            builder.append_value(v)
        }
    }

    let ca = builder.finish();
    let s = ca.into_series();
    Ok(s.into())
}
  1. 至此进入polars-core这个crate
crates/polars-core/src/chunked_array/builder/primitive.rsGitHub
#[derive(Clone)]
pub struct PrimitiveChunkedBuilder<T>
where
    T: PolarsNumericType,
{
    array_builder: MutablePrimitiveArray<T::Native>,
    pub(crate) field: Field,
}

impl<T> ChunkedBuilder<T::Native, T> for PrimitiveChunkedBuilder<T>
where
    T: PolarsNumericType,
{
    /// Appends a value of type `T` into the builder
    #[inline]
    fn append_value(&mut self, v: T::Native) {
        self.array_builder.push(Some(v))
    }

    /// Appends a null slot into the builder
    #[inline]
    fn append_null(&mut self) {
        self.array_builder.push(None)
    }

    fn finish(mut self) -> ChunkedArray<T> {
        let arr = self.array_builder.as_box();
        ChunkedArray::new_with_compute_len(Arc::new(self.field), vec![arr])
    }

    fn shrink_to_fit(&mut self) {
        self.array_builder.shrink_to_fit()
    }
}

impl<T> PrimitiveChunkedBuilder<T>
where
    T: PolarsNumericType,
{
    pub fn new(name: PlSmallStr, capacity: usize) -> Self {
        let array_builder = MutablePrimitiveArray::<T::Native>::with_capacity(capacity)
            .to(T::get_static_dtype().to_arrow(CompatLevel::newest()));

        PrimitiveChunkedBuilder {
            array_builder,
            field: Field::new(name, T::get_static_dtype()),
        }
    }
}

再往下的部分都是rust代码,至此,梳理清楚了从py到底层rust代码调用的全流程。

flowchart TD A["py-polars/src/polars"] B["py-polars/runtime"] C["crates/polars-python"] D["crates/polars-core"] A --> B B -- 进入rust --> C C --> D


Previous Post
02-polars数据类型