Polars, 旬の13のお役立ち機能

import polars as pl
pl.show_versions()
---Version info---
Polars: 0.16.6
Index type: UInt32
Platform: Linux-5.15.79.1-microsoft-standard-WSL2-x86_64-with-glibc2.31
Python: 3.10.9 (main, Dec 27 2022, 12:45:52) [Clang 15.0.6 ]
---Optional dependencies---
pyarrow: 10.0.1
pandas: 1.5.3
numpy: 1.24.2
fsspec: <not installed>
connectorx: <not installed>
xlsx2csv: <not installed>
deltalake: <not installed>
matplotlib: <not installed>

df = pl.DataFrame(
    {
        "a": [1, 2, 3],
        "b": [1, 2, 2 << 32],
        "c": [-1, 2, 1 << 30],
        "d": [-112, 2, 112],
        "e": [-112, 2, 129],
        "f": ["a", "b", "c"],
        "g": [0.1, 1.32, 0.12],
        "h": [True, None, False],
    }
)
df

shape: (3, 8)
┌─────┬────────────┬────────────┬──────┬──────┬─────┬──────┬───────┐
│ a   ┆ b          ┆ c          ┆ d    ┆ e    ┆ f   ┆ g    ┆ h     │
│ --- ┆ ---        ┆ ---        ┆ ---  ┆ ---  ┆ --- ┆ ---  ┆ ---   │
│ i64 ┆ i64        ┆ i64        ┆ i64  ┆ i64  ┆ str ┆ f64  ┆ bool  │
╞═════╪════════════╪════════════╪══════╪══════╪═════╪══════╪═══════╡
│ 1   ┆ 1          ┆ -1         ┆ -112 ┆ -112 ┆ a   ┆ 0.1  ┆ true  │
│ 2   ┆ 2          ┆ 2          ┆ 2    ┆ 2    ┆ b   ┆ 1.32 ┆ null  │
│ 3   ┆ 8589934592 ┆ 1073741824 ┆ 112  ┆ 129  ┆ c   ┆ 0.12 ┆ false │
└─────┴────────────┴────────────┴──────┴──────┴─────┴──────┴───────┘

df.select(pl.all().shrink_dtype())

shape: (3, 8)
┌─────┬────────────┬────────────┬──────┬──────┬─────┬──────┬───────┐
│ a   ┆ b          ┆ c          ┆ d    ┆ e    ┆ f   ┆ g    ┆ h     │
│ --- ┆ ---        ┆ ---        ┆ ---  ┆ ---  ┆ --- ┆ ---  ┆ ---   │
│ i8  ┆ i64        ┆ i32        ┆ i8   ┆ i16  ┆ str ┆ f32  ┆ bool  │
╞═════╪════════════╪════════════╪══════╪══════╪═════╪══════╪═══════╡
│ 1   ┆ 1          ┆ -1         ┆ -112 ┆ -112 ┆ a   ┆ 0.1  ┆ true  │
│ 2   ┆ 2          ┆ 2          ┆ 2    ┆ 2    ┆ b   ┆ 1.32 ┆ null  │
│ 3   ┆ 8589934592 ┆ 1073741824 ┆ 112  ┆ 129  ┆ c   ┆ 0.12 ┆ false │
└─────┴────────────┴────────────┴──────┴──────┴─────┴──────┴───────┘

df = pl.DataFrame({
    "a": [1,2,3],
    "b": list("abc"),
})
# df.with_columns(pl.lit(99).alias("c")) より記述短くて済む。
df.with_columns(c=99)

shape: (3, 3)
┌─────┬─────┬─────┐
│ a   ┆ b   ┆ c   │
│ --- ┆ --- ┆ --- │
│ i64 ┆ str ┆ i32 │
╞═════╪═════╪═════╡
│ 1   ┆ a   ┆ 99  │
│ 2   ┆ b   ┆ 99  │
│ 3   ┆ c   ┆ 99  │
└─────┴─────┴─────┘

df1 = pl.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
print(df1.with_columns(baz=pl.col("foo") * 2, qux=pl.col("bar") + 1))
print(df1.with_columns([(pl.col("foo") * 2).alias("baz"), (pl.col("bar") + 1).alias("qux")]))

shape: (3, 4)
┌─────┬─────┬─────┬─────┐
│ foo ┆ bar ┆ baz ┆ qux │
│ --- ┆ --- ┆ --- ┆ --- │
│ i64 ┆ i64 ┆ i64 ┆ i64 │
╞═════╪═════╪═════╪═════╡
│ 1   ┆ 4   ┆ 2   ┆ 5   │
│ 2   ┆ 5   ┆ 4   ┆ 6   │
│ 3   ┆ 6   ┆ 6   ┆ 7   │
└─────┴─────┴─────┴─────┘
shape: (3, 4)
┌─────┬─────┬─────┬─────┐
│ foo ┆ bar ┆ baz ┆ qux │
│ --- ┆ --- ┆ --- ┆ --- │
│ i64 ┆ i64 ┆ i64 ┆ i64 │
╞═════╪═════╪═════╪═════╡
│ 1   ┆ 4   ┆ 2   ┆ 5   │
│ 2   ┆ 5   ┆ 4   ┆ 6   │
│ 3   ┆ 6   ┆ 6   ┆ 7   │
└─────┴─────┴─────┴─────┘

df = pl.DataFrame({
    "a": pl.arange(0, 5e7, eager=True),
}).with_columns(
    b=pl.col("a") // 10000,
    c=pl.col("a").shuffle(seed=1)
)
df

shape: (50000000, 3)
┌──────────┬──────┬──────────┐
│ a        ┆ b    ┆ c        │
│ ---      ┆ ---  ┆ ---      │
│ i64      ┆ i64  ┆ i64      │
╞══════════╪══════╪══════════╡
│ 0        ┆ 0    ┆ 27454110 │
│ 1        ┆ 0    ┆ 2309916  │
│ 2        ┆ 0    ┆ 15065100 │
│ 3        ┆ 0    ┆ 12766444 │
│ ...      ┆ ...  ┆ ...      │
│ 49999996 ┆ 4999 ┆ 17167194 │
│ 49999997 ┆ 4999 ┆ 9092583  │
│ 49999998 ┆ 4999 ┆ 1929693  │
│ 49999999 ┆ 4999 ┆ 35668469 │
└──────────┴──────┴──────────┘

%time pd_df1 = df.to_pandas()
CPU times: user 478 ms, sys: 968 ms, total: 1.45 s
Wall time: 943 ms

%time pd_df2 = df.to_pandas(use_pyarrow_extension_array=True)
CPU times: user 1.88 ms, sys: 0 ns, total: 1.88 ms
Wall time: 2.01 ms

df_1 = pl.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
df_2 = pl.DataFrame({"foo": [10, 20, 30], "bar": [40, 50, 60]})

pl.concat([df_1, df_2]) # immutabble
df_1.vstack(df_2) # immutable
df_1.extend(df_2) # in-place change for df_1

df = pl.DataFrame({"s": ["Café", None, "345", "東京"]}).with_columns(
    [
        pl.col("s").str.n_chars().alias("nchars"),
        pl.col("s").str.lengths().alias("length"),
    ]
)
df

shape: (4, 3)
┌──────┬────────┬────────┐
│ s    ┆ nchars ┆ length │
│ ---  ┆ ---    ┆ ---    │
│ str  ┆ u32    ┆ u32    │
╞══════╪════════╪════════╡
│ Café ┆ 4      ┆ 5      │
│ null ┆ null   ┆ null   │
│ 345  ┆ 3      ┆ 3      │
│ 東京 ┆ 2      ┆ 6      │
└──────┴────────┴────────┘

python -m pip install duckdb[polars]

import duckdb
import polars as pl

df = pl.DataFrame(
    {
        "A": [1, 2, 3, 4, 5],
        "fruits": ["banana", "banana", "apple", "apple", "banana"],
        "B": [5, 4, 3, 2, 1],
        "cars": ["beetle", "audi", "beetle", "beetle", "beetle"],
    }
)
duckdb.sql('SELECT * FROM df').show()

df2 = df.lazy()
duckdb.sql('SELECT * FROM df2').show()

df = pl.DataFrame({str(i): [i] for i in range(20)})
print(df)
with pl.Config() as cfg:
    cfg.set_tbl_cols(-1)  
    print(df)

shape: (1, 20)
┌─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┐
│ 0   ┆ 1   ┆ 2   ┆ 3   ┆ ... ┆ 16  ┆ 17  ┆ 18  ┆ 19  │
│ --- ┆ --- ┆ --- ┆ --- ┆     ┆ --- ┆ --- ┆ --- ┆ --- │
│ i64 ┆ i64 ┆ i64 ┆ i64 ┆     ┆ i64 ┆ i64 ┆ i64 ┆ i64 │
╞═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╡
│ 0   ┆ 1   ┆ 2   ┆ 3   ┆ ... ┆ 16  ┆ 17  ┆ 18  ┆ 19  │
└─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┘
shape: (1, 20)
┌─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┬─────┐
│ 0   ┆ 1   ┆ 2   ┆ 3   ┆ 4   ┆ 5   ┆ 6   ┆ 7   ┆ 8   ┆ 9   ┆ 10  ┆ 11  ┆ 12  ┆ 13  ┆ 14  ┆ 15  ┆ 16  ┆ 17  ┆ 18  ┆ 19  │
│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │
│ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │
╞═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╪═════╡
│ 0   ┆ 1   ┆ 2   ┆ 3   ┆ 4   ┆ 5   ┆ 6   ┆ 7   ┆ 8   ┆ 9   ┆ 10  ┆ 11  ┆ 12  ┆ 13  ┆ 14  ┆ 15  ┆ 16  ┆ 17  ┆ 18  ┆ 19  │
└─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┴─────┘

pl.Config.state() # check current status
pl.Config.restore_defaults() # reset to default
str = pl.Config.save()
pl.Config.load(str)