Python Dask库处理大规模数据集的强大功能实战

import dask
import dask.array as da

x = da.ones((1000, 1000), chunks=(100, 100))
y = (x + x.T).mean(axis=0)
result = y.compute()

import dask.array as da

x = da.ones((1000, 1000), chunks=(100, 100))
y = x + x.T
result = y.compute()

from dask.distributed import Client
import dask.array as da
client = Client()
x = da.ones((1000, 1000), chunks=(100, 100))
y = (x + x.T).mean(axis=0)
result = y.compute()

from dask.distributed import Client
client = Client()
# 在此处执行Dask任务

import dask.dataframe as dd

df = dd.read_csv('large_dataset.csv', blocksize=25e6)
result = df.groupby('column').mean().compute()

import dask.array as da
from dask_ml.linear_model import LinearRegression
X = da.random.random((100000, 10), chunks=(1000, 10))
y = X.dot(da.random.random((10, 1), chunks=(10, 1))) + da.random.random((100000,), chunks=(1000,))
model = LinearRegression()
model.fit(X, y)

x = da.ones((1000, 1000), chunks=(200, 200))  # 调整块大小

x = da.ones((1000, 1000), chunks=(200, 200))
y = x + x.T

# 分阶段执行
result = y.mean(axis=0).compute()