pyspark实现本地简单Kmeans代码及数据集

以下是一个简单的本地KMeans实现的示例代码：

from pyspark import SparkConf, SparkContext
from numpy import array
from math import sqrt

# 定义计算欧几里得距离的函数
def euclidean_distance(point1, point2):
    return sqrt(sum([(a - b) ** 2 for a, b in zip(point1, point2)]))

# 定义KMeans函数
def kmeans(data, k, converge_dist):
    k_points = data.takeSample(False, k, 1)
    temp_dist = 1.0

    while temp_dist > converge_dist:
        closest = data.map(
            lambda point: (min([(i, euclidean_distance(point, k_points[i])) for i in range(k)], key=lambda x: x[1])[0], (point, 1))
        )
        point_stats = closest.reduceByKey(
            lambda point1, point2: (array(point1[0]) + array(point2[0]), point1[1] + point2[1])
        )
        new_points = point_stats.map(
            lambda st: (st[0], st[1][0] / st[1][1])
        ).collect()

        temp_dist = sum(
            [euclidean_distance(k_points[ik], p) for ik, p in new_points]
        )

        for ik, p in new_points:
            k_points[ik] = p

    return k_points

# 创建Spark环境
conf = SparkConf().setAppName("KMeans").setMaster("local")
sc = SparkContext(conf=conf)

# 加载数据集
data = sc.textFile("data.txt").map(lambda line: array([float(x) for x in line.split(' ')]))

# 调用KMeans函数
centroids = kmeans(data, 3, 0.1)

# 输出聚类中心
for c in centroids:
    print(c)

数据集示例（data.txt）：

1.0 2.0 3.0
4.0 5.0 6.0
7.0 8.0 9.0
10.0 11.0 12.0
13.0 14.0 15.0
16.0 17.0 18.0

运行后会输出3个聚类中心的坐标值。注意，这个实现是在本地运行的，而不是分布式的，因此适用于小规模数据集。