用java,scala,python三种语言实现spark rdd的glom算子

Java实现：

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.List;

public class GlomExample {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("GlomExample").setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);

        List<Integer> data = new ArrayList<>();
        for (int i = 1; i <= 10; i++) {
            data.add(i);
        }

        JavaRDD<Integer> rdd = sc.parallelize(data, 2);
        JavaRDD<List<Integer>> glomRdd = rdd.glom();

        List<List<Integer>> result = glomRdd.collect();
        for (int i = 0; i < result.size(); i++) {
            System.out.println("Partition " + i + ": " + result.get(i));
        }

        sc.stop();
    }
}

Scala实现：

import org.apache.spark.{SparkConf, SparkContext}

object GlomExample {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("GlomExample").setMaster("local")
    val sc = new SparkContext(conf)

    val data = 1 to 10
    val rdd = sc.parallelize(data, 2)
    val glomRdd = rdd.glom()

    val result = glomRdd.collect()
    for (i <- result.indices) {
      println(s"Partition $i: ${result(i).toList}")
    }

    sc.stop()
  }
}

Python实现：

from pyspark import SparkConf, SparkContext

conf = SparkConf().setAppName("GlomExample").setMaster("local")
sc = SparkContext(conf=conf)

data = range(1, 11)
rdd = sc.parallelize(data, 2)
glom_rdd = rdd.glom()

result = glom_rdd.collect()
for i in range(len(result)):
    print(f"Partition {i}: {list(result[i])}")

sc.stop()