使用java、scala、python三种语言实现spark rdd的flatmap算子的使用

Java实现：

JavaRDD<String> lines = sc.textFile("path/to/file");
JavaRDD<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
    public Iterable<String> call(String s) {
        return Arrays.asList(s.split(" "));
    }
});

Scala实现：

val lines = sc.textFile("path/to/file")
val words = lines.flatMap(line => line.split(" "))

Python实现：

lines = sc.textFile("path/to/file")
words = lines.flatMap(lambda line: line.split(" "))

相关问题

使用java、scala、python三种语言实现spark rdd的reduce算子的使用

Java:

JavaRDD<Integer> numbers = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5));
int sum = numbers.reduce((a, b) -> a + b);
System.out.println(sum);

Scala:

val numbers = sc.parallelize(Seq(1, 2, 3, 4, 5))
val sum = numbers.reduce(_ + _)
println(sum)

Python:

numbers = sc.parallelize([1, 2, 3, 4, 5])
sum = numbers.reduce(lambda a, b: a + b)
print(sum)

用java,scala,python三种语言实现spark rdd的treeReduce算子

Java实现：

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function2;

public class TreeReduceExample {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("TreeReduceExample").setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(conf);

        JavaRDD<Integer> rdd = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10));

        Integer result = rdd.treeReduce(new Function2<Integer, Integer, Integer>() {
            public Integer call(Integer a, Integer b) throws Exception {
                return a + b;
            }
        });

        System.out.println("Result: " + result);

        sc.close();
    }
}

Scala实现：

import org.apache.spark.{SparkConf, SparkContext}

object TreeReduceExample {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("TreeReduceExample").setMaster("local")
    val sc = new SparkContext(conf)

    val rdd = sc.parallelize(Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 10))

    val result = rdd.treeReduce((a: Int, b: Int) => a + b)

    println("Result: " + result)

    sc.stop()
  }
}

Python实现：

from pyspark import SparkConf, SparkContext

conf = SparkConf().setAppName("TreeReduceExample").setMaster("local")
sc = SparkContext(conf=conf)

rdd = sc.parallelize([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])

result = rdd.treeReduce(lambda a, b: a + b)

print("Result: " + str(result))

sc.stop()