hadoop倒排索引的程序

以下是一个简单的Hadoop倒排索引程序示例：

1. Map阶段：

public static class InvertedIndexMapper extends Mapper<LongWritable, Text, Text, Text> {
    private final static Text word = new Text();
    private final static Text location = new Text();

    public void map(LongWritable key, Text val, Context context) throws IOException, InterruptedException {
        String line = val.toString();
        String[] parts = line.split(",");
        location.set(parts[0]); // 文档ID
        StringTokenizer itr = new StringTokenizer(parts[1]);
        while (itr.hasMoreTokens()) {
            word.set(itr.nextToken());
            context.write(word, location); // 输出键值对
        }
    }
}

2. Reduce阶段：

public static class InvertedIndexReducer extends Reducer<Text, Text, Text, Text> {
    private final static Text result = new Text();

    public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        StringBuilder sb = new StringBuilder();
        for (Text val : values) {
            sb.append(val.toString()).append(","); // 将文档ID拼接成一个字符串
        }
        result.set(sb.toString());
        context.write(key, result); // 输出键值对
    }
}

3. Driver代码：

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "Inverted Index");
    job.setJarByClass(InvertedIndex.class);
    job.setMapperClass(InvertedIndexMapper.class);
    job.setReducerClass(InvertedIndexReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}

以上代码实现了一个简单的Hadoop倒排索引程序，可以通过命令行参数指定输入文件路径和输出文件路径。