MapReduce 单词统计案例编程

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.util.Tool;

public class WordCountMapReduce extends Configuration
implements Tool {

}

@Override

public void map(LongWritable key, Text value, Context context)

throws IOException,
InterruptedException {

String line = value.toString();

String[] cols = line.split("\t");

for (String col : cols) {

context.write(new Text(col),
new IntWritable(1));

}

}

// step 2: Reducer Class

@Override

protected void cleanup(Context context) throws IOException,

InterruptedException {

// TODO

}

@Override

protected void setup(Context context) throws IOException,

InterruptedException {

// TODO

}

}

public static class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable> {
    @Override
    public void reduce(Text key, Iterable<IntWritable> values,
                       Context context) throws IOException, InterruptedException {
        int sum=0;
        for (IntWritable count :values)
        {
            sum+=count.get();
        }
        context.write(new Text(key),new IntWritable(sum));
    }

    @Override
    public void cleanup(Context context) throws IOException,
            InterruptedException {
        // TODO

    }

    @Override
    public void setup(Context context) throws IOException,
            InterruptedException {
        // TODO

    }
}

public int run(String[] args) throws Exception {

    Configuration configuration = this.getConf();
    Job job = Job.getInstance(configuration, this.getClass()
            .getSimpleName());

    job.setJarByClass(this.getClass());
    Path inpath = new Path(args[0]);
    FileInputFormat.addInputPath(job, inpath);

    // output:
    Path outpath = new Path(args[1]);
    FileOutputFormat.setOutputPath(job, outpath);

    // mapper
    job.setMapperClass(WordCountMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    // ================shuffle====================
    // 1.分区
    // job.setPartitionerClass(cls);

    // 2.排序
    // job.setSortComparatorClass(cls);

    // 3.combiner优化
    // job.setCombinerClass(WordCountCombiner.class);

    // 4.compress 压缩

    // 5.group分组
    // job.setGroupingComparatorClass(cls);

    // ================shuffle====================

    // reducer
    job.setReducerClass(WordCountReduce.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    //reduce个数可以指定
    //job.setNumReduceTasks(tasks);

    // submit job
    boolean isSucces = job.waitForCompletion(true);
    return isSucces ? 0 : 1;
}

public static void main(String[] args) throws Exception {
    args = new String[] {
            "hdfs://bigdata-senior01.chybinmy.com:8020/wordcountdemo/input/wordcount.input",
            "hdfs://bigdata-senior01.chybinmy.com:8020/wordcountdemo/output1" };

    Configuration configuration = new Configuration();
    int status = ToolRunner.run(configuration, new WordCountMapReduce(), args);
    System.exit(status);
}

hadoop  3

hbase   1

hive   
2

mapreduce      
1

spark   2

sqoop   1

storm   1