Hadoop之——数据类型

转载请注明出处:http://blog.csdn.net/l1028386804/article/details/46039055

一.  Hadoop内置的数据类型

• BooleanWritable:标准布尔型数值
• ByteWritable:单字节数值
• DoubleWritable:双字节数值
• FloatWritable:浮点数
• IntWritable:整型数
• LongWritable:长整型数
• Text:使用UTF8格式存储的文本
• NullWritable:当<key, value>中的key或value为空时使用

二、Hadoop自定义数据类型实例

把后面的URLString 封装成 URL类型。代码如下

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;

import org.apache.hadoop.io.Writable;
/**
 * @author liuyazhuang
 */
public class URLWritable implements Writable {

	protected URL url;

	public URLWritable() {

	}

	public URLWritable(URL url) {
		this.url = url;
	}

	@Override
	public void write(DataOutput out) throws IOException {
		out.writeUTF(url.toString());
	}

	@Override
	public void readFields(DataInput in) throws IOException {
		this.url = new URL(in.readUTF());
	}

	public void set(String string) {
		try {
			this.url = new URL(string);
		} catch (MalformedURLException e) {
			throw new RuntimeException("Should not have happened " + e.toString());
		}
	}
}

import java.io.IOException;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;
/**
 * @author liuyazhuang
 */
public class TimeUrlLineRecordReader extends RecordReader<Text, URLWritable> {
	public static final String Time_URL_SEPERATOR =
	    "mapreduce.input.keyvaluelinerecordreader.key.value.separator";

	private final LineRecordReader lineRecordReader;

	private byte separator = (byte) '\t';

	private Text innerValue;

	private Text key;

	private URLWritable value;

	public static int findSeparator(byte[] utf, int start, int length, byte sep) {
		for (int i = start; i < (start + length); i++) {
			if (utf[i] == sep) {
				return i;
			}
		}
		return -1;
	}

	public static void setKeyValue(Text key, URLWritable value, byte[] line,
			int lineLen, int pos) {
		if (pos == -1) {
			key.set(line, 0, lineLen);
			value.set(StringUtils.EMPTY);
		} else {
			key.set(line, 0, pos);
			String url = null;
			System.arraycopy(line, pos + 1,url , 0, lineLen - pos - 1);
			value.set(url);
		}
	}

	public TimeUrlLineRecordReader(Configuration conf) throws IOException {
		lineRecordReader = new LineRecordReader();
		String sepStr = conf.get(Time_URL_SEPERATOR, "\t");
	    this.separator = (byte) sepStr.charAt(0);
	}

	@Override
	public void initialize(InputSplit split, TaskAttemptContext context)
			throws IOException, InterruptedException {
		 lineRecordReader.initialize(split, context);
	}

	@Override
	public boolean nextKeyValue() throws IOException, InterruptedException {
		byte[] line = null;
		int lineLen = -1;
		if (lineRecordReader.nextKeyValue()) {
			innerValue = lineRecordReader.getCurrentValue();
			line = innerValue.getBytes();
			lineLen = innerValue.getLength();
		} else {
			return false;
		}
		if (line == null) {
			return false;
		}
		if (key == null) {
			key = new Text();
		}
		if (value == null) {
			value = new URLWritable();
		}
		int pos = findSeparator(line, 0, lineLen, this.separator);
		setKeyValue(key, value, line, lineLen, pos);
	    return true;
	}

	@Override
	public Text getCurrentKey() throws IOException, InterruptedException {
		return key;
	}

	@Override
	public URLWritable getCurrentValue() throws IOException,
			InterruptedException {
		return value;
	}

	@Override
	public float getProgress() throws IOException, InterruptedException {
		return lineRecordReader.getProgress();
	}

	@Override
	public void close() throws IOException {
		lineRecordReader.close();
	}
}

import java.io.IOException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
/**
 * @author liuyazhuang
 */
public class TimeUrlTextInputFormat extends FileInputFormat<Text, URLWritable>{

	@Override
	protected boolean isSplitable(JobContext context, Path file) {
		final CompressionCodec codec = new CompressionCodecFactory(
				context.getConfiguration()).getCodec(file);
		return codec == null;
	}

	@Override
	public RecordReader<Text, URLWritable> createRecordReader(InputSplit split,
			TaskAttemptContext context) throws IOException, InterruptedException {
		context.setStatus(split.toString());
		return new TimeUrlLineRecordReader(context.getConfiguration());
	}
}

import java.io.IOException;

import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/**
 * @author liuyazhuang
 */
public class CustomTimeUrl extends Configured implements Tool {

	public static class CustomTimeUrlMapper extends Mapper<Text, URLWritable, Text, URLWritable> {

		@Override
		protected void map(Text key, URLWritable value, Context context)
				throws IOException, InterruptedException {
			context.write(key, value);
		}

	}

	public static class CustomTimeUrlReducer extends Reducer<Text, URLWritable, Text, URLWritable> {

		@Override
		protected void reduce(Text key, Iterable<URLWritable> values,Context context)throws IOException, InterruptedException {
			for (URLWritable value : values) {
				context.write(key, value);
			}
		}

	}

	    @Override
	     public int run(String[] args) throws Exception {
		Job job = new Job(getConf());
		job.setJarByClass(getClass());
		job.setJobName("CustomTimeUrl");

		job.setInputFormatClass(TimeUrlTextInputFormat.class);
		job.setOutputFormatClass(TextOutputFormat.class);

		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(URLWritable.class);

		job.setMapperClass(CustomTimeUrlMapper.class);
		job.setReducerClass(CustomTimeUrlReducer.class);

		FileInputFormat.setInputPaths(job, new Path("/timeurl/input/"));
		FileOutputFormat.setOutputPath(job, new Path("/timeurl/output"));

		boolean success = job.waitForCompletion(true);
		return success ? 0 : 1;
	}

	public static void main(String[] args) throws Exception {
		int result = ToolRunner.run(new TimeUrl(), args);
		System.exit(result);
	}
}