hadoop与mysql数据库相连读出数据

转自：http://blog.csdn.net/qwertyu8656/article/details/6426054

用0.20.2版本 有些类已经过时 但必须要用 因为新版本对数据库连接支持不够

运行mysql创建数据库School，建立teacher表，并自行填写值

[c-sharp] view plaincopy

1. DROP TABLE IF EXISTS `school`.`teacher`;
2. CREATE TABLE  `school`.`teacher` (
3. `id` int(11) default NULL,
4. `name` char(20) default NULL,
5. `age` int(11) default NULL,
6. `departmentID` int(11) default NULL
7. ) ENGINE=InnoDB DEFAULT CHARSET=latin1;

在eclipse中运行编译通过 但要加入必须的库 以及 hadoop0.20.2的eclipse的插件

[c-sharp] view plaincopy

1. import java.io.IOException;
2. import org.apache.hadoop.fs.Path;
3. import org.apache.hadoop.io.LongWritable;
4. import org.apache.hadoop.io.Text;
5. import org.apache.hadoop.mapred.FileOutputFormat;
6. import org.apache.hadoop.mapred.JobClient;
7. import org.apache.hadoop.mapred.JobConf;
8. import org.apache.hadoop.mapred.lib.IdentityReducer;
9. import org.apache.hadoop.mapred.lib.db.DBConfiguration;
10. import org.apache.hadoop.mapred.lib.db.DBInputFormat;
11. public class DBAccess2 {
12. public static void main(String[] args) throws IOException {
13. JobConf conf = new JobConf(DBAccess2.class);
14. conf.setOutputKeyClass(LongWritable.class);
15. conf.setOutputValueClass(Text.class);
16. conf.setInputFormat(DBInputFormat.class);
17. FileOutputFormat.setOutputPath(conf, new Path("hdfs://localhost:9000/dbout"));
18. DBConfiguration.configureDB(conf,"com.mysql.jdbc.Driver",
19. "jdbc:mysql://localhost:3306/school","root","zxcvbnm");
20. String [] fields = {"id", "name", "age", "departmentID"};
21. DBInputFormat.setInput(conf, TeacherRecord.class, "teacher",
22. null, "id", fields);
23. conf.setMapperClass(DBAccessMapper.class);
24. conf.setReducerClass(IdentityReducer.class);
25. JobClient.runJob(conf);
26. }
27. }

注：请自行修改数据库连接语句 用户名 密码 等等。

[c-sharp] view plaincopy

1. import java.io.IOException;
2. import org.apache.hadoop.io.LongWritable;
3. import org.apache.hadoop.io.Text;
4. import org.apache.hadoop.mapred.MapReduceBase;
5. import org.apache.hadoop.mapred.Mapper;
6. import org.apache.hadoop.mapred.OutputCollector;
7. import org.apache.hadoop.mapred.Reporter;
8. public class DBAccessMapper extends MapReduceBase implements
9. Mapper<LongWritable, TeacherRecord, LongWritable, Text> {
10. @Override
11. public void map(LongWritable key, TeacherRecord value,
12. OutputCollector<LongWritable, Text> collector, Reporter reporter)
13. throws IOException {
14. // TODO Auto-generated method stub
15. collector.collect(new LongWritable(value.id),
16. new Text(value.toString()));
17. }
18. }

[c-sharp] view plaincopy

1. import java.io.DataInput;
2. import java.io.DataOutput;
3. import java.io.IOException;
4. import java.sql.PreparedStatement;
5. import java.sql.ResultSet;
6. import java.sql.SQLException;
7. import org.apache.hadoop.io.Text;
8. import org.apache.hadoop.io.Writable;
9. import org.apache.hadoop.mapred.lib.db.DBWritable;
10. public class TeacherRecord implements Writable, DBWritable{
11. int id;
12. String name;
13. int age;
14. int departmentID;
15. @Override
16. public void readFields(DataInput in) throws IOException {
17. // TODO Auto-generated method stub
18. this.id = in.readInt();
19. this.name = Text.readString(in);
20. this.age = in.readInt();
21. this.departmentID = in.readInt();
22. }
23. @Override
24. public void write(DataOutput out) throws IOException {
25. // TODO Auto-generated method stub
26. out.writeInt(this.id);
27. Text.writeString(out, this.name);
28. out.writeInt(this.age);
29. out.writeInt(this.departmentID);
30. }
31. @Override
32. public void readFields(ResultSet result) throws SQLException {
33. // TODO Auto-generated method stub
34. this.id = result.getInt(1);
35. this.name = result.getString(2);
36. this.age = result.getInt(3);
37. this.departmentID = result.getInt(4);
38. }
39. @Override
40. public void write(PreparedStatement stmt) throws SQLException {
41. // TODO Auto-generated method stub
42. stmt.setInt(1, this.id);
43. stmt.setString(2, this.name);
44. stmt.setInt(3, this.age);
45. stmt.setInt(4, this.departmentID);
46. }
47. @Override
48. public String toString() {
49. // TODO Auto-generated method stub
50. return new String(this.name + " " + this.age + " " + this.departmentID);
51. }
52. }

hadoop与mysql数据库相连读出数据