mapreduce 读取mysql

package com.jsptpd.mysqlintolocal;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Main {
	private static String PATH_TO_CORE_SITE_XML = "D:\\workspace_scala1\\test1314\\conf\\core-site.xml";

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		System.setProperty("HADOOP_USER_NAME", "hdfs");
		Configuration conf = new Configuration();
		conf.addResource(new Path(PATH_TO_CORE_SITE_XML));
		//conf.set("mapreduce.application.framework.path", "file:///D:/mapreduce.tar.gz");
		DBConfiguration.configureDB(conf,"com.mysql.jdbc.Driver", "jdbc:mysql://xxx.xxx.xxxx.xxxx:3306/hive?useUnicode=true", "pig", "123456");
		//Job job = Job.getInstance(conf);
		Job job = Job.getInstance(conf);
        job.setJarByClass(Main.class);
        job.setJobName("Helloworld");

        job.setMapperClass(Db2hdfsMapper.class);

        job.setReducerClass(Db2hdfsReducer.class);

        //job.setOutputKeyClass(LongWritable.class);

        job.setOutputValueClass(Text.class);

        job.setInputFormatClass(DBInputFormat.class);

        FileOutputFormat.setOutputPath(job, new Path("file:///D:/wjj1314"));
        String[] fields = {"ver_id","schema_version","version_comment"};
        DBInputFormat.setInput(job, User.class,"VERSION", "", "", fields);
        System.exit(job.waitForCompletion(true)? 0 : 1);
		

	}

}

以上是主函数

package com.jsptpd.mysqlintolocal;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.lib.db.DBWritable;

public class User implements DBWritable,WritableComparable{
	private Integer ver_id;
	private String  schema_version;
	private String version_comment;
	public User() {
		
	}
	
	public User(Integer ver_id,String  schema_version,String version_comment) {
		this.ver_id = ver_id;
		this.schema_version = schema_version;
		this.version_comment = version_comment;
	}

	public Integer getVer_id() {
		return ver_id;
	}

	public void setVer_id(Integer ver_id) {
		this.ver_id = ver_id;
	}

	public String getSchema_version() {
		return schema_version;
	}

	public void setSchema_version(String schema_version) {
		this.schema_version = schema_version;
	}

	public String getVersion_comment() {
		return version_comment;
	}

	public void setVersion_comment(String version_comment) {
		this.version_comment = version_comment;
	}

	public String toString() {
		return ver_id+"|"+schema_version+"|"+version_comment;
		
	}
	
	public void readFields(DataInput datainput) throws IOException {
		// TODO Auto-generated method stub
		ver_id=datainput.readInt();
		schema_version=datainput.readUTF();
		version_comment=datainput.readUTF();
		
	}

	public void write(DataOutput dataoupt) throws IOException {
		dataoupt.writeInt(ver_id);
		dataoupt.writeUTF(schema_version);
		dataoupt.writeUTF(version_comment);
		
	}

	public void readFields(ResultSet resultset) throws SQLException {
		ver_id=resultset.getInt(1);
		schema_version=resultset.getString(2);
		version_comment=resultset.getString(3);
		
	}

	public void write(PreparedStatement preparestatement) throws SQLException {
		// TODO Auto-generated method stub
		preparestatement.setInt(1, ver_id);
		preparestatement.setString(2, schema_version);
		preparestatement.setString(3, version_comment);
	}
	
	public int compareTo(User o) {
		return this.ver_id.compareTo(o.getVer_id());
	}

}

数据函数

package com.jsptpd.mysqlintolocal;
import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class Db2hdfsMapper extends Mapper{

	@Override
	protected void map(LongWritable key, User value, Mapper.Context context)
			throws IOException, InterruptedException {
		context.write(new LongWritable(value.getVer_id()), new Text(value.toString()));
	}
	

}

map函数

package com.jsptpd.mysqlintolocal;

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class Db2hdfsReducer extends Reducer{

	protected void reduce(LongWritable key, Iterablevalues, Context context)
			throws IOException, InterruptedException {
		for (Iteratoritr = values.iterator(); itr.hasNext();) {
			context.write(key, itr.next());
		}
	}

}

reduce函数