说明
在MR中经常会使用的是join，而join分为两种：一是ReduceJoin；二是MapJoin。
ReduceJoin
ReduceJoin工作原理
Map端的主要工作：为来自不同表或文件的key/value时，打标签以区别不同来源的记录。然后用连接字段作为key，其余部分和新加的标志作为value，最后进行输出。
Reduce端的主要工作：在Reduce端以连接字段作为key的分组已经完成，我们只需要在每一个分组当中将那些来源于不同文件的记录（在Map阶段已经打标志）分开，最后进行合并就ok了。
示例
输入文件：
order.txt
pd.txt
OrderBean:订单对象
package com.xing.MapReduce.ReduceJoin;
import org.apache.hadoop.io.Writable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class OrderBean implements Writable {
private String id;
private String pid;
private String num;
private String adress;
private String name;
private String type;
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getPid() {
return pid;
}
public void setPid(String pid) {
this.pid = pid;
}
public String getNum() {
return num;
}
public void setNum(String num) {
this.num = num;
}
public String getAdress() {
return adress;
}
public void setAdress(String adress) {
this.adress = adress;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeUTF(id);
dataOutput.writeUTF(pid);
dataOutput.writeUTF(num);
dataOutput.writeUTF(adress);
dataOutput.writeUTF(name);
dataOutput.writeUTF(type);
}
public void readFields(DataInput dataInput) throws IOException {
this.id= dataInput.readUTF();
this.pid= dataInput.readUTF();
this.num= dataInput.readUTF();
this.adress= dataInput.readUTF();
this.name= dataInput.readUTF();
this.type=dataInput.readUTF();
}
@Override
public String toString() {
return id+"\t"+pid+"\t"+num+"\t"+adress+"\t"+name+"\t"+type;
}
}ReduceJoinMapper：mapper处理类
package com.xing.MapReduce.ReduceJoin;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.IOException;
public class ReduceJoinMapper extends Mapper<LongWritable,Text,Text,OrderBean> {
private OrderBean orderBean = new OrderBean();
private Text k = new Text();
private String fileName;
@Override
protected void setup(Context context) throws IOException, InterruptedException {
FileSplit inputSplit = (FileSplit) context.getInputSplit();
fileName = inputSplit.getPath().getName();
}
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String s = value.toString();
String[] split = s.split("\t", -1);
// 根据不同的输入文件 做不同的操作 最终输出为pid orderbean的kv键值对
if (fileName.startsWith("order")){
// id pid adress num
orderBean.setId(split[0]);
orderBean.setPid(split[1]);
orderBean.setAdress(split[2]);
orderBean.setNum(split[3]);
orderBean.setName("");
orderBean.setType("1");
k.set(split[1]);
}else {
orderBean.setId("");
orderBean.setPid(split[0]);
orderBean.setAdress("");
orderBean.setNum("");
orderBean.setName(split[1]);
orderBean.setType("2");
k.set(split[0]);
}
context.write(k,orderBean);
}
}ReduceJoinReducer：reducer处理类
package com.xing.MapReduce.ReduceJoin;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
public class ReduceJoinReducer extends Reducer<Text,OrderBean,OrderBean,NullWritable> {
@Override
protected void reduce(Text key, Iterable<OrderBean> values, Context context) throws IOException, InterruptedException {
List<OrderBean> list = new ArrayList<OrderBean>();
OrderBean orderBeansTmp = new OrderBean();
// 判断类型 如果是产品类型 则复制一份给orderBeansTmp 否则就添加到list
for (OrderBean orderBean : values) {
if (orderBean.getType().equals("1")){
/**
* 注意这里不能直接添加到list 要先复制一份 如果直接添加都list 呢么添加进去的地址是同一个地址 最后的结果都是当前循环的最后一个
*/
OrderBean orderBean1 = new OrderBean();
try {
BeanUtils.copyProperties(orderBean1,orderBean );
list.add(orderBean1);
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
}else {
try {
BeanUtils.copyProperties(orderBeansTmp,orderBean );
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
}
}
// 循环遍历填充名称
for (OrderBean orderBean : list) {
System.out.println("打印："+orderBean);
orderBean.setName(orderBeansTmp.getName());
context.write(orderBean,NullWritable.get());
}
}
}ReduceJoinDriver：驱动类
package com.xing.MapReduce.ReduceJoin;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class ReduceJoinDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
System.setProperty("hadoop.home.dir", "E:\\hadoop-2.7.1");
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Job job = Job.getInstance(configuration);
job.setJobName("ReduceJoin");
job.setJarByClass(ReduceJoinDriver.class);
job.setMapperClass(ReduceJoinMapper.class);
job.setReducerClass(ReduceJoinReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(OrderBean.class);
job.setOutputKeyClass(OrderBean.class);
job.setOutputValueClass(NullWritable.class);
Path input = new Path("E:\\hdfs\\data\\reducejoin\\input");
Path output = new Path("E:\\hdfs\\data\\reducejoin\\output");
// 删除输出目录数据
if (fs.exists(output)){
fs.delete(output,true );
}
FileInputFormat.setInputPaths(job,input);
FileOutputFormat.setOutputPath(job,output );
boolean b = job.waitForCompletion(true);
System.exit(b?0:-1);
}
}输出结果：
缺陷
reduceJoin容易造成数据倾斜，合并的操作都是在Reduce阶段完成，Reduce端的处理压力很大，Map节点的运算负载很低，资源利用率不高，且在Reduce阶段易产生数据倾斜。
怎么解决？
Map端实现数据合并，这就是MapJoin
MapJoin
使用前提： 缓存的文件必须是小文件，文件是直接加载到内存中，如果文件太大，容易内存不足。
优点：
在Map端缓存多张表，提前处理业务逻辑，这样增加Map端业务，减少Reduce端数据的压力，尽可能的减少数据倾斜。
具体办法：
采用DistributedCache
（1）在Mapper的setup阶段，将文件读取到缓存集合中。
（2）在驱动函数中加载缓存。job.addCacheFile(new URI("XXXXX"));
示例
MapJoinDriver: 驱动类
package com.xing.MapReduce.MapJoin;
import com.xing.MapReduce.ReduceJoin.ReduceJoinReducer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class MapJoinDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
System.setProperty("hadoop.home.dir", "E:\\hadoop-2.7.1");
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.get(configuration);
Job job = Job.getInstance(configuration);
job.setJobName("MapJoin");
job.setJarByClass(MapJoinDriver.class);
job.setMapperClass(MapJoinMapper.class);
job.setReducerClass(ReduceJoinReducer.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
// 添加缓存文件
job.addCacheFile(new Path("E:\\hdfs\\data\\reducejoin\\input1\\pd.txt").toUri());
job.setNumReduceTasks(0);
Path input = new Path("E:\\hdfs\\data\\reducejoin\\input");
Path output = new Path("E:\\hdfs\\data\\reducejoin\\output");
// 删除输出目录数据
if (fs.exists(output)){
fs.delete(output,true );
}
FileInputFormat.setInputPaths(job,input);
FileOutputFormat.setOutputPath(job,output );
boolean b = job.waitForCompletion(true);
System.exit(b?0:-1);
}
}MapJoinMapper: map端处理
package com.xing.MapReduce.MapJoin;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import java.io.*;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
public class MapJoinMapper extends Mapper<LongWritable,Text,Text,NullWritable> {
private Map<String,String> map = new HashMap<String, String>();
private Text k = new Text();
@Override
protected void setup(Context context) throws IOException, InterruptedException {
// 读取缓存文件
URI[] cacheFiles = context.getCacheFiles();
String cacheFile = cacheFiles[0].getPath();
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(cacheFile),"UTF-8"));
String line;
while (StringUtils.isNotEmpty(line = reader.readLine())){
String[] strings = line.split("\t",-1);
map.put(strings[0],strings[1]);
}
reader.close();
}
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String s = value.toString();
String[] split = s.split("\t", -1);
String name = map.get(split[1]);
s =s.concat("\t"+name);
k.set(s);
context.write(k,NullWritable.get());
}
}重点看setup方法。
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
