2

My MapReduce program is as follows:

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import static java.lang.Math.sqrt;

public class WordCount {

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "word count");
        job.setJarByClass(WordCount.class);
        job.setMapperClass(WordCountMapper.class);
        job.setCombinerClass(WordCountReducer.class);
        job.setReducerClass(WordCountReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }

    public static class WordCountMapper extends Mapper<Object, Text, Text, Text> {

        private final static IntWritable one = new IntWritable(1);
        private Text word = new Text();

        public void map(Object key, Text value, Context context) throws IOException,InterruptedException {
            String[] line = value.toString().split(";");
            int Classe= Integer.parseInt(line[5]);
            String F_Name=line[0];
            int dx= Integer.parseInt(line[1])-Integer.parseInt(line[3]);
            int dy= Integer.parseInt(line[4])-Integer.parseInt(line[2]);;
            int q= (int) sqrt((dx*dx)+(dy*dy));
            String Name_Classe=F_Name+","+Classe;
            String res=1+","+q;
            context.write(new Text(Name_Classe),new Text(res));

        }
    }

    public static class WordCountReducer extends Reducer<Text, Text, Text, Text> {
        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<String> values, Context context) throws IOException, InterruptedException {
            int d=0;
            int in=0;
            Iterator<Text> it=context.getValues().iterator();
            while (it.hasNext()){

                String value=it.next().toString();
                d = d + Integer.parseInt(value.split(" ")[0]);
                in = in + Integer.parseInt(value.split(" ")[1]);
            }
            String vars2 = context.getCurrentKey().toString();
            String F_Name=vars2.split(" ")[0];
            int An=Integer.parseInt(vars2.split(" ")[1]);
            //std::string result_Key=context.getInputKey();
            String result_value=d+","+An+","+in;
            context.write(new Text(F_Name), new Text(result_value));
        }
    }

}

My data looks like:

Gr-1;8;8;8;8;0
Gr-1;24;8;24;8;0
Gr-1;40;8;40;8;0
Gr-1;56;8;56;8;0
Gr-2;72;8;72;8;0
Gr-2;88;8;88;8;0
Gr-2;104;8;104;8;0
Gr-2;120;8;120;8;0

The error I get is:

Error: java.io.IOException: Type mismatch in value from map: expected org.apache.hadoop.io.IntWritable, received org.apache.hadoop.io.Text


    at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1077)
        at org.apache.hadoop.mapred.MapTask$NewOutputCollector.write(MapTask.java:715)
        at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
        at org.apache.hadoop.mapreduce.lib.map.WrappedMapper$Context.write(WrappedMapper.java:112)
        at WordCount$WordCountMapper.map(WordCount.java:46)
        at WordCount$WordCountMapper.map(WordCount.java:32)
        at org.apache.hadoop.mapreduce.Mapper.run(Mapper.java:146)
        at org.apache.hadoop.mapred.MapTask.runNewMapper(MapTask.java:787)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:341)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:422)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)

Thank you

1 Answer 1

2

You are setting the output value class as IntWritable but your output values from mapper and reducer are all Text. So hadoop is expecting IntWritable but getting Text instead

So please set the following outputValueClass

job.setOutputValueClass(IntWritable.class);

to

job.setOutputValueClass(Text.class);

in your main function

Sign up to request clarification or add additional context in comments.

3 Comments

My pleasure @MEHDISAOUDI. You can upvote my answer as well ;)
I need 15 reputation to upvote your answer , thanks Ramesh
You wished and your wish is granted :)

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.