ArrayWritable을 반복합니다 - NoSuchMethodException

Question

방금 ​​MapReduce로 작업하기 시작했습니다. Google을 통해 대답 할 수없는 이상한 버그가 발생했습니다. 나는 ArrayWritable를 사용하여 기본 프로그램을 만들고 있어요,하지만 난 그것을 실행할 때, 나는 감소하는 동안 다음과 같은 오류가 발생합니다 : 내가 하둡 1.2.1을 사용하고

java.lang.RuntimeException: 
java.lang.NoSuchMethodException:org.apache.hadoop.io.ArrayWritable.<init>() 
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115) 
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:62) 
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:40) 
at org.apache.hadoop.mapred.Task$ValuesIterator.readNextValue(Task.java:1276) 
at org.apache.hadoop.mapred.Task$ValuesIterator.next(Task.java:1214) 
at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.moveToNext(ReduceTask.java:250) 
at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.next(ReduceTask.java:246) 
at PageRank$Reduce.reduce(Unknown Source) 
at PageRank$Reduce.reduce(Unknown Source) 
at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:522) 
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:421) 
at org.apache.hadoop.mapred.Child$4.run(Child.java:255) 

합니다.

 //while(values.hasNext()){ 
     // tmp = values.next(); 
      output.collect(key, tmp); 
     //} 

모든 것이 확인 될 것입니다 : 내가 선 (클래스를 줄) 아래에 의견을 경우

import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.mapred.*; 
import org.apache.hadoop.io.*; 
import org.apache.hadoop.mapred.join.*; 
import java.io.IOException; 
import java.util.Iterator; 

public class TempClass { 

    public static class MapClass extends MapReduceBase 
    implements Mapper<LongWritable, Text, Text, ArrayWritable> { 
    public void map(LongWritable key, Text value, 
     OutputCollector<Text, ArrayWritable> output, 
     Reporter reporter) throws IOException { 

     String[] arr_str = new String[]{"a","b","c"}; 
     for(int i=0; i<3; i++) 
     output.collect(new Text("my_key"), new ArrayWritable(arr_str)); 
    } 
    }  

    public static class Reduce extends MapReduceBase 
    implements Reducer<Text, ArrayWritable, Text, ArrayWritable> { 

    public void reduce(Text key, Iterator<ArrayWritable> values, 
     OutputCollector<Text, ArrayWritable> output, 
     Reporter reporter) throws IOException { 

     ArrayWritable tmp; 

     while(values.hasNext()){ 
      tmp = values.next(); 
      output.collect(key, tmp); 
     } 
    } 
    } 

    public static void main(String[] args) throws Exception { 
    Configuration conf = new Configuration(); 

    JobConf job = new JobConf(conf, TempClass.class); 

    job.setOutputKeyClass(Text.class); 
    job.setOutputValueClass(ArrayWritable.class); 
    job.setOutputFormat(TextOutputFormat.class); 
    job.setInputFormat(TextInputFormat.class); 

    job.setMapperClass(MapClass.class); 
    job.setReducerClass(Reduce.class); 

    FileInputFormat.setInputPaths(job, new Path(args[0])); 
    FileOutputFormat.setOutputPath(job, new Path(args[1])); 

    job.setJobName("TempClass"); 

    JobClient.runJob(job); 
    } 
} 

: 여기에 내 코드입니다. 아이디어가 있습니까?

Answer 1

A 클래스의 인스턴스를 포함하는 배열에 대해 쓸 수 있습니다. 이 요소는 모두 동일한 클래스의 인스턴스 여야합니다. 이 writable이 Reducer의 입력 값이 될 경우 값을 적절한 유형으로 설정하는 하위 클래스를 만들어야합니다. 예 : 공용 클래스 IntArrayWritable extends ArrayWritable {public IntArrayWritable() {super (IntWritable.class); }}

여기는 ArrayWritable의 문서에 있습니다. 일반적으로 Writable에는 매개 변수가없는 생성자가 있어야합니다.

import java.io.IOException; 
import java.util.Iterator; 

import org.apache.hadoop.conf.Configuration; 
import org.apache.hadoop.fs.Path; 
import org.apache.hadoop.io.ArrayWritable; 
import org.apache.hadoop.io.LongWritable; 
import org.apache.hadoop.io.Text; 
import org.apache.hadoop.mapred.FileInputFormat; 
import org.apache.hadoop.mapred.FileOutputFormat; 
import org.apache.hadoop.mapred.JobClient; 
import org.apache.hadoop.mapred.JobConf; 
import org.apache.hadoop.mapred.MapReduceBase; 
import org.apache.hadoop.mapred.Mapper; 
import org.apache.hadoop.mapred.OutputCollector; 
import org.apache.hadoop.mapred.Reducer; 
import org.apache.hadoop.mapred.Reporter; 
import org.apache.hadoop.mapred.TextInputFormat; 
import org.apache.hadoop.mapred.TextOutputFormat; 

public class TempClass { 

    public static class TextArrayWritable extends ArrayWritable { 
     public TextArrayWritable() { 
      super(Text.class); 
     } 

     public TextArrayWritable(String[] strings) { 
      super(Text.class); 
      Text[] texts = new Text[strings.length]; 
      for (int i = 0; i < strings.length; i++) { 
       texts[i] = new Text(strings[i]); 
      } 
      set(texts); 
     } 
    } 

    public static class MapClass extends MapReduceBase implements 
      Mapper<LongWritable, Text, Text, ArrayWritable> { 
     public void map(LongWritable key, Text value, 
       OutputCollector<Text, ArrayWritable> output, Reporter reporter) 
       throws IOException { 

      String[] arr_str = new String[] { 
        "a", "b", "c" }; 
      for (int i = 0; i < 3; i++) 
       output.collect(new Text("my_key"), new TextArrayWritable(
         arr_str)); 
     } 
    } 

    public static class Reduce extends MapReduceBase implements 
      Reducer<Text, TextArrayWritable, Text, TextArrayWritable> { 

     public void reduce(Text key, Iterator<TextArrayWritable> values, 
       OutputCollector<Text, TextArrayWritable> output, 
       Reporter reporter) throws IOException { 

      TextArrayWritable tmp; 

      while (values.hasNext()) { 
       tmp = values.next(); 
       output.collect(key, tmp); 
      } 
     } 
    } 

    public static void main(String[] args) throws Exception { 
     Configuration conf = new Configuration(); 

     JobConf job = new JobConf(conf, TempClass.class); 

     job.setOutputKeyClass(Text.class); 
     job.setOutputValueClass(TextArrayWritable.class); 
     job.setOutputFormat(TextOutputFormat.class); 
     job.setInputFormat(TextInputFormat.class); 

     job.setMapperClass(MapClass.class); 
     job.setReducerClass(Reduce.class); 

     FileInputFormat.setInputPaths(job, new Path(args[0])); 
     FileOutputFormat.setOutputPath(job, new Path(args[1])); 

     job.setJobName("TempClass"); 

     JobClient.runJob(job); 
    } 
} 

:

난 그냥 당신의 코드를 수정