MapReduce代码示例

  1. import com.emc.greenplum.gpdb.hadoop.io.GPDBWritable;
  2. import com.emc.greenplum.gpdb.hadoop.mapreduce.lib.input.GPDBInputFormat;
  3. import com.emc.greenplum.gpdb.hadoop.mapreduce.lib.output.GPDBOutputFormat;
  4. import java.io.*;
  5. import java.util.*;
  6. import org.apache.hadoop.fs.Path;
  7. import org.apache.hadoop.conf.*;
  8. import org.apache.hadoop.io.*;
  9. import org.apache.hadoop.mapreduce.*;
  10. import org.apache.hadoop.mapreduce.lib.output.*;
  11. import org.apache.hadoop.mapreduce.lib.input.*;
  12. import org.apache.hadoop.util.*;
  14. public class demoMR {
  16. /*
  17.  * Helper routine to create our generic record. This section shows the
  18.  * format of the data. Modify as necessary. 
  19.  */
  20.  public static GPDBWritable generateGenericRecord() throws
  21.       IOException {
  22.  int[] colType = new int[3];
  23.  colType[0] = GPDBWritable.BIGINT;
  24.  colType[1] = GPDBWritable.BOOLEAN;
  25.  colType[2] = GPDBWritable.VARCHAR;
  27.   /*
  28.    * This section passes the values of the data. Modify as necessary. 
  29.    */ 
  30.   GPDBWritable gw = new GPDBWritable(colType); 
  31.   gw.setLong (0, (long)12345);  
  32.   gw.setBoolean(1, true); 
  33.   gw.setString (2, "abcdef");
  34.   return gw; 
  35. } 
  37. /* 
  38.  * DEMO Map/Reduce class test1
  39.  * -- Regardless of the input, this section dumps the generic record
  40.  * into GPDBFormat/
  41.  */
  42.  public static class Map_test1 
  43.      extends Mapper<LongWritable, Text, LongWritable, GPDBWritable> {
  45.   private LongWritable word = new LongWritable(1);
  47.   public void map(LongWritable key, Text value, Context context) throws
  48.        IOException { 
  49.     try {
  50.       GPDBWritable gw = generateGenericRecord();
  51.       context.write(word, gw); 
  52.       } 
  53.       catch (Exception e) { 
  54.         throw new IOException (e.getMessage()); 
  55.       } 
  56.     }
  57.   }
  59.   Configuration conf = new Configuration(true);
  60.   Job job = new Job(conf, "test1");
  61.   job.setJarByClass(demoMR.class);
  62.   job.setInputFormatClass(TextInputFormat.class);
  63.   job.setOutputKeyClass (LongWritable.class);
  64.   job.setOutputValueClass (GPDBWritable.class);
  65.   job.setOutputFormatClass(GPDBOutputFormat.class);
  66.   job.setMapperClass(Map_test1.class);
  67.   FileInputFormat.setInputPaths (job, new Path("/demo/data/tmp"));
  68.   GPDBOutputFormat.setOutputPath(job, new Path("/demo/data/MRTest1"));
  69.   job.waitForCompletion(true);
  70. }

上级主题： 一次性HDFS协议安装