Hadoop自定義輸出排序方式
package com.hgs; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class AvgValue { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { if(args.length!=2) { System.err.println("Usage: MaxTemperature <input path> <output path>"); System.exit(1); } Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "avg of grades"); job.setJarByClass(AvgValue.class); job.setMapperClass(InputClass.class); job.setReducerClass(OutputClass.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(DoubleWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true)?0:1); } } class InputClass extends Mapper<LongWritable, Text, Text, DoubleWritable>{ @Override protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, DoubleWritable>.Context context) throws IOException, InterruptedException { String line = value.toString(); if(line.length()>0){ String[] array = line.split("\t"); if(array.length==2){ String name=array[0]; int grade = Integer.parseInt(array[1]); context.write(new Text(name), new DoubleWritable(grade)); } } } } class OutputClass extends Reducer<Text, DoubleWritable, NameKey, DoubleWritable>{ @Override protected void reduce(Text text, Iterable<DoubleWritable> iterable, Reducer<Text, DoubleWritable, NameKey, DoubleWritable>.Context context) throws IOException, InterruptedException { int sum = 0; int cnt= 0 ; for(DoubleWritable iw : iterable) { sum+=iw.get(); cnt++; } NameKey nk = new NameKey(text,new DoubleWritable(sum/cnt)); context.write(nk, new DoubleWritable(sum/cnt)); } } //該處透過將輸出記過封裝為一個bean並且實現WritableComparable類,重寫compareTo,來實現對自定義排序 class NameKey implements WritableComparable<NameKey>{ private Text name ; private DoubleWritable grade ; public NameKey(Text name,DoubleWritable grade) { this.name = name; this.grade = grade; } public Text getName() { return name; } public void setName(Text name) { this.name = name; } public DoubleWritable getGrade() { return grade; } public void setGrade(DoubleWritable grade) { this.grade = grade; } @Override public void write(DataOutput out) throws IOException { name.write(out); grade.write(out); } @Override public void readFields(DataInput in) throws IOException { name.readFields(in); grade.readFields(in); } @Override public String toString() { return name.toString(); } @Override public int compareTo(NameKey o) { double me = grade.get(); double other = o.getGrade().get(); int slid = (int)(me-other); return slid; } } //class Maxreducer extends Reducer
來自 “ ITPUB部落格 ” ,連結:http://blog.itpub.net/31506529/viewspace-2213391/,如需轉載,請註明出處,否則將追究法律責任。
相關文章
- hadoop 自定義格式化輸出Hadoop
- MySQL自定義排序MySql排序
- SQL自定義排序SQL排序
- 表單驗證自定義格式輸出
- DOClever 4.2.0 版本釋出,支援介面自定義排序排序
- 要點3:輸入函式對比與自定義輸入方式函式
- Python自定義排序Python排序
- hadoop streaming 按欄位排序與輸出分割詳解Hadoop排序
- Java 對映 自定義排序Java排序
- js:陣列自定義排序JS陣列排序
- Java 列表元素自定義排序Java排序
- hadoop mapreducez自定義分割槽Hadoop
- hadoop自定義許可權Hadoop
- 排序,檔案輸入輸出排序
- log4j自定義輸出多個檔案
- Artisan 自定義輸出格式
- 自定義UITextView輸入框UITextView
- [2]自定義Lua解析方式
- Hadoop中自定義計數器Hadoop
- hadoop輸出設定Hadoop
- 自定義 ocelot 中介軟體輸出自定義錯誤資訊
- SD--如何在輸出控制中增加自定義欄位
- Javascript陣列排序sort方法和自定義排序方法JavaScript陣列排序
- VS code 自定義快捷輸入
- mysql自定義排序順序語句MySql排序
- YTUOJ-學生成績輸入和輸出(使用者自定義型別)型別
- spark:自定義分割槽,自定義排序,spark與jdbc,廣播變數等Spark排序JDBC變數
- AndroidKeyboard(自定義輸入法)Android
- VSCode自定義快捷輸入VSCode
- JAVA Comparator 自定義排序 原始碼分析Java排序原始碼
- Java中使用lambda表示式自定義排序Java排序
- Request 增加自定義欄位的方式
- awk多行日誌排序輸出排序
- IDEA自定義設定快捷鍵輸出你想要的語句!Idea
- 操作日誌記錄(包括輸出至自定義日誌檔案)
- 自定義popup彈出框
- Artisan 進度條 自定義輸出格式
- 自定義數字格式字串輸出示例字串