Hadoop自定義輸出排序方式
package com.hgs; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DoubleWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; public class AvgValue { public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { if(args.length!=2) { System.err.println("Usage: MaxTemperature <input path> <output path>"); System.exit(1); } Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "avg of grades"); job.setJarByClass(AvgValue.class); job.setMapperClass(InputClass.class); job.setReducerClass(OutputClass.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(DoubleWritable.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); System.exit(job.waitForCompletion(true)?0:1); } } class InputClass extends Mapper<LongWritable, Text, Text, DoubleWritable>{ @Override protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, DoubleWritable>.Context context) throws IOException, InterruptedException { String line = value.toString(); if(line.length()>0){ String[] array = line.split("\t"); if(array.length==2){ String name=array[0]; int grade = Integer.parseInt(array[1]); context.write(new Text(name), new DoubleWritable(grade)); } } } } class OutputClass extends Reducer<Text, DoubleWritable, NameKey, DoubleWritable>{ @Override protected void reduce(Text text, Iterable<DoubleWritable> iterable, Reducer<Text, DoubleWritable, NameKey, DoubleWritable>.Context context) throws IOException, InterruptedException { int sum = 0; int cnt= 0 ; for(DoubleWritable iw : iterable) { sum+=iw.get(); cnt++; } NameKey nk = new NameKey(text,new DoubleWritable(sum/cnt)); context.write(nk, new DoubleWritable(sum/cnt)); } } //該處透過將輸出記過封裝為一個bean並且實現WritableComparable類,重寫compareTo,來實現對自定義排序 class NameKey implements WritableComparable<NameKey>{ private Text name ; private DoubleWritable grade ; public NameKey(Text name,DoubleWritable grade) { this.name = name; this.grade = grade; } public Text getName() { return name; } public void setName(Text name) { this.name = name; } public DoubleWritable getGrade() { return grade; } public void setGrade(DoubleWritable grade) { this.grade = grade; } @Override public void write(DataOutput out) throws IOException { name.write(out); grade.write(out); } @Override public void readFields(DataInput in) throws IOException { name.readFields(in); grade.readFields(in); } @Override public String toString() { return name.toString(); } @Override public int compareTo(NameKey o) { double me = grade.get(); double other = o.getGrade().get(); int slid = (int)(me-other); return slid; } } //class Maxreducer extends Reducer
來自 “ ITPUB部落格 ” ,連結:http://blog.itpub.net/31506529/viewspace-2213391/,如需轉載,請註明出處,否則將追究法律責任。
相關文章
- hadoop 自定義格式化輸出Hadoop
- Python自定義排序Python排序
- MySQL自定義排序MySql排序
- 表單驗證自定義格式輸出
- hadoop mapreducez自定義分割槽Hadoop
- hadoop自定義許可權Hadoop
- 要點3:輸入函式對比與自定義輸入方式函式
- Java 列表元素自定義排序Java排序
- js:陣列自定義排序JS陣列排序
- Java 對映 自定義排序Java排序
- 排序,檔案輸入輸出排序
- Artisan 自定義輸出格式
- VSCode自定義快捷輸入VSCode
- [2]自定義Lua解析方式
- Javascript陣列排序sort方法和自定義排序方法JavaScript陣列排序
- JAVA Comparator 自定義排序 原始碼分析Java排序原始碼
- mysql自定義排序順序語句MySql排序
- VS code 自定義快捷輸入
- 1122. 陣列的相對排序(計數排序 / 自定義排序)陣列排序
- 自定義 ocelot 中介軟體輸出自定義錯誤資訊
- hadoop 自定義GroupComparator實現求最大值Hadoop
- IDEA自定義設定快捷鍵輸出你想要的語句!Idea
- spark:自定義分割槽,自定義排序,spark與jdbc,廣播變數等Spark排序JDBC變數
- Java中使用lambda表示式自定義排序Java排序
- 織夢dedecms自定義文章排序weight用法排序
- Request 增加自定義欄位的方式
- springbootredis自定義序列化方式(fastJson)Spring BootRedisASTJSON
- [20200317]NULL與排序輸出.txtNull排序
- 操作日誌記錄(包括輸出至自定義日誌檔案)
- 使用Guava的ComparisonChain實現自定義的排序GuavaAI排序
- Artisan 進度條 自定義輸出格式
- Shader從入門到跑路:顏色自定義輸出、紋理取樣
- Laravel Admin 自定義 JavaScript 的正確方式?LaravelJavaScript
- PHP 自定義session儲存 FILE 方式類PHPSession
- PHP 安全輸入輸出方式 「防止 XSS 注入」PHP
- C# 泛型集合的自定義型別排序C#泛型型別排序
- map自定義排序,根據鍵或者值排隊排序
- python的幾種輸出方式Python