Hadoop Streaming 讀ORC檔案

【背景】

hadoop Streaming的處理流程是先通過inputFormat讀出輸入檔案內容，將其傳遞mapper，再將mapper返回的key，value傳給reducer，最後將reducer返回的值通過outputformat寫入輸出檔案。
目前有個需求是通過hadoop streaming讀取roc檔案。使用正常的org.apache.orc.mapred.OrcInputFormat讀orc檔案時每行返回的值是：

null    {"name":"123","age":"456"}
null    {"name":"456","age":"789"}

返回這種資料的原因是OrcInputFormat讀取檔案返回的值是<NullWritable, OrcStruct>, NullWritable toString的返回值是null， OrcStruct toString的返回值是一個json串。
需要開發一個轉換器，只返回OrcInputFormat返回的json串的value即可。即返回：

123 456
456 789

【重寫InputFormat，單檔案讀取】

package is.orc;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.*;
import org.apache.orc.TypeDescription;
import org.apache.orc.mapred.OrcInputFormat;
import org.apache.orc.mapred.OrcMapredRecordReader;
import org.apache.orc.mapred.OrcStruct;
import org.apache.orc.Reader;
import org.apache.orc.Reader.Options;
import java.io.IOException;

public class OrcInputAsTextInputFormat extends org.apache.hadoop.mapred.FileInputFormat<Text, Text> {
    //真正讀檔案的還是OrcInputFormat
    protected OrcInputFormat<OrcStruct> orcInputFormat = new OrcInputFormat();

    public RecordReader<Text, Text> getRecordReader(InputSplit split, JobConf job, Reporter reporter) throws IOException {
        OrcMapredRecordReader realReader = (OrcMapredRecordReader) orcInputFormat.getRecordReader(split, job, reporter);
        return new TextRecordReaderWrapper
                (realReader);
    }

    public static boolean[] parseInclude(TypeDescription schema, String columnsStr) {
        return OrcInputFormat.parseInclude(schema, columnsStr);
    }

    public static void setSearchArgument(Configuration conf, SearchArgument sarg, String[] columnNames) {
        OrcInputFormat.setSearchArgument(conf, sarg, columnNames);
    }

    public static Options buildOptions(Configuration conf, Reader reader, long start, long length) {
        return OrcInputFormat.buildOptions(conf, reader, start, length);
    }

    protected static class TextRecordReaderWrapper implements RecordReader<Text, Text> {

        private OrcMapredRecordReader realReader;
        private OrcStruct orcVal ;
        private StringBuilder buffer;
        private final int numOfFields;
        public TextRecordReaderWrapper(OrcMapredRecordReader realReader) throws IOException{
            this.realReader = realReader;
            this.orcVal = (OrcStruct)realReader.createValue();
            this.buffer = new StringBuilder();
            this.numOfFields = this.orcVal.getNumFields();
        }

        public boolean next(Text key, Text value) throws IOException {
            // 將第一個欄位作為key，剩餘的欄位以\t為分隔符組成字串作為value
            if (realReader.next(NullWritable.get(), orcVal)){
                buffer.setLength(0); //清空buffer
                key.set(orcVal.getFieldValue(0).toString());
                //以\t為分隔符，組裝返回值
                for(int i = 1; i < numOfFields; ++i) {
                    buffer.append("\t");
                    WritableComparable curField = orcVal.getFieldValue(i);
                    if (curField != null && ! curField.equals(NullWritable.get())){
                        buffer.append(curField.toString());
                    }

                }
                value.set(buffer.substring(1)); //去掉開始新增的\t
                return Boolean.TRUE;
            }

            return Boolean.FALSE;
        }

        public Text createKey() {
            return new Text();
        }

        public Text createValue() {
            return new Text();
        }

        public long getPos() throws IOException {
            return realReader.getPos();
        }

        public void close() throws IOException {
            realReader.close();
        }

        public float getProgress() throws IOException {
            return realReader.getProgress();
        }
    }
}

【多檔案讀取】

MapReduce在讀資料的時候可以通過合併小檔案的方式減少map個數，比如說CombineSequenceFileInputFormat。如果不合並小檔案，可能出現map數過大的情況,資源消耗過多，且執行效率很慢。對應到orc格式時沒找到官方提供的包，只能自己寫一個。具體程式碼如下：

package is.orc;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapred.lib.CombineFileRecordReader;
import org.apache.hadoop.mapred.lib.CombineFileRecordReaderWrapper;
import org.apache.hadoop.mapred.lib.CombineFileSplit;

import java.io.IOException;

public class CombineOrcInputAsTextInputFormat
        extends CombineFileInputFormat<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> {
    @SuppressWarnings({ "rawtypes", "unchecked" })
    public RecordReader<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> getRecordReader(InputSplit split, JobConf conf,
                                             Reporter reporter) throws IOException {
        return new CombineFileRecordReader(conf, (CombineFileSplit)split, reporter,
                ORCFileRecordReaderWrapper.class);
    }

    /**
     * A record reader that may be passed to <code>CombineFileRecordReader</code>
     * so that it can be used in a <code>CombineFileInputFormat</code>-equivalent
     * for <code>SequenceFileInputFormat</code>.
     *
     * @see CombineFileRecordReader
     * @see CombineFileInputFormat
     * @see SequenceFileInputFormat
     */
    private static class ORCFileRecordReaderWrapper
            extends CombineFileRecordReaderWrapper<org.apache.hadoop.io.Text, org.apache.hadoop.io.Text> {
        // this constructor signature is required by CombineFileRecordReader
        public ORCFileRecordReaderWrapper(CombineFileSplit split,
                                               Configuration conf, Reporter reporter, Integer idx) throws IOException {
            //只需配置此處的InputFormat為第一部分編寫的OrcInputAsTextInputFormat即可。具體的合併操作，CombineFileInputFormat已幫我們實現
            super(new OrcInputAsTextInputFormat(), split, conf, reporter, idx);
        }
    }
}