Mapreduce原始碼分析分片、處理流程

破棉襖發表於2014-08-12
原始碼

InputFormat（程式碼以TextInputFormat為例）：

FileinputFormat中有三個重要方法：
            1).isSplitable
            2).getSplits
            3).createRecordReader

一.isSplitable方法：
                改方法返回值為bool型別，判斷是否進行分片。
  
二.getSplits方法：
         該方法返回值為List。如果isSplitable返回值為ture，則該方法返回的為分塊之後的檔案，否則為沒有分塊的檔案。
          InputSplit是一個抽象類，FileSplit繼承該類，FileSplit有以下屬性：
              private Path file;               //檔案路徑
              private long start;            //該塊儲存的檔案的起始下標
              private long length;         //該塊儲存塊大小
              private String[] hosts;    //儲存該塊的主機

     getSplits方法原始碼：                 
  1. /**
  2.    * Generate the list of files and make them into FileSplits.
  3.    * @param job the job context
  4.    * @throws IOException
  5.    */
  6.   public List<InputSplit> getSplits(JobContext job) throws IOException {
  7.     long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
  8.     long maxSize = getMaxSplitSize(job);

  10.     // generate splits
  11.     List<InputSplit> splits = new ArrayList<InputSplit>();
  12.     List<FileStatus> files = listStatus(job);                //該方法會遍歷輸入目錄和目錄的子目錄，將檔案資訊儲存到List中。
  13.     for (FileStatus file: files) {                           //遍歷該List將檔案放入到 splits 中。
  14.       Path path = file.getPath();
  15.       long length = file.getLen();
  16.       if (length != 0) {
  17.         BlockLocation[] blkLocations;
  18.         if (file instanceof LocatedFileStatus) {
  19.           blkLocations = ((LocatedFileStatus) file).getBlockLocations();
  20.         } else {
  21.           FileSystem fs = path.getFileSystem(job.getConfiguration());
  22.           blkLocations = fs.getFileBlockLocations(file, 0, length);
  23.         }
  24.         if (isSplitable(job, path)) {                                  //如果為分片
  25.           long blockSize = file.getBlockSize();
  26.           long splitSize = computeSplitSize(blockSize, minSize, maxSize); //獲取分片大小

  28.           long bytesRemaining = length;
  29.           while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {  // SPLIT_SLOP值為1.1 如果 檔案總大小/分片大小<1.1 即使該檔案大小大於塊大小，那該檔案也不會分割。
  30.             int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
  31.             splits.add(makeSplit(path, length-bytesRemaining, splitSize,blkLocations[blkIndex].getHosts()));    
  32.              // makeSplit方法： protected FileSplit makeSplit(Path file, long start, long length, String[] hosts) {
  33.              //            return new FileSplit(file, start, length, hosts);
             
  34.          
    1. bytesRemaining -= splitSize;                                   //          bytesRemaining 減去 已經分片的大小splitSize
  35.           }

  37.           if (bytesRemaining != 0) {                                     
  38.             int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
  39.             splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
  40.                        blkLocations[blkIndex].getHosts()));
  41.           }
  42.         } else { // not splitable                                            //如果不分片，則直接將檔案放入到List中，起始位置為0，大小為檔案總大小。                                                                            
  43.           splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts()));
  44.         }
  45.       } else {
  46.         //Create empty hosts array for zero length files
  47.         splits.add(makeSplit(path, 0, length, new String[0]));
  48.       }
  49.     }
  50.     // Save the number of input files for metrics/loadgen
  51.     job.getConfiguration().setLong(NUM_INPUT_FILES, files.size());
  52.     LOG.debug("Total # of splits: " + splits.size());
            return splits;
      }
 三.createRecordReader方法：
       該方法返回一個RecordReader物件。以支援分片的LineRecordReader為例：
       該物件有四個重要方法：
        1).initialize
        2).nextKeyValue
        3).getCurrentKey
        4).getCurrentValue

     1.initialize方法：     
  1. public void initialize(InputSplit genericSplit,
  2.                          TaskAttemptContext context) throws IOException {
  3.     FileSplit split = (FileSplit) genericSplit;
  4.     Configuration job = context.getConfiguration();
  5.     this.maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE);
  6.     start = split.getStart();                                            //獲取起始下標
  7.     end = start + split.getLength();                                     //獲取大小
  8.     final Path file = split.getPath();

  10.     // open the file and seek to the start of the split
  11.     final FileSystem fs = file.getFileSystem(job);
  12.     fileIn = fs.open(file);                      
  13.     
  14.     CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);   //根據檔案字尾名獲取相應解碼器。 如果輸入檔案為壓縮檔案則會自動獲取。
  15.     if (null!=codec) {                                                         //如果是壓縮檔案
  16.       isCompressedInput = true;    
  17.       decompressor = CodecPool.getDecompressor(codec);
  18.       if (codec instanceof SplittableCompressionCodec) {
  19.         final SplitCompressionInputStream cIn =
  20.           ((SplittableCompressionCodec)codec).createInputStream(
  21.             fileIn, decompressor, start, end,
  22.             SplittableCompressionCodec.READ_MODE.BYBLOCK);
  23.         if (null == this.recordDelimiterBytes){                       //recordDelimiterBytes為終止符，如果讀取一行時遇到該符號則結束。
  24.           in = new LineReader(cIn, job);                              //獲取到LineReader   LineReader封裝了一個InputStream
  25.         } else {
  26.           in = new LineReader(cIn, job, this.recordDelimiterBytes);
  27.         }

  29.         start = cIn.getAdjustedStart();
  30.         end = cIn.getAdjustedEnd();
  31.         filePosition = cIn;
  32.       } else {
  33.         if (null == this.recordDelimiterBytes) {
  34.           in = new LineReader(codec.createInputStream(fileIn, decompressor),
  35.               job);
  36.         } else {
  37.           in = new LineReader(codec.createInputStream(fileIn,
  38.               decompressor), job, this.recordDelimiterBytes);
  39.         }
  40.         filePosition = fileIn;
  41.       }
  42.     } else {
  43.       fileIn.seek(start);
  44.       if (null == this.recordDelimiterBytes){
  45.         in = new LineReader(fileIn, job);
  46.       } else {
  47.         in = new LineReader(fileIn, job, this.recordDelimiterBytes);
  48.       }

  50.       filePosition = fileIn;
  51.     }
  52.     // If this is not the first split, we always throw away first record
  53.     // because we always (except the last split) read one extra line in
  54.     // next() method.
  55.     if (start != 0) {
  56.       start += in.readLine(new Text(), 0, maxBytesToConsume(start));
  57.     }
  58.     this.pos = start;
  59.   }
  2.nextKeyValue方法：
 
  1. public boolean nextKeyValue() throws IOException {
  2.     if (key == null) {
  3.       key = new LongWritable();
  4.     }
  5.     key.set(pos);                                                       //pos預設值為FileSplit的start，也就是行號。
  6.     if (value == null) {
  7.       value = new Text();
  8.     }
  9.     int newSize = 0;
  10.     // We always read one extra line, which lies outside the upper
  11.     // split limit i.e. (end - 1)
  12.     while (getFilePosition() <= end) {
  13.       newSize = in.readLine(value, maxLineLength,                      //獲取到value
  14.           Math.max(maxBytesToConsume(pos), maxLineLength));
  15.       pos += newSize;
  16.       if (newSize < maxLineLength) {
  17.         break;
  18.       }

  20.       // line too long. try again
  21.       LOG.info("Skipped line of size " + newSize + " at pos " + 
                   (pos - newSize));
        }
        if (newSize == 0) {
          key = null;
          value = null;
          return false;
        } else {
          return true;
        }
      }
 3.getCurrentKey 和 getCurrentValue 為獲取 key value

看一下map的run方法：
  1. public void run(Context context) throws IOException, InterruptedException {
  2.     setup(context);
  3.     try {
  4.       while (context.nextKeyValue()) {
  5.         map(context.getCurrentKey(), context.getCurrentValue(), context);
  6.       }
  7.     } finally {
  8.       cleanup(context);
  9.     }
  10.   }



可能不是很全面，主要目的為個人備忘



       

來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/29754888/viewspace-1249907/，如需轉載，請註明出處，否則將追究法律責任。

相關文章