Java 多執行緒讀取檔案並統計詞頻 例項 出神入化的《ThreadPoolExecutor》

重在展示多執行緒ThreadPoolExecutor的使用，和執行緒同步器CountDownLatch，以及相關CAS的原子操作和執行緒安全的Map/佇列。

ThreadPool主執行緒

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;

/**
 * ClassName: ThreadPool
 * Description:
 * date: 2021/1/16 18:24
 *
 * @author hewei
 */
public class ThreadPool {
    /**儲存詞的佇列**/
    private static ArrayBlockingQueue<String> oneWordQueue = new ArrayBlockingQueue<String>(Integer.MAX_VALUE / 4);
    /**儲存詞的次數**/
    private static ConcurrentHashMap<String, AtomicInteger> wordCount = new ConcurrentHashMap<String, AtomicInteger>();
    /**判斷中英文的正則**/
    private static String[] englishOrChinaWordCompile = {"[a-zA-Z]+", "[\\u4e00-\\u9fa5]"};
    private static long waitTimeSecond = 5;
    /**已知會建立10個執行緒池**/
    private static CountDownLatch countDownLatch = new CountDownLatch(10);
    /**儲存詞的次數排序**/
    private static AtomicReferenceArray wordCountArray=new AtomicReferenceArray<String>(Integer.MAX_VALUE/16);

    public static void main(String[] args) {
        BlockingQueue<Runnable> blockingQueue = new LinkedBlockingDeque<>(10000);
        ThreadPoolExecutor readFilePool = new ThreadPoolExecutor(5, 5,
                waitTimeSecond, TimeUnit.SECONDS, blockingQueue, new UserThreadFactory("Study-WordCountReadFile-"),
                new UserRejectHandler());
        //executor.allowCoreThreadTimeOut(true);
        Long beginTime = System.currentTimeMillis();
        //讀取D盤的檔案
        File file = new File("E:\\tmp");
        if (file.isDirectory()) {
            File[] files = file.listFiles();
            // 定義檔案型別
            for (File one : files) {
                String fileName = one.getName();
                readFilePool.execute(new WordCountReadFileTask(countDownLatch, "E:\\tmp\\" + fileName, oneWordQueue, englishOrChinaWordCompile));
            }
        }
        ThreadPoolExecutor sortWordPool = new ThreadPoolExecutor(5, 5,
                waitTimeSecond, TimeUnit.SECONDS, blockingQueue, new UserThreadFactory("Study-WordCount-"),
                new UserRejectHandler());
        //executor1.allowCoreThreadTimeOut(true);
        for (int i = 0; i < 5; i++) {
            sortWordPool.execute(new WordCountTask(countDownLatch, wordCount, oneWordQueue, waitTimeSecond,null));
        }
        try {
            countDownLatch.await();
            readFilePool.shutdown();
            sortWordPool.shutdown();
            // 寫出到檔案
            List<Map.Entry<String, AtomicInteger>> list = new ArrayList(wordCount.entrySet());
            Comparator com = new Comparator<Map.Entry<String, AtomicInteger>>(){
                @Override
                public int compare(Map.Entry<String, AtomicInteger> o1, Map.Entry<String, AtomicInteger> o2) {
                    return ((Integer)o2.getValue().get()).compareTo((Integer) o1.getValue().get());
                }
            };
            list.sort(com);
            // 寫出到檔案
            BufferedWriter bw = new BufferedWriter(new FileWriter("E:\\read.txt"));
            for(int i=0;i<list.size();i++){
                if(i<10) {
                    System.out.println("單詞  " + list.get(i).getKey() + ",次數  " + list.get(i).getValue());
                }
                bw.write("單詞  "+ list.get(i).getKey()+",次數  "+ list.get(i).getValue());
                bw.newLine();
            }
            bw.flush();
            bw.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

 

UserThreadFactory

import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * ClassName: UserThreadFactory
 * Description:自定義執行緒建立工廠
 * date: 2021/1/16 18:26
 *
 * @author hewei
 */
public class UserThreadFactory implements ThreadFactory {
    /**
     * 自定義執行緒名稱字首
     **/
    private final String prefixName;
    /**
     * 執行緒計數器 從1開始
     */
    private final AtomicInteger threadNumber = new AtomicInteger(1);

    public UserThreadFactory(String prefixName) {
        this.prefixName = prefixName;
    }

    @Override
    public Thread newThread(Runnable runnable) {
        //建立執行緒
        String name = prefixName + threadNumber.getAndIncrement();
        return new WorkThread(runnable,name);
    }
    /**
     *自定義工作執行緒，定義執行緒名稱有助於對jvm問題排查
     */
    class WorkThread extends Thread {
        /**
         * 執行緒名稱
         */
        private String name;

        /**
         * @param target  執行的方法
         * @param name 執行緒的名稱
         */
        public WorkThread(Runnable target, String name) {
            super(target);
            super.setName(name);
            this.name=name;
            System.out.println("建立："+name);
        }

        @Override
        public void run() {
            try {
                /**
                 * super.run()等同於target.run()
                 */
                super.run();
            } finally {
                System.out.println("結束執行緒:" + name);
            }
        }
    }
}

WordCountReadFileTask

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * ClassName: WordCountTask
 * Description:
 * date: 2021/1/17 19:48
 *
 * @author hewei
 */
public class WordCountReadFileTask implements Runnable {
    private String filePathAndName;
    private ArrayBlockingQueue<String> oneWordQueue;
    private  String[] englishOrChinaWordCompile;
    private  CountDownLatch countDownLatch;

    public WordCountReadFileTask(CountDownLatch countDownLatch,String filePathAndName, ArrayBlockingQueue<String> oneWordQueue, String[] englishOrChinaWordCompile) {
        this.countDownLatch=countDownLatch;
        this.filePathAndName = filePathAndName;
        this.oneWordQueue = oneWordQueue;
        this.englishOrChinaWordCompile = englishOrChinaWordCompile;
    }

    @Override
    public void run() {
        try {
            BufferedReader br = new BufferedReader(new FileReader(filePathAndName));
            StringBuffer sb = new StringBuffer();
            List<String> strList=new ArrayList<String>();
            String line = "";
            while((line=br.readLine())!=null){
                sb.append(line);
                /**
                 * 為了保證不超過Integer.max_value
                 */
                if(sb.length()>50000000) {
                    strList.add(sb.toString());
                    /**
                     * 清空StringBuffer
                     * 1.delete,從到到尾
                     * 2.new 新的物件。但會丟棄老物件加速gc到來
                     * 3.setlength=0,不符合這裡的場景
                     */
                    sb.delete(0,sb.length());
                }
            }
            if(sb!=null){
                strList.add(sb.toString());
            }
            br.close();
            for(String words:strList) {
                for (String oneCompile : englishOrChinaWordCompile) {
                    //正則
                    Pattern p = Pattern.compile(oneCompile);
                    Matcher matcher = p.matcher(words);
                    while (matcher.find()) {
                        /**
                         * 新增一個元素，如果佇列滿，則阻塞等待佇列被消費騰出空間來
                         */
                        oneWordQueue.put(matcher.group());
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            countDownLatch.countDown();
        }
    }
}

WordCountTask

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * ClassName: WordCountTask
 * Description:
 * date: 2021/1/17 19:48
 *
 * @author hewei
 */
public class WordCountTask implements Runnable {
    private ArrayBlockingQueue<String> oneWordQueue;
    private ConcurrentHashMap<String, AtomicInteger> wordCount;
    private long waitTimeSecond;
    private  CountDownLatch countDownLatch;
    private static AtomicReferenceArray wordCountArray;

    public WordCountTask(CountDownLatch countDownLatch,ConcurrentHashMap<String,AtomicInteger> wordCount,
                         ArrayBlockingQueue<String> oneWordQueue,long waitTimeSecond,AtomicReferenceArray wordCountArray) {
        this.wordCountArray=wordCountArray;
        this.countDownLatch=countDownLatch;
        this.oneWordQueue = oneWordQueue;
        this.wordCount=wordCount;
        this.waitTimeSecond=waitTimeSecond;
    }

    @Override
    public void run() {
        try {
            String oneWord;
            AtomicInteger nowCount;
            while ((oneWord=oneWordQueue.poll(waitTimeSecond, TimeUnit.SECONDS))!=null) {
                /**
                 * 迴圈從佇列裡取出元素，然後加入到map中
                 * 在加入map過程中，程式碼塊會有指令重排問題。所以每一步都需要加判斷。
                 * 所以需要每一個操作都要具有原子性。
                 */
                if((nowCount=wordCount.get(oneWord))==null){
                    nowCount=new AtomicInteger(1);
                    AtomicInteger ifExistCount=wordCount.putIfAbsent(oneWord,nowCount);
                    if(ifExistCount!=null) {
                        ifExistCount.getAndIncrement();
                    }
                }else{
                    nowCount.getAndIncrement();
                }
                /**
                 * 實時排序，該排序依賴執行緒安全
                 * 略
                 */
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            countDownLatch.countDown();
        }
    }
}