資料遷移方案 + Elasticsearch在綜合搜尋列表實現

王阿汪?發表於2018-08-29
Elasticsearch

1 UML

基本類圖

2 遷移實現

2.1 從資料庫讀取進件id存到檔案中

    @RequestMapping(value = "***", method = {RequestMethod.GET})
    @ResponseBody
    public void writeFile(Integer id) {
        if (null == id) {
            pieceSearchService.writePieceFile();
        } else {
            pieceSearchService.writePieceFile(id);
        }
    }
    
    // PieceSearchServiceImpl
    public void writePieceFile() {
        dataMoveManager.startWriteFile();
    }

複製程式碼
    // AbstractTemplate
    public void startWriteFile() {
        readDataAndWriteFile();
    }
    // DataMoveManager
    protected void readDataAndWriteFile() {
        pieceInfoWriteFileTask.start();
    }
    
    // AbstractTaskTemplate
    public void start() {
        List<? extends Callable<String>> task = createTask();
        ExecutorService executorService = createExecutorService();
        List list = runTaskForResult(executorService, task);
        writeLogFile(list);
    }
    
    // PieceInfoWriteFileTask
    protected List<? extends Callable<String>> createTask() {
        //獲取進件起始id 並構建任務
        //查詢所有進件資料（500W+）
        List<PieceIdAndIdDTO> pieceIdAndIdDTOS = pieceSearchDao.queryAllPieceInfo();
        List<MyTask> tasks = this.getTaskFromPieceId(pieceIdAndIdDTOS);
        return tasks;
    }
    
    // PieceInfoWriteFileTask
    List<MyTask> getTaskFromPieceId(List<PieceIdAndIdDTO> pieceIdAndIdDTOS) {
        //任務集合
        List<MyTask> result = Lists.newArrayList();
        Integer index = 1;
        int fileName = 1;
        List<PieceIdAndIdDTO> ids = Lists.newArrayList();
        for (PieceIdAndIdDTO pieceIdAndIdDTO : pieceIdAndIdDTOS) {
            ids.add(pieceIdAndIdDTO);
            //每 條建立一個任務
            if (index % FILE_MAX_NUM == 0) {
                //建立任務
                MyTask myTask = new MyTask();
                myTask.setIds(ids);
                myTask.setFileName(Integer.toString(fileName));
                result.add(myTask);
                ids = Lists.newArrayList(); //一定要new
                fileName++;
            }
            index++;
        }
        if (!ids.isEmpty()) {
            MyTask myTask = new MyTask();
            myTask.setIds(ids);
            myTask.setFileName(Integer.toString(fileName));
            result.add(myTask);
        }
        return result;
    }
    // PieceInfoWriteFileTask
    protected ExecutorService createExecutorService() {
        //建立執行緒池
        ExecutorService executorService = Executors.newFixedThreadPool(MAX_THREAD_NUM);
        return executorService;
    }
    
    // AbstractTaskTemplate
    private List runTaskForResult(ExecutorService executorService, List<? extends Callable<String>> tasks) {
        //執行緒池執行任務
        List<Future<String>> futures = new ArrayList<>();
        try {
            futures = executorService.invokeAll(tasks);
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            //關閉執行緒池
            executorService.shutdown();
        }
        //組裝返回結果
        List<String> results = Lists.newArrayList();
        for (Future<String> future : futures) {
            try {
                results.add(future.get());
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }
        return results;
    }
    
    // PieceInfoWriteFileTask
    protected void writeLogFile(List results) {
        //日誌檔名稱
        String fileName = "writeFile.log";
        //檔案路徑
        String filePath = ABSOLUTE_PATH + FILE_ROOT_PATH + fileName;
        //按行寫出
        try {
            FileKit.writeLines(results, filePath, "UTF-8");
        } catch (IOException e) {
            e.printStackTrace();
            log.error("class:PieceInfoWriteFileTask  method: writeLogFile()" +
                " 日誌資訊寫出失敗, 失敗原因：" + e.getMessage());
        }
    }
複製程式碼
    // PieceInfoWriteFileTask
    class MyTask implements Callable<String> {

        private List<PieceIdAndIdDTO> ids;

        private String fileName;

        @Override
        public String call() {
            String message = "檔案生成失敗";
            try {
                message = writeFile(ids, fileName);
                long l2 = System.currentTimeMillis();
            } catch (Exception e) {
                return message + "----失敗原因：" + e.getMessage();
            }
            return message;
        }
    }
    
    // PieceInfoWriteFileTask
    private String writeFile(List<PieceIdAndIdDTO> pieceIdAndId, String fileName) {
        //檔案路徑+檔名
        String filePath = ABSOLUTE_PATH + FILE_ROOT_PATH + fileName + ".txt";
        try {
            log.info("檔案輸出路徑：" + filePath);
            //輸出檔案
            FileKit.writeLines(pieceIdAndId, filePath, "UTF-8");
        } catch (IOException e) {
            return "此檔案生成失敗：" + fileName + "失敗原因：" + e.getMessage();
        }
        return "此檔案生成成功：" + fileName;
    }
複製程式碼

2.2 資料遷移

2.2.1 從檔案讀取id存到佇列中

    @RequestMapping(value = "***", method = {RequestMethod.GET})
    @ResponseBody
    public void dataMove() {
        pieceSearchService.pieceDataMove();
    }
    
    // PieceSearchServiceImpl
    public void pieceDataMove() {
        dataMoveManager.startReadForDb();
    }
    
    
複製程式碼
    // AbstractTemplate
    public void startReadForDb() {
        readFileForDb();
    }
    // DataMoveManager
    protected void readFileForDb() {
        pieceInfoDataMoveTask.start();
    }
    
    // AbstractTaskTemplate
    public void start() {
        List<? extends Callable<String>> task = createTask();
        ExecutorService executorService = createExecutorService();
        List list = runTaskForResult(executorService, task);
        writeLogFile(list);
    }
    
    // PieceInfoDataMoveTask
    protected List<? extends Callable<String>> createTask() {

        //獲取此路徑下的所有.txt檔案
        List<File> files = null;
        try {
            files = FileKit.listFilesWithSuffix(new File(ABSOLUTE_PATH + FILE_ROOT_PATH), ".txt");
        } catch (Exception e) {
            e.printStackTrace();
            log.error("class:PieceInfoDataMoveTask  method: createTask()" +
                " 建立任務組失敗, 失敗原因：" + e.getMessage());
        }

        //根據檔案構建任務
        List<MyTask> tasks = getTaskWithFile(files);
        return tasks;
    }
    
    // PieceInfoDataMoveTask
     private List<MyTask> getTaskWithFile(List<File> files) {
        ArrayList<MyTask> tasks = Lists.newArrayList();
        for (File file : files) {
            MyTask myTask = new MyTask();
            myTask.setFile(file);
            tasks.add(myTask);
        }
        return tasks;
    }
    
    // PieceInfoDataMoveTask
    private class MyTask implements Callable<String> {

        private File file;

        @Override
        public String call() throws Exception {
            //每次儲存的進件Id
            ArrayList<String> ids = new ArrayList<>();
            //開始時間
            long beginTime = System.currentTimeMillis();
            //檔名稱
            String fileName = file.getName();
            BufferedReader reader = null;
            //行資料
            String line;
            int a = 1;
            try {
                reader = FileKit.getReader(file, "UTF-8");
                while (true) {
                    line = reader.readLine();
                    if (Func.isEmpty(line)) {
                        break;
                    }
                    //每行資料格式為  id:1121  pieceId:ZPTE1213
                    String id = line.split("\\s+")[0].split(":")[1];
                    ids.add(id);
                    //每多少行  執行插入操作
                    if (a % MAX_SAVE_NUM == 0) {
                        //存入佇列
                        QueueUtil.QUEUE_IDS.put(ids);
                        ids = Lists.newArrayList();
                    }
                    a++;
                }
            } catch (Exception e) {
                return "讀取" + fileName + "檔案失敗原因：" + e.getMessage();
            } finally {
                if (!ids.isEmpty()) {
                    //存入佇列
                    QueueUtil.QUEUE_IDS.put(ids);
                }
                IoKit.close(reader);
            }
            long useTime = (System.currentTimeMillis() - beginTime) / 1000;
            return "讀取" + fileName + "檔案完畢，耗時：" + useTime + "秒 , 檔案共：" + (a - 1) + "條資料";
        }
    }
    
    // QueueUtil
    public class QueueUtil {

    /**
     *佇列一：存放從檔案中讀取的id集合
     */
    public static final LinkedBlockingQueue<List<String>> QUEUE_IDS = new LinkedBlockingQueue<List<String>>();

    /**
     * 佇列二：存放組裝資料過後的List<JsonObject>
     */
    public static final  LinkedBlockingQueue<List<JSONObject>> QUEUE_JSON_OBJECTS = new LinkedBlockingQueue<List<JSONObject>>();


    /**
     *佇列三：存放所有失敗id集合
     */
    public static final LinkedBlockingQueue<List<String>> QUEUE_FAIL_IDS = new LinkedBlockingQueue<List<String>>();

    /**
     *佇列四：存放所有校驗過的集合
     */
    public static final LinkedBlockingQueue<SaveToMysqlDTO> QUEUE_SAVETOMYSQL = new LinkedBlockingQueue<SaveToMysqlDTO>();

    /**
     *佇列五：存放所有校驗過的集合
     */
    public static final LinkedBlockingQueue<List<Document>> QUEUE_SAVETOMONGO = new LinkedBlockingQueue<List<Document>>();


    /**
     *佇列六：存放所有校驗過的集合
     */
    public static final LinkedBlockingQueue<Map<Long,PieceJsonDTO>> QUEUE_SAVETOES = new LinkedBlockingQueue< Map<Long,PieceJsonDTO>>();
}

複製程式碼
    // PieceInfoDataMoveTask
    protected ExecutorService createExecutorService() {
        //建立執行緒池
        ExecutorService executorService = Executors.newFixedThreadPool(Integer.valueOf(MAX_THREAD_NUM));
        return executorService;
    }
    
    // AbstractTaskTemplate
    private List runTaskForResult(ExecutorService executorService, List<? extends Callable<String>> tasks) {
        //執行緒池執行任務
        List<Future<String>> futures = new ArrayList<>();
        try {
            futures = executorService.invokeAll(tasks);
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            //關閉執行緒池
            executorService.shutdown();
        }
        //組裝返回結果
        List<String> results = Lists.newArrayList();
        for (Future<String> future : futures) {
            try {
                results.add(future.get());
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }
        return results;
    }
    
    // PieceInfoDataMoveTask
    protected void writeLogFile(List results) {
    }

複製程式碼

2.2.2 消費進件id佇列（QUEUE_IDS）

    //spring 配置檔案載入bean 呼叫init() 方法
    <bean  class="com.migration.eagle.manager.PieceSearchManager" init-method="init" />
    /**
     * 啟動執行緒
     */
    // PieceSearchManager
    private void init() {
        //組裝JsonObject
        readQueueForAssembleJsonThread.start();
        log.info("執行緒1啟動完畢，正在讀取ids佇列！！！！");
        //校驗資料
        readQueueForValidateThread.start();
        log.info("執行緒2啟動完畢，正在讀取List<JsonObject>佇列！！！！");
        //入庫資料
        readQueueForSaveMysqlThread.start();
        log.info("執行緒3啟動完畢，正在讀取saveToMysql佇列！！！！");
        //入庫Mongo
        readQueueForSaveMongoThread.start();
        log.info("執行緒4啟動完畢，正在讀取saveToMongo佇列！！！！");
        //入庫es
        readQueueForSaveESThread.start();
        log.info("執行緒5啟動完畢，正在讀取saveToES佇列！！！！");
        //寫出失敗日誌
        readQueueForWriteLogThread.start();
        log.info("執行緒6啟動完畢，正在讀取失敗ids佇列！！！！");
    }   
複製程式碼
public class ReadQueueForAssembleJsonThread extends Thread {

    @Autowired
    PieceAssembleManager pieceAssembleManager;

    @Override
    public void run() {
        //建立執行緒池
        //任務數
        int countThread = Integer.valueOf(new Config().getMaxAssembleJsonThreadNum());
        ExecutorService executorService = Executors.newFixedThreadPool(countThread);
        for (int i = 0; i < countThread; i++) {
            MyTask task = new MyTask();
            executorService.execute(task);
        }
        executorService.shutdown();
    }


    @Data
    class MyTask implements Runnable {

        @Override
        public void run() {
            while (true) {
                try {
                    //讀取佇列中的ids
                    List<String> ids = QueueUtil.QUEUE_IDS.take();
                    if (null != ids) {
                        long l1 = System.currentTimeMillis();
                        //組裝資料
                        Map<String, List> pieceJson_map = pieceAssembleManager.getPieceJson(ids);
                        //存入佇列
                        QueueUtil.QUEUE_JSON_OBJECTS.put(pieceJson_map.get("success"));
                        //存入失敗ids佇列
                        QueueUtil.QUEUE_FAIL_IDS.put(pieceJson_map.get("fail"));
                        long l2 = System.currentTimeMillis();
                        log.info("組裝JSON耗時：" + (l2 - l1)," ，佇列大小： "+QueueUtil.QUEUE_IDS.size());
                        ids=null;
                        pieceJson_map=null;
                    }
                } catch (Exception e) {
                    log.error("組裝資料失敗，失敗原因： "+e.getMessage());
                    e.printStackTrace();
                }
            }
        }
    }


}
複製程式碼

2.2.3 消費JsonObject佇列（QUEUE_JSON_OBJECTS）

public class ReadQueueForValidateThread extends Thread {

    @Autowired
    PieceSearchManager pieceSearchManager;


    @Override
    public void run() {
        //建立執行緒池
        int countThread = Integer.valueOf(new Config().getMaxValidateThreadNum());
        ExecutorService executorService = Executors.newFixedThreadPool(countThread);
        List<JSONObject> jsonObjects = Lists.newArrayList();
        while (true) {
            try {
                //從佇列中讀取List<JsonObject>
                List<JSONObject> poll = QueueUtil.QUEUE_JSON_OBJECTS.take();
                if (null != poll) {
                    //建立任務
                    for (int i = 1; i <= poll.size(); i++) {
                        jsonObjects.add(poll.get(i - 1));
                        if (i % 100 == 0) {
                            MyTask myTask = new MyTask();
                            myTask.setJsonObjects(jsonObjects);
                            executorService.execute(myTask);
                            jsonObjects = Lists.newArrayList();
                        }
                    }
                    if (!jsonObjects.isEmpty()) {
                        MyTask myTask = new MyTask();
                        myTask.setJsonObjects(jsonObjects);
                        executorService.execute(myTask);
                        jsonObjects = Lists.newArrayList();
                    }
                    log.info("校驗佇列大小： "+QueueUtil.QUEUE_JSON_OBJECTS.size());
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }


    }


    @Data
    class MyTask implements Runnable {
        private List<JSONObject> jsonObjects;

        @Override
        public void run() {
            long l = System.currentTimeMillis();
            SaveToMysqlDTO dto = new SaveToMysqlDTO();
            //mongo佇列所需資料
            List<Document> toMongos = Lists.newArrayList();
            //es佇列所需資料
            Map<Long,PieceJsonDTO>  id_PieceJsonDTO =Maps.newHashMap();

            List<InvestigatePieceEntity> investigatePieceDTOS = Lists.newArrayList();
            List<PieceDetailEntity> pieceDetailDTOS = Lists.newArrayList();
            List<PieceContactEntity> pieceContactDTOS = Lists.newArrayList();
            //失敗結果
            List<String> failedJson = Lists.newArrayList();
            try {
                for (JSONObject finalObj : jsonObjects) {
                    Map<String, JSONObject> map = pieceSearchManager.validatePiece(finalObj);
                    if (map.containsKey("successData")) {
                        pieceSearchManager.savePiecesLast(map.get("successData"), investigatePieceDTOS, pieceContactDTOS, pieceDetailDTOS,id_PieceJsonDTO);
                        //增加id(存入mongo使用)
                        JsonObject jsonObject = JsonKit.parseJsonObject(finalObj.toString());
                        JsonElement appkey = JsonKit.getValueByPath(jsonObject, MongoPieceManagerImpl.PIECE_CODE_JSON_PATH);
                        jsonObject.add(MongoPieceManagerImpl.MONGO_ID, appkey);
                        toMongos.add(Document.parse(jsonObject.toString()));
                    } else {
                        failedJson.add(((Map) ((Map) map.get("failed").get("data")).get("applicant")).get("intpc_id").toString());
                    }
                    map=null;
                }
                dto.setInvestigatePieceDTOS(investigatePieceDTOS);
                dto.setPieceContactDTOS(pieceContactDTOS);
                dto.setPieceDetailDTOS(pieceDetailDTOS);

                //存入失敗佇列
                QueueUtil.QUEUE_FAIL_IDS.put(failedJson);
                //存入入庫佇列
                QueueUtil.QUEUE_SAVETOMYSQL.put(dto);
                QueueUtil.QUEUE_SAVETOMONGO.put(toMongos);
                QueueUtil.QUEUE_SAVETOES.put(id_PieceJsonDTO);

                long l1 = System.currentTimeMillis();
                log.info("校驗耗時：" + (l1 - l));
                dto=null;
                toMongos=null;
                id_PieceJsonDTO=null;
                failedJson=null;
                jsonObjects=null;
            } catch (Exception e) {
                log.error("校驗失敗，失敗原因：  "+e.getMessage());
                e.printStackTrace();
            }
        }
    }

}
複製程式碼

2.2.4 消費儲存到資料庫資料佇列（QUEUE_SAVETOMYSQL）

public class ReadQueueForSaveMysqlThread extends Thread {

    @Autowired
    PieceSearchManager pieceSearchManager;

    @Override
    public void run() {
        //建立執行緒池
        //任務數
        int countThread = Integer.valueOf(new Config().getMaxSaveMysqlThreadNum());
        ExecutorService executorService = Executors.newFixedThreadPool(countThread);
        for (int i = 0; i < countThread; i++) {
            MyTask task = new MyTask();
            executorService.execute(task);
        }
        executorService.shutdown();
    }


    @Data
    class MyTask implements Runnable {

        @Override
        public void run() {
            while (true) {
                try {
                    //從佇列中讀取校驗過後的資料
                    SaveToMysqlDTO dto = QueueUtil.QUEUE_SAVETOMYSQL.take();
                    if (null != dto) {
                        //資料入庫
                        long l = System.currentTimeMillis();
                        pieceSearchManager.savePieceToSql(dto);
                        long l1 = System.currentTimeMillis();
                        log.info("入庫Mysql耗時：" + (l1 - l)+" , 佇列大小： "+QueueUtil.QUEUE_SAVETOMYSQL.size());
                    }
                } catch (InterruptedException e) {
                    log.error("插入Mysql失敗，失敗原因：  "+e.getMessage());
                    e.printStackTrace();
                }
            }
        }
    }

}
複製程式碼

2.2.5 消費儲存到mongoDB資料佇列（QUEUE_SAVETOMONGO）

public class ReadQueueForSaveMongoThread extends Thread {

    @Autowired
    MongoUtil mongoUtil;

    private MongoCollection mongoCollection;

    @Override
    public void run() {
        //獲取mongo 連結
        mongoCollection = mongoUtil.mongoClient();
        //建立執行緒池
        //任務數
        int countThread = Integer.valueOf(new Config().getMaxSaveMongoThreadNum());
        ExecutorService executorService = Executors.newFixedThreadPool(countThread);
        for (int i = 0; i < countThread; i++) {
            MyTask task = new MyTask();
            executorService.execute(task);
        }
        executorService.shutdown();
    }


    @Data
    class MyTask implements Runnable {

        @Override
        public void run() {
            while (true) {
                try {
                    //從佇列中讀取校驗過後的資料
                    List<Document> documents = QueueUtil.QUEUE_SAVETOMONGO.take();
                    if (null != documents) {
                        //資料入庫
                        long l = System.currentTimeMillis();
                        mongoCollection.insertMany(documents);
                        long l1 = System.currentTimeMillis();
                        log.info("Mongo耗時：" + (l1 - l)+", 佇列大小： "+ QueueUtil.QUEUE_SAVETOMONGO.size());
                    }
                } catch (Exception e) {
                    log.error("插入mongo失敗，失敗原因：  "+e.getMessage());
                    e.printStackTrace();
                }
            }
        }
    }

}

複製程式碼

2.2.6 消費儲存到Elasticsearch資料佇列（QUEUE_SAVETOES）

public class ReadQueueForSaveESThread extends Thread {

    @Autowired
    ElasticsearchManager elasticsearchManager;

    @Override
    public void run() {
        //建立執行緒池
        //任務數
        int countThread = Integer.valueOf(new Config().getMaxSaveESThreadNum());
        ExecutorService executorService = Executors.newFixedThreadPool(countThread);
        for (int i = 0; i < countThread; i++) {
            MyTask task = new MyTask();
            executorService.execute(task);
        }
        executorService.shutdown();
    }


    @Data
    class MyTask implements Runnable {

        @Override
        public void run() {
            BulkProcessor bulkProcessor = elasticsearchManager.createBulkProcessor();
            while (true) {
                try {
                    //從佇列中讀取校驗過後的資料
                    Map<Long, PieceJsonDTO> map = QueueUtil.QUEUE_SAVETOES.take();
                    if (null != map) {
                        for (Long id : map.keySet()) {
                            //建立request  blukprocessor
                            PieceJsonDTO pieceJsonDTO = map.get(id);
                            String s = JsonKit.toJsonSerializeNulls(pieceJsonDTO);
                            bulkProcessor.add(elasticsearchManager.createPieceIndexRequest(Long.toString(id),s));
                            pieceJsonDTO=null;
                        }
                        map = null;
                        log.info("es佇列大小："+QueueUtil.QUEUE_SAVETOES.size());
                    }
                } catch (Exception e) {
                    log.error("插入es失敗，失敗原因：  "+e.getMessage());
                    e.printStackTrace();
                }
            }
        }
    }

}
複製程式碼

2.2.7 消費失敗佇列儲存到檔案中（QUEUE_FAIL_IDS）

public class ReadQueueForWriteLogThread extends Thread {

    /**
     * 檔案根路徑
     */
    private final static String FILE_ROOT_PATH = new Config().getFileRootPath();
    /**
     * 專案路徑
     */
    private final static String ABSOLUTE_PATH = FileKit.getAbsolutePath("");


    @Override
    public void run() {
        //日誌檔名稱
        String fileName = "datamove.log";
        //檔案路徑
        String filePath = ABSOLUTE_PATH + FILE_ROOT_PATH + fileName;
        while (true) {
            try {
                List<String> fail_ids = QueueUtil.QUEUE_FAIL_IDS.take();
                if (null != fail_ids) {
                    ArrayList<String> result = Lists.newArrayList();
                    String s = "失敗個數：" + fail_ids.size() + " , 失敗id: " + fail_ids;
                    result.add(s);
                    //按行寫出
                    FileKit.appendLines(result, filePath, "UTF-8");
                }
            } catch (Exception e) {
                log.error("寫出失敗ids失敗，失敗原因：  "+e.getMessage());
                e.printStackTrace();
            }
        }
    }
}
複製程式碼

2.3 失敗資料處理

    @RequestMapping(value = "***", method = {RequestMethod.GET})
    @ResponseBody
    public void readLog() {
        pieceSearchService.pieceLogFile();
    }
    
    // PieceSearchServiceImpl
    public void pieceLogFile() {
        dataMoveManager.startReadLogForDb();
    }
    
    // AbstractTemplate
    public void startReadLogForDb() {
        readLogFileForDb();
    }
    
    // DataMoveManager
    protected void readLogFileForDb() {
        pieceInfoLogFileTask.start();
    }
    
    // AbstractTaskTemplate
    public void start() {
        List<? extends Callable<String>> task = createTask();
        ExecutorService executorService = createExecutorService();
        List list = runTaskForResult(executorService, task);
        writeLogFile(list);
    }
    
    // PieceInfoLogFileTask
    protected List<? extends Callable<String>> createTask() {
        //獲取 datamove.log路徑
        String fileName = "datamove.log";
        String filePath = ABSOLUTE_PATH + FILE_ROOT_PATH + fileName;

        //按行讀取  獲取檔案中 失敗id:
        List<String> ids = Lists.newArrayList();
        try {
            ids = readLines(new File(filePath), "UTF-8", ids);
        } catch (IOException e) {
            e.printStackTrace();
        }

        //建立任務組
        if (!Func.isEmpty(ids)) {
            List<MyTask> taskWithIds = getTaskWithIds(ids);
            return taskWithIds;
        }
        return null;
    }
    
    // PieceInfoLogFileTask
    private List<MyTask> getTaskWithIds(List<String> ids) {
        //一百個ids 起一個任務
        ArrayList<MyTask> tasks = Lists.newArrayList();
        int a = 1;
        int beginIndex = 0;
        while (true) {
            if (a * 100 >= ids.size()) {
                MyTask myTask = new MyTask();
                myTask.setIds(ids.subList(beginIndex, ids.size()));
                tasks.add(myTask);
                break;
            }
            MyTask myTask = new MyTask();
            //subList() 包左不包右
            myTask.setIds(ids.subList(beginIndex, beginIndex + 100));
            tasks.add(myTask);
            beginIndex += 100;
            a++;
        }
        return tasks;
    }
    
    private class MyTask implements Callable<String> {

        private List<String> ids;

        @Override
        public String call() throws Exception {
            //每次儲存的進件Id
            ArrayList<String> pieceIds = new ArrayList<>();
            //開始時間
            long beginTime = System.currentTimeMillis();
            for (int i = 0; i < ids.size(); i++) {
                pieceIds.add(ids.get(i));
                //每多少行  執行插入操作
                if (i % MAX_SAVE_NUM == 0) {
                    //執行查詢儲存操作
                    System.out.println(ids);
                    pieceIds.clear();
                }
            }
            if (!pieceIds.isEmpty()) {
                //執行查詢儲存操作

            }
            long useTime = (System.currentTimeMillis() - beginTime) / 1000;
            return "日誌檔案失敗進件重新入庫完畢，耗時：" + useTime + "秒 , 檔案共：" + ids.size() + "條資料， 失敗個數： , 失敗id:";
        }
    }
    
    // PieceInfoLogFileTask
    protected ExecutorService createExecutorService() {
        //建立執行緒池
        ExecutorService executorService = Executors.newFixedThreadPool(Integer.valueOf(MAX_THREAD_NUM));
        return executorService;
    }
    
    // AbstractTaskTemplate
    private List runTaskForResult(ExecutorService executorService, List<? extends Callable<String>> tasks) {
        //執行緒池執行任務
        List<Future<String>> futures = new ArrayList<>();
        try {
            futures = executorService.invokeAll(tasks);
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            //關閉執行緒池
            executorService.shutdown();
        }
        //組裝返回結果
        List<String> results = Lists.newArrayList();
        for (Future<String> future : futures) {
            try {
                results.add(future.get());
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
        }
        return results;
    }
    
    // PieceInfoLogFileTask
    protected void writeLogFile(List results) {
        //日誌檔名稱
        String fileName = "datamove.log";
        //檔案路徑
        String filePath = ABSOLUTE_PATH + FILE_ROOT_PATH + fileName;
        //按行寫出
        try {
            FileKit.writeLines(results, filePath, "UTF-8");
        } catch (IOException e) {
            e.printStackTrace();
            log.error("class:PieceInfoDataMoveTask  method: writeLogFile()" +
                " 日誌資訊寫出失敗, 失敗原因：" + e.getMessage());
        }
    }

複製程式碼

2.4 資料遷移流程圖

資料遷移流程圖

3 Elasticsearch在綜合搜尋列表實現

public Page<InvestigatePiecePageResDTO> pagination(final InvestigatePiecePageReqDTO investigatepiecePageReqDTO, final Integer pageNo, final Integer pageSize) {
        //建立es組合查詢器
        BoolQueryBuilder mustQuery = QueryBuilders.boolQuery();
        //進件編號
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getPieceCode())) {
            mustQuery.must(QueryBuilders.wildcardQuery("pieceCode.keyword", "*" + investigatepiecePageReqDTO.getPieceCode() + "*"));
        }
        //手機號
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getMobile())) {
            mustQuery.must(QueryBuilders.wildcardQuery("pieceContacts.mobile.keyword", "*" + investigatepiecePageReqDTO.getMobile() + "*"));
        }
        //身份證號
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getIdCard())) {
            mustQuery.must(QueryBuilders.wildcardQuery("idCard.keyword", "*" + investigatepiecePageReqDTO.getIdCard() + "*"));
        }
        //服務網點
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getServicePoint())) {
            mustQuery.must(QueryBuilders.wildcardQuery("servicePoint.keyword", "*" + investigatepiecePageReqDTO.getServicePoint() + "*"));
        }
        //座機號
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getPhone())) {
            mustQuery.must(QueryBuilders.wildcardQuery("pieceContacts.phone.keyword", "*" + investigatepiecePageReqDTO.getPhone() + "*"));
        }
        //單位名稱
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getCompanyName())) {
            mustQuery.must(QueryBuilders.wildcardQuery("companyName.keyword", "*" + investigatepiecePageReqDTO.getCompanyName() + "*"));
        }
        //工單編號
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getOrderCode())) {
            mustQuery.must(QueryBuilders.wildcardQuery("orderCode.keyword", "*" + investigatepiecePageReqDTO.getOrderCode() + "*"));
        }
        //案件編號
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getCaseCode())) {
            mustQuery.must(QueryBuilders.wildcardQuery("caseCode.keyword", "*" + investigatepiecePageReqDTO.getCaseCode() + "*"));
        }
        //規則搜尋
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getRuleCode())) {
            mustQuery.must(QueryBuilders.wildcardQuery("ruleName.keyword", "*" + investigatepiecePageReqDTO.getRuleCode() + "*"));
        }
        //客戶經理
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getCustomerManager())) {
            mustQuery.must(QueryBuilders.wildcardQuery("customerManager.keyword", "*" + investigatepiecePageReqDTO.getCustomerManager() + "*"));
        }
        //進件大區
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getPieceFromArea())) {
            mustQuery.must(QueryBuilders.wildcardQuery("pieceFromArea.keyword", "*" + investigatepiecePageReqDTO.getPieceFromArea() + "*"));
        }
        //進件狀態
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getBusinessStatus())) {
            mustQuery.must(QueryBuilders.termQuery("businessStatus.keyword", investigatepiecePageReqDTO.getBusinessStatus()));
        }
        //產品型別
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getProductCode())) {
            mustQuery.must(QueryBuilders.termQuery("productName.keyword", investigatepiecePageReqDTO.getProductCode()));
        } else {
            //預設查詢當前使用者許可權下的所有產品型別
            List<String> productCodes = getProductCodeByCurrentUser();
            mustQuery.must(QueryBuilders.termsQuery("productName.keyword", productCodes));
        }
        //反欺詐處理人
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getDecisionMaker())) {
            mustQuery.must(QueryBuilders.wildcardQuery("decisionMaker.keyword", "*" + investigatepiecePageReqDTO.getDecisionMaker() + "*"));
        }
        //單位地址
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getCompanyAddress())) {
            mustQuery.must(QueryBuilders.wildcardQuery("companyAddress.keyword", "*" + investigatepiecePageReqDTO.getCompanyAddress() + "*"));
        }
        //欺詐報警
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getFraudAlarmLevelCode())) {
            DictionaryDTO dictionaryDTO = DictionaryKit.QZJGLB_MAP().get(investigatepiecePageReqDTO.getFraudAlarmLevelCode());
            mustQuery.must(QueryBuilders.termQuery("fraudAlarmLevelName.keyword", dictionaryDTO.getName()));
        }
        //決策結果
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getResultCode())) {
            DictionaryDTO dictionaryDTO = DictionaryKit.JCJG_MAP().get(investigatepiecePageReqDTO.getResultCode());
            mustQuery.must(QueryBuilders.termQuery("decisionResult.keyword", dictionaryDTO.getName()));
        }
        //反欺詐處理狀態
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getDecisionStatus())) {
            DictionaryDTO dictionaryDTO = DictionaryKit.JCZT_MAP().get(investigatepiecePageReqDTO.getDecisionStatus());
            mustQuery.must(QueryBuilders.termQuery("decisionStatus.keyword", dictionaryDTO.getName()));
        }
        //客服人員
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getCustomerService())) {
            mustQuery.must(QueryBuilders.wildcardQuery("serviceUserName.keyword", "*" + investigatepiecePageReqDTO.getCustomerService() + "*"));
        }
        //客戶姓名
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getCustomerName())) {
            mustQuery.must(QueryBuilders.wildcardQuery("customerName.keyword", "*" + investigatepiecePageReqDTO.getCustomerName() + "*"));
        }
        //團隊經理
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getTeamManager())) {
            mustQuery.must(QueryBuilders.wildcardQuery("teamManager.keyword", "*" + investigatepiecePageReqDTO.getTeamManager() + "*"));
        }
        //進件時間
        try {
            if (Func.isNotEmpty(investigatepiecePageReqDTO.getEnterCreditTimeStart()) && Func.isNotEmpty(investigatepiecePageReqDTO.getEnterCreditTimeEnd())) {
                mustQuery.must(QueryBuilders.rangeQuery("enterCreditTime").from(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getEnterCreditTimeStart()).getTime()).to(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getEnterCreditTimeEnd()).getTime()));
            } else if (Func.isNotEmpty(investigatepiecePageReqDTO.getEnterCreditTimeStart())) {
                mustQuery.must(QueryBuilders.rangeQuery("enterCreditTime").from(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getEnterCreditTimeStart()).getTime()));
            } else if (Func.isNotEmpty(investigatepiecePageReqDTO.getEnterCreditTimeEnd())) {
                mustQuery.must(QueryBuilders.rangeQuery("enterCreditTime").to(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getEnterCreditTimeEnd()).getTime()));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        //決策時間
        try {
            if (Func.isNotEmpty(investigatepiecePageReqDTO.getDecisionTimeStart()) && Func.isNotEmpty(investigatepiecePageReqDTO.getDecisionTimeEnd())) {
                mustQuery.must(QueryBuilders.rangeQuery("decisionTime").from(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getDecisionTimeStart()).getTime()).to(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getDecisionTimeEnd()).getTime()));
            } else if (Func.isNotEmpty(investigatepiecePageReqDTO.getDecisionTimeStart())) {
                mustQuery.must(QueryBuilders.rangeQuery("decisionTime").from(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getDecisionTimeStart()).getTime()));
            } else if (Func.isNotEmpty(investigatepiecePageReqDTO.getDecisionTimeEnd())) {
                mustQuery.must(QueryBuilders.rangeQuery("decisionTime").to(SIMPLE_DATE_FORMAT.parse(investigatepiecePageReqDTO.getDecisionTimeEnd()).getTime()));
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        //是否初始化
        if (Boolean.valueOf(investigatepiecePageReqDTO.getIfInit())) {
            //第一次進入，設定時間為今天
            mustQuery.must(QueryBuilders.rangeQuery("enterCreditTime").from(getTodayStartTime()).to(getTodayEndTime()));
        }
        //組裝分頁 排序條件
        SearchSourceBuilder source = new SearchSourceBuilder();
        source.query(mustQuery);
        //排序欄位
        if (Func.isNotEmpty(investigatepiecePageReqDTO.getSortChangeKey())) {
            SortOrder sortOrder = "asc".equals(investigatepiecePageReqDTO.getSortChangeWay()) ? SortOrder.ASC : SortOrder.DESC;
            if (Func.equals(investigatepiecePageReqDTO.getSortChangeKey(), InvestigatePieceEntity.DB_COL_ENTER_CREDIT_TIME)) {
                source.sort("enterCreditTime", sortOrder);
            } else if (Func.equals(investigatepiecePageReqDTO.getSortChangeKey(), InvestigatePieceEntity.DB_COL_DECISION_TIME)) {
                source.sort("decisionTime", sortOrder);
            } else if (Func.equals(investigatepiecePageReqDTO.getSortChangeKey(), InvestigatePieceEntity.DB_COL_DECISION_MAKER)) {
                source.sort("decisionMaker.keyword", sortOrder);
            } else {
                source.sort("enterCreditTime", sortOrder);
            }
        } else {
            source.sort("enterCreditTime", SortOrder.DESC);
        }
        //設定分頁  注意：es分頁預設從0頁開始
        source.from(pageNo - 1);
        source.size(pageSize);

        //查詢結果
        Page<InvestigatePiecePageResDTO> page = getResponseFromEs(source);
        return page;
    }
複製程式碼
private Page<InvestigatePiecePageResDTO> getResponseFromEs(SearchSourceBuilder source) {
        //建立查詢es請求
        SearchHits hits;
        try {
            SearchResponse response = elasticsearchManager.getPieceResponseWithBuilder(source);
            hits = response.getHits();
        } catch (Exception e) {
            throw new ServiceException(500, "es搜尋引擎連線失敗");
        }
        //獲取命中資料
        SearchHit[] hitsHits = hits.getHits();
        //總命中數
        Long totalHits = hits.getTotalHits();
        //組裝前端返回DTO
        List<InvestigatePiecePageResDTO> result = Lists.newArrayList();
        for (SearchHit hitsHit : hitsHits) {
            InvestigatePiecePageResDTO investigatePiecePageResDTO = new InvestigatePiecePageResDTO();
            Map<String, Object> sourceAsMap = hitsHit.getSourceAsMap();
            investigatePiecePageResDTO.setId(Long.valueOf(hitsHit.getId()));
            investigatePiecePageResDTO.setPieceCode((String) sourceAsMap.get("pieceCode"));
            investigatePiecePageResDTO.setAppKey((String) sourceAsMap.get("appKey"));
            Integer decisionId = (Integer) sourceAsMap.get("decisionId");
            if (null != decisionId) {
                investigatePiecePageResDTO.setDecisionId(decisionId.longValue());
            }
            List<String> orderCode = (List<String>) sourceAsMap.get("orderCode");
            if (Func.isNotEmpty(orderCode)) {
                investigatePiecePageResDTO.setOrderCode(((List<String>) sourceAsMap.get("orderCode")).get(0));
            }
            investigatePiecePageResDTO.setCustomerName((String) sourceAsMap.get("customerName"));
            investigatePiecePageResDTO.setIdCard((String) sourceAsMap.get("idCard"));
            investigatePiecePageResDTO.setProductName(DictionaryKit.CPLX_MAP().get(sourceAsMap.get("productName")).getName());
            investigatePiecePageResDTO.setEnterCreditTime(SIMPLE_DATE_FORMAT.format(new Date((Long) sourceAsMap.get("enterCreditTime"))));
            investigatePiecePageResDTO.setBusinessStatus((String) sourceAsMap.get("businessStatus"));
            Long decisionTime = (Long) sourceAsMap.get("decisionTime");
            if (null != decisionTime) {
                investigatePiecePageResDTO.setDecisionTime(SIMPLE_DATE_FORMAT.format(new Date(decisionTime)));
            }
            investigatePiecePageResDTO.setDecisionResult((String) sourceAsMap.get("decisionStatus"));
            investigatePiecePageResDTO.setFraudAlarmLevelName((String) sourceAsMap.get("fraudAlarmLevelName"));
            investigatePiecePageResDTO.setDecisionMaker((String) sourceAsMap.get("decisionMaker"));
            investigatePiecePageResDTO.setServicePoint((String) sourceAsMap.get("servicePoint"));
            String decisionStatus = (String) sourceAsMap.get("decisionStatus");
            if ("已決策".equals(decisionStatus)) {
                investigatePiecePageResDTO.set_disabled(false);
            } else {
                investigatePiecePageResDTO.set_disabled(true);
            }
            result.add(investigatePiecePageResDTO);
        }

        //組裝前端分頁DTO
        Page<InvestigatePiecePageResDTO> page = new Page<>();
        page.setRecords(result);
        page.setTotal(totalHits.intValue());
        return page;
    }
複製程式碼

相關文章