easyExcel分批匯入檔案

原文地址 ： https://blog.csdn.net/qq_42060055/article/details/116139016

一些關於easyExcel匯入檔案操作

需求: 匯入大資料量檔案 其中資料達到萬級、十萬級, 錯誤檔案進行錯誤單元格標紅, 可匯出修改完繼續匯入

由於資料量多大 一次行全部讀到記憶體中可能會導致記憶體溢位問題

使用easyExcel poi的監聽器進行操作

三步曲:

1、解析excel為inputStream流, 讀取流,解析excel

2、判斷excel中每條資料的格式, 正確和錯誤相對記錄

3、透過監聽器每解析150條資料, 進行入庫操作, 錯誤資料存在記憶體中(考慮錯誤資料不多的情況)

// 這裡用到ossfs 反正就是讀取excel為input流,
涉及到兩個系統之間流的傳輸, 這裡直接把檔案上傳到oss
try {
    in = new FileInputStream(localFileName);
} catch (FileNotFoundException e) {
    in = HttpUtil.io(HttpUtil.Atom.builder().url(diseaseDto.getFileUrl()).build());
}

　　

// 這裡解析excel其中
OltHosIcdDiseaseListener為自定義監聽器
try {
    LoggerUtil.info(LOGGER, "開始解析IcdDisease");
    OltHosIcdDiseaseListener oltHosIcdDiseaseListener = new OltHosIcdDiseaseListener(isCfgPrd, icdCodeList, delIcdCodeList, diseaseDto, oltConfigService, exportTaskHandler);
    excelReader = EasyExcel.read(in, oltHosIcdDiseaseListener).build();
    ReadSheet readSheet = EasyExcel.readSheet(0).build();
    excelReader.read(readSheet);
} finally {
    try {
        if (in != null) {
            in.close();
        }
        if (excelReader != null) {
            // 這裡千萬別忘記關閉，讀的時候會建立臨時檔案，到時磁碟會崩的
            excelReader.finish();
        }
    } catch (Exception e) {
        LoggerUtil.error(LOGGER, "{0},{1}", e.getMessage(), e);
    }
}

　　

// 透過建構函式, 初始化一些list與物件
// 這個是匯入的核心方法, 所有的匯入邏輯, 判斷邏輯與入庫都在這裡操作
// 採用無物件方式
@Slf4j
public class OltHosIcdDiseaseListener extends AnalysisEventListener<Map<Integer, String>> {
    private OltConfigService oltConfigService;
    private ExportTaskHandler exportTaskHandler;
    private static final int batchCount = 150;
    private int countNum = 0;
    private boolean isCfgPrd;
    private int successCount = 0;
    private int errorCount = 0;
 
    private List<String> checkRepeatCode = new ArrayList<>();
    private List<String> icdCodeList;
    private List<String> delIcdCodeList;
    private OltHosIcdDiseaseDto diseaseDto;
    private List<OltHosIcdDiseaseDto> successList = new ArrayList<>();
    private List<OltHosIcdDiseaseDto> errorList = new ArrayList<>();
    private List<OltHosIcdDiseaseDto> tempErrorList = new ArrayList<>();
 
 
    public OltHosIcdDiseaseListener(boolean isCfgPrd, List<String> icdCodeList, List<String> delIcdCodeList, OltHosIcdDiseaseDto diseaseDto,
                                    OltConfigService oltConfigService, ExportTaskHandler exportTaskHandler) {
        this.isCfgPrd = isCfgPrd;
        this.icdCodeList = icdCodeList;
        this.delIcdCodeList = delIcdCodeList;
        this.diseaseDto = diseaseDto;
        this.oltConfigService = oltConfigService;
        this.exportTaskHandler = exportTaskHandler;
    }
 
/**
  * 這個每一條資料解析都會來呼叫
  * data --> 實體類
  * analysisContext excel資訊
 */
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
    int rouNumber = context.readRowHolder().getRowIndex() + 1;
 
    // 這裡是因為表頭在第二行
    if (rouNumber == 2) {
        // 這裡是校驗表頭
        checkExcelHead(data);
    } else if (rouNumber > 2) {
        // 這裡是校驗資料
        checkReadData(data);
    }
 
    // 超過150條就先入庫
    if (countNum >= batchCount) {
        // 處理excel匯出的正確資料
        batchOperateData();
    }
    countNum++;
}
 
/**
 * @author songhc
 * @create 
 * @desc 呼叫完成監聽, 確保資料已全部處理完
 **/
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
    // 這裡也要儲存資料，確保最後遺留的資料也儲存到資料庫
    Map<String, Object> objMap = new HashMap<>();
    // 處理excel匯出的正確資料
    batchOperateData();
 
    // 錯誤資料填充oss表格
    Object object = uploadErrorData(errorList, diseaseDto);
 
    objMap.put("errorInfo", object);
    objMap.put("successCount", successCount);
    objMap.put("errorCount", errorCount);
 
    // 錯誤資料記錄redis, 下次使用
    RedisStringHandler.set(String.format(RedisKeyConstants.EXPORT_ERROR_RESULT, "disease" + diseaseDto.getUserId() + "_" + diseaseDto.getRgId() + "_" + diseaseDto.getHosId()), JSONObject.toJSONString(objMap));
}
 
// 這裡是封裝所有的錯誤資料
// 包括封裝單元格
private Object uploadErrorData (List<OltHosIcdDiseaseDto> errorList, OltHosIcdDiseaseDto dto) {
    Map<Integer, List<Integer>> map = new HashMap<>();
    LinkedList<OltHosIcdDiseaseDto> newErrorList = new LinkedList<>();
    if (CollectionUtils.isNotEmpty(errorList)) {
        for (int i = 0; i < errorList.size(); i++) {
            OltHosIcdDiseaseDto e = errorList.get(i);
            List<Integer> integerList = new ArrayList<>();
            if (e.getErrorReasonMap() != null && !e.getErrorReasonMap().isEmpty()) {
                List<String> reasonList = new ArrayList<>();
                for (Integer key: e.getErrorReasonMap().keySet()) {
                    // 標紅單元格
                    integerList.add(key);
                    reasonList.add(e.getErrorReasonMap().get(key));
                }
                map.put(i + 2, integerList);
                e.setErrorReason(String.join("、", reasonList));
            }
            newErrorList.add(e);
        }
    }
 
 
    // 封裝匯出服務入參
    String uuid = UUIDUtil.create();
    String errorFileName = dto.getHosName() + "(待處理診斷資料)" + dto.getStatDataStr() + ".xlsx";
    SysExportRecordDto sysExportRecordDto = SysExportRecordDto.builder().batchId(uuid).userId(dto.getUserId()).pfCode(dto.getPfCode())
            .source(dto.getSource()).fileName(errorFileName).creator(dto.getCreator()).operator(dto.getCreator()).build();
    // 建立匯出記錄
    QueueHandler.createTaskRecord(sysExportRecordDto);
    // 獲取url
    // 虛擬碼
    String fileName = "aaa.xlsx";
    String BUCKET_NAME = "bbb";
    String fileUrl = String.format(OssClientConfig.OSS_REAL_PATH, BUCKET_NAME,
            UploadFileType.getFolderByType(UploadFileType.REPORT)).concat(fileName);
    // 加入非同步執行緒任務
    this.exportTaskHandler.exportIcdErrorDiseaseData(OltErrorResult.builder().map(map).errorList(newErrorList)
                    .fileName(errorFileName).source(dto.getSource()).build(),
            uuid, errorFileName, fileUrl);
    // 構建返回佇列資訊
    return QueueHandler.buildQueueInfo(sysExportRecordDto);
}
 
private void batchOperateData() {
    checkErrorExcelList(tempErrorList, icdCodeList);
    checkSuccessExcelList(successList, tempErrorList, icdCodeList);
 
    // 將臨時錯誤資料儲存到所有錯誤資料列表
    this.errorList.addAll(tempErrorList);
    // 清理list
    this.successList.clear();
    this.tempErrorList.clear();
    this.countNum = 0;
}
 
private void checkExcelHead(Map<Integer, String> data) {
    boolean templateFlag = true;
    // 第二行  校驗excel標題
    try {
        String diseaseCategoryStr = data.get(0);
        if (StringUtils.isBlank(diseaseCategoryStr) || !"診eee（必填）".equals(diseaseCategoryStr)) {
            templateFlag = false;
        }
    } catch (Exception e) {
        templateFlag = false;
    }
    try {
        String icdNameStr = data.get(1);
        if (StringUtils.isBlank(icdNameStr) || !"醫vv稱（必填）".equals(icdNameStr)) {
            templateFlag = false;
        }
    } catch (Exception e) {
        templateFlag = false;
    }
    try {
        String icdCodeStr = data.get(2);
        if (StringUtils.isBlank(icdCodeStr) || !"醫aa（必填）".equals(icdCodeStr)) {
            templateFlag = false;
        }
    } catch (Exception e) {
        templateFlag = false;
    }
    if (!templateFlag) {
        throw new PlatException("檔案模版不匹配");
    }
}
 
private void checkReadData(Map<Integer, String> data) {
    // 迴圈cell
    OltHosIcdDiseaseDto temDisDto = OltHosIcdDiseaseDto.buildDefault();
    temDisDto.setHosId(diseaseDto.getHosId());
    // key為所在的列, value為錯誤原因
    Map<Integer, String> map = new HashMap<>();
    boolean flag = true;
    try {
        // 解析第二列
        String diseaseCategory = data.get(0);
        if (StringUtils.isBlank(diseaseCategory)) {
            temDisDto.setDiseaseCategoryStr(StringUtils.EMPTY);
            map.put(0, "aaa為空");
            flag = false;
        } else {
            temDisDto.setDiseaseCategoryStr(diseaseCategory);
        }
    } catch (Exception e) {
        temDisDto.setDiseaseCategoryStr(StringUtils.EMPTY);
        map.put(0, "bbb為空");
        flag = false;
    }
 
    try {
        String icdName = data.get(1);
        if (StringUtils.isBlank(icdName)) {
            temDisDto.setIcdName(StringUtils.EMPTY);
            map.put(1, "為空");
            flag = false;
        } else {
            temDisDto.setIcdName(icdName);
        }
    } catch (Exception e) {
        temDisDto.setIcdName(StringUtils.EMPTY);
        map.put(1, "ccc稱為空");
        flag = false;
    }
 
    try {
        String icdCode = data.get(2);
        if (StringUtils.isBlank(icdCode)) {
            temDisDto.setIcdCode(StringUtils.EMPTY);
            map.put(2, "ddd為空");
            flag = false;
        } else {
            temDisDto.setIcdCode(icdCode);
        }
    } catch (Exception e) {
        temDisDto.setIcdCode(StringUtils.EMPTY);
        map.put(2, "ddd為空");
        flag = false;
    }
 
 
    try {
        if (!DiseaseCategory.TCM_SYNDROME.getDesc().equals(temDisDto.getDiseaseCategoryStr())) {
            String standardIcdName = data.get(3);
            if (isCfgPrd && StringUtils.isBlank(standardIcdName)) {
                temDisDto.setStandardIcdName(StringUtils.EMPTY);
                map.put(3, "vvv為空");
                flag = false;
            } else {
                temDisDto.setStandardIcdName(standardIcdName);
            }
        }
    } catch (Exception e) {
        temDisDto.setStandardIcdName(StringUtils.EMPTY);
        map.put(3, "vvv為空");
        flag = false;
    }
 
    try {
        if (!DiseaseCategory.TCM_SYNDROME.getDesc().equals(temDisDto.getDiseaseCategoryStr())) {
            String standardIcdCode = data.get(4);
            if (isCfgPrd && StringUtils.isBlank(standardIcdCode)) {
                temDisDto.setStandardIcdCode(StringUtils.EMPTY);
                map.put(4, "eee為空");
                flag = false;
            } else {
                temDisDto.setStandardIcdCode(standardIcdCode);
            }
        }
    } catch (Exception e) {
        temDisDto.setStandardIcdCode(StringUtils.EMPTY);
        map.put(4, "eee為空");
        flag = false;
    }
    temDisDto.setErrorReasonMap(map);
 
 
    // 如果flag為 false 說明資料有問題
    if (!flag) {
        tempErrorList.add(temDisDto);
    } else {
        successList.add(temDisDto);
    }
}
 
private void checkErrorExcelList(List<OltHosIcdDiseaseDto> errorList, List<String> icdCodeList) {
    if (CollectionUtils.isNotEmpty(errorList)) {
        // 錯誤就往裡加, 正確重新定義列表
        errorList.forEach(e -> {
            Map<Integer, String> map = new HashMap<>();
            if (!DiseaseCategory.belongTo(e.getDiseaseCategoryStr())) {
                map.put(0, "aaa不正確");
            } else {
                e.setDiseaseCategory(DiseaseCategory.getCodeByDesc(e.getDiseaseCategoryStr()));
            }
            
            // excel是否存在重複資料
            if (checkRepeatCode.contains(e.getIcdCode())) {
                map.put(2, "bbb重複");
            }
 
            if (CollectionUtils.isNotEmpty(icdCodeList) && icdCodeList.contains(e.getIcdCode())) {
                map.put(2, "ttt重複");
            }
            if (e.getErrorReasonMap() != null && !e.getErrorReasonMap().isEmpty()) {
                Map<Integer, String> errorReasonMap = e.getErrorReasonMap();
                errorReasonMap.putAll(map);
                e.setErrorReasonMap(errorReasonMap);
            }
            errorCount++;
        });
    }
}
 
/**
 * 侵入式給errorList賦值
 * @param list
 * @param errorList
 * @param icdCodeList
 */
private void checkSuccessExcelList(List<OltHosIcdDiseaseDto> list, List<OltHosIcdDiseaseDto> errorList,
                                   List<String> icdCodeList) {
    List<OltHosIcdDiseaseDto> newList = new ArrayList<>();
 
    if (CollectionUtils.isNotEmpty(list)) {
        // 錯誤就往裡加, 正確重新定義列表
        list.forEach(e -> {
            Map<Integer, String> map = new HashMap<>();
            boolean flag = false;
            // 判
            if (!DiseaseCategory.belongTo(e.getDiseaseCategoryStr())) {
                map.put(0, "不正確");
                flag = true;
            } else {
                e.setDiseaseCategory(DiseaseCategory.getCodeByDesc(e.getDiseaseCategoryStr()));
            }
 
            // excel是否存在重複資料
            if (checkRepeatCode.contains(e.getIcdCode())) {
                map.put(2, "重複");
                flag = true;
            } else {
                // 判斷診斷編碼
                if (CollectionUtils.isNotEmpty(icdCodeList) && icdCodeList.contains(e.getIcdCode())) {
                    map.put(2, "重複");
                    flag = true;
                }
            }
            e.setErrorReasonMap(map);
            if (flag) {
                errorCount++;
                errorList.add(e);
            } else {
                e.setIcdPinyin(HzUtils.getPinyinCap(e.getIcdName(), HzUtils.CaseType.UPPERCASE));
                e.setIcdWb(HzUtils.getWbCap(e.getIcdName(), HzUtils.CaseType.UPPERCASE));
                newList.add(e);
                checkRepeatCode.add(e.getIcdCode());
                successCount++;
            }
        });
    }
 
    // 正確資料入庫
    if (CollectionUtils.isNotEmpty(newList)) {
        oltConfigService.batchAddHosIcdDisease(delIcdCodeList, newList);
    }
}

　　其中,匯入錯誤資料用了easyExcel的模版填充方式, 模版存於oss上

/**
 * @author songhc
 * @create
 * @desc 匯出錯誤資料
 **/
@Async
public void exportIcdErrorDiseaseData(OltErrorResult dto, String fileBatch, String fileName, String fileUrl) {
    Map<Integer, List> map = new HashMap<>();
    map.put(0, dto.getErrorList());
    Map<Integer, Map<Integer, List<Integer>>> styleMap = new HashMap<>();
    styleMap.put(0, dto.getMap());
    ExportExistHandler.exportExistTemplateData(map, styleMap, fileBatch, fileName, fileUrl);
}

　　接下來就是填充錯誤模版的實現

/**
 * @param errorMap key為sheetNo, value為填充的資料
 * @param styleMap key為sheetNo, value為錯誤資料座標
 * @param fileBatch 批次號
 * @param fileName 檔名
 * @param fileUrl 檔案路徑
 * @description 匯出服務封裝方法(無需分頁查詢, 資料為動態傳入)
 * @className exportNoModelData
 */
public static void exportExistTemplateData(Map<Integer, List> errorMap, Map<Integer, Map<Integer, List<Integer>>> styleMap, String fileBatch, String fileName, String fileUrl) {
    String ossFileName = fileName.substring(0, fileName.lastIndexOf('.'))
            .concat("-").concat(LocalDateTime.now()
                    .format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"))).concat(fileName.substring(fileName.lastIndexOf('.')));
    InputStream inputStream = HttpUtil.io(HttpUtil.Atom.builder().url(fileUrl).build());
    if (null == inputStream) {
        return;
    }
    String localFileName = String.format(TaskNoteHandler.staticExportConfig.getExportPath(), ossFileName);
    ExcelWriter excelWriter = null;
    int resultCount = 0;
    try {
        if (errorMap != null && !errorMap.isEmpty()) {
            excelWriter = EasyExcel.write(localFileName)
                        .withTemplate(inputStream)
                        .build();
            // for迴圈是一個excel可能有多個sheet的相容寫法
            for (Integer i: errorMap.keySet()) {
                // 這裡使用easyExcel的 registerWriteHandler 方法, 自定義CellColorSheetWriteHandler實現, 給每一個單元格填充顏色
                WriteSheet writeSheet = EasyExcel.writerSheet(i).registerWriteHandler(new CellColorSheetWriteHandler(styleMap.get(i),
                        IndexedColors.RED1.getIndex())).build();
                excelWriter.fill(errorMap.get(i), writeSheet);
            }
        }
    } catch (Exception e){
        LoggerUtil.error(LOGGER, "檔案寫入異常，error{0}", e);
        // 檔案匯出失敗
        TaskNoteHandler.doUploadFailed(fileBatch, resultCount);
        return;
    } finally {
        // 關閉流
        if (excelWriter != null) {
            excelWriter.finish();
        }
    }
    // 1、上傳檔案（多種方案）；2、更新記錄
    TaskNoteHandler.doUploadAndNote(fileBatch, ossFileName, localFileName, resultCount);
}

　　

/**
 * @description 自定義單元格格式攔截器
 * @className CellColorSheetWriteHandler
 * @package 
 * @Author songhc
 */
public class CellColorSheetWriteHandler implements CellWriteHandler {
 
    /**
     * map
     * key：第i行
     * value：第i行中單元格索引集合
     */
    private Map<Integer, List<Integer>> map;
    /**
     * 顏色
     */
    private Short colorIndex;
 
    /**
     * 有參構造
     */
    public CellColorSheetWriteHandler(Map<Integer, List<Integer>> map, Short colorIndex) {
        this.map = map;
        this.colorIndex = colorIndex;
    }
 
    /**
     * 無參構造
     */
    public CellColorSheetWriteHandler() {
    }
 
    @Override
    public void beforeCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Row row, Head head, Integer integer, Integer integer1, Boolean aBoolean) {
    }
 
    /**
     * 在單元格建立後呼叫
     */
    @Override
    public void afterCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Cell cell, Head head, Integer integer, Boolean aBoolean) {
    }
 
    @Override
    public void afterCellDataConverted(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, CellData cellData, Cell cell, Head head, Integer integer, Boolean aBoolean) {
    }
 
    /**
     * 在單元上的所有操作完成後呼叫
     */
    @Override
    public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List<CellData> cellDataList, Cell cell, Head head, Integer relativeRowIndex, Boolean isHead) {
        //當前行的第i列
        int i = cell.getColumnIndex();
        // 根據單元格獲取workbook
        Workbook workbook = cell.getSheet().getWorkbook();
        //不處理第一行
        if (0 != cell.getRowIndex()) {
            List<Integer> integerList = map.get(cell.getRowIndex());
            // 自定義單元格樣式 
            if (CollectionUtils.isNotEmpty(integerList)) {
                if (integerList.contains(i)) {
                    // 單元格策略
                    WriteCellStyle contentWriteCellStyle = new WriteCellStyle();
                    // 設定背景顏色白色
                    contentWriteCellStyle.setFillForegroundColor(colorIndex);
                    // 設定垂直居中為居中對齊
                     contentWriteCellStyle.setVerticalAlignment(VerticalAlignment.CENTER);
                    // 設定左右對齊為中央對齊
                     contentWriteCellStyle.setHorizontalAlignment(HorizontalAlignment.RIGHT);
                    // 設定單元格上下左右邊框為細邊框
                      contentWriteCellStyle.setBorderBottom(BorderStyle.THIN);
                      contentWriteCellStyle.setBorderLeft(BorderStyle.THIN);
                     contentWriteCellStyle.setBorderRight(BorderStyle.THIN);
                     contentWriteCellStyle.setBorderTop(BorderStyle.THIN);
                    // 建立字型例項
                     WriteFont cellWriteFont = new WriteFont();
                    // 設定字型大小
                     cellWriteFont.setFontName("宋體");
                     cellWriteFont.setFontHeightInPoints((short) 10);
                    //設定字型顏色
                    // cellWriteFont.setColor(IndexedColors.BLACK1.getIndex());
                    //單元格顏色
                    //contentWriteCellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
                    contentWriteCellStyle.setWriteFont(cellWriteFont);
                    CellStyle cellStyle = StyleUtil.buildHeadCellStyle(workbook, contentWriteCellStyle);
                    //設定當前行第i列的樣式
                    cell.getRow().getCell(i).setCellStyle(cellStyle);
                }
            }
        }
    }
}

　　對於一個excel多sheet, 操作也是一樣

// 不同的是這裡可以定義多個監聽器
// readSheet(0)  —-> 這裡的資料代表sheet的位置
OltDrugFrequencyListener oltDrugFrequencyListener = new OltDrugFrequencyListener(isCfgPrd, dfCodeList, frequencyDto, oltConfigService);
OltDrugUsageListener oltDrugUsageListener = new OltDrugUsageListener(isCfgPrd, dUCodeList, OltDrugUsageDto.builder().hosId(frequencyDto.getHosId()).build(), oltConfigService);
OltDrugDurationListener oltDrugDurationListener = new OltDrugDurationListener(durationCodeList, OltDrugDurationDefDto.builder().hosId(frequencyDto.getHosId()).build(), oltConfigService);
 
ReadSheet readSheet = EasyExcel.readSheet(0).registerReadListener(oltDrugFrequencyListener).build();
ReadSheet readSheet2 = EasyExcel.readSheet(2).registerReadListener(oltDrugUsageListener).build();
ReadSheet readSheet4 = EasyExcel.readSheet(4).registerReadListener(oltDrugDurationListener).build();
excelReader.read(readSheet, readSheet2, readSheet4);

　　經過測試, 該方法匯入2W條資料差不多需要10秒, 也不會影響記憶體