一個分詞指令碼

import jieba

target_file = "stress_words.txt"

# 讀取空白行之前的句子為個性化詞典
custom_dict = {}
with open(target_file, "r", encoding="utf-8") as f:
    for line in f.readlines():
        # 之前做的標記，跳過此行
        if(line.strip() == "||"):
            continue
        # 新做的標記，結束個性詞典
        if(line.strip() == ""):
            break
        # 切分出前半部分，將長度大於1的詞加入詞典
        line = line.split("||")[0]
        for word in line.strip().split(" "):
            if(len(word) > 1):
                freq = custom_dict.get(word,0)
                custom_dict[word] = freq+1

# 將詞典儲存
special_dict = """
布藝 床品 家紡 窗簾
"""
for w in special_dict.replace("/n","").strip().split(" "):
    custom_dict[w] = 20000000

with open("format_dict.txt", "w", encoding="utf-8") as out:
    out.write("\n".join([k+" "+str(custom_dict[k]) for k in custom_dict.keys()]))

# 載入詞典
jieba.load_userdict("format_dict.txt")

# 根據已經整理的詞典重新分詞
words = []
cut_start = False
with open(target_file, "r", encoding="utf-8") as f:
    for line in f.readlines():
        # 之前做的標記，跳過此行
        if(line.strip() == "||"):
            continue

        # 遇到空白標記行，則改變標記，跳過該空白行，並開始切詞
        if(line.strip == ""):
            cut_start = True
            words.append("   ||   ")
            continue

        if "||" in line:
            line = line.split("||")[0]

        res = []
        if(not cut_start):
            # 遇到空白行之前，不重新切詞，只分析
            res = line.strip().split(" ")
        else:
            # 遇到空白行之後，進行重新切詞
            res = list(jieba.cut(line.strip().replace(" ", "")))
        # 記錄改行詞典中未出現的詞
        unique_word = [w for w in res if w not in custom_dict]
        cut_res = " ".join(res)
        true_len = 60 - (len(cut_res)*2 - len(res) + 1)
        words.append(cut_res + true_len*" " + "||  " + " ".join(unique_word))

with open(target_file, "w", encoding="utf-8") as out:
    out.write("\n".join(words))