【web服務】耗時任務基於API與worker模式

Tony_xiao發表於2024-10-15
WebAPI模式
  • 場景：當我們爬蟲或者獲取資料需要很長的時間計算或者等待，這種情況情況下基於API與worker模式完成交付
  • 1.API端:用flask框架
import json
import uuid
import redis
from flask import Flask, request, jsonify

app = Flask(__name__)
pool = redis.ConnectionPool(host='127.0.0.1', port=6379, password='123456',encoding='utf-8', max_connections=1000)

@app.route("/task",method=["POST"])
def task():
    # 構建引數
    total_dict = {}
    user_dict = request.json
    total_dict.update(user_dict)
    total_dict.update({"token":"1231312","sign":"dsfewfwef"})

    # 建立任務ID
    task_id = str(uuid.uuid4())

    # 放入redis
    conn = redis.Redis(connection_pool=pool)
    task_info = {"uid":task_id,"info":total_dict}
    conn.lpush("task_queue",json.dumps(task_info))

    return jsonify({"status":True,"data":task_id})

@app.route("/fetch",method=["GET"])
def fetch():
    # 透過url傳遞任務id
    task_id = request.args.get("task_id")
    conn = redis.Redis(connection_pool=pool)
    result = conn.hget("task_result",task_id)
    if not result:
        return jsonify({"status":False,"data":"任務還在執行中"})

    conn.hdel("task_result",task_id)
    result_string = result.decode('utf-8')
    return jsonify({"status":True,"data":result_string})

if __name__ == '__main__':
    app.run(host="127.0.0.1", port=9999)
  • 2.客戶端
import time

import requests

# 1.傳送請求（建立任務）
res = requests.post(
    url="http://127.0.0.1:9999/task",
    json={
        "aid": 1,
        "bfc": True,
        "ctime": int(time.time())
    }
)
# {"status": True, "data": "fa5d4f0d-9d29-4008-8097-d1a7d5f9c2cd"}
res_dict = res.json()
task_id = res_dict['data']

# 2.查詢結果
while True:
    time.sleep(5)
    res = requests.get(
        url="http://127.0.0.1:9999/fetch",
        params={
            "task_id": task_id
        }
    )
    # {"status": False, "data": "任務還在執行中"}
    # {"status": True, "data": result_string}
    res_dict = res.json()
    if not res_dict['status']:
        print(res_dict['data'])
        continue

    print("執行完畢", res_dict['data'])
    break
  • 3.worker
import random
import time

import redis
import json

POOL = redis.ConnectionPool(host='127.0.0.1', port=6379, password='123456', encoding='utf-8', max_connections=1000)

def run():
    while True:
        conn = redis.Redis(connection_pool=POOL)
        res = conn.brpop(["task_queue"], timeout=30)  # ("task_queue",b"{}")
        print("取任務：", res)
        if not res:
            continue
        task_dict = json.loads(res[-1].decode('utf-8'))
        task_id = task_dict['uid']

        # 根據任務資訊去執行任務，耗時：5分鐘、1小時
        time.sleep(random.randint(5, 10))

        # 結算結果放入結果佇列
        conn = redis.Redis(connection_pool=POOL)
        conn.hset("task_result", task_id, "11121123123123123")


if __name__ == '__main__':
    run()

  • 4.執行效果

  • 5.此場景適用於計算密集型與長時間耗時任務，我們將API部署在遠端伺服器上，使用本地電腦當做worker，可以節約成本

相關文章