批次非同步上傳aws圖片指令碼（python）

第二个卿老师發表於2024-06-16
非同步指令碼Python

背景

工作中需要上傳一些測試圖片，於是網上找找資料（官方說明），前置步驟如下。

  1. python需要3.8以上，安裝最新的boto3庫： 
    pip install boto3
  2. 有一個S3許可權的aws賬戶，得到訪問金鑰ACCESS_KEY與SECRET_KEY，以及上傳圖片的儲存桶位置
  3. 安裝非同步程式設計asyncio，aiohttp庫，方便本地非同步上傳圖片

程式碼實現

 1 # -*- coding: utf-8 -*-
 2 """
 3 # @Author : qgc
 4 # @Time : 2024/3/13 14:28
 5 # @File : upload_to_aws.py
 6 # Description : 檔案說明
 7 """
 8 import os
 9 import boto3
10 import asyncio
11 import aiohttp
12 
13 aws_access_key_id = 'xxxxxx'
14 aws_secret_access_key = 'xxxxxxx'
15 bucket_name = 'deme-normal'   # 儲存桶的具體路由，路由不存在會上傳失敗，但不會報錯
16 
17 async def upload_to_aws(local_file, s3_file):
18     s3 = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
19     try:
20         s3.upload_file(local_file, bucket_name, s3_file)
21         print(f"{local_file} upload to s3://{bucket_name}/{s3_file} successed")
22         return True
23     except FileNotFoundError:
24         print('本地檔案未找到')
25         return False
26 
27 async def upload_task(url, session, semaphore):
28     try:
29         if url:
30             local_filename = url
31             s3_path = 'qgctest/' + os.path.basename(url)
32             await upload_to_aws(local_filename, s3_path)
33     except Exception as e:
34         print(f'處理{url}:{e}')
35 
36 def get_image_files(directory):
37     image_files = []
38     if directory:
39         for file in os.listdir(directory):
40             if file.endswith('.jpg') or file.endswith('.jpeg') or file.endswith('.png') or file.endswith('.gif'):
41                 image_files.append(os.path.join(directory, file))
42     return image_files
43 
44 async def main(image_files):
45     semaphore = asyncio.Semaphore(10)
46     connector = aiohttp.TCPConnector(limit=10)
47     async with aiohttp.ClientSession(connector=connector) as session:
48         tasks = [upload_task(url, session, semaphore) for url in image_files]
49         await asyncio.gather(*tasks)
50 
51 
52 if __name__ == '__main__':
53     directory = f'E:\圖片\images2'  # 本地圖片路徑
54     image_files = get_image_files(directory)
55     # image_files = ['E:\\圖片\\images2\\0.jpg']
56     print(f'共找到{len(image_files)}個圖片')
57     asyncio.get_event_loop().run_until_complete(main(image_files))

相關文章