實現基於zoom平臺上的oss額外儲存

夢想改造家發表於2018-05-14
OOM

OSS的擴充套件性和超大的儲存空間，使得在一些第三方昂貴的儲存成本極大的壓縮，譬如ZOOM。但如何實現將ZOOM上的視訊轉移儲存到OSS這個第三方儲存介質上呢？其實很簡單，具體實現方式是先將ZOOM上的雲錄製平臺中所有視訊的地址獲取到，然後分批下載到跟OSS繫結的檔案目錄下，繫結OSS後的目錄能實現和bucket的同步。繫結OSS和目錄的方法可以通過ossfs工具實現。

以下是ZOOM視訊的爬蟲指令碼的程式碼，基於python實現 ，這也是基礎爬蟲的程式碼實現，觀眾老爺們也可以當看個爬蟲入門

# -*- coding=utf8 -*-
import requests 
import bs4
import urllib
import datetime
import download as ds
import dd


from lxml import html


session_requests = requests.session()
login_url = "https://www.zoom.us/signin"
#登入頁面的賬號密碼
payload = {
    "email": "xxx@qq.com",
    "password": "password"
}
#傳送seesion請求
result = session_requests.get(login_url)

tree = html.fromstring(result.text)

result = session_requests.post(
    login_url,
    data = payload,
    headers = dict(referer=login_url)
)

#第一頁訪問
url = `https://www.zoom.us/recording/management`
#獲取到的頁面值
result = session_requests.get(
    url,
    headers = dict(referer = url)
)
#獲取第一頁整個頁面，轉成BeautifulSoup
soup = bs4.BeautifulSoup(result.text, `lxml`)
#獲取到頁數
page=soup.find_all(class_="pagination")
page1=bs4.BeautifulSoup(page.__str__(),`lxml`);#將收縮結果轉成bs格式在進行篩選，篩選出list集合
pp=page1.select("li");


num=1
# 獲取a下的href
hrefs = []
videoHrefs=[]
while(num<=pp.__len__()-2):
    url = `https://www.zoom.us/recording/management?p=`+num.__str__()
    #重新請求頁面
    result = session_requests.get(
        url,
        headers=dict(referer=url)
    )
    soup = bs4.BeautifulSoup(result.text, `lxml`)
    # 獲取指定class的<div></div>
    dv = soup.find_all(class_="list-col rec-filesize")
    # 將a記錄篩選出來
    a = bs4.BeautifulSoup(dv.__str__(), `lxml`)
    ahref = a.find_all("a")

    #獲取到所有的href-meeting地址
    for i in ahref:
        hrefs.append(i[`href`])
        href=i[`href`]
        #訪問href地址獲取視訊地址
        resultForVideo = session_requests.get(
            href,
            headers=dict(referer=url)
        )
        #獲取class=btn play-id的節點
        play = bs4.BeautifulSoup(resultForVideo.text, `lxml`)

        # t通過新的soup獲取指定class的<a></a>
        aPlay = play.find_all(class_="btn play-id")
        # 將a記錄篩選出來
        str=aPlay[0][`href`].replace("play", "download");
        videoHrefs.append(str)
        print str
        filename = "D:/url/" + num.__str__() + ".mp4"
        print filename
        nowTime = datetime.datetime.now().strftime(`%Y-%m-%d %H:%M:%S`)  # 現在
        print nowTime
        # ds.do_load_media(str, filename)
        #dd._downloader(str,filename)
        urllib.urlretrieve(str, filename)
        # urllib.request.urlretrieve(url, filename, Schedule)
    #計數值增加
    num=num+1

print videoHrefs
print hrefs.__len__()

以上是基於程式碼層的爬蟲實現，因為儲存的地址是filename，而現在的filename為 filename = “D:/url/” + num.__str__() + “.mp4”

但是該檔案地址與OSS沒有關係，所以需修改為OSS繫結下的目錄地址，OSS繫結目錄地址的操作方式如下：

1.通過下載ossfs
2.通過安裝命令執行安裝

  • 對於Ubuntu，安裝命令為：

sudo apt-get update
sudo apt-get install gdebi-core
sudo gdebi your_ossfs_package

  • 對於CentOS6.5及以上，安裝命令為：

sudo yum localinstall your_ossfs_package

  • 對於CentOS5，安裝命令為：

sudo yum localinstall your_ossfs_package –nogpgcheck

3.設定bucket name 和 AccessKeyId/Secret資訊，將其存放在/etc/passwd-ossfs 檔案中，注意這個檔案的許可權必須正確設定，建議設為640。

echo my-bucket:my-access-key-id:my-access-key-secret > /etc/passwd-ossfs
chmod 640 /etc/passwd-ossfs

4.將OSS bucket mount到指定目錄。
ossfs my-bucket my-mount-point -ourl=my-oss-endpoint

具體的掛載目錄名稱有觀眾老爺們自行決定，掛載完成後只需要將程式碼中的filename賦值於指定的目錄就大功告成了~~

2018/06/04 update 增加了oss上傳功能，改成了讀取配置檔案:


# -*- coding=utf8 -*-
import requests
import bs4
import urllib
import datetime
import os
import time
import oss2
from lxml import html

#建立存放目錄地址
filedir="C:/"+time.strftime("%Y-%m-%d")
folder=os.path.exists(filedir)
if not folder:
    os.mkdir(filedir)
else:
    print "---  There is this folder!  ---"
    filedir=filedir+"-bk"
    os.mkdir(filedir)
# 開啟a.txt配置檔案
with open(`C:/Users/Administrator/Desktop/a.txt`) as file_object:
    lines = file_object.readlines()
    for line in lines:
        str = line.split(" ");
        username = str[0]
        password = str[1]
        print("當前賬號", username)
        session_requests = requests.session()

        login_url = "https://www.zoom.us/signin"
        payload = {
            "email": username,
            "password": password
        }
        result = session_requests.get(login_url)

        tree = html.fromstring(result.text)
        result = session_requests.post(
            login_url,
            data = payload,
            headers = dict(referer=login_url)
        )
        #第一頁訪問
        url = `https://www.zoom.us/recording/management`
        print url
        result = session_requests.get(
            url,
            headers = dict(referer = url)
        )
        #獲取第一頁整個頁面
        soup = bs4.BeautifulSoup(result.text, `lxml`)
        #獲取到頁數
        page=soup.find_all(class_="pagination")
        page1=bs4.BeautifulSoup(page.__str__(),`lxml`);#將收縮結果轉成bs格式在進行篩選，篩選出list集合
        pp=page1.select("li");
        num=1
        vnum=1
        page=0
        if pp.__len__()==0:
           page=1
        else:
            pa=bs4.BeautifulSoup(pp[-2].__str__(), `lxml`)
            page=int(pa.find(`a`).text)
        print(`page:`, page)
        # 獲取a下的href
        hrefs = []
        videoHrefs=[]
        while(num<=page):
            url = `https://www.zoom.us/recording/management?p=`+num.__str__()
            #重新請求頁面
            result = session_requests.get(
                url,
                headers=dict(referer=url)
            )
            soup = bs4.BeautifulSoup(result.text, `lxml`)
            # 獲取指定class的<div></div>
            dv = soup.find_all(class_="list-col rec-filesize")
            # 將a記錄篩選出來
            a = bs4.BeautifulSoup(dv.__str__(), `lxml`)
            ahref = a.find_all("a")

            #獲取到所有的href-meeting地址
            for i in ahref:
                hrefs.append(i[`href`])
                href=i[`href`]
                #訪問href地址獲取視訊地址
                resultForVideo = session_requests.get(
                    href,
                    headers=dict(referer=url)
                )
                #獲取class=btn play-id的節點
                play = bs4.BeautifulSoup(resultForVideo.text, `lxml`)

                # t通過新的soup獲取指定class的<a></a>
                aPlay = play.find_all(class_="btn play-id")
                # 將a記錄篩選出來
                str=aPlay[0][`href`].replace("play", "download");
                #videoHrefs.append(str)
                print str
                # 獲取到會議ID
                metID = play.find_all(class_="control-label meeting-topic")
                st = metID.__str__().index(`>`) + 1
                en = metID.__str__().index(`</`)
                metID = metID.__str__()[st:en]
                # 判斷是否之前已經有相同檔名有的話就加編號
                echo = ""
                if metID in videoHrefs:
                    # for fnum in range(videoHrefs.__len__()):
                    #     ss = videoHrefs[fnum][0:videoHrefs[i].__len__() - 3]
                    #     if ss == metID:
                    #         echo=videoHrefs[fnum]
                    # metID=echo[0:echo.__len__()-3]+(int(echo[echo.__len__()-3:echo.__len__()])+1).__str__()
                    metID = metID + "-002"
                # 獲取到新的metID的命名
                videoHrefs.append(metID)

                filename = filedir + "/" + metID.__str__() + ".mp4"
                print("fileNum", vnum)
                vnum=vnum+1
                print filename
                nowTime = datetime.datetime.now().strftime(`%Y-%m-%d %H:%M:%S`)  # 現在
                print nowTime
                # ds.do_load_media(str, filename)
                #dd._downloader(str,filename)
                urllib.urlretrieve(str, filename)
                # urllib.request.urlretrieve(url, filename, Schedule)
            #頁碼計數值增加
            num=num+1
            print(`page`,num)

print videoHrefs
print hrefs.__len__()


#通過oss2 將視訊上傳到bucket
#使用者aky
auth = oss2.Auth(`aid`, `keys`)
bucket = oss2.Bucket(auth, `oss-cn-shanghai-internal.aliyuncs.com`, `bucketName`)
#獲取當前時間命名的目錄
nowTimeDir=time.strftime("%Y-%m-%d")
#迴圈上傳視訊到oss上
ossNum=1
for videoPath in videoHrefs:
    ossPath=nowTimeDir+`/`+videoPath+".mp4"
    localPath=filedir+"/"+ videoPath.__str__() + ".mp4"
    bucket.put_object_from_file(ossPath, localPath)
    print("ossFile",ossNum)
    ossNum=ossNum+1


相關文章