影片去重和鏡頭智慧分割是現代影片處理中的重要技術，尤其是在內容創作和管理中。下面我將為你詳細解釋這一過程中的關鍵概念、相關技術以及如何利用大模型TransNetV2來實現自動剪輯，並給出一個Python自動指令碼的示例。
關鍵概念解釋
- 影片去重：指的是從一段影片中識別並刪除重複的片段，以減少冗餘內容，提高影片的觀賞性和資訊密度。
- 鏡頭智慧分割：是將影片分割成多個鏡頭（場景），每個鏡頭代表一個獨立的場景或事件，這樣可以更方便地進行分析和編輯。
- TransNetV2：是一種基於深度學習的模型，專門用於影片分析和處理。它透過學習影片中的時間和空間特徵，能夠有效地識別鏡頭的起始和結束位置。
- Python自動指令碼：使用Python編寫的程式，可以自動執行影片處理任務，如鏡頭分割和去重，減少人工干預。
去重手段
市面上影片去重軟體有很多，但我們要把去重功能整合到自己的指令碼中，所以不能使用其他軟體，換言之要自己實現去重功能。
去重的手段主要有以下：
- 抽幀
- 插幀
- 裁剪
- 改解析度
- 改畫面比例
- 新增片頭片尾
- 修改幀率
- 修改位元率
- 修改亮度、對比度、飽和度
- 變速
- 新增影片蒙版
- 新增圖片蒙版
- 新增貼紙
新增轉場- 新增背景圖片/影片
- 掃光
- 修改背景音樂音量
- 新增額外音軌
- 映象反轉
- 畫中畫
- 修改後設資料
- 加水印
- 打亂鏡頭順序
Python 自動指令碼示例
以下是一個使用TransNetV2模型進行影片鏡頭智慧分割的Python自動指令碼示例：
main.py
import os
from moviepy.editor import VideoFileClip
from transnetv2 import TransNetV2
if __name__ == '__main__':
video_path = input('請輸入影片檔案路徑\n')
while not os.path.isfile(video_path):
video_path = input('請輸入正確的影片檔案路徑\n')
video_name = os.path.basename(video_path)
video_name_without_ext = os.path.splitext(video_name)[0]
video_folder = os.path.dirname(video_path)
output_folder = os.path.join(video_folder, video_name_without_ext)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
model = TransNetV2()
video_frames, single_frame_predictions, all_frame_predictions = model.predict_video_2(video_path)
scenes = model.predictions_to_scenes(single_frame_predictions)
video_clip = VideoFileClip(video_path)
for i, (start, end) in enumerate(scenes):
start_time = start / video_clip.fps
end_time = end / video_clip.fps
segment_clip = video_clip.subclip(start_time, end_time)
output_path = os.path.join(output_folder, f'{video_name_without_ext}_{i+1}.mp4')
segment_clip.write_videofile(output_path, codec='libx264', fps=video_clip.fps)
video_clip.close()
input('\n任務已完成，按Enter鍵退出……')
main.py
import os
from moviepy.editor import VideoFileClip
from transnetv2 import TransNetV2
if __name__ == '__main__':
video_path = input('請輸入影片檔案路徑\n')
while not os.path.isfile(video_path):
video_path = input('請輸入正確的影片檔案路徑\n')
video_name = os.path.basename(video_path)
video_name_without_ext = os.path.splitext(video_name)[0]
video_folder = os.path.dirname(video_path)
output_folder = os.path.join(video_folder, video_name_without_ext)
if not os.path.exists(output_folder):
os.makedirs(output_folder)
model = TransNetV2()
video_frames, single_frame_predictions, all_frame_predictions = model.predict_video_2(video_path)
scenes = model.predictions_to_scenes(single_frame_predictions)
video_clip = VideoFileClip(video_path)
for i, (start, end) in enumerate(scenes):
start_time = start / video_clip.fps
end_time = end / video_clip.fps
segment_clip = video_clip.subclip(start_time, end_time)
output_path = os.path.join(output_folder, f'{video_name_without_ext}_{i+1}.mp4')
segment_clip.write_videofile(output_path, codec='libx264', fps=video_clip.fps)
video_clip.close()
input('\n任務已完成，按Enter鍵退出……')
transnetv2.py
import math
import os
os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
import numpy as np
import tensorflow as tf
from moviepy.editor import VideoFileClip
class TransNetV2:
def __init__(self, model_dir=None):
if model_dir is None:
# model_dir = os.path.join(os.path.dirname(__file__), "transnetv2-weights/")
model_dir = "transnetv2-weights/"
if not os.path.isdir(model_dir):
raise FileNotFoundError(f"[TransNetV2] ERROR: {model_dir} is not a directory.")
else:
print(f"[TransNetV2] Using weights from {model_dir}.")
self._input_size = (27, 48, 3)
try:
self._model = tf.saved_model.load(model_dir)
except OSError as exc:
raise IOError(f"[TransNetV2] It seems that files in {model_dir} are corrupted or missing. "
f"Re-download them manually and retry. For more info, see: "
f"https://github.com/soCzech/TransNetV2/issues/1#issuecomment-647357796") from exc
def predict_raw(self, frames: np.ndarray):
assert len(frames.shape) == 5 and frames.shape[2:] == self._input_size, \
"[TransNetV2] Input shape must be [batch, frames, height, width, 3]."
frames = tf.cast(frames, tf.float32)
logits, dict_ = self._model(frames)
single_frame_pred = tf.sigmoid(logits)
all_frames_pred = tf.sigmoid(dict_["many_hot"])
return single_frame_pred, all_frames_pred
def predict_frames(self, frames: np.ndarray):
assert len(frames.shape) == 4 and frames.shape[1:] == self._input_size, \
"[TransNetV2] Input shape must be [frames, height, width, 3]."
def input_iterator():
# return windows of size 100 where the first/last 25 frames are from the previous/next batch
# the first and last window must be padded by copies of the first and last frame of the video
no_padded_frames_start = 25
no_padded_frames_end = 25 + 50 - (len(frames) % 50 if len(frames) % 50 != 0 else 50) # 25 - 74
start_frame = np.expand_dims(frames[0], 0)
end_frame = np.expand_dims(frames[-1], 0)
padded_inputs = np.concatenate(
[start_frame] * no_padded_frames_start + [frames] + [end_frame] * no_padded_frames_end, 0
)
ptr = 0
while ptr + 100 <= len(padded_inputs):
out = padded_inputs[ptr:ptr + 100]
ptr += 50
yield out[np.newaxis]
predictions = []
for inp in input_iterator():
single_frame_pred, all_frames_pred = self.predict_raw(inp)
predictions.append((single_frame_pred.numpy()[0, 25:75, 0],
all_frames_pred.numpy()[0, 25:75, 0]))
print("\r[TransNetV2] Processing video frames {}/{}".format(
min(len(predictions) * 50, len(frames)), len(frames)
), end="")
print("\n")
single_frame_pred = np.concatenate([single_ for single_, all_ in predictions])
all_frames_pred = np.concatenate([all_ for single_, all_ in predictions])
return single_frame_pred[:len(frames)], all_frames_pred[:len(frames)] # remove extra padded frames
def predict_video(self, video_fn: str):
try:
import ffmpeg
except ModuleNotFoundError:
raise ModuleNotFoundError("For `predict_video` function `ffmpeg` needs to be installed in order to extract "
"individual frames from video file. Install `ffmpeg` command line tool and then "
"install python wrapper by `pip install ffmpeg-python`.")
print("[TransNetV2] Extracting frames from {}".format(video_fn))
video_stream, err = ffmpeg.input(video_fn).output(
"pipe:", format="rawvideo", pix_fmt="rgb24", s="48x27"
).run(capture_stdout=True, capture_stderr=True)
video = np.frombuffer(video_stream, np.uint8).reshape([-1, 27, 48, 3])
return (video, *self.predict_frames(video))
def predict_video_2(self, video_fn: str):
print("[TransNetV2] Extracting frames from {}".format(video_fn))
clip = VideoFileClip(video_fn, target_resolution=(27, 48))
duration = math.floor(clip.duration * 10) / 10
fps = clip.fps # 影片的幀率
frames = []
for t in range(0, int(duration * fps)):
frame = clip.get_frame(t / fps) # 獲取當前時間點的幀
if len(frame) != 0: # 如果幀的長度不為零
frames.append(frame) # 將幀新增到 frames 列表中
video = np.array(frames)
return video, *self.predict_frames(video)
@staticmethod
def predictions_to_scenes(predictions: np.ndarray, threshold: float = 0.5):
predictions = (predictions > threshold).astype(np.uint8)
scenes = []
t, t_prev, start = -1, 0, 0
for i, t in enumerate(predictions):
if t_prev == 1 and t == 0:
start = i
if t_prev == 0 and t == 1 and i != 0:
scenes.append([start, i])
t_prev = t
if t == 0:
scenes.append([start, i])
# just fix if all predictions are 1
if len(scenes) == 0:
return np.array([[0, len(predictions) - 1]], dtype=np.int32)
return np.array(scenes, dtype=np.int32)
@staticmethod
def visualize_predictions(frames: np.ndarray, predictions):
from PIL import Image, ImageDraw
if isinstance(predictions, np.ndarray):
predictions = [predictions]
ih, iw, ic = frames.shape[1:]
width = 25
# pad frames so that length of the video is divisible by width
# pad frames also by len(predictions) pixels in width in order to show predictions
pad_with = width - len(frames) % width if len(frames) % width != 0 else 0
frames = np.pad(frames, [(0, pad_with), (0, 1), (0, len(predictions)), (0, 0)])
predictions = [np.pad(x, (0, pad_with)) for x in predictions]
height = len(frames) // width
img = frames.reshape([height, width, ih + 1, iw + len(predictions), ic])
img = np.concatenate(np.split(
np.concatenate(np.split(img, height), axis=2)[0], width
), axis=2)[0, :-1]
img = Image.fromarray(img)
draw = ImageDraw.Draw(img)
# iterate over all frames
for i, pred in enumerate(zip(*predictions)):
x, y = i % width, i // width
x, y = x * (iw + len(predictions)) + iw, y * (ih + 1) + ih - 1
# we can visualize multiple predictions per single frame
for j, p in enumerate(pred):
color = [0, 0, 0]
color[(j + 1) % 3] = 255
value = round(p * (ih - 1))
if value != 0:
draw.line((x + j, y, x + j, y - value), fill=tuple(color), width=1)
return img
def main():
import sys
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("files", type=str, nargs="+", help="path to video files to process")
parser.add_argument("--weights", type=str, default=None,
help="path to TransNet V2 weights, tries to infer the location if not specified")
parser.add_argument('--visualize', action="store_true",
help="save a png file with prediction visualization for each extracted video")
args = parser.parse_args()
model = TransNetV2(args.weights)
for file in args.files:
if os.path.exists(file + ".predictions.txt") or os.path.exists(file + ".scenes.txt"):
print(f"[TransNetV2] {file}.predictions.txt or {file}.scenes.txt already exists. "
f"Skipping video {file}.", file=sys.stderr)
continue
video_frames, single_frame_predictions, all_frame_predictions = \
model.predict_video(file)
predictions = np.stack([single_frame_predictions, all_frame_predictions], 1)
np.savetxt(file + ".predictions.txt", predictions, fmt="%.6f")
scenes = model.predictions_to_scenes(single_frame_predictions)
np.savetxt(file + ".scenes.txt", scenes, fmt="%d")
if args.visualize:
if os.path.exists(file + ".vis.png"):
print(f"[TransNetV2] {file}.vis.png already exists. "
f"Skipping visualization of video {file}.", file=sys.stderr)
continue
pil_image = model.visualize_predictions(
video_frames, predictions=(single_frame_predictions, all_frame_predictions))
pil_image.save(file + ".vis.png")
if __name__ == "__main__":
main()
完整原始碼即刻收集