多线程、进程下载视频
一、结果展示二、使用的工具三、线程进程方法化四、多进程视频下载五、存在的问题
一、结果展示
二、使用的工具
视频下载使用you-get, 多线程使用threading、 多进程multiprocessing 以下为三个库的简介 you-get:
threading:
multiprocessing
三、线程进程方法化
from concurrent.futures import ThreadPoolExecutor as e
import time,sys
from multiprocessing.pool import Pool
from threading import Thread as t
class Thread_Pool():
"""需要输入线程数量，函数，参数"""
def __init__(self,thread_,fun,param=""):
"""传入list，通过map一个一个迭代"""
self.thread = thread_
self.fun = fun
if type(param).__name__ == 'list':
self.param = param
else:
option = input("param须为列表格式,是否继续？？？")
if option == "是":
self.param = [i for i in range(self.thread)]
else:
sys.exit()
def concurrent_Thread_package(self):
"""concurrent线程"""
global results
starttime = time.time()
pool = e(self.thread) # 一次同时多少个
try:
results = pool.map(self.fun, self.param)
except:
pass
for i, result in enumerate(results):
print('done ! result {}: {}'.format(i, result))
endtime = time.time()
across = (endtime-starttime)/60
print("花费时间{time}分钟-------------------------------------".format(time=across))
def multiprocessing_pool(self):
print("这是进程>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
"""multiprocessing.pool方法,必须在if__name__=="__main__"中运行"""
starttime = time.time()
pool = Pool(self.thread)
pool.map(self.fun, tuple(self.param))
endtime = time.time()
dtime = endtime - starttime
print("程序运行时间：%.8s s" % dtime)
def thread_module(self):
print("线程单个，没有建立线程池")
start = time.time()
for i in self.param:
t_ = t(target=self.fun, args=(i,)) # args传参
t_.start()
t_.join()
end = time.time()
dtime =end-start
print("程序运行时间：%.8s s" % dtime)
def processing_thread(self,i):
pool = e(self.thread) # 一次同时多少个
results = pool.map(self.fun, self.param)
for i_, result in enumerate(results):
print('第{}个进程下的第{}线程! result : {}done '.format(i, i_,result))
def processing_and_threading(self):
"""进程上附件线程,线程只能异步，进程可以并发"""
pool = Pool(2)
print()
i = 1
for i in range(10):
pool.apply_async(self.processing_thread,(i,))
i+=1
pool.close()
pool.join()
四、多进程视频下载
import os,glob,time
from 常用设置.进程与线程.线程方法化 import Thread_Pool as t
def download_mp4(i):
path = os.path.dirname(__file__)+"//下载"
os.system(f"you-get -o {path} https://www.bilibili.com/video/BV1Wy4y1B7vg?p={i}")
if __name__ =="__main__":
index = []
for i in range(1,20):
index.append(i)
t(20,download_mp4,index).multiprocessing_pool()#concurrent_Thread_package
path = os.path.dirname(__file__)+"//下载"
for i in glob.glob(os.path.join(path,"*")):
if i.split(".")[-1] == "xml":
os.remove(i)
五、存在的问题
即使多进程、多线程下载速度依旧没有快多少，问题在于后续视频下载网速提不上来，比如一开始能有4M/s，后续只有80K/s
