大数据测试---CSV文件分段读取上传

分段上传数据–CSV上传

def upload_stream(file, num=None): upload_url = “http://XXXXXXX” read_csv_file = pd.read_csv(file, iterator=True, low_memory=False) count = 0 if num is None: cycles_numbers = 1000 else: cycles_numbers = num // 1000 if num % 1000 == 0 else num // 1000 + 1 for i in range(cycles_numbers): # 设置循环次数 try: user = read_csv_file.get_chunk(1000) # 设置每次上传的条数 count += user.shape[0] data = user.to_json(orient="records", force_ascii=False) d = requests.post(upload_url, data.encode(encoding='utf-8'), headers=get_header(), verify=False) print(d.text) except StopIteration: print("传完了:" + str(count)) break print("上传数据行数:", count) print("上传数据列数:", user.shape[1])