import pickle import math import numpy as np from torch.utils.data import DataLoader, TensorDataset def sample_data(filepath, idx, path): f = open(filepath, 'rb') bi = pickle.load(f) data = np.arange(0, idx) data = data.tolist() samples = [] bs = [] a = path r = math.ceil(len(data) / a) for i in range(a): if len(data) > r: sample = np.random.choice(data, r, replace=False) else: sample = np.random.choice(data, len(data), replace=False) b = [bi[j] for j in list(sample)] bs.append(b) samples.append(sample) for s in sample: data.remove(s) samples = [np.array(s) for s in samples] samples = np.concatenate(samples, axis=0) bs = [np.array(e) for e in bs] bs = np.concatenate(bs, axis=0) return samples, bs如何让每一次for循环中的b值都能单独进行一次深度学习的dataloader,最后遍历完for循环中所有的b值
时间: 2024-03-20 08:41:06 浏览: 138
ECGdata.zip_ECG data.pickle_ECGData.zip_ECG_read_ECGdata如何使用_用电数
你可以在for循环中将每个b值转换为一个TensorDataset对象,并将它们放入一个列表中,最后使用DataLoader将它们组成一个batch。以下是示例代码:
```
import pickle
import math
import numpy as np
import torch
from torch.utils.data import DataLoader, TensorDataset
def sample_data(filepath, idx, path, batch_size):
f = open(filepath, 'rb')
bi = pickle.load(f)
data = np.arange(0, idx)
data = data.tolist()
samples = []
bs = []
dataloader_list = [] # 用于存放每个b值的dataloader
a = path
r = math.ceil(len(data) / a)
for i in range(a):
if len(data) > r:
sample = np.random.choice(data, r, replace=False)
else:
sample = np.random.choice(data, len(data), replace=False)
b = [bi[j] for j in list(sample)]
bs.append(b)
samples.append(sample)
for s in sample:
data.remove(s)
# 将b值转换为TensorDataset对象
dataset = TensorDataset(torch.Tensor(b))
# 创建dataloader
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
dataloader_list.append(dataloader)
samples = [np.array(s) for s in samples]
samples = np.concatenate(samples, axis=0)
bs = [np.array(e) for e in bs]
bs = np.concatenate(bs, axis=0)
# 遍历dataloader_list中的所有dataloader
for dataloader in dataloader_list:
for batch in dataloader:
# 在这里对每个batch进行深度学习操作
pass
return samples, bs
```
在上面的代码中,我们使用`TensorDataset`将每个b值转换为一个数据集对象。然后,使用`DataLoader`将每个数据集组成一个batch,并将它们放入一个列表中。最后,我们遍历dataloader_list中的所有dataloader并对它们进行深度学习操作。注意,你需要根据你的具体情况设置batch_size参数。
阅读全文