from __future__ import print_function from pandas import DataFrame,Series import pandas as pd datafile='/root/dataset/air_customer_Data/air_data.csv' data=pd.read_csv(datafile,encoding='utf-8') cleanedfile='cleaned.csv' data1=data[data['SUM_YR_1'].notnull() & data['SUM_YR_2'].notnull()] data1 index1=data['SUM_YR_1']!=0 index2=data['SUM_YR_2']!=0 index3=data['SEG_KM_SUM']>0 data1 = data1[(index1 | index2) & index3] data1.to_csv(cleanedfile) data2=data1[['LOAD_TIME','FFP_DATE','LAST_TO_END','FLIGHT_COUNT','SEG_KM_SUM','avg_discount']] data2.to_csv('datadecrese.csv') 3 import numpy as np data=pd.read_csv('datadecrese.csv') data['L']=pd.to_datetime(data['LOAD_TIME'])-pd.to_datetime(data['FFP_DATE']) data['L'] =data['L'].astype("str").str.split().str[0] # 去除数据中的days字符,只输出数字,再转化为数值型数据 data['L'] = data['L'].astype("int") / 30 data.drop(columns=['LOAD_TIME','FFP_DATE'], inplace=True) data.rename(columns = {'LAST_TO_END':'R','FLIGHT_COUNT':'F','SEG_KM_SUM':'M','avg_discount':'C'},inplace=True) data.drop(columns=['Unnamed: 0'], inplace=True) 4 data.describe() 5 P108 data=(data-data.mean())/data.std() 6 import pandas as pd from pandas import DataFrame,Series from sklearn.cluster import KMeans k=5 kmodel=KMeans(n_clusters=k,random_state=3) kmodel.fit(data) 7 import matplotlib import matplotlib.pyplot as plt clu=kmodel.cluster_centers_ x=[1,2,3,4,5] plt.rcParams['font.sans-serif'] = 'SimHei' plt.rcParams['axes.unicode_minus'] = False for i in range(5): plt.plot(x,clu[i]) plt.show
时间: 2024-01-25 16:02:44 浏览: 22
这段代码是用Python的Pandas和Scikit-learn库进行数据预处理和聚类分析的。首先,从一个csv文件中读取数据,然后进行数据清洗,去除缺失值和异常值,再将数据保存为新的csv文件。接着,读取新的csv文件,对数据进行标准化处理,再使用KMeans算法进行聚类分析。最后,使用Matplotlib库对聚类结果进行可视化展示。
相关问题
class myDataset(Dataset): def __init__(self, csv_file, txt_file, root_dir, other_file): self.csv_data = pd.read_csv(csv_file) with open(txt_file, 'r') as f: data_list = f.readlines() self.txt_data = data_list self.root_dir = root_dir def __ien__(self): return len(self.csv_data) def __getitem__(self, idx): data = (self.csv_data[idx], self.txt_data[idx]) return data
在您提供的代码中,有两个地方出现了错误。
首先,`__ien__` 应该是 `__len__`,表示获取数据集的长度。修正后的代码如下:
```python
import pandas as pd
from torch.utils.data import Dataset
class myDataset(Dataset):
def __init__(self, csv_file, txt_file, root_dir, other_file):
self.csv_data = pd.read_csv(csv_file)
with open(txt_file, 'r') as f:
data_list = f.readlines()
self.txt_data = data_list
self.root_dir = root_dir
def __len__(self):
return len(self.csv_data)
def __getitem__(self, idx):
data = (self.csv_data.iloc[idx], self.txt_data[idx])
return data
```
其次,在 `__getitem__` 方法中,您正在尝试使用索引访问 `self.csv_data` 和 `self.txt_data`,这样会导致错误。要正确访问数据,请使用 `.iloc` 用于 DataFrame 的索引,以及使用索引访问列表。修正后的代码如下:
```python
import pandas as pd
from torch.utils.data import Dataset
class myDataset(Dataset):
def __init__(self, csv_file, txt_file, root_dir, other_file):
self.csv_data = pd.read_csv(csv_file)
with open(txt_file, 'r') as f:
data_list = f.readlines()
self.txt_data = data_list
self.root_dir = root_dir
def __len__(self):
return len(self.csv_data)
def __getitem__(self, idx):
data = (self.csv_data.iloc[idx], self.txt_data[idx])
return data
```
这样,您的代码应该能够正常运行了。请注意,我假设您使用了 PyTorch 的 `Dataset` 类来实现您自己的数据集类。如果您使用其他深度学习框架或库,可能需要相应地调整代码。
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--path', type=str, default=r"data/UCI HAR Dataset/UCI HAR Dataset", help='UCI dataset data path') parser.add_argument('--save', type=str, default='data/UCI_Smartphone_Raw.csv', help='save file name') args = parser.parse_args() data_path = args.path # read train subjects train_subjects = pd.read_csv(os.path.join(data_path, 'train/subject_train.txt'), header=None, names=['subject']) # read test subjects test_subjects = pd.read_csv(os.path.join(data_path, 'test/subject_test.txt'), header=None, names=['subject']) # concat subjects = pd.concat([train_subjects, test_subjects], axis=0) # read train labels train_labels = pd.read_csv(os.path.join(data_path, 'train/y_train.txt'), header=None, names=['label']) # read train labels test_labels = pd.read_csv(os.path.join(data_path, 'test/y_test.txt'), header=None, names=['label']) # labels labels = pd.concat([train_labels, test_labels], axis=0) final_dataframe = pd.concat([subjects, labels], axis=1) data = [] for name in COLUMNS: final_dataframe = pd.concat([final_dataframe, read_txt(name)], axis=1) final_dataframe.to_csv(args.save,index=False) 如何将文中txt文件改成mnist数据集数据,其他不做大修改
要将代码修改为使用MNIST数据集而不是txt文件,你需要进行以下修改:
1. 导入所需的库和模块:
```python
import numpy as np
import pandas as pd
from sklearn.datasets import fetch_openml
```
2. 加载MNIST数据集:
```python
mnist = fetch_openml('mnist_784', version=1, cache=True)
```
3. 将图像矩阵和标签分别存储为numpy数组:
```python
X = mnist.data
y = mnist.target
```
4. 创建一个包含所有数据的Pandas数据框:
```python
df = pd.DataFrame(X)
df['label'] = y
```
5. 将数据框保存为CSV文件:
```python
df.to_csv(args.save, index=False)
```
注意,这里我们使用了`args.save`作为保存文件名,这是根据你之前的代码中的`--save`命令行参数来决定的。
除了以上修改,你还需要删除以下代码块,因为MNIST数据集已经包含了图像数据和标签,不需要再读取其他文件:
```python
# read train subjects
train_subjects = pd.read_csv(os.path.join(data_path, 'train/subject_train.txt'), header=None, names=['subject'])
# read test subjects
test_subjects = pd.read_csv(os.path.join(data_path, 'test/subject_test.txt'), header=None, names=['subject'])
# concat
subjects = pd.concat([train_subjects, test_subjects], axis=0)
# read train labels
train_labels = pd.read_csv(os.path.join(data_path, 'train/y_train.txt'), header=None, names=['label'])
# read train labels
test_labels = pd.read_csv(os.path.join(data_path, 'test/y_test.txt'), header=None, names=['label'])
# labels
labels = pd.concat([train_labels, test_labels], axis=0)
final_dataframe = pd.concat([subjects, labels], axis=1)
data = []
for name in COLUMNS:
final_dataframe = pd.concat([final_dataframe, read_txt(name)], axis=1)
```
这些修改后的代码将直接将MNIST数据集转换为CSV文件,并保存到指定的路径中。