mindspore_yolo/tf02_data_generate_csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n",
      "sys.version_info(major=3, minor=6, micro=9, releaselevel='final', serial=0)\n",
      "matplotlib 3.3.4\n",
      "numpy 1.19.5\n",
      "pandas 1.1.5\n",
      "sklearn 0.24.2\n",
      "tensorflow 2.2.0\n",
      "tensorflow.keras 2.3.0-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, tf, keras:\n",
    "    print(module.__name__, module.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "housing = fetch_california_housing()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11610, 8) (11610,)\n",
      "(3870, 8) (3870,)\n",
      "(5160, 8) (5160,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train_all, x_test, y_train_all, y_test = train_test_split(\n",
    "    housing.data, housing.target, random_state = 7)\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(\n",
    "    x_train_all, y_train_all, random_state = 11)\n",
    "print(x_train.shape, y_train.shape)\n",
    "print(x_valid.shape, y_valid.shape)\n",
    "print(x_test.shape, y_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "x_train_scaled = scaler.fit_transform(x_train)\n",
    "x_valid_scaled = scaler.transform(x_valid)\n",
    "x_test_scaled = scaler.transform(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf generate_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf01-dataset_basic_api.ipynb  tf03-tfrecord_basic_api.ipynb\r\n",
      "tf02_data_generate_csv.ipynb  tf04_data_generate_tfrecord.ipynb\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(x_train_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'tf02_data_generate_csv.ipynb',\n",
       " 'tf04_data_generate_tfrecord.ipynb',\n",
       " 'tf03-tfrecord_basic_api.ipynb',\n",
       " 'tf01-dataset_basic_api.ipynb']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [0 1 2 3 4]\n",
      "1 [5 6 7 8 9]\n",
      "2 [10 11 12 13 14]\n",
      "3 [15 16 17 18 19]\n"
     ]
    }
   ],
   "source": [
    "#为了把数据分好\n",
    "for file_idx, row_indices in enumerate(np.array_split(np.arange(20), 4)):\n",
    "    print(file_idx,row_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MedInc,HouseAge,AveRooms,AveBedrms,Population,AveOccup,Latitude,Longitude,MidianHouseValue\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#下面要把特征工程后的数据存为csv文件\n",
    "output_dir = \"generate_csv\"\n",
    "if not os.path.exists(output_dir):\n",
    "    os.mkdir(output_dir)\n",
    "\n",
    "#save_to_csv是工作可以直接复用的\n",
    "def save_to_csv(output_dir, data, name_prefix,\n",
    "                header=None, n_parts=10):\n",
    "    #生成文件名 格式generate_csv/{}_{:02d}.csv\n",
    "    path_format = os.path.join(output_dir, \"{}_{:02d}.csv\") \n",
    "    filenames = []\n",
    "    #把数据分为n_parts部分，写到文件中去\n",
    "    for file_idx, row_indices in enumerate(\n",
    "        np.array_split(np.arange(len(data)), n_parts)):\n",
    "        #print(file_idx,row_indices)\n",
    "        #生成子文件名\n",
    "        part_csv = path_format.format(name_prefix, file_idx)\n",
    "        filenames.append(part_csv) #文件名添加到列表\n",
    "        with open(part_csv, \"w\", encoding=\"utf-8\") as f:\n",
    "            #先写头部\n",
    "            if header is not None:\n",
    "                f.write(header + \"\\n\")\n",
    "            for row_index in row_indices:\n",
    "                #把字符串化后的每个字符串用逗号拼接起来\n",
    "                f.write(\",\".join(\n",
    "                    [repr(col) for col in data[row_index]]))\n",
    "                f.write('\\n')\n",
    "    return filenames\n",
    "#np.c_把x和y合并起来,按轴1合并\n",
    "train_data = np.c_[x_train_scaled, y_train]\n",
    "valid_data = np.c_[x_valid_scaled, y_valid]\n",
    "test_data = np.c_[x_test_scaled, y_test]\n",
    "#头部,特征，也有目标\n",
    "header_cols = housing.feature_names + [\"MidianHouseValue\"]\n",
    "#把列表变为字符串\n",
    "header_str = \",\".join(header_cols)\n",
    "print(header_str)\n",
    "print('-'*50)\n",
    "train_filenames = save_to_csv(output_dir, train_data, \"train\",\n",
    "                              header_str, n_parts=20)\n",
    "valid_filenames = save_to_csv(output_dir, valid_data, \"valid\",\n",
    "                              header_str, n_parts=10)\n",
    "test_filenames = save_to_csv(output_dir, test_data, \"test\",\n",
    "                             header_str, n_parts=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_array=np.array([[1,2,3],[4,5,6]])\n",
    "np.savetxt(\"temp.csv\",temp_array)  #savetxt会自动将整型数或者浮点数转为字符串存储"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.000000000000000000e+00 2.000000000000000000e+00 3.000000000000000000e+00\r\n",
      "4.000000000000000000e+00 5.000000000000000000e+00 6.000000000000000000e+00\r\n"
     ]
    }
   ],
   "source": [
    "!cat temp.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['generate_csv/train_00.csv', 'generate_csv/train_01.csv', 'generate_csv/train_02.csv', 'generate_csv/train_03.csv', 'generate_csv/train_04.csv', 'generate_csv/train_05.csv', 'generate_csv/train_06.csv', 'generate_csv/train_07.csv', 'generate_csv/train_08.csv', 'generate_csv/train_09.csv', 'generate_csv/train_10.csv', 'generate_csv/train_11.csv', 'generate_csv/train_12.csv', 'generate_csv/train_13.csv', 'generate_csv/train_14.csv', 'generate_csv/train_15.csv', 'generate_csv/train_16.csv', 'generate_csv/train_17.csv', 'generate_csv/train_18.csv', 'generate_csv/train_19.csv']\n",
      "train filenames:\n",
      "['generate_csv/train_00.csv',\n",
      " 'generate_csv/train_01.csv',\n",
      " 'generate_csv/train_02.csv',\n",
      " 'generate_csv/train_03.csv',\n",
      " 'generate_csv/train_04.csv',\n",
      " 'generate_csv/train_05.csv',\n",
      " 'generate_csv/train_06.csv',\n",
      " 'generate_csv/train_07.csv',\n",
      " 'generate_csv/train_08.csv',\n",
      " 'generate_csv/train_09.csv',\n",
      " 'generate_csv/train_10.csv',\n",
      " 'generate_csv/train_11.csv',\n",
      " 'generate_csv/train_12.csv',\n",
      " 'generate_csv/train_13.csv',\n",
      " 'generate_csv/train_14.csv',\n",
      " 'generate_csv/train_15.csv',\n",
      " 'generate_csv/train_16.csv',\n",
      " 'generate_csv/train_17.csv',\n",
      " 'generate_csv/train_18.csv',\n",
      " 'generate_csv/train_19.csv']\n",
      "valid filenames:\n",
      "['generate_csv/valid_00.csv',\n",
      " 'generate_csv/valid_01.csv',\n",
      " 'generate_csv/valid_02.csv',\n",
      " 'generate_csv/valid_03.csv',\n",
      " 'generate_csv/valid_04.csv',\n",
      " 'generate_csv/valid_05.csv',\n",
      " 'generate_csv/valid_06.csv',\n",
      " 'generate_csv/valid_07.csv',\n",
      " 'generate_csv/valid_08.csv',\n",
      " 'generate_csv/valid_09.csv']\n",
      "test filenames:\n",
      "['generate_csv/test_00.csv',\n",
      " 'generate_csv/test_01.csv',\n",
      " 'generate_csv/test_02.csv',\n",
      " 'generate_csv/test_03.csv',\n",
      " 'generate_csv/test_04.csv',\n",
      " 'generate_csv/test_05.csv',\n",
      " 'generate_csv/test_06.csv',\n",
      " 'generate_csv/test_07.csv',\n",
      " 'generate_csv/test_08.csv',\n",
      " 'generate_csv/test_09.csv']\n"
     ]
    }
   ],
   "source": [
    "#看下生成文件的文件名\n",
    "print(train_filenames)\n",
    "import pprint  #为了打印美观性\n",
    "print(\"train filenames:\")\n",
    "pprint.pprint(train_filenames)\n",
    "print(\"valid filenames:\")\n",
    "pprint.pprint(valid_filenames)\n",
    "print(\"test filenames:\")\n",
    "pprint.pprint(test_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(b'generate_csv/train_13.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_01.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_14.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_11.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_12.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_06.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_15.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_10.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_05.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_02.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_00.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_07.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_16.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_09.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_19.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_03.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_04.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_18.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_17.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_08.csv', shape=(), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "# 1. filename -> dataset\n",
    "# 2. read file -> dataset -> datasets -> merge\n",
    "# 3. parse csv\n",
    "#list_files把文件名搞为一个dataset\n",
    "# list_files默认行为是按不确定的随机混排顺序返回文件名\n",
    "filename_dataset = tf.data.Dataset.list_files(train_filenames)\n",
    "for filename in filename_dataset:\n",
    "    print(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(b'generate_csv/train_00.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_01.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_02.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_03.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_04.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_05.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_06.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_07.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_08.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_09.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_10.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_11.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_12.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_13.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_14.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_15.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_16.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_17.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_18.csv', shape=(), dtype=string)\n",
      "tf.Tensor(b'generate_csv/train_19.csv', shape=(), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "filename_mydataset=tf.data.Dataset.from_tensor_slices(train_filenames)\n",
    "filename_mydataset=filename_mydataset.repeat(1)\n",
    "for i in filename_mydataset:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把数据从文件中拿出来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(b'0.801544314532886,0.27216142415910205,-0.11624392696666119,-0.2023115137272354,-0.5430515742518128,-0.021039615516440048,-0.5897620622908205,-0.08241845654707416,3.226', shape=(), dtype=string)\n",
      "tf.Tensor(b'-0.2980728090942217,0.3522616607867429,-0.10920507530549702,-0.25055520947444,-0.034064024638222286,-0.006034004264459185,1.080554840130013,-1.0611381656679573,1.514', shape=(), dtype=string)\n",
      "tf.Tensor(b'0.8115083791797953,-0.04823952235146133,0.5187339067174729,-0.029386394873127775,-0.034064024638222286,-0.05081594842905086,-0.7157356834231196,0.9162751241885168,2.147', shape=(), dtype=string)\n",
      "tf.Tensor(b'-0.6906143291679195,-0.1283397589791022,7.0201810347470595,5.624287386169439,-0.2663292879200034,-0.03662080416157129,-0.6457503383496215,1.2058962626018372,1.352', shape=(), dtype=string)\n",
      "tf.Tensor(b'0.401276648075221,-0.9293421252555106,-0.05333050451405854,-0.1865945262276826,0.6545661895448709,0.026434465728210874,0.9312527706398824,-1.4406417263474771,2.512', shape=(), dtype=string)\n",
      "tf.Tensor(b'-0.8757754235423053,1.874166156711919,-0.9487499555702599,-0.09657184824705009,-0.7163432355284542,-0.07790191228558485,0.9825753570271144,-1.4206678547327694,2.75', shape=(), dtype=string)\n",
      "tf.Tensor(b'0.15782311132800697,0.43236189741438374,0.3379948076652917,-0.015880306122244434,-0.3733890577139493,-0.05305245634489608,0.8006134598360177,-1.2359095422966828,3.169', shape=(), dtype=string)\n",
      "tf.Tensor(b'2.2878417437355094,-1.8905449647872008,0.6607106467795992,-0.14964778023694128,-0.06672632728722275,0.44788055801575993,-0.5337737862320228,0.5667323709310584,3.59', shape=(), dtype=string)\n",
      "tf.Tensor(b'-1.0591781535672364,1.393564736946074,-0.026331968874673636,-0.11006759528831847,-0.6138198966579805,-0.09695934953589447,0.3247131133362288,-0.037477245413977976,0.672', shape=(), dtype=string)\n",
      "tf.Tensor(b'-0.2223565745313433,1.393564736946074,0.02991299565857307,0.0801452044790158,-0.509481985418118,-0.06238599304952824,-0.86503775291325,0.8613469772480595,2.0', shape=(), dtype=string)\n",
      "tf.Tensor(b'-0.03058829290446139,-0.9293421252555106,0.2596214817762415,-0.00601274044096368,-0.5004091235711734,-0.030779867916061836,1.5984463936739026,-1.8151518191233238,1.598', shape=(), dtype=string)\n",
      "tf.Tensor(b'1.9063832474401923,0.5124621340420246,0.44758280183798754,-0.276721775345798,-0.6310583341671753,-0.07081146722873086,-0.7064043040799849,0.7464972154634646,5.00001', shape=(), dtype=string)\n",
      "tf.Tensor(b'-0.9868720801669367,0.832863080552588,-0.18684708416901633,-0.14888949288707784,-0.4532302419670616,-0.11504995754593579,1.6730974284189664,-0.7465496877362412,1.138', shape=(), dtype=string)\n",
      "tf.Tensor(b'0.29422955783115173,1.874166156711919,0.004626028663628252,-0.28479278487900694,-0.5602900117610076,-0.1196496378702887,1.3558305307524392,-0.9512818717870428,1.625', shape=(), dtype=string)\n",
      "tf.Tensor(b'0.7751155655229017,1.874166156711919,0.15645971958808144,-0.18905190538070707,-0.6292437617977863,-0.08791603438866835,-0.7483955111240856,0.5717258388347319,4.851', shape=(), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "#一访问list_files的dataset对象就随机了文件顺序\n",
    "# for filename in filename_dataset:\n",
    "#     print(filename)\n",
    "n_readers = 5\n",
    "dataset = filename_mydataset.interleave(\n",
    "    #前面1行是header\n",
    "#     lambda filename: tf.data.TextLineDataset(filename),\n",
    "    #不带header,把特征名字去掉\n",
    "    lambda filename: tf.data.TextLineDataset(filename).skip(1),\n",
    "    cycle_length = n_readers,  #cycle_length和block_length增加获取了数据的随机性\n",
    "    block_length=2\n",
    ")\n",
    "for line in dataset.take(15):\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把每一行数据切分为对应类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor: shape=(), dtype=int32, numpy=1>, <tf.Tensor: shape=(), dtype=int32, numpy=2>, <tf.Tensor: shape=(), dtype=float32, numpy=3.0>, <tf.Tensor: shape=(), dtype=string, numpy=b'4'>, <tf.Tensor: shape=(), dtype=float32, numpy=5.0>]\n"
     ]
    }
   ],
   "source": [
    "#parse csv 解析csv，通过decode_csv\n",
    "# tf.io.decode_csv(str, record_defaults)\n",
    "\n",
    "sample_str = '1,2,3,4,5'\n",
    "record_defaults = [\n",
    "    tf.constant(0, dtype=tf.int32),\n",
    "    0,\n",
    "    np.nan,\n",
    "    \"hello1\",\n",
    "    tf.constant([])#没有固定类型，默认是float32\n",
    "]\n",
    "#sample_str数据格式化，按照record_defaults进行处理\n",
    "parsed_fields = tf.io.decode_csv(sample_str, record_defaults)\n",
    "print(parsed_fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(), dtype=int32, numpy=0>,\n",
       " <tf.Tensor: shape=(), dtype=int32, numpy=0>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=nan>,\n",
       " <tf.Tensor: shape=(), dtype=string, numpy=b'hello1'>,\n",
       " <tf.Tensor: shape=(), dtype=float32, numpy=1.0>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#我们传一个空的字符串测试\n",
    "#最后一个为1是可以转换的\n",
    "try:\n",
    "    parsed_fields = tf.io.decode_csv(',,,,1', record_defaults)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)\n",
    "parsed_fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expect 5 fields but have 7 in record 0 [Op:DecodeCSV]\n"
     ]
    }
   ],
   "source": [
    "#我们给的值过多的情况\n",
    "try:\n",
    "    parsed_fields = tf.io.decode_csv('1,2,3,4,5,6,', record_defaults)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(8,), dtype=float32, numpy=\n",
       " array([-0.9868721 ,  0.8328631 , -0.18684709, -0.1488895 , -0.45323023,\n",
       "        -0.11504996,  1.6730974 , -0.74654967], dtype=float32)>,\n",
       " <tf.Tensor: shape=(1,), dtype=float32, numpy=array([1.138], dtype=float32)>)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#解析一行\n",
    "def parse_csv_line(line, n_fields = 9):\n",
    "    #先写一个默认的格式，就是9个nan,如果从csv中读取缺失数据，就会变为nan\n",
    "    defs = [tf.constant(np.nan)] * n_fields\n",
    "    #使用decode_csv解析\n",
    "    parsed_fields = tf.io.decode_csv(line, record_defaults=defs)\n",
    "    #前8个是x，最后一个是y\n",
    "    x = tf.stack(parsed_fields[0:-1])\n",
    "    y = tf.stack(parsed_fields[-1:])\n",
    "    return x, y\n",
    "\n",
    "parse_csv_line(b'-0.9868720801669367,0.832863080552588,-0.18684708416901633,-0.14888949288707784,-0.4532302419670616,-0.11504995754593579,1.6730974284189664,-0.7465496877362412,1.138',\n",
    "               n_fields=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 8), (None, 1)), types: (tf.float32, tf.float32)>\n",
      "--------------------------------------------------\n",
      "x:\n",
      "<tf.Tensor: shape=(4, 8), dtype=float32, numpy=\n",
      "array([[ 0.15782312,  0.4323619 ,  0.3379948 , -0.01588031, -0.37338907,\n",
      "        -0.05305246,  0.80061346, -1.2359096 ],\n",
      "       [-1.0591781 ,  1.3935647 , -0.02633197, -0.1100676 , -0.6138199 ,\n",
      "        -0.09695935,  0.3247131 , -0.03747724],\n",
      "       [-0.82195884,  1.8741661 ,  0.1821235 , -0.03170019, -0.6011179 ,\n",
      "        -0.14337493,  1.0852206 , -0.8613995 ],\n",
      "       [ 0.63034356,  1.8741661 , -0.06713215, -0.12543367, -0.19737554,\n",
      "        -0.02272263, -0.69240725,  0.72652334]], dtype=float32)>\n",
      "y:\n",
      "<tf.Tensor: shape=(4, 1), dtype=float32, numpy=\n",
      "array([[3.169],\n",
      "       [0.672],\n",
      "       [1.054],\n",
      "       [2.419]], dtype=float32)>\n",
      "x:\n",
      "<tf.Tensor: shape=(4, 8), dtype=float32, numpy=\n",
      "array([[ 0.48530516, -0.8492419 , -0.06530126, -0.02337966,  1.4974351 ,\n",
      "        -0.07790658, -0.90236324,  0.78145146],\n",
      "       [ 2.2878418 , -1.890545  ,  0.66071063, -0.14964779, -0.06672633,\n",
      "         0.44788057, -0.5337738 ,  0.56673235],\n",
      "       [-0.22235657,  1.3935647 ,  0.029913  ,  0.0801452 , -0.50948197,\n",
      "        -0.06238599, -0.86503774,  0.86134696],\n",
      "       [-0.46794146, -0.92934215,  0.11909926, -0.06047011,  0.30344644,\n",
      "        -0.02185189,  1.8737221 , -1.0411643 ]], dtype=float32)>\n",
      "y:\n",
      "<tf.Tensor: shape=(4, 1), dtype=float32, numpy=\n",
      "array([[2.956],\n",
      "       [3.59 ],\n",
      "       [2.   ],\n",
      "       [1.012]], dtype=float32)>\n"
     ]
    }
   ],
   "source": [
    "# 1. filename -> dataset\n",
    "# 2. read file -> dataset -> datasets -> merge\n",
    "# 3. parse csv\n",
    "#完成整个流程\n",
    "def csv_reader_dataset(filenames, n_readers=5,\n",
    "                       batch_size=32, n_parse_threads=5,\n",
    "                       shuffle_buffer_size=10000):\n",
    "    #把文件名类别变为dataset tensor\n",
    "    dataset = tf.data.Dataset.list_files(filenames)\n",
    "    #变为repeat dataset可以让读到最后一个样本时，从新去读第一个样本\n",
    "    dataset = dataset.repeat()\n",
    "    dataset = dataset.interleave(\n",
    "        #skip(1)是因为每个文件存了特征名字，target名字\n",
    "        lambda filename: tf.data.TextLineDataset(filename).skip(1),\n",
    "        cycle_length = n_readers\n",
    "    )\n",
    "    dataset.shuffle(shuffle_buffer_size) #对数据进行洗牌，混乱\n",
    "    #map，通过parse_csv_line对数据集进行映射，map只会给函数传递一个参数，这个参数\n",
    "    #就是dataset中的tensor\n",
    "    dataset = dataset.map(parse_csv_line,\n",
    "                          num_parallel_calls=n_parse_threads)\n",
    "    dataset = dataset.batch(batch_size)\n",
    "    return dataset\n",
    "#这里是一个测试,写4是为了大家理解\n",
    "train_set = csv_reader_dataset(train_filenames, batch_size=4)\n",
    "print(train_set)\n",
    "print('-'*50)\n",
    "i=0\n",
    "#是csv_reader_dataset处理后的结果，\n",
    "for x_batch, y_batch in train_set.take(2):\n",
    "#     i=i+1\n",
    "    print(\"x:\")\n",
    "    pprint.pprint(x_batch)\n",
    "    print(\"y:\")\n",
    "    pprint.pprint(y_batch)\n",
    "# print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 137 ms, sys: 40.3 ms, total: 177 ms\n",
      "Wall time: 160 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "batch_size = 32\n",
    "train_set = csv_reader_dataset(train_filenames,\n",
    "                               batch_size = batch_size)\n",
    "valid_set = csv_reader_dataset(valid_filenames,\n",
    "                               batch_size = batch_size)\n",
    "test_set = csv_reader_dataset(test_filenames,\n",
    "                              batch_size = batch_size)\n",
    "\n",
    "# print(train_set)\n",
    "# print(valid_set)\n",
    "# print(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 1.1306 - val_loss: 0.9811\n",
      "Epoch 2/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 2.4388 - val_loss: 0.5692\n",
      "Epoch 3/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 0.5545 - val_loss: 0.6181\n",
      "Epoch 4/100\n",
      "348/348 [==============================] - 1s 4ms/step - loss: 0.6097 - val_loss: 0.4497\n",
      "Epoch 5/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 0.4277 - val_loss: 0.4555\n",
      "Epoch 6/100\n",
      "348/348 [==============================] - 1s 4ms/step - loss: 0.3998 - val_loss: 0.3870\n",
      "Epoch 7/100\n",
      "348/348 [==============================] - 1s 4ms/step - loss: 0.3889 - val_loss: 0.4119\n",
      "Epoch 8/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 0.3831 - val_loss: 0.3941\n",
      "Epoch 9/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 0.3870 - val_loss: 0.4068\n",
      "Epoch 10/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 0.3689 - val_loss: 0.3801\n",
      "Epoch 11/100\n",
      "348/348 [==============================] - 1s 3ms/step - loss: 0.3804 - val_loss: 0.3957\n"
     ]
    }
   ],
   "source": [
    "#我们知道长度为8\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation='relu',\n",
    "                       input_shape=[8]),\n",
    "    keras.layers.Dense(1),\n",
    "])\n",
    "model.compile(loss=\"mean_squared_error\", optimizer=\"sgd\")\n",
    "callbacks = [keras.callbacks.EarlyStopping(\n",
    "    patience=5, min_delta=1e-2)]\n",
    "\n",
    "#当是BatchDataset,必须制定steps_per_epoch，validation_steps\n",
    "history = model.fit(train_set,\n",
    "                    validation_data = valid_set,\n",
    "                    steps_per_epoch = 11160 // batch_size, #每epoch训练的步数\n",
    "                    validation_steps = 3870 // batch_size,\n",
    "                    epochs = 100,\n",
    "                    callbacks = callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "161/161 [==============================] - 0s 2ms/step - loss: 0.3995\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.39946985244750977"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(test_set, steps = 5160 // batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(4,), dtype=int64, numpy=array([0, 1, 2, 3])>,\n",
       " <tf.Tensor: shape=(4,), dtype=int64, numpy=array([4, 5, 6, 7])>]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = tf.data.Dataset.range(8)\n",
    "dataset = dataset.batch(4)  #把tensor组合到一起，就是分了batch\n",
    "list(dataset)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}