Tensorflow加载与预处理数据详解实现方法

dataset = dataset.map(lambda x: x * 2)

filepath_dataset = tf.data.Dataset.list_files(train_filepaths, seed=42)

n_readers = 5
dataset = filepath_dataset.interleave(
	lambda filepath: tf.data.TextLineDataset(filepath).skip(1),
	cycle_length=n_readers)

X_mean, X_std = [...] # 每个特征的均值和标准差
n_inputs = 8 # 对应8个特征
def preprocess(line):
	defs = [0.] * n_inputs + [tf.constant([], dtype=tf.float32)] # 包含csv中每一列的默认值的数组
	fields = tf.io.decode_csv(line, record_defaults=defs) 
	# line 是要解析的行，record_defaults 是一个包含CSV文件每一列的默认值的数组
	x = tf.stack(fields[:-1])
	y = tf.stack(fields[-1:])
	return (x - X_mean) / X_std, y

def csv_reader_dataset(filepaths, repeat=1, n_readers=5,
					   n_read_threads=None, shuffle_buffer_size=10000,
					   n_parse_threads=5, batch_size=32):
	dataset = tf.data.Dataset.list_files(filepaths)
	dataset = filepath_dataset.interleave(
		lambda filepath: tf.data.TextLineDataset(filepath).skip(1),
		cycle_length=n_readers, num_parallel_calls=n_read_threads)
	dataset = dataset.map(preprocess, num_parallel_calls=n_parse_threads)
	dataset = dataset.shuffle(shuffle_buffer_size).repeat(repeat)
	return dataset.batch(batch_size).prefetch(1)