Python读取HDFS目录下的所有文件的实现示例

from hdfs import InsecureClient
hdfs_client = InsecureClient('http://your-hdfs-namenode-url:50070')

file_list = hdfs_client.list('/path/to/directory')
for file_name in file_list:
    print(file_name)

with hdfs_client.read('/path/to/file', encoding='utf-8') as reader:
    file_contents = reader.read()
    print(file_contents)

def recursive_list(client, path):
    results = []
    for element in client.list(path, include_toplevel=False):
        full_path = path + "/" + element
        if client.status(full_path, strict=False)['type'] == 'DIRECTORY':
            results.extend(recursive_list(client, full_path))
        else:
            results.append(full_path)
    return results
file_list = recursive_list(hdfs_client, '/path/to/directory')
for file_name in file_list:
    with hdfs_client.read(file_name, encoding='utf-8') as reader:
        file_contents = reader.read()
        print(file_name, file_contents)