Python实现批量下载SMAP数据

pip install requests

import requests
from requests.auth import HTTPBasicAuth
from datetime import datetime, timedelta

def download_smap_data(username, password, data_urls, save_path):
    for url in data_urls:
        response = requests.get(url, auth=HTTPBasicAuth(username, password), stream=True)
        if response.status_code == 200:
            # 解析文件名
            filename = url.split('/')[-1]
            file_path = f"{save_path}/{filename}"
            
            # 保存文件
            with open(file_path, 'wb') as file:
                    for chunk in response.iter_content(chunk_size=1024):
                        if chunk:
                            file.write(chunk)
                    print(f"下载成功: {filename}")
         else:
            print(f"下载失败: {url}")

# 示例数据下载链接
data_urls = [
    "https://example.com/smap_data_1.zip",
    "https://example.com/smap_data_2.zip",
    # 添加更多数据链接
]

# 设置保存路径
save_path = "./smap_data"

# 替换为你的NASA Earthdata账户信息
username = "your_username"
password = "your_password"

# 执行数据下载
download_smap_data(username, password, data_urls, save_path)

def generate_date_range(start_date, end_date):
    current_date = start_date
    while current_date <= end_date:
        yield current_date
        current_date += timedelta(days=1)

def download_smap_data_with_time_range(username, password, data_urls_template, save_path, start_date, end_date):
    for date in generate_date_range(start_date, end_date):
        formatted_date = date.strftime("%Y%m%d")
        data_url = data_urls_template.format(date=formatted_date)
        download_smap_data(username, password, [data_url], save_path)

# 示例时间范围
start_date = datetime(2023, 1, 1)
end_date = datetime(2023, 1, 5)

# 示例数据下载链接模板
data_urls_template = "https://example.com/smap_data_{date}.zip"

# 执行带时间范围的数据下载
download_smap_data_with_time_range(username, password, data_urls_template, save_path, start_date, end_date)

import threading
import queue

def download_worker(username, password, url_queue, save_path):
    while True:
        url = url_queue.get()
        if url is None:
            break

        response = requests.get(url, auth=HTTPBasicAuth(username, password), stream=True)
        if response.status_code == 200:
            filename = url.split('/')[-1]
            file_path = f"{save_path}/{filename}"
            
            with open(file_path, 'wb') as file:
                for chunk in response.iter_content(chunk_size=1024):
                    if chunk:
                        file.write(chunk)
                print(f"下载成功: {filename}")
        else:
            print(f"下载失败: {url}")

def download_smap_data_multithread(username, password, data_urls, save_path, num_threads=4):
    url_queue = queue.Queue()

    # 将数据链接放入队列
    for url in data_urls:
        url_queue.put(url)

    # 创建线程池
    threads = []
    for _ in range(num_threads):
        thread = threading.Thread(target=download_worker, args=(username, password, url_queue, save_path))
        thread.start()
        threads.append(thread)

    # 等待所有线程完成
    for thread in threads:
        thread.join()

# 示例：多线程数据下载
download_smap_data_multithread(username, password, data_urls, save_path, num_threads=4)

import logging

# 配置日志记录
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def download_worker_with_logging(username, password, url_queue, save_path):
    while True:
        url = url_queue.get()
        if url is None:
            break

        try:
            response = requests.get(url, auth=HTTPBasicAuth(username, password), stream=True)
            response.raise_for_status()

            filename = url.split('/')[-1]
            file_path = f"{save_path}/{filename}"

            with open(file_path, 'wb') as file:
                for chunk in response.iter_content(chunk_size=1024):
                    if chunk:
                        file.write(chunk)
                logger.info(f"下载成功: {filename}")
        except Exception as e:
            logger.error(f"下载失败: {url}, 错误信息: {str(e)}")

# 示例：带错误处理与日志记录的多线程数据下载
download_smap_data_multithread_with_logging(username, password, data_urls, save_path, num_threads=4)