Python使用正则表达式实现从日志中精准提取关键字段

import re

# 示例日志行
log_line = '127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /index.html HTTP/1.1" 200 4824'

# 正则表达式
pattern = r'"(GET|POST|PUT|DELETE) ([^\s]+)'

# 匹配
match = re.search(pattern, log_line)

if match:
    request_method = match.group(1)  # 提取请求方法
    url = match.group(2)            # 提取URL
    print(f"请求方法: {request_method}, URL: {url}")

import re
# 示例日志列表
log_lines = [
    '127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /index.html HTTP/1.1" 200 4824',
    '127.0.0.1 - - [10/Oct/2023:13:55:40 +0000] "POST /login HTTP/1.1" 302 552',
    '127.0.0.1 - - [10/Oct/2023:13:55:45 +0000] "GET /about.html HTTP/1.1" 200 2345'
]
# 正则表达式
pattern = r'POST'
# 过滤包含特定关键词的日志行
filtered_lines = [line for line in log_lines if re.search(pattern, line)]
for line in filtered_lines:
    print(line)

import re

# 示例日志行
log_line = '127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /index.html HTTP/1.1" 200 4824'

# 正则表达式
pattern = r'\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} [+\-]\d{4})\]'

# 匹配
match = re.search(pattern, log_line)

if match:
    timestamp = match.group(1)
    print(f"时间戳: {timestamp}")

import re

# 示例多行日志
log_data = '''127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /index.html HTTP/1.1" 200 4824
127.0.0.1 - - [10/Oct/2023:13:55:40 +0000] "POST /login HTTP/1.1" 302 552
127.0.0.1 - - [10/Oct/2023:13:55:45 +0000] "GET /about.html HTTP/1.1" 200 2345'''

# 正则表达式
pattern = r'"(GET|POST|PUT|DELETE) ([^\s]+) \S+" (\d{3})'

# 处理多行日志
for match in re.finditer(pattern, log_data):
    request_method = match.group(1)
    url = match.group(2)
    status_code = match.group(3)
    print(f"请求方法: {request_method}, URL: {url}, 状态码: {status_code}")

import re

# 示例复杂日志行
log_line = '127.0.0.1 - - [10/Oct/2023:13:55:36 +0000] "GET /api/v1/users/123?query=abc HTTP/1.1" 200 4824 - "Mozilla/5.0"'

# 正则表达式
pattern = r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} [+\-]\d{4})\] "(\w+) (\S+)' \
          r'(\S+)" (\d{3}) (\d+) - "(.*)"'

# 匹配
match = re.match(pattern, log_line)

if match:
    ip_address = match.group(1)
    timestamp = match.group(2)
    request_method = match.group(3)
    request_url = match.group(4)
    request_protocol = match.group(5)
    status_code = match.group(6)
    response_size = match.group(7)
    user_agent = match.group(8)
    
    print(f"IP地址: {ip_address}, 时间戳: {timestamp}, 请求方法: {request_method}, 请求URL: {request_url}, "
          f"请求协议: {request_protocol}, 状态码: {status_code}, 响应大小: {response_size}, User-Agent: {user_agent}")

import re

# 示例日志行
log_line = 'ERROR [10/Oct/2023:13:55:36 +0000] [Thread-2] Exception in thread "main" java.lang.NullPointerException: Cannot invoke "java.util.List.get(int)" because the return value of "java.util.List.stream()" is null'

# 正则表达式
pattern = r'ERROR \[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} [+\-]\d{4})\] \[(\w+)\] (.*)'

# 匹配
match = re.search(pattern, log_line)

if match:
    timestamp = match.group(1)
    thread = match.group(2)
    exception_info = match.group(3)
    print(f"时间戳: {timestamp}, 线程: {thread}, 异常信息: {exception_info}")

import re
import requests

# 定义日志处理函数
def extract_log_data():
    # 从服务器获取日志数据
    log_data = requests.get('http://example.com/log.txt').text
    
    # 正则表达式
    pattern = r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - \[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2} [+\-]\d{4})\] "(\w+) (\S+)' \
              r'(\S+)" (\d{3}) (\d+) - "(.*)"'
    
    # 处理多行日志
    for match in re.finditer(pattern, log_data):
        ip_address = match.group(1)
        timestamp = match.group(2)
        request_method = match.group(3)
        request_url = match.group(4)
        request_protocol = match.group(5)
        status_code = match.group(6)
        response_size = match.group(7)
        user_agent = match.group(8)
        
        print(f"IP地址: {ip_address}, 时间戳: {timestamp}, 请求方法: {request_method}, 请求URL: {request_url}, "
              f"请求协议: {request_protocol}, 状态码: {status_code}, 响应大小: {response_size}, User-Agent: {user_agent}")

# 使用Hey Cron设置定时任务
# Hey Cron可以帮助你定期执行Python脚本，确保日志处理任务的自动化
# 例如，设置每5分钟执行一次日志处理函数
# Hey Cron语法示例：*/5 * * * * python3 /path/to/your/script.py

if __name__ == "__main__":
    extract_log_data()