开源 python 应用 开发（六）网络爬虫

pip install requests -i https://mirrors.aliyun.com/pypi/simple/

pip install beautifulsoup4 -i https://mirrors.aliyun.com/pypi/simple/

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

class WebsiteLogin:
    def __init__(self, base_url):
        self.base_url = base_url
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        })
    
    def find_hidden_inputs(self, html):
        """查找表单中的隐藏字段"""
        soup = BeautifulSoup(html, 'html.parser')
        hidden_inputs = {}
        for input_tag in soup.find_all('input', type='hidden'):
            hidden_inputs[input_tag.get('name')] = input_tag.get('value', '')
        return hidden_inputs
    
    def login(self, login_url, username, password, extra_data=None):
        """登录网站"""
        # 获取登录页面
        try:
            login_page = self.session.get(urljoin(self.base_url, login_url))
            login_page.raise_for_status()
        except requests.RequestException as e:
            print(f"访问登录页面失败: {e}")
            return False
        
        # 获取隐藏字段
        hidden_inputs = self.find_hidden_inputs(login_page.text)
        
        # 准备登录数据
        login_data = {
            'inputName': username,
            'inputPwd': password,
            **hidden_inputs
        }
        
        # 添加额外的表单数据
        if extra_data:
            login_data.update(extra_data)
        
        # 提交登录表单
        try:
            response = self.session.post(
                urljoin(self.base_url, login_url),
                data=login_data,
                headers={'Referer': urljoin(self.base_url, login_url)}
            )
            response.raise_for_status()
        except requests.RequestException as e:
            print(f"提交登录表单失败: {e}")
            return False
        
        # 验证登录是否成功（根据实际网站调整）
        #if 'logout' not in response.text.lower() and 'sign out' not in response.text.lower():
        #    print("登录失败，请检查凭据")
        #    return False

        if 'login' in response.url.lower():
            return False

        print("登录成功！")
        return True
    
    def get_page(self, page_url):
        """获取指定页面的内容"""
        try:
            response = self.session.get(urljoin(self.base_url, page_url))
            response.raise_for_status()
            return response.text
        except requests.RequestException as e:
            print(f"获取页面失败: {e}")
            return None

# 使用示例
if __name__ == "__main__":
    # 配置信息
    config = {
        'base_url': 'http://xxx.com',
        'login_url': '',
        'username': 'xxx',
        'password': 'xxx',
        'target_page': '/xxx/xxx'
    }
    
    # 创建登录实例
    web_login = WebsiteLogin(config['base_url'])
    
    # 尝试登录
    if web_login.login(config['login_url'], config['username'], config['password']):
        # 登录成功后获取目标页面
        target_content = web_login.get_page(config['target_page'])
        
        if target_content:
            print(f"成功获取目标页面内容，长度: {len(target_content)}")
            # 保存内容
            with open('target_page.html', 'w', encoding='utf-8') as f:
                f.write(target_content)
        else:
            print("未能获取目标页面内容")
    else:
        print("登录失败，程序终止")