简介

Colly 是一个用于构建网络爬虫的 Golang 框架。使用 Colly，您可以构建各种复杂的网络爬虫，从简单的爬虫到处理数百万网页的复杂异步网站爬虫。Colly 提供了一个 API 用于执行网络请求和处理接收到的内容（例如与 HTML 文档的 DOM 树交互）。

  1. import json
  2. import logging
  3. import threading
  4. import time
  6. import requests
  9. class IBoxNFT:
  10.     def __init__(self, phone, sms):
  11.         self.phone = phone
  12.         self.sms = sms
  15. class Crawler(IBoxNFT):
  17.     def __init__(self, basic, suffix, phone, sms):
  18.         self.basic = basic
  19.         self.link = f'{self.basic}/{suffix}'
  20.         self.un_know = False
  21.         self.start = time.perf_counter()
  22.         super(Crawler, self).__init__(phone, sms)
  24.     def login_ibox(self, version=None, new_suffix=None):
  25.         """
  26.         实现IBox的登录功能，登录请求的版本和后缀有所不同，需要对其进行更新
  27.         :param version: 版本号，初始版本为v1.2
  28.         :param new_suffix: 登录的后缀
  29.         :return: 返回登录完成信息
  30.         """
  31.         if not (version, new_suffix):
  32.             v, s = "v1.1", "/user/login"
  34.         header = Crawler.public_params("headers")[0]
  35.         resp = Crawler.crawler_link(
  36.             'post',
  37.             self.suffix("v1.1", "/user/login"),
  38.             headers=header,
  39.             json={"phoneNumber": self.phone, "code": str(self.sms)},
  40.         )
  42.         # if resp.get('code') != 1:
  43.         #     print("登录失败，检查登录信息")
  44.         print("登录信息\n", resp.json())
  46.     def suffix(self, version, new_suffix):
  47.         link = f"{self.link.replace('http', 'https').replace('v1.2', str(version))[:46]}{new_suffix}"
  48.         return link
  50.     @staticmethod
  51.     def public_params(param) -> dict:
  52.         """通过闭包返回公共的params参数，其中包括
  53.            （请求、构造、公共请求头参数）
  54.         """
  56.         def inner():
  57.             if "crawler_min_nft" in param:
  58.                 return {
  59.                     "classifyId": "",
  60.                     "origin": "0",
  61.                     "pageSize": "20",
  62.                     "sort": "1",
  63.                     "page": "1",
  64.                     "type": "0"
  65.                 }
  66.             if "crawler_new_nft" in param:
  67.                 return {
  68.                     "classifyId": "",
  69.                     "origin": "0",
  70.                     "pageSize": "20",
  71.                     "sort": "0",
  72.                     "page": "1",
  73.                     "type": "0"
  74.                 }
  76.             if "headers" in param:
  77.                 return {
  78.                            'Accept-Encoding': 'gzip',
  79.                            'IB-PLATFORM-TYPE': 'android',
  80.                            'Host': 'api-app.ibox.art',
  81.                            'language': 'zh-CN',
  82.                            'ib-app-version':' 1.1.4',
  83.                            'Content-Type': 'application/json; charset=UTF-8',
  84.                            'Accept-Language': 'zh-CN',
  85.                            'Connection': 'Keep-Alive',
  86.                            'user-agent': 'iBoxApp209'
  87.                        },
  89.         return inner()
  91.     @property
  92.     def get_links(self):
  93.         return self.link
  95.     @get_links.setter
  96.     def upgrade_links(self, new_link):
  97.         """
  98.         ibox版本更新迭代很快, {upgrade_links} 用于更新IBox后续的请求URL
  99.         :param new_link: 更新后的ibox基本链接
  100.         """
  101.         self.link = new_link
  103.     @property
  104.     def check_status(self) -> bool:
  105.         return self.un_know
  107.     def crawler_book_nft(self, payload: dict):
  108.         pass
  110.     def crawler_min_nft(self):
  111.         """获取ibox的最低价格nft, 实时更新"""
  112.         resp = Crawler.crawler_link(
  113.             'get',
  114.             self.link,
  115.             headers=Crawler.public_params("headers")[0],
  116.             params=Crawler.public_params(self.crawler_min_nft.__name__)
  117.         )
  119.         nft = dict()
  120.         for i in self.get_min_price(resp.json()):
  121.             nft = {
  122.                 '图藏系列': i['albumName'],
  123.                 '价格': i['priceCny'],
  124.                 '图藏URL': "www.ibox.art{}".format(i['thumbPic'])
  125.             }
  126.             if not i:
  127.                 self.un_know = True
  128.         return nft
  130.     @staticmethod
  131.     def get_min_price(args):
  132.         """监测价格数据，如果满足条件，触发通知"""
  133.         for k, v in enumerate(args['data']['list']):
  134.             if args['data']['list'][k]['albumName'] == 'iBox礼遇系列' \
  135.                     and int(args['data']['list'][k]['priceCny']) < 800:
  136.                 yield args['data']['list'][k]
  138.             if args['data']['list'][k]['albumName'] == 'iBox纪念系列' \
  139.                     and int(args['data']['list'][k]['priceCny']) < 600:
  140.                 yield args['data']['list'][k]
  142.             if args['data']['list'][k]['albumName'] == 'iBox赛博生肖系列' \
  143.                     and int(args['data']['list'][k]['priceCny']) > 800:
  144.                 yield args['data']['list'][k]
  146.             if args['data']['list'][k]['albumName'] == '异星夜袭' \
  147.                     and int(args['data']['list'][k]['priceCny']) > 2000:
  148.                 yield args['data']['list'][k]
  150.     @staticmethod
  151.     def crawler_link(methods, link, headers, params=None, data=None, json=None, proxy=None):
  152.         try:
  153.             if 'get' in methods:
  154.                 r = requests.get(link, params=params, headers=headers, proxies=proxy)
  155.                 return r
  157.             if 'post' in methods:
  158.                 r = requests.post(link, data=data, json=json, proxies=proxy)
  159.                 return r
  161.         except (requests.exceptions.ConnectionError, requests.exceptions.RequestException) as e:
  162.             logging.error("The Request is fail", e, r.status_code)
  163.         except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as e:
  164.             logging.error(e, r.status_code)
  166.     @staticmethod
  167.     def genrator_wtoken():
  168.         pass
  172.     def run(self):
  173.         print(self.crawler_min_nft())
  176. if __name__ == '__main__':
  177.     print("启动爬虫获取ibox")
  178.     #while True:
  179.     a = Crawler("http://api-app.ibox.art", "nft-mall-web/v1.2/nft/product/getResellList", "13420280437", 516254)
  180.     a.login_ibox()
  181.         # if a.check_status:
  182.         #     break


基本配置

初始化Collector采集实例

colly.NewCollector(options ...func(*Collector)) *Collector
采集器可以设置多个可选参数：

  • 限制爬取的域名

    • colly.AllowedDomains(“www.baidu.com”, “baidu.com”)

  • 设置请求头

    • colly.UserAgent(“xy”)

  • 启动异步

    • colly.Async(true)

  • 启动调试器

    • colly.Debugger(&debug.LogDebugger{})

  • 设置爬取页面的深度, 如果设置为1，只抓取Visit()中的url

    • colly.MaxDepth(1)

https://lk668.github.io/2021/04/05/2021-04-05-%E6%89%8B%E6%8A%8A%E6%89%8B%E6%95%99%E4%BD%A0%E5%A6%82%E4%BD%95%E7%94%A8golang%E5%AE%9E%E7%8E%B0%E4%B8%80%E4%B8%AAtimewheel/

回调函数

在发起请求前被调用

func (c *Collector) OnRequest(f RequestCallback) {}

在请求过程中发生错误被调用

func (c *Collector) OnError(f RequestCallback) {}

收到请求后被调用

func (c *Collector) OnResponse(f RequestCallback) {}

收到内容按html的jquery进行数据清洗

func (c *Collector) OnHTML(goquerySelector string,f HTMLCallback) {}

收到内容按XML的xpath进行数据清洗

func (c *Collector) OnXML(xpathQuery string,f XMLCallback) {}

在OnHTML之后被调用

func (c *Collector) OnScraped(f ScrapedCallback) {}

并行爬取