爬取白鲸nft排名前25项目,持有nft大户地址数据。
作者:互联网
https://moby.gg/rankings?tab=Market
SELECT address '钱包地址', COUNT (1) '持有nft项目数', SUM (balance) '持有nft个数', MAX (ct) '爬取时间' FROM `nft_analytics` WHERE time_type = '1d' AND ct = '2022-06-09' GROUP BY address, ct ORDER BY COUNT (1) DESC, SUM (balance) DESC LIMIT 100;
#coding=utf-8 import requests import time import json import math import datetime from requests.packages.urllib3 import disable_warnings data_12h = '' from selenium_chrome.MySqlUtils import getMysql disable_warnings() def spider_nft(time_type): ''' 12h 1d 3d :param time_type: :return: ''' time_12h = f'https://moby-api.onrender.com/market/rank/{time_type}' time_12h_resp = requests.get(time_12h,timeout=600,verify=False) mysql = getMysql() if(time_12h_resp.status_code == 200 and time_12h_resp.reason == 'OK'): nft_address_list = json.loads(time_12h_resp.text)['data'] for nft_obj in nft_address_list: try: nft_address = nft_obj['contract']['address'] holder_url = f'https://ethplorer.io/service/service.php?data={nft_address}&page=tab=tab-holders%26pageSize=500%26holders=1&showTx=all' holder_resp = requests.get(holder_url,timeout=60,verify=False) if(holder_resp.status_code == 200): time.sleep(5) resp_data = json.loads(holder_resp.text) total = resp_data['pager']['holders']['total'] holder1 = resp_data['holders'] ''' id bigint(20) (NULL) NO PRI (NULL) auto_increment select,insert,update,references name varchar(500) utf8_general_ci YES (NULL) select,insert,update,references address varchar(500) utf8_general_ci YES (NULL) select,insert,update,references balance varchar(500) utf8_general_ci YES (NULL) select,insert,update,references contract_address varchar(500) utf8_general_ci YES (NULL) select,insert,update,references owner varchar(500) utf8_general_ci YES (NULL) select,insert,update,references time_type varchar(100) utf8_general_ci YES (NULL) select,insert,update,references ct datetime (NULL) YES (NULL) select,insert,update,references ''' name = resp_data['token']['name'] contract_address = resp_data['token']['address'] owner = resp_data['token']['owner'] time_type = time_type ct = datetime.datetime.now().strftime('%Y-%m-%d') num = math.ceil(total/500) for n in range(2,num+1): holder_url = f'https://ethplorer.io/service/service.php?data={nft_address}&page=tab=tab-holders%26pageSize=500%26holders={n}&showTx=all' holder_resp = requests.get(holder_url,timeout=60,verify=False) if (holder_resp.status_code == 200): holder1 += json.loads(holder_resp.text)['holders'] time.sleep(5) for h in holder1: address = h['address'] balance = h['balance'] insert_sql = f'insert into nft_analytics (name,address,balance,contract_address,owner,time_type,ct) values ("'+name+'","'+address+'",'+str(balance)+',"'+contract_address+'","'+owner+'","'+time_type+'","'+ct+'")' print(insert_sql) mysql.execute_db(insert_sql) except BaseException as e: print(e) if __name__ == '__main__': spider_nft('1d')
import pymysql class MysqlDb(): def __init__(self, host, port, user, passwd, db): # 建立数据库连接 self.conn = pymysql.connect( host=host, port=port, user=user, passwd=passwd, db=db ) # 通过 cursor() 创建游标对象,并让查询结果以字典格式输出 self.cur = self.conn.cursor(cursor=pymysql.cursors.DictCursor) def __del__(self): # 对象资源被释放时触发,在对象即将被删除时的最后操作 # 关闭游标 self.cur.close() # 关闭数据库连接 self.conn.close() def select_db(self, sql): """查询""" # 使用 execute() 执行sql self.cur.execute(sql) # 使用 fetchall() 获取查询结果 data = self.cur.fetchall() return data def execute_db(self, sql): """更新/插入/删除""" try: # 使用 execute() 执行sql self.cur.execute(sql) # 提交事务 self.conn.commit() except Exception as e: print("操作出现错误:{}".format(e)) # 回滚所有更改 self.conn.rollback() def getMysql(): try: db = MysqlDb("127.0.0.1", 3306, "root", "root", "coin_project") except BaseException as e: print('初始化mysql失败:'+e) return db
if __name__ == '__main__': db = getMysql()
/*表: nft_analytics*/---------------------- /*列信息*/-----------
自增id id
nft名称 name
地址 address
持有nft数量 balance
nft合约地址 contract_address
nft合约创建地址 owner
时间类型 time_type
创建时间 ct Field Type Collation Null Key Default Extra Privileges Comment ---------------- ------------ --------------- ------ ------ ------- -------------- ------------------------------- --------- id bigint(20) (NULL) NO PRI (NULL) auto_increment select,insert,update,references name varchar(500) utf8_general_ci YES (NULL) select,insert,update,references address varchar(500) utf8_general_ci YES (NULL) select,insert,update,references balance int(255) (NULL) YES (NULL) select,insert,update,references contract_address varchar(500) utf8_general_ci YES (NULL) select,insert,update,references owner varchar(500) utf8_general_ci YES (NULL) select,insert,update,references time_type varchar(100) utf8_general_ci YES (NULL) select,insert,update,references ct datetime (NULL) YES (NULL) select,insert,update,references /*索引信息*/-------------- Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment ------------- ---------- -------- ------------ ----------- --------- ----------- -------- ------ ------ ---------- ------- --------------- nft_analytics 0 PRIMARY 1 id A 230789 (NULL) (NULL) BTREE /*DDL 信息*/------------ CREATE TABLE `nft_analytics` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `name` varchar(500) DEFAULT NULL, `address` varchar(500) DEFAULT NULL, `balance` int(255) DEFAULT NULL, `contract_address` varchar(500) DEFAULT NULL, `owner` varchar(500) DEFAULT NULL, `time_type` varchar(100) DEFAULT NULL, `ct` datetime DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=241992 DEFAULT CHARSET=utf8
标签:25,insert,address,白鲸,nft,time,NULL,select 来源: https://www.cnblogs.com/cuinima/p/16364199.html