python 将爬取的数据存入mysql

需求：将爬到的数据存入mysql

思路：1、从文件中读取数据

           2、将数据存入到mysql中

完整代码如下

import pymysql
from os import path 
import time

def readFile(filePath):
	fileStr = open(filePath,encoding='utf-8')
	dataArray = list() 
	for lineStr in fileStr.readlines():
		lineStr = lineStr.strip() 
		itemArray = lineStr.split(",")
		if len(itemArray[1]) != 0:
			print(lineStr+"---"+itemArray[1])
			dataArray.append(itemArray)
	fileStr.close()  
	return dataArray

def insetMysql(dataArray):
	# 打开数据库连接
	db = pymysql.connect(host = "localhost",user = "root",passwd = "123456",port = 3306,db = "magiclife", charset='utf8' )
	# 使用cursor()方法获取操作游标 
	cursor = db.cursor()
	insertCountry = "中国"
	insertTime = time.strftime('%Y-%m-%d',time.localtime(time.time()))
	insertBz="SQL添加"
	for dataItem in dataArray:     # 第一个实例
		# SQL 插入语句
		sql = "INSERT INTO ml_phone_info\
		(ml_p_mobile_prefix,ml_p_operators,ml_p_country,ml_p_provice,ml_p_city,ml_p_addtime,ml_p_changetime,ml_p_bz)\
		VALUES (%s, %s,'%s',%s,%s,'%s','%s','%s')" % (dataItem[0],dataItem[3],insertCountry,dataItem[1],dataItem[2],insertTime,insertTime,insertBz)
		try:
			# 执行sql语句
			print("-----执行------"+sql)
			cursor.execute(sql)
			# 提交到数据库执行
			db.commit()
		except Exception :
			# Rollback in case there is any error
			print("-----错误------"+Exception )
			db.rollback()
		# 关闭数据库连接
	db.close()

def main():
	#1、读取100条数据
	#2、存数据
	filePath = path.dirname('.')+"phone1.csv"
	dataArray = readFile(filePath)
	insetMysql(dataArray)

if __name__ == '__main__':
	main()

文件中数据格式如下

"1453573","福建","厦门","联通"
"1453574","福建","厦门","联通"
"1453575","福建","漳州","联通"
"1453576","福建","漳州","联通"
"1453577","福建","龙岩","联通"
"1453578",,,
"1453579",,,
"1453580","福建","福州","联通"
"1453581","福建","福州","联通"
"1453582","福建","福州","联通"
"1453583","福建","福州","联通"
"1453584","福建","福州","联通"

遇到的问题：1、数据中有垃圾数据需要做判空处理

                      2、数据存储时如果存入的事String,需要在占位符上加上单引号，不然会报类型异常

标签：联通,dataItem,python,ml,db,存入,福建,mysql,dataArray
来源： https://blog.csdn.net/jiangyou4/article/details/100996399