import requests
import json
import re
import time
import simplejson
import xlsxwriter
workbook = xlsxwriter.Workbook('kebiao_test.xlsx')
worksheet = workbook.add_worksheet()
worksheet.set_column('A:A', 10)
worksheet.set_column('B:B', 20)
worksheet.set_column('C:C', 5)
worksheet.set_column('D:D', 20)
worksheet.set_column('E:E', 20)
worksheet.set_column('F:F', 500)
worksheet.write(0,0,'学校')
worksheet.write(0,1,'主题')
worksheet.write(0,2,'性别')
worksheet.write(0,3,'日期')
worksheet.write(0,4,'时间')
worksheet.write(0,5,'内容')
# 登陆部分
# 用requests.Session()记录cookie
s = requests.Session()
loginurl = 'http://120.55.151.61/V2/StudentSkip/loginCheckV4.action'
logindata = 'phoneBrand=Xiaomi&platform=1&deviceCode=867711022104024
&account=6603135B883F9B40DED6374A22593&phoneVersion=19
&password=98A09B5680DF25A934ACF9B3614AF4EA&channel=XiaoMiMarket
&phoneModel=HM+2A&versionNumber=7.4.1&'
headers = {
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'User-Agent': 'Dalvik/1.6.0 (Linux; U; Android 4.4.4;
HM 2A MIUI/V7.3.1.0.KHLCNDD)',
'Host': '120.55.151.61',
'Connection': 'Keep-Alive',
'Accept-Encoding': 'gzip',
'Content-Length': '213',
}
# 提交登陆信息，开始登陆
data = s.post(url=loginurl, data=logindata, headers=headers,
stream=True, verify=False)
loginResult = data.text
# 打印登陆信息，检查是否登陆成功
# 成功后返回个人信息
print(loginResult)
# 定义一个函数，获取信息并保存
# 传入要POST的表单数据，即req_data
def get_data(req_data):
req_url = 'http://120.55.151.61/Treehole/V4/Cave/getList.action'
data_r = s.post(url=req_url, data=req_data, headers=headers)
data_r = data_r.text
# 返回的布尔值为小写，这里转化为真正的布尔值
true = True
false = False
# 简单处理json数据，直接解析好像有编码问题，不太清楚。。。
data_j = eval(data_r)
data_js = json.dumps(data_j)
data_dict = simplejson.loads(data_js)
# 获取时间戳
data = data_dict['data']
timestampLong = data['timestampLong']
#print(timestampLong)
messageBO = data['messageBOs']
# 处理抓到的json数据，拿到目标数据并保存
for each in messageBO:
if 'studentBO' in each:
print(each)
#topicDict = {}
if each.get('content', False):
schoolNamex = each["schoolName"]
worksheet.write(i,0,schoolNamex)
tag = each['moodTagBO']['moodTagName']
worksheet.write(i,1,tag)
genderx = each['studentBO']['gender']
worksheet.write(i,2,genderx)
contentx = each['content']
#print(contentx)
worksheet.write(i,5,contentx)
#topicDict['messageId'] = each['messageId'
time_f = list(time.localtime(int(str(each['issueTime'])[:-3])))
time_s = str(time_f[0])+'/'+str(time_f[1])+'/'+str(time_f[2])
h = time_f[3]
m = time_f[4]
sec = time_f[5]
if h < 10:
h = '0'+str(h)
if m < 10:
m = '0'+str(m)
if sec < 10:
sec = '0'+str(sec)
time_t = str(h)+':'+str(m)+':'+str(sec)
datex = time_s
# print(datex)
worksheet.write(i,3,datex)
worksheet.write(i,4,time_t)
i += 1
global i
# 用获取的时间戳构建新的POST表单数据，持续循环抓取
new_req_data = 'timestamp='+str(timestampLong)+
'&preMoodTimestap=1461641185406&phoneBrand=Xiaomi
&platform=1&phoneVersion=19&channel=XiaoMiMarket
&phoneModel=HM+2A&versionNumber=7.4.1&'
print('--------------------------new page---------')
#print(new_req_data)
# 定义抓取发帖条数
if i <= 20000:
try:
get_data(new_req_data)
except:
print('fail')
finally:
workbook.close()
else:
workbook.close()
print('全部抓取成功了！！！')
# i 是抓取条数的标识
i = 1
# 此处传入第一个页面的POST数据，之后展开循环
get_data(req_data='timestamp=146646454916&preMoodTimestap=1461641185406
&phoneBrand=Xiaomi&platform=1&phoneVersion=19&channel=XiaoMiMarket