web爬虫学习（四）——手机APP爬取

import requests

import json

import re

import time

import simplejson

import xlsxwriter

 

workbook = xlsxwriter.Workbook('kebiao_test.xlsx')

worksheet = workbook.add_worksheet()

worksheet.set_column('A:A', 10)

worksheet.set_column('B:B', 20)

worksheet.set_column('C:C', 5)

worksheet.set_column('D:D', 20)

worksheet.set_column('E:E', 20)

worksheet.set_column('F:F', 500)

 

worksheet.write(0,0,'学校')

worksheet.write(0,1,'主题')

worksheet.write(0,2,'性别')

worksheet.write(0,3,'日期')

worksheet.write(0,4,'时间')

worksheet.write(0,5,'内容')

 

# 登陆部分

# 用requests.Session()记录cookie

s = requests.Session()

loginurl = 'http://120.55.151.61/V2/StudentSkip/loginCheckV4.action'

logindata = 'phoneBrand=Xiaomi&platform=1&deviceCode=867711022104024

&account=6603135B883F9B40DED6374A22593&phoneVersion=19

&password=98A09B5680DF25A934ACF9B3614AF4EA&channel=XiaoMiMarket

&phoneModel=HM+2A&versionNumber=7.4.1&'

headers = {

   'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',

   'User-Agent': 'Dalvik/1.6.0 (Linux; U; Android 4.4.4;

HM 2A MIUI/V7.3.1.0.KHLCNDD)',

   'Host': '120.55.151.61',

   'Connection': 'Keep-Alive',

   'Accept-Encoding': 'gzip',

   'Content-Length': '213',

}

 

# 提交登陆信息，开始登陆

data = s.post(url=loginurl, data=logindata, headers=headers,

stream=True, verify=False)

loginResult = data.text

# 打印登陆信息，检查是否登陆成功

# 成功后返回个人信息

print(loginResult)

 

# 定义一个函数，获取信息并保存

# 传入要POST的表单数据，即req_data

def get_data(req_data):

   req_url = 'http://120.55.151.61/Treehole/V4/Cave/getList.action'

   data_r = s.post(url=req_url, data=req_data, headers=headers)

   data_r = data_r.text

   # 返回的布尔值为小写，这里转化为真正的布尔值

   true = True

   false = False

    

   # 简单处理json数据，直接解析好像有编码问题，不太清楚。。。

   data_j = eval(data_r)

   data_js = json.dumps(data_j)

   data_dict = simplejson.loads(data_js)

 

   # 获取时间戳

   data = data_dict['data']

   timestampLong = data['timestampLong']

   #print(timestampLong)

   messageBO = data['messageBOs']

    

   # 处理抓到的json数据，拿到目标数据并保存

   for each in messageBO:

      if 'studentBO' in each:

         print(each)

         #topicDict = {}

         if each.get('content', False):

            schoolNamex = each["schoolName"]

            worksheet.write(i,0,schoolNamex)

            tag = each['moodTagBO']['moodTagName']

            worksheet.write(i,1,tag)

            genderx = each['studentBO']['gender']

            worksheet.write(i,2,genderx)

            contentx = each['content']

            #print(contentx)

            worksheet.write(i,5,contentx)

            #topicDict['messageId'] = each['messageId'

            time_f = list(time.localtime(int(str(each['issueTime'])[:-3])))

            time_s = str(time_f[0])+'/'+str(time_f[1])+'/'+str(time_f[2])

 

            h = time_f[3]

            m = time_f[4]

            sec = time_f[5]

            if h < 10:

               h = '0'+str(h)

            if m < 10:

               m = '0'+str(m)

            if sec < 10:

               sec = '0'+str(sec)

            time_t = str(h)+':'+str(m)+':'+str(sec)

            datex = time_s

            # print(datex)

            worksheet.write(i,3,datex)

            worksheet.write(i,4,time_t)

 

            i += 1

            global i

   # 用获取的时间戳构建新的POST表单数据，持续循环抓取

   new_req_data = 'timestamp='+str(timestampLong)+

                              '&preMoodTimestap=1461641185406&phoneBrand=Xiaomi

                              &platform=1&phoneVersion=19&channel=XiaoMiMarket

                              &phoneModel=HM+2A&versionNumber=7.4.1&'

   print('--------------------------new page---------')

   #print(new_req_data)

   # 定义抓取发帖条数

   if i <= 20000:

      try:

         get_data(new_req_data)

      except:

         print('fail')

      finally:

         workbook.close()

   else:

      workbook.close()

      print('全部抓取成功了！！！')

# i 是抓取条数的标识  

i = 1

# 此处传入第一个页面的POST数据，之后展开循环

get_data(req_data='timestamp=146646454916&preMoodTimestap=1461641185406

&phoneBrand=Xiaomi&platform=1&phoneVersion=19&channel=XiaoMiMarket