R语言实现多线程：百度贴吧复仇之矛吧数据抓取源码解析

### 
library(RCurl) 
library(stringr) 
library(rvest) 
#读入母页数据和子页数据 
file_list=dir(getwd()) 
all_mother=read.csv(file_list[2],header = T) 
all_sons=read.csv(file_list[3],header = T) 
all_sons=all_sons[-1,] 
all_mother=paste(rep("https://tieba.baidu.com/f?kw=复仇之矛&ie=utf-8&pn=",211),50*c(0:210),sep = "") 
# 
zhutie_data_frame=read.csv(file_list[4],header = T) 
###Rvest的办法 
#empty_df=cbind(rep(0,1),rep(0,1),rep(0,1),rep(0,1),rep(0,1),rep(0,1),rep(0,1),rep(0,1)) 
#colnames(empty_df)=c("page_id","temp_tiezi_id","temp_reply","temp_main","temp_louzhu","temp_vice","temp_last_reply","temp_last_time") 
#zhutie_data_frame=empty_df 
for(i in 1:1)#length(all_mother)) 
{ 
temp_url=getURL(all_mother[i]) 
temp_html=read_html(temp_url) 
all_data=html_nodes(temp_html,".j_reply_data , .j_user_card , .threadlist_abs_onlyline , .j_th_tit .j_th_tit , .center_text") 
tiezi_id_temp=gsub("title.+","",all_data[grep("<a href=./p/",all_data)]) 
num_tiezi=length(tiezi_id_temp) 
all_data_text=html_text(all_data) 
temp_tiezi_id=tiezi_id_temp 
temp_reply=all_data_text[grep("title=.回复.",all_data)] 
temp_main=all_data_text[grep("<a href=..p.",all_data)] 
temp_all_user=all_data_text[grep("frs.author.name",all_data)] 
temp_louzhu=temp_all_user[2*c(1:num_tiezi)-1] 
#page1 

#temp_last_reply=c(NA,NA,temp_all_user[2*c(1:num_tiezi)]) 
#page2~211 
#temp_last_reply=temp_all_user[2*c(1:num_tiezi)] 
temp_vice=all_data_text[grep("class=.threadlist.abs",all_data)] 
temp_last_time=all_data_text[grep("最后回复时间",all_data)] 
page_id=rep(i,num_tiezi) 
temp_data_frame=cbind(page_id,temp_tiezi_id,temp_reply,temp_main,temp_louzhu,temp_vice,temp_last_reply,temp_last_time) 
zhutie_data_frame=rbind(zhutie_data_frame,temp_data_frame) 
print(i) 
} 
zhutie_data_frame=zhutie_data_frame[-1,] 
write.csv(zhutie_data_frame,"zhutie_data_frame_V3.csv") 



多线程 
setwd("~/Desktop/数据抓取小练习") 
file_list=dir(getwd()) 
#install.packages("data.table") 
fun_curl=function(y){ 
library(data.table) 
library(devtools) 
#install_github("rdom","cpsievert") 
library(rdom) 
library(Rwebdriver) 
library(devtools) 
#install_github("Rwebdriver","crubba") 
#aa=rdom("https://tieba.baidu.com/p/5389638505",".lzl_content_main") 
#stopifnot(Sys.which("phantomjs")!="") 
library(RSelenium)