骨子里hive的udf
是比较容易完结的,只必要世袭UDF,完结其evaluate(卡塔 尔(英语:State of Qatar)方法,代码如下。

encoding=utf8

import pymysql
import time
import sys
import requests
import os
import types

import requests
from lxml import html

  1. @Description(name = “decoder_url”, value = “_FUNC_(url [,code][,count]) – decoder a URL from a String for count times using code as encoding scheme “, extended = “”  

将html实体化

import cgi
import urllib2
import warnings

http_headers = { ‘Accept’: ‘/‘,’Connection’: ‘keep-alive’,
‘User-Agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116
Safari/537.36’}

正则

import re
reload(sys)
sys.setdefaultencoding(‘utf-8’)
from pyquery import PyQuery as pq
from lxml import etree

def get_real_url(url,try_count = 1):
if try_count > 3:
return url
try:
rs = requests.get(url,headers=http_headers,timeout=10)
if rs.status_code > 400:
return get_real_url(url,try_count+1)
return rs.url
except:
return get_real_url(url, try_count + 1)

屏蔽错误

warnings.filterwarnings(“ignore”)

def getContent(num):
url=’http://kaoyan.eol.cn/html/ky/zyml/’
response=requests.get(url).content
selector=html.fromstring(response)
for x in selector.xpath(“//span[@class=’font10′]/strong/text()”):
print ‘#’+x
temp_x=x[0:2]
for i in selector.xpath(“//span[@class=’font10′]/text()”):
if(i[0:2]==temp_x):
print ‘##’+i
titleText=i[0:4]
for j in selector.xpath(“//div[@align=’left’]/a/text()”):
tempText=j
if(j[0:4]==titleText):
print ‘###’+j
urll=”https://baike.baidu.com/item/”
urlll=”http://baike.baidu.com/search/word?word=”
str=j[6:].strip()
try:
urlll=get_real_url(urlll+str)
respl=requests.get(urlll,headers=http_headers,allow_redirects=False).content
select=html.fromstring(respl)
h=select.xpath(“//div[@class=’para’]/text()”)
print h[0].encode(“latin1”).decode(“utf8”)
except:
print “null”

下载图片

def dowloadPic(imageUrl,filePath):
r = requests.get(imageUrl,timeout=60)
status=r.status_code
if status == 404:
return 404
with open(filePath, “wb”) as code:
code.write(r.content)
def getData(final_url):
print final_url
file_open=open(‘./url.txt’, ‘w’)
file_open.write(final_url)
file_open.close()
#链接数据库
conn = pymysql.connect(host=’127.0.0.1′, port=3306, user=’root’,
passwd=’root’, db=’bookinfo’, charset=’utf8′)
#设置浮标
cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)
#布署数据库来源地址
detail_url=final_url
r=requests.get(final_url)
d=pq(r.content.decode(‘gbk’,”ignore”))
#赢得大图地址
none_img=’http://images.zxhsd.com/photo/book_m//nologo.gif’
#大图地址前缀
bigtop=’http://images.zxhsd.com/photo/book_m’
#小图地址前缀
smalltop=’http://www.zxhsd.com/photo/book_m’
big_path=d(‘.pic’).find(‘a’).attr(‘href’)
if big_path is None:
return ‘back’
elif big_path == none_img:
big_path=”
small_path=”
else:
small_path=big_path.replace(bigtop,smalltop)
#获取a标签html
ahtml=d(‘.top’).html()
#解析a标签HTML
cate=pq(ahtml)
#获得分类的尾声八个分拣
#收获最终三个标签必需先获得a标签的HTML分析后本事这样写
category=cate(‘a:last’).text()
#赢得书名
name=d(‘.topbarcon’).find(‘h3’).eq(0).text()
i=0
while i<3:
k=0
while k<4:
text=d(‘.ProductsInfo’).find(‘ul’).eq(i).find(‘li’).eq(k).text()
#将获取的从头到尾的经过转变到字符串类型
text=str(text)
#获取isbn
if ‘ISBN:’ in text:
isbn=text.replace(‘ISBN:’,”)
if isbn == None or isbn==”:
return ‘back’
isbn=isbn.strip()
sel=’select count() from bi_book where isbn =’+isbn
cursor.execute(sel)
result=cursor.fetchone()
count=result[‘count(
)’]
if count != 0 :
print u’已存在’
return ‘back’
#拿到价格
if ‘定价:’ in text:
price=text.replace(‘定价: ¥’,”)
price=price.replace(‘ ‘,”).encode(‘utf8’)
if ‘作者:’ in text:
author=text.replace(‘作者:’,”)
author=author.strip().encode(‘utf8’)
#print author
if ‘出版社:’ in text:
press_name=text.replace(‘出版社:’,”)
press_name=press_name.strip().decode(‘utf8’)
if ‘页数:’ in text:
page_num=text.replace(‘页数:’,”)
page_num=page_num.strip().decode(‘utf8’)
if ‘开 本:’ in text:
#开本和包裹在同步收获总共数据
cont=text.replace(‘开 本:’,”)
cont=cont.replace(‘ ‘,”)
#收获开本
size=cont[0:5]
#获得包装
packaging=cont.replace(size,”).decode(‘utf8’)
size=size.decode(‘utf8’)
#获得出版时间
#正则相配 search冲全部字符串中找 match只找第多少个
press_res=re.search(r’第d版’, text)
if press_res:
#获得版次
edition=press_res.group()
#要深入分析成utf8不然乱码和真是字符串不等同替换不了
edition=edition.decode(‘utf8’)
text=text.decode(‘utf8’)
press_time=text.replace(edition,”)
press_time=press_time.strip()

        #获取印刷时间
        #正则匹配 search冲全部字符串中找 match只找第一个
        print_res=re.search(r'第d次印刷', text)
        if print_res:
            #获取印次
            impressions=print_res.group()
            #要解析成utf8不然乱码和真是字符串不一样替换不了
            impressions=impressions.decode('utf8')
            text=text.decode('utf8')
            print_time=text.replace(impressions,'')
            #截取掉首尾空格
            print_time=print_time.strip() 
        k+=1
    i+=1
if ('isbn' in locals().keys()) == False:
    return 'back'
if ('author' in locals().keys()) == False:
    author = ''
if ('press_time' in locals().keys()) == False:
    press_time = '1970-01-01'
if ('print_time' in locals().keys()) == False:
    print_time = '1970-01-01' 
if ('impressions' in locals().keys()) == False:
    impressions = ''
if ('edition' in locals().keys())== False:
    edition = ''
if ('page_num' in locals().keys())== False:
    page_num = ''
# if ('word_num' in locals().keys())== False:
#   word_num = ''
if ('packaging' in locals().keys())== False:
    packaging = ''
if ('size' in locals().keys())== False:
    size = ''
if ('press_name' in locals().keys())== False:
    press_name = ''
y=0
while y<8:
    title=d('.t').find('.title').eq(y).text()
    text=str(title)
    if '目录' in text:
        directory=d('.t').find('p').eq(y).html()
        if directory==None:
            directory=''
        else:
            directory=directory.encode("utf8", "ignore")
    if '内容提要' in text:
        content=d('.t').find('p').eq(y).html()
        if content==None:
            content=''
        else:
            #截取掉首尾空格
            content=content.strip()
            content=content.encode("utf8", "ignore")
    y+=1
#如果内容简介和目录没有的时候指定为空字符串
if ('content' in locals().keys())== False:
    content = ''
if ('directory' in locals().keys())== False:
    directory = ''
if ('content' in locals().keys())== False and ('directory' in locals().keys())== False:
    details=''
else:
    a='内容简介<br>'.encode("utf8", "ignore")
    b='<br><br>目录<br>'.encode("utf8", "ignore")
    details=a+content+b+directory
    details=cgi.escape(details)
#录入时间
add_time = time.strftime('%Y-%m-%d',time.localtime(time.time()))
#下载小图
#文件根目录
root_path=sys.path[0]
#创建isbn文件夹路径
root_path=root_path.replace('\','/')
isbn_path=root_path+'/download/'+isbn
if big_path != '' and small_path !='' :
    #创建isbn目录
    if os.path.isdir(isbn_path) ==False :
        os.mkdir(isbn_path)
        #组合下载后图片保存路径
        down_img_small = isbn_path+"/small"+isbn+".jpg"
        down_img_big   = isbn_path+'/big'+isbn+".jpg"
        #调用下载图片方法
        small_res=dowloadPic(small_path,down_img_small)
        #大图保存数据库路径
        big_res=dowloadPic(big_path,down_img_big)
        #小图保存数据库路径
        if small_res==404 :
            img_small = 'none-picture/none-small.jpg'
        else :
            img_small = 'download/'+isbn+'/small'+isbn+'.jpg'
        if big_res==404 :
            img_big = 'none-picture/none-big.jpg'
        else :
            img_big   = 'download/'+isbn+'/big'+isbn+'.jpg'

    else :
        #组合保存数据库中的图片路径
        img_small = 'download/'+isbn+'/small'+isbn+'.jpg'
        img_big   = 'download/'+isbn+'/big'+isbn+'.jpg'
else :
    img_big = 'none-picture/none-big.jpg'
    img_small = 'none-picture/none-small.jpg'
source_type = 5
try :
    #要插入的列表
    li=[0,source_type,category,details,detail_url,price,add_time,packaging,print_time,impressions,name,author,press_name,isbn,edition,size,press_time,page_num,img_big,img_small]
    #执行sql
    sql="insert into bi_book (book_id,source_type,category,details,detail_url,price,add_time,packaging,print_time,impressions,name,author,press_name,isbn,edition,size,press_time,page_num,img_big,img_small) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
    aaa=cursor.execute(sql,li)
    if aaa==1:
        print u'插入成功'
    conn.commit()

except Exception, e :
    return 'back'

getContent(0)

p=3757000

while p<3758429:

p=0
while p<1:
#string=str(p)
home_url=”http://www.zxhsd.com/kgsm/ts/2017/03/04/3760868.shtml”
# file_open=open(‘./number.txt’, ‘w’)
# file_open.write(string)
# file_open.close()
getData(home_url)
p+=1

相关文章