本帖最后由 xiaoxue 于 2019-3-4 20:51 编辑 7 N4 e& I0 G3 F; G0 X$ t % L j r. I: v: F # -*- coding: UTF-8 -*-& P, G) A$ l& {( ]1 C3 B7 Z: z& Q import requests& m; k' G& Y* ? import os8 Q, Y& [% L+ [ t: J! z8 h: i; N! S import subprocess! t9 M* X. H9 N4 N' J import sys. f y. X$ u" A3 p from tqdm import tqdm1 d# q1 ^* b( o3 ]# x$ }, L4 a import time,random6 i2 Z6 {% I; \& `2 ]6 A from HTMLParser import HTMLParser: s2 I. i8 c; ]+ l$ \# g$ b; r+ J: u import sys * ~, l8 s- M1 {0 G9 }# ] reload(sys) . u- p: g, r& t, Y9 g sys.setdefaultencoding('utf8') 5 V( B5 W* x1 T5 Q4 ~" ^0 a 5 \& t- n3 O! m" I9 K( L class WebRequest(object): G0 L5 {$ m8 k+ \' ]% h. x! X def __init__(self, *args, **kwargs): r. B' j5 `0 J0 d. L pass! I7 T7 H8 E3 z7 C3 r 9 c+ g8 Y# k7 |. V% z. y0 ~ @property: E! L# m8 \$ }* l$ i( I4 u* l2 Q def user_agent(self):- x: c7 f$ j% ] """0 k% {' {3 q- S2 ?, ] return an User-Agent at random* z+ n& J7 C# U8 q" ?: w" b :return:0 r/ {# i, z6 u/ V: j9 y i """' p* L+ d* E Y! R# ? ua_list = [+ [8 ^! K4 \' j2 D; J 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',7 `3 q" ^" M5 r4 [ 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',' F5 x0 z2 X# K' L6 w% f3 C* C 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',5 b& T* W3 c D8 N+ x/ e) i+ X9 u 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',1 k# @" v1 o- o. o: d 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',& W6 x! M) J- H+ C$ C 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',/ A$ {0 y S# b+ K* a/ K8 L( ` 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',1 }$ f! v2 n2 ?; ]1 A" P$ z 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',* J& b* B8 Z# g# j; d ], Z$ L* B9 k# @# P X; { | return random.choice(ua_list)' l7 s- U( ^2 Q8 m9 w/ m# H9 ] 9 V4 P' Z _1 b2 K% H# { @property, m6 \7 ^9 v1 @+ x" p def header(self):' t; `0 f0 r( a4 b! d' X """5 E; J4 g$ I; c2 U) J basic header1 H5 d+ i) G7 {+ c, [ :return:, f; n' F) ^. N5 _* E: ^- C """- [$ [5 b% U# T; Z return {'User-Agent': self.user_agent,: b: p+ x ] m/ B 'Accept': '*/*',: W. y# o) r1 M+ k! i8 g 'Connection': 'keep-alive',/ }$ ?( i7 J$ j0 |1 v4 t5 Y! { 'Accept-Language': 'zh-CN,zh;q=0.8'}% U! E' C& ]7 h, b2 y " }) }5 K1 C O9 ?1 v def get(self, url, header=None, retry_time=5, timeout=30,/ f/ B9 o. a$ E5 Y retry_flag=list(), retry_interval=5, *args, **kwargs):% z1 v) J+ W7 i """ V" k; e# v, `! {3 J8 F, T get method8 n7 m/ b* i1 H4 U :param url: target url' @+ l+ \' x) [' ~ :param header: headers# x) ]0 L! m9 `0 B; e :param retry_time: retry time when network error$ e y* p/ h0 @3 a6 X f6 h :param timeout: network timeout: X3 V( x5 R, R1 Z :param retry_flag: if retry_flag in content. do retry& c8 k) ~1 w& H5 e. m :param retry_interval: retry interval(second)# k: D: d6 E8 R :param args:1 b, V* ~1 o* r# {6 C- s :param kwargs:0 @- s$ @9 C' H :return:8 E& H/ c, F+ g8 k """+ E! J3 \4 Y4 W- M- S/ t/ P+ Q2 M headers = self.header0 I! h: n& k9 b1 E5 r/ @* d if header and isinstance(header, dict):6 V) ?/ ?- y# @; @ headers.update(header); V: J. C9 m A$ m# ]( r# C2 S8 `0 t7 x while True:* s$ ]) [! y& T- B! ?; n try:6 F; g9 G0 I6 O7 L& y' C/ a3 }' x html = requests.get(url, headers=headers, timeout=timeout, stream=True)" a, G6 U' s! r( F2 V print 'content size: %d' % len(html.content)0 x; @8 U$ c1 Y9 ?2 E5 g7 H if any(f in html.content for f in retry_flag):( e) N: @3 E& b& w" ], a raise Exception& h& o5 {4 V% P, D, O# U/ k8 m$ ^ if not html.content:/ G+ _6 [* _9 j3 ] print 'content is Null,retry...~' + url# @ A4 L# t' E) O. O7 P, b+ `3 Z raise Exception0 U6 v9 Z2 K5 _9 U& i return html' Z( C8 Z L1 v: G W/ P# B& c: N' e except Exception as e: U* H" U' w, d" [' }1 u print(e)# a1 B2 x4 a3 d7 f7 G: f e ) F2 S6 u- I) _ % K7 F5 e4 T6 R' z/ n' _" G: j def DownloadAudio(url, path, refer=None):/ }) P7 l; n6 _ with open(path, 'wb') as handle:4 M0 K8 T! p- {3 n8 p if refer:* k1 h; s0 d' Z# Q$ c hdr = {: l2 @2 `5 T$ R, g% C8 } 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',& o3 M4 U* T7 L& s7 \ 'Referer': refer}2 @. L* H6 u4 L5 B; ^1 q& D3 t else:! }4 ~+ ]9 G2 Y' B9 V4 {' Z hdr = {6 z0 I4 g7 L. ?0 F6 e 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',% {0 t$ ?- @6 G' U% S9 M/ i 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',) ~7 w. j! @ l: E 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'," v3 ^0 x9 {# X" a+ }; k 'Accept-Encoding': 'none',- W0 d/ s) ^! ^ i 'Accept-Language': 'en-US,en;q=0.8',2 k7 O) Z8 l: x7 t 'Connection': 'keep-alive'}7 z% C# w; z9 \' _3 t3 D1 u2 I wr = WebRequest() R1 ]: ~7 Z+ @; `2 a #print hdr, _% c! R3 K" u" \5 ^! K response = wr.get(url,header=hdr)# u: r' f( o( v0 l( w& w6 w' M# ]6 u #html = requests.get('https://www.example.com', proxies={"http": "http://{}".format(proxy)})" M2 j3 K9 W0 M" l& Q. ~ #response = requests.get(url, stream=True, headers={'User-agent': 'Mozilla/5.0'})& l+ a' Z* g3 I8 b s; l1 @' }* N #print url# Z1 m* g( N6 @ d for block in tqdm(response.iter_content(),ascii=True, desc='ImageDownload'):" f& K1 M4 L; R V" g5 Z if not block:* t7 Z* n" L& ? h3 b, f break) u. _8 a8 c* j) D* Y3 X S handle.write(block); e0 ~, q) R H% G. S 3 m* F5 H' o) j) ^ def translate(to_translate, to_langage="auto", langage="auto"):8 [, ~4 [ @/ r4 W- _1 M '''Return the translation using google translate4 a7 I& N& x: O/ J7 k you must shortcut the langage you define (French = fr, English = en, Spanish = es, etc...)% v% P- m6 P( L3 ~% W! Y if you don't define anything it will detect it or use english by default1 z, B+ m* n+ J6 T" S2 T L Example:, m! \8 B6 R2 q- e print(translate("salut tu vas bien?", "en"))! l' [# T/ m- D5 W* Q hello you alright?'''$ o7 {5 @: ?& ^, _* q( v agents = {'User-Agent':"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)"}( {, S; S7 b6 ]$ F. \( ^ before_trans = 'class="t0">'( l8 D. M. Y2 }6 m6 \ link = "http://translate.google.com/m?ie=UTF-8&hl=%s&sl=%s&q=%s" % (to_langage, langage, to_translate.replace(" ", "+"))' o$ _% I8 @! r( c request = requests.get(link, headers=agents)0 y& [7 A/ ^6 A( B- O: N$ A page = request.content' f7 @6 Y! x( p #print page6 B H2 i& e7 l: T l# u! u result = page[page.find(before_trans)+len(before_trans):]; z5 y# R' c( M# f result = result.split("<")[0]- m8 S f4 j, x. \/ s& J return result* u. f i7 S, Y j A" L7 R6 E tsxt_sen = []9 P8 k) g( a: S- X from nltk.tokenize import sent_tokenize2 `( i- @2 Z/ g' |2 I9 L4 v* p* t def sentence_split(sentence): # 编写分句函数$ O% I' ^! w% c& W4 u0 a: q" z text_sen = []$ e- a( X. O/ e! ] for s in sentence.split(','):* o4 F+ _+ Y8 j # if '?' in s:" P j! H" j% N, T L- S # tsxt_sen.extend(s.split('?'))" e4 S) r& n2 \$ s! ~4 ~$ w; E # elif ',' in s:- u, Y: s9 B8 B! x # tsxt_sen.extend(s.split(','))9 L9 x* T; h# l( o0 N" a( c # else:' q/ x9 d5 V& N X8 D) O" W6 O& n: S text_sen.append(s)3 h p- u( _$ u return tsxt_sen" A/ k; z' p8 |- x 6 t: G; l2 E9 N) i! @( C 8 F. @5 X5 E0 S9 c- G3 l z& k2 u" U str = '''# f2 e" ~& M( V6 ^ It has not quite been greeted with the enthusiasm of the bottles of Scotch in the novel Whisky Galore, but the arrival of a cargo of pineapples on the shores of northern Scotland has sent scavengers rushing to the beach.; q2 Z3 Y( T9 m5 s D) _ ! k0 o. _# _' J* E& w; j The fruit, which has appeared on Shetland and on beaches across the Western Isles, is thought to have come from several containers lost overboard in the Atlantic last October by cargo ship MV Lombok Strait.: A+ J$ s9 J. ~* K& J2 x 2 U, \' K. C+ z4 U4 J/ Z+ ` Shetland islanders have now shared photos of their fruitful haul, in a story that echoes the plot of Whisky Galore, the 1947 novel by Sir Compton Mackenzie, later made and remade for the big screen.$ K( ?9 v3 Y8 q7 o ; u! R9 W: J2 f% q& ]1 Z$ y7 l The story tells of how the locals find their island awash with whisky after a ship loaded with liquor is wrecked off a fictional Scottish island. 7 l& M# y7 p( p" v- P& D 4 `9 ^, j6 o* y- k. B The plot was based on the real-life salvage operations carried out by locals when the SS Politician, carrying 280,000 bottles of malt whisky, ran aground on Eriskay in 1943.. w6 Y4 e; G& l$ n) Z / T' ^7 r+ z2 \- x. { The arrival of the pineapples will be far less lucrative for beachcombers, and not as profitable, either, as the grounding of the cargo ship MSC Napoli off Branscombe beach in Devon in 2007.& s1 v$ G& V4 @# d" F! m& a3 |8 V( J n4 i- J' O# q7 p Then scavengers ignored police advice and arrived in their droves to explore shipping containers, which contained motorbikes, pet food, wine barrels and anti-wrinkle cream.; Z! l" j' N1 _9 K8 p8 v '''7 b2 |1 x9 Y; n2 w! b1 ? 4 } P; H- o4 g+ H : F) e. O8 R% f3 A text = ''.join(str).strip().lstrip().rstrip().replace('\n',' '): K+ G1 }( m# X3 r% V with open('~/text.txt', 'wb') as handle:) n( {+ l# | }; W+ X handle.write(text)0 l* n" N& V, D4 h0 \) T" \ 4 k; ]1 M+ b8 [* T* d. Z str = str.replace('”','"'). Y3 t' r5 `' x) S E( I" M' G1 s str = str.replace('“','"')3 K0 {6 K7 Q9 ?! L4 X/ d5 B # {5 m: L% E: L6 t1 k# F # N2 V( M) o2 a/ d def splitStr(str):! c! {( d* l- w( g/ ] arr = str.split(' ')0 u5 ~5 ]- m) ?5 E ~. F8 A sz = len(arr)5 {: m6 m* K: m5 D return [' '.join(arr[0:sz/2]),' '.join(arr[(sz/2):])]5 r8 B0 m6 S' B+ d; Z; R" F4 j ) t4 y0 d( Q' D v / @* X9 e; n0 s0 h- ~& ?, h #print splitStr('with wording from North Korea about getting rid of its nuclear weapons and a guarantee from the United States that it would not interfere with the North’s regime or demand redress for human rights abuses.')) ^8 I/ _& D. I, h 6 v: M5 H. O, M( {+ C0 F keyArr = []) i* A0 I8 M5 @# _9 C% } sent_tokenize_list = sent_tokenize(str)3 F8 ?* i7 j2 p; V; i: K7 i% H for s in sent_tokenize_list:, `: h" }# _( p% {1 F& _ sz = len(' '.join(s.split())). a3 Y0 Q ~7 I4 h4 T- H if sz <= 190:: L# s( N! b$ d keyArr.append(' '.join(s.split()))/ }% T/ X& ~- g3 V' U6 s% ` #print '[ '+' '.join(s.split())+' ]'0 Z( A& }* i9 r% }9 j' y else: x9 |+ G9 h A7 M8 I4 s' ~) B: x! G for ses in s.split(','):7 X5 a9 Q2 h: {+ t* h L" L if len(ses) > 190:8 ?+ w' r! N3 b. W' _, M4 P keyArr.extend(splitStr(ses))' P5 W* A( U1 M* S; Y( \ I else:+ o1 y; S4 N1 M* _ keyArr.append(' '.join(ses.split()))3 `$ P% n. P$ Z& q# B1 V5 D 9 S- s, N: }1 f* ^ / W# `2 j! m* W( D API_URL = "http://translate.google.com/translate_tts?ie=UTF-8&tl=en-us"1 R# Z9 i0 \1 k( x& e& K! p #key = HTMLParser.unescape.__func__(HTMLParser, translate(str,'en'))2 D* S& a+ g5 `8 _6 n #keyArr = sentence_split(str)0 U, `; F) O3 _9 P- |4 S7 z ! S! h6 B; N6 v, f1 M7 p$ e & D$ B( q: v% a" G4 L3 u #print([text[i:i+n] for i in xrange(0, len(text), n)])+ E% K! R5 o5 T2 Z4 V os.system('rm ~/tmp/audio/*.mp3'); G0 \5 c- B5 }+ _# B8 u namei = 0, [3 g5 u/ C% p4 V/ v s for i in keyArr:: F7 v9 h8 g$ ^5 Y4 X juzi = ' '.join(i.split())" P+ _1 }. q1 W; M' k sz = len(juzi)% m6 j1 `0 C0 _ link = API_URL+"&q="+juzi+"&client=tw-ob"" L5 |) I( T( O% g print link9 }7 C7 c8 M& m+ @9 ?$ T- C path = "~/tmp/audio/%d.mp3" % namei3 r- B1 w' E+ i# B #print path3 ?3 J) |) Z" v8 y7 d DownloadAudio(link,path). p) h8 [: [; t: r os.system('file '+path)$ z9 J' v0 I* J6 G8 P: V! n! M( N! ~ time.sleep(1)% L/ f/ B/ M2 o+ i' v4 M! O( ^8 } namei = namei + 1* r2 \& S% a5 D( ]3 u 复制代码 0 C- Q2 R. H5 W( f1 T) Z
查看全部评分
相关帖子
发表于 2018-8-28 13:42:09
楼主
发表于 2018-8-28 22:30:07
厉害啊，python高手啊！
发表于 2018-8-29 21:31:47
