|
发表于 2020-7-20 20:12:52
|
显示全部楼层
- * A) w) M5 h1 K; J, o
- # -*- coding: UTF-8 -*-- D! X! a( S, h1 ~* s
- import requests4 y; P* v3 d, g% c! u
- import os6 k1 A( [) J) y" g* H4 e' s
- import subprocess9 z! O5 |$ _; I F+ u+ B: O: z$ z
- import sys
, j) k' H; r7 t0 _, k8 c - from tqdm import tqdm5 U( B) X6 K$ h' |! b, p4 g
- import time,random, E% x- x3 Y. _3 {, M
- from HTMLParser import HTMLParser
( P* L8 \( S, W3 [4 N - import sys
2 P$ o2 C) s# S7 b- l' y! r - reload(sys) : _$ y3 ^ ^. b
- sys.setdefaultencoding('utf8')
7 [- E4 M; L- |# P$ w
$ z3 `1 Q. l2 q* K3 v2 J- class WebRequest(object):
' Z% h: I# U9 {4 i7 p6 K4 |% I - def __init__(self, *args, **kwargs):
9 C, ^4 }5 E) s1 o: x5 C/ W - pass
6 ]0 E5 ]' P# _' l: A - ) a1 X& Y5 B9 Q- e f0 d
- @property1 \2 q0 _1 Q6 Q: |- Y+ {
- def user_agent(self):
1 w) ?# d! t, J5 j6 s8 |/ r - """
0 V* _) B. Q: N! [! f - return an User-Agent at random( Y {/ [( R" K
- :return:
) U+ H4 U( q" B5 T9 i+ ] - """
! D$ a+ T$ T; }0 z8 o, n - ua_list = [
, k) Z. I; \& N9 {4 U f" i8 W - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101'," ?, |/ [; r; ]1 i, j1 m6 R* u
- 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
7 U) Q5 c% @1 F+ Z" ^/ W8 _ - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',7 M; u1 [; @! t5 f
- 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
* a" r, t& U* x+ F2 Z2 F - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
$ n2 R+ ]) [5 Y9 t( I8 Q! y4 [ - 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
' \; b) W8 ~( H8 q$ g - 'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',9 Y: q# E! U* [9 L8 A, {
- 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
- ?) N) l# v: m& y7 I- \9 b$ S - ]
, {, @1 a' W4 \ - return random.choice(ua_list)
& ^) f7 Y, R4 Z" x: |" p; r ^7 W+ _
?. {. f# B3 v5 l5 m. `. B- @property
/ r9 @# ?5 v; V' F6 k: z1 ~ - def header(self):
) F: f, ?6 N9 s* Z - """
2 A& _* M5 a* k; ^: t7 X - basic header5 g! q. |5 L1 p+ J7 n4 s6 f+ C
- :return:$ P' N w' e6 o3 L Y) `4 A5 N4 p
- """
$ M) Y+ b9 ~- y% u5 ^ - return {'User-Agent': self.user_agent,$ J* S( c" W. O. P7 q( m
- 'Accept': '*/*',; I( R2 _' ?. X3 _# g! L
- 'Connection': 'keep-alive',4 Q7 ]7 t D* f; @# t
- 'Accept-Language': 'zh-CN,zh;q=0.8'}+ @$ E. k9 L; j& k- y
- % b- I8 [5 j+ c& J: m+ y
- def get(self, url, header=None, retry_time=5, timeout=30,
4 @4 t7 u0 h8 c% @5 t& X& i( f( k4 g - retry_flag=list(), retry_interval=5, *args, **kwargs):( {! l- K; K- R9 v( @. S0 k
- """$ K" u6 ~+ W, n/ L1 @$ y0 g
- get method, m2 z2 @2 h! c. m" |- c) I7 {: q
- :param url: target url4 \8 G8 B$ y& S
- :param header: headers
. ^2 C, m2 D+ z9 w - :param retry_time: retry time when network error, J1 ]' R3 X9 p9 [) a2 I
- :param timeout: network timeout9 U x+ F6 P: W' V6 k
- :param retry_flag: if retry_flag in content. do retry
" Z& k. e4 q" q9 W* l @. }6 h - :param retry_interval: retry interval(second)
O; ^1 G/ n9 F% C q+ ^2 ~ - :param args:
/ E5 z6 m9 @3 n, o. ` - :param kwargs:) ~: p1 j B; b# |. y" `% G" H
- :return:2 S7 g- E6 ~4 ?( V) z/ J2 [7 r1 r
- """4 l) U& {& k, M( N( i- I2 h
- headers = self.header
# m. Q( V% r& Q" p6 k - if header and isinstance(header, dict):
% P: m- w1 v8 f: J - headers.update(header)
: K6 ~1 `/ e9 k/ y; C - while True:: h& B+ K/ ~% M. N6 J+ }# D+ \
- try:3 K* v) v8 `0 ], _3 r4 D
- html = requests.get(url, headers=headers, timeout=timeout, stream=True)' a4 i& ^4 C8 \% W% ]
- print 'content size: %d' % len(html.content)
. Q; l) H3 l1 Z( X* c, ^' F - if any(f in html.content for f in retry_flag):( L) \3 V: g t" ?+ P9 B; @
- raise Exception4 m& C; T* u9 g/ R
- if not html.content:
8 T/ L( x- ~5 H J) v4 d! e3 o - print 'content is Null,retry...~' + url1 l1 y- w4 F6 c7 [* `2 K: K2 l
- raise Exception
) n2 ]/ }) y# L* W' ]' j7 E* Y8 M - return html# L& Z# e0 f5 ]! t. b
- except Exception as e:
5 R7 u I {. i. M1 j5 X - print(e)0 b. b: X% F9 m/ f& d
) m3 N# p# J( N" N7 s' i7 Z0 y
9 K: X4 \# i9 h9 y8 D- def DownloadAudio(url, path, refer=None):- @' I, D% t8 L4 S% `" a9 V
- with open(path, 'wb') as handle: ~& l# t) b% k" K" R
- if refer:% P' b7 y9 N) J" u: w; w6 k
- hdr = {9 f) Z: v7 O2 g9 b% v
- 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',2 h& x/ n( _* u" s* O5 N' h
- 'Referer': refer}5 e& K4 a1 \) \/ s) A F6 R
- else:
8 E- i' g0 V- Q2 b" J+ o - hdr = {: s2 y% R' J3 ~% b: l
- 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
$ k3 P1 r o3 Q - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
7 x+ H/ k- k! ~. m0 S0 H2 c - 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',/ e" S2 H7 g6 P
- 'Accept-Encoding': 'none',
2 e% ~' {) B; `# t# s& p - 'Accept-Language': 'en-US,en;q=0.8',
; x6 N+ c4 K5 t! X8 z4 R& {- V! [ [ - 'Connection': 'keep-alive'}8 r, t; u( n/ G$ p+ H
- wr = WebRequest()5 c9 d( b, c, k
- #print hdr, A* x# r$ n: L1 J, C0 j0 o& [
- response = wr.get(url,header=hdr)
6 s" P$ g" ~6 w - #html = requests.get('https://www.example.com', proxies={"http": "http://{}".format(p!r!o!x!y)})
. r } u% @7 R( Z* z6 ~ - #response = requests.get(url, stream=True, headers={'User-agent': 'Mozilla/5.0'})8 q3 S$ r* C- `1 ^
- #print url- }3 U9 Q1 [/ b3 V+ |$ l( f
- for block in tqdm(response.iter_content(),ascii=True, desc='ImageDownload'):
, T' q$ k M" Z7 [' v - if not block:
. e0 D/ u+ C" |' \/ _ - break
+ t. L( t, k: H0 ?: ~( `) ^ - handle.write(block); e4 w/ X3 M( o
- ; `" a5 O' y, P9 V
- def translate(to_translate, to_langage="auto", langage="auto"):
0 W* J- o3 D% |) G! m4 g - '''Return the translation using google translate* }# ], Z+ ]% _: b- B3 W* D. t
- you must shortcut the langage you define (French = fr, English = en, Spanish = es, etc...)
$ M0 R# T' g! P9 |! l1 e - if you don't define anything it will detect it or use english by default
( v. y( h; i, w/ h. u) J. J7 j3 s+ i - Example:
7 ~7 h) e' y4 R( N& s% { - print(translate("salut tu vas bien?", "en"))" u, Y, v& h( I* ~" u& _& m& X
- hello you alright?'''
: c' k, j9 z! F# ` - agents = {'User-Agent':"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)"}
) o; K/ A# T+ {# D - before_trans = 'class="t0">'; g+ {5 y. c7 H0 Y
- link = "http://translate.google.com/m?ie=UTF-8&hl=%s&sl=%s&q=%s" % (to_langage, langage, to_translate.replace(" ", "+"))
$ I5 J( }2 J$ c( p4 ]2 K. C - request = requests.get(link, headers=agents)1 ~% i3 a0 c$ G
- page = request.content
4 I {+ C( q' f# g - #print page) D$ D- n$ ]# V4 p' R) A) u
- result = page[page.find(before_trans)+len(before_trans):]
- s- C- y$ t; q' S/ K$ w2 L/ J* g - result = result.split("<")[0]/ g! G2 n+ K [
- return result2 d/ k0 J3 ?1 L( u" ?
# D6 }% ?1 c8 v( @; j- tsxt_sen = []
; @0 }0 e+ q6 o1 L - from nltk.tokenize import sent_tokenize. `2 x% L! C! V8 L( z) a" k
- def sentence_split(sentence): # 编写分句函数9 t* B9 @+ d" K0 s! b, Z
- text_sen = []
) y2 n( ^8 w) U7 }4 g - for s in sentence.split(','):
; L) {$ }; p7 ]9 _2 u - # if '?' in s: U. s8 ^8 L1 j" t/ A7 n
- # tsxt_sen.extend(s.split('?'))) w# a8 s$ t0 K+ v1 p
- # elif ',' in s:
; c K, W; a/ x/ {3 @( s2 e - # tsxt_sen.extend(s.split(','))
- v$ v7 j v9 }: U, V- b6 g - # else:
, D F; l, w4 M: D! R( {6 H$ m - text_sen.append(s)
7 m- |, X, K$ k a& o' T8 P% N3 H - return tsxt_sen6 h4 s: b0 t- }6 l( g( D! J
! A3 z9 n& @( T6 p1 }2 O
# |: \; `) v* R* m- str = '''
5 O9 y( x- t6 g, W$ i$ } - It has not quite been greeted with the enthusiasm of the bottles of Scotch in the novel Whisky Galore, but the arrival of a cargo of pineapples on the shores of northern Scotland has sent scavengers rushing to the beach.
! y# a. |7 p( F5 `3 \2 d - 3 G @/ M. S( E0 h! i# x4 R
- The fruit, which has appeared on Shetland and on beaches across the Western Isles, is thought to have come from several containers lost overboard in the Atlantic last October by cargo ship MV Lombok Strait.- i" \ T4 x7 }
, t' N( \ X& B! \- Shetland islanders have now shared photos of their fruitful haul, in a story that echoes the plot of Whisky Galore, the 1947 novel by Sir Compton Mackenzie, later made and remade for the big screen.. z( O" Q8 j D. ? k
- $ c& _# S' Y( ^4 T" u3 i2 j% A
- The story tells of how the locals find their island awash with whisky after a ship loaded with liquor is wrecked off a fictional Scottish island. 7 a7 m7 f( H+ U" |
! |7 I& O; K) g7 a. k- The plot was based on the real-life salvage operations carried out by locals when the SS Politician, carrying 280,000 bottles of malt whisky, ran aground on Eriskay in 1943.
4 g' X5 r: W2 g0 g5 D6 A - ! M3 |% ~8 K/ Z0 s( E. [# e! Z
- The arrival of the pineapples will be far less lucrative for beachcombers, and not as profitable, either, as the grounding of the cargo ship MSC Napoli off Branscombe beach in Devon in 2007.' U- v3 T( z6 [( x8 \+ L, x
% z2 Q) r7 m9 Y. N: j: d' ]- Then scavengers ignored police advice and arrived in their droves to explore shipping containers, which contained motorbikes, pet food, wine barrels and anti-wrinkle cream.
. d; W& r0 y* j2 x1 N. a - '''
5 ^0 A+ }" B; u
' {8 [% B% h p% a# c0 T( G& H0 p( L
1 W$ M3 T2 o% G. i0 l: [- text = ''.join(str).strip().lstrip().rstrip().replace('\n',' ')6 G/ ?4 Y/ M9 m0 w
- with open('~/text.txt', 'wb') as handle:
& a7 K. j" n8 k+ L - handle.write(text). L% L$ M( j5 X
- 0 ]6 q6 e7 G. Q0 [ O
- str = str.replace('”','"')- E+ f+ r0 G0 q1 m; `" e* D( B6 h4 k. y2 V
- str = str.replace('“','"'). q4 K M$ U7 q+ S
- 1 U# x* R% ]4 }6 F. v5 o& C4 u& j
0 r3 T0 D1 k9 B! ~; B7 [- def splitStr(str):
, a: K7 \+ Y0 k d% J - arr = str.split(' ')% r X8 ]7 ]- p5 j! F9 `4 i
- sz = len(arr)
( f& j4 a# r% P5 |2 _6 A - return [' '.join(arr[0:sz/2]),' '.join(arr[(sz/2):])]
6 n0 t+ `# S4 r! N2 I, l -
6 Y7 ?; U. g, v% ], m2 H - . H0 \) r+ `. I- t, d! n
- #print splitStr('with wording from North Korea about getting rid of its nuclear weapons and a guarantee from the United States that it would not interfere with the North’s regime or demand redress for human rights abuses.')/ G- J5 Z Y0 n) K$ A
- : U6 m, F) X$ \- i9 C& m8 q, [
- keyArr = []
( _7 j( v+ j- x5 n: i - sent_tokenize_list = sent_tokenize(str), D8 j& j! p E& Y
- for s in sent_tokenize_list:, f, X) @ F) `5 v
- sz = len(' '.join(s.split()))
3 p! l3 W; r4 i, H0 w5 B5 A0 |; g# B% X - if sz <= 190:
" u# ]6 J3 v4 B8 q8 A4 d - keyArr.append(' '.join(s.split()))
- F! X7 U; @6 L: z: J# s - #print '[ '+' '.join(s.split())+' ]'' `& a/ c. p+ V8 X) V. b9 J
- else:; _4 b, e- @6 D" r: o" _
- for ses in s.split(','):
' \9 C( l# n% F# `+ R. l - if len(ses) > 190:
" n) R9 S4 J1 w( B4 e5 n8 ` - keyArr.extend(splitStr(ses))
3 _0 n% i+ P' o& r - else:
+ X+ G% D; n% r! m - keyArr.append(' '.join(ses.split()))
: y. O0 `7 b0 |, l" }1 I) B - 5 d. t! o% n6 U3 k. R" N" r! N: r r
- $ Z, a; o) b$ M# e' U8 E
- API_URL = "http://translate.google.com/translate_tts?ie=UTF-8&tl=en-us"7 [$ R% Y i, W* f9 T0 i
- #key = HTMLParser.unescape.__func__(HTMLParser, translate(str,'en'))
- D# ]8 A+ l9 Z% ^3 s5 V$ g - #keyArr = sentence_split(str)( y7 o$ U) i1 C; Y
- - s' M8 b' P' T. o( O, S+ t4 b
- % S# B5 X. d2 m
- #print([text[i:i+n] for i in xrange(0, len(text), n)])5 e; Q% L# L/ N
- os.system('rm ~/tmp/audio/*.mp3')3 c* s& Z) {) g
- namei = 0$ D7 q- b# X7 M
- for i in keyArr:# A3 O; z; h b O' K
- juzi = ' '.join(i.split())9 O" s# @* s& y' J. ?
- sz = len(juzi)
4 M+ P8 }: C: [! y7 B' p$ E - link = API_URL+"&q="+juzi+"&client=tw-ob"4 g0 T0 ~. m6 d2 K7 Q- [$ J
- print link
9 ?* [5 P, ~9 w# _- m* l - path = "~/tmp/audio/%d.mp3" % namei
& s# F; H- A7 g* ?2 ^5 p6 V - #print path: Q3 }! J7 I: W+ n$ Y$ k$ e/ n* N5 s/ q
- DownloadAudio(link,path)
`- H! [7 P* V4 M3 V; a% t/ @2 A0 \* a: k - os.system('file '+path)
- I3 P: Y* D+ _4 { - time.sleep(1)
& }9 e8 c1 \- s. | - namei = namei + 1
7 }- @8 r4 @) }
复制代码 |
|