]>
jfr.im git - z_archive/twitter.git/blob - twitter/util.py
2 Internal utility functions.
4 `htmlentitydecode` came from here:
5 http://wiki.python.org/moin/EscapingHtml
8 from __future__
import print_function
16 from html
.entities
import name2codepoint
18 import urllib
.request
as urllib2
19 import urllib
.parse
as urlparse
21 from htmlentitydefs
import name2codepoint
25 def htmlentitydecode(s
):
27 '&(%s);' % '|'.join(name2codepoint
),
28 lambda m
: unichr(name2codepoint
[m
.group(1)]), s
)
30 def smrt_input(globals_
, locals_
, ps1
=">>> ", ps2
="... "):
37 inputs
.append(input(prompt
))
39 ret
= eval('\n'.join(inputs
), globals_
, locals_
)
46 def printNicely(string
):
47 if hasattr(sys
.stdout
, 'buffer'):
48 sys
.stdout
.buffer.write(string
.encode('utf8'))
51 print(string
.encode('utf8'))
53 __all__
= ["htmlentitydecode", "smrt_input"]
56 print(msg
, file=sys
.stderr
)
59 """A class to count fails during a repetitive task.
62 maximum: An integer for the maximum of fails to allow.
63 exit: An integer for the exit code when maximum of fail is reached.
66 count: Count a fail, exit when maximum of fails is reached.
67 wait: Same as count but also sleep for a given time in seconds.
69 def __init__(self
, maximum
=10, exit
=1):
76 err("Too many consecutive fails, exiting.")
77 raise SystemExit(self
.exit
)
79 def wait(self
, delay
=0):
86 """Find all links in the given line. The function returns a sprintf style
87 format string (with %s placeholders for the links) and a list of urls."""
88 l
= line
.replace("%", "%%")
89 regex
= "(https?://[^ )]+)"
91 re
.sub(regex
, "%s", l
),
92 [m
.group(1) for m
in re
.finditer(regex
, l
)])
94 def follow_redirects(link
, sites
= None):
95 """Follow directs for the link as long as the redirects are on the given
96 sites and return the resolved link."""
98 return sites
== None or urlparse
.urlparse(url
).hostname
in sites
100 class RedirectHandler(urllib2
.HTTPRedirectHandler
):
103 def redirect_request(self
, req
, fp
, code
, msg
, hdrs
, newurl
):
104 self
.last_url
= newurl
105 if not follow(newurl
):
107 r
= urllib2
.HTTPRedirectHandler
.redirect_request(
108 self
, req
, fp
, code
, msg
, hdrs
, newurl
)
109 r
.get_method
= lambda : 'HEAD'
114 redirect_handler
= RedirectHandler()
115 opener
= urllib2
.build_opener(redirect_handler
)
116 req
= urllib2
.Request(link
)
117 req
.get_method
= lambda : 'HEAD'
119 with contextlib
.closing(opener
.open(req
)) as site
:
121 except (urllib2
.HTTPError
, urllib2
.URLError
):
122 return redirect_handler
.last_url
if redirect_handler
.last_url
else link
124 def expand_line(line
, sites
):
125 """Expand the links in the line for the given sites."""
127 msg_format
, links
= find_links(l
)
128 args
= tuple(follow_redirects(l
, sites
) for l
in links
)
129 return msg_format
% args
131 def parse_host_list(list_of_hosts
):
132 """Parse the comma separated list of hosts."""
134 m
.group(1) for m
in re
.finditer("\s*([^,\s]+)\s*,?\s*", list_of_hosts
))