]>
jfr.im git - z_archive/twitter.git/blob - twitter/util.py
2 Internal utility functions.
4 `htmlentitydecode` came from here:
5 http://wiki.python.org/moin/EscapingHtml
8 from __future__
import print_function
18 from html
.entities
import name2codepoint
20 import urllib
.request
as urllib2
21 import urllib
.parse
as urlparse
23 from htmlentitydefs
import name2codepoint
27 def htmlentitydecode(s
):
29 '&(%s);' % '|'.join(name2codepoint
),
30 lambda m
: unichr(name2codepoint
[m
.group(1)]), s
)
32 def smrt_input(globals_
, locals_
, ps1
=">>> ", ps2
="... "):
39 inputs
.append(input(prompt
))
41 ret
= eval('\n'.join(inputs
), globals_
, locals_
)
48 def printNicely(string
):
49 if hasattr(sys
.stdout
, 'buffer'):
50 sys
.stdout
.buffer.write(string
.encode('utf8'))
53 print(string
.encode('utf8'))
55 __all__
= ["htmlentitydecode", "smrt_input"]
58 print(msg
, file=sys
.stderr
)
61 """A class to count fails during a repetitive task.
64 maximum: An integer for the maximum of fails to allow.
65 exit: An integer for the exit code when maximum of fail is reached.
68 count: Count a fail, exit when maximum of fails is reached.
69 wait: Same as count but also sleep for a given time in seconds.
71 def __init__(self
, maximum
=10, exit
=1):
78 err("Too many consecutive fails, exiting.")
79 raise SystemExit(self
.exit
)
81 def wait(self
, delay
=0):
88 """Find all links in the given line. The function returns a sprintf style
89 format string (with %s placeholders for the links) and a list of urls."""
90 l
= line
.replace("%", "%%")
91 regex
= "(https?://[^ )]+)"
93 re
.sub(regex
, "%s", l
),
94 [m
.group(1) for m
in re
.finditer(regex
, l
)])
96 def follow_redirects(link
, sites
= None):
97 """Follow directs for the link as long as the redirects are on the given
98 sites and return the resolved link."""
100 return sites
== None or urlparse
.urlparse(url
).hostname
in sites
102 class RedirectHandler(urllib2
.HTTPRedirectHandler
):
105 def redirect_request(self
, req
, fp
, code
, msg
, hdrs
, newurl
):
106 self
.last_url
= newurl
107 if not follow(newurl
):
109 r
= urllib2
.HTTPRedirectHandler
.redirect_request(
110 self
, req
, fp
, code
, msg
, hdrs
, newurl
)
111 r
.get_method
= lambda : 'HEAD'
116 redirect_handler
= RedirectHandler()
117 opener
= urllib2
.build_opener(redirect_handler
)
118 req
= urllib2
.Request(link
)
119 req
.get_method
= lambda : 'HEAD'
121 with contextlib
.closing(opener
.open(req
,timeout
=1)) as site
:
124 return redirect_handler
.last_url
if redirect_handler
.last_url
else link
126 def expand_line(line
, sites
):
127 """Expand the links in the line for the given sites."""
130 msg_format
, links
= find_links(l
)
131 args
= tuple(follow_redirects(l
, sites
) for l
in links
)
132 line
= msg_format
% args
133 except Exception as e
:
135 err("expanding line %s failed due to %s" % (line
, unicode(e
)))
140 def parse_host_list(list_of_hosts
):
141 """Parse the comma separated list of hosts."""
143 m
.group(1) for m
in re
.finditer("\s*([^,\s]+)\s*,?\s*", list_of_hosts
))
147 def align_text(text
, left_margin
=17, max_width
=160):
149 for line
in text
.split('\n'):
150 temp_lines
= textwrap
.wrap(line
, max_width
- left_margin
)
151 temp_lines
= [(' ' * left_margin
+ line
) for line
in temp_lines
]
152 lines
.append('\n'.join(temp_lines
))
153 ret
= '\n'.join(lines
)