1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
|
#-- vim:sw=2:et
#++
#
# :title: rbot utilities provider
#
# Author:: Tom Gilbert <tom@linuxbrit.co.uk>
# Author:: Giuseppe "Oblomov" Bilotta <giuseppe.bilotta@gmail.com>
#
# Copyright:: (C) 2002-2006 Tom Gilbert
# Copyright:: (C) 2007 Giuseppe Bilotta
#
# TODO some of these Utils should be rewritten as extensions to the approriate
# standard Ruby classes and accordingly be moved to extends.rb
require 'tempfile'
require 'set'
# Try to load htmlentities, fall back to an HTML escape table.
begin
require 'htmlentities'
rescue LoadError
gems = nil
begin
gems = require 'rubygems'
rescue LoadError
gems = false
end
if gems
retry
else
module ::Irc
module Utils
UNESCAPE_TABLE = {
'laquo' => '«',
'raquo' => '»',
'quot' => '"',
'apos' => '\'',
'micro' => 'µ',
'copy' => '©',
'trade' => '™',
'reg' => '®',
'amp' => '&',
'lt' => '<',
'gt' => '>',
'hellip' => '…',
'nbsp' => ' ',
'Agrave' => 'À',
'Aacute' => 'Á',
'Acirc' => 'Â',
'Atilde' => 'Ã',
'Auml' => 'Ä',
'Aring' => 'Å',
'AElig' => 'Æ',
'OElig' => 'Œ',
'Ccedil' => 'Ç',
'Egrave' => 'È',
'Eacute' => 'É',
'Ecirc' => 'Ê',
'Euml' => 'Ë',
'Igrave' => 'Ì',
'Iacute' => 'Í',
'Icirc' => 'Î',
'Iuml' => 'Ï',
'ETH' => 'Ð',
'Ntilde' => 'Ñ',
'Ograve' => 'Ò',
'Oacute' => 'Ó',
'Ocirc' => 'Ô',
'Otilde' => 'Õ',
'Ouml' => 'Ö',
'Oslash' => 'Ø',
'Ugrave' => 'Ù',
'Uacute' => 'Ú',
'Ucirc' => 'Û',
'Uuml' => 'Ü',
'Yacute' => 'Ý',
'THORN' => 'Þ',
'szlig' => 'ß',
'agrave' => 'à',
'aacute' => 'á',
'acirc' => 'â',
'atilde' => 'ã',
'auml' => 'ä',
'aring' => 'å',
'aelig' => 'æ',
'oelig' => 'œ',
'ccedil' => 'ç',
'egrave' => 'è',
'eacute' => 'é',
'ecirc' => 'ê',
'euml' => 'ë',
'igrave' => 'ì',
'iacute' => 'í',
'icirc' => 'î',
'iuml' => 'ï',
'eth' => 'ð',
'ntilde' => 'ñ',
'ograve' => 'ò',
'oacute' => 'ó',
'ocirc' => 'ô',
'otilde' => 'õ',
'ouml' => 'ö',
'oslash' => 'ø',
'ugrave' => 'ù',
'uacute' => 'ú',
'ucirc' => 'û',
'uuml' => 'ü',
'yacute' => 'ý',
'thorn' => 'þ',
'yuml' => 'ÿ'
}
end
end
end
end
begin
require 'hpricot'
module ::Irc
module Utils
AFTER_PAR_PATH = /^(?:div|span)$/
AFTER_PAR_EX = /^(?:td|tr|tbody|table)$/
AFTER_PAR_CLASS = /body|message|text/i
end
end
rescue LoadError
gems = nil
begin
gems = require 'rubygems'
rescue LoadError
gems = false
end
if gems
retry
else
module ::Irc
module Utils
# Some regular expressions to manage HTML data
# Title
TITLE_REGEX = /<\s*?title\s*?>(.+?)<\s*?\/title\s*?>/im
# H1, H2, etc
HX_REGEX = /<h(\d)(?:\s+[^>]*)?>(.*?)<\/h\1>/im
# A paragraph
PAR_REGEX = /<p(?:\s+[^>]*)?>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
# Some blogging and forum platforms use spans or divs with a 'body' or 'message' or 'text' in their class
# to mark actual text
AFTER_PAR1_REGEX = /<\w+\s+[^>]*(?:body|message|text)[^>]*>.*?<\/?(?:p|div|html|body|table|td|tr)(?:\s+[^>]*)?>/im
# At worst, we can try stuff which is comprised between two <br>
AFTER_PAR2_REGEX = /<br(?:\s+[^>]*)?\/?>.*?<\/?(?:br|p|div|html|body|table|td|tr)(?:\s+[^>]*)?\/?>/im
end
end
end
end
module ::Irc
# Miscellaneous useful functions
module Utils
@@bot = nil unless defined? @@bot
@@safe_save_dir = nil unless defined?(@@safe_save_dir)
# The bot instance
def Utils.bot
@@bot
end
# Set up some Utils routines which depend on the associated bot.
def Utils.bot=(b)
debug "initializing utils"
@@bot = b
@@safe_save_dir = "#{@@bot.botclass}/safe_save"
end
# Seconds per minute
SEC_PER_MIN = 60
# Seconds per hour
SEC_PER_HR = SEC_PER_MIN * 60
# Seconds per day
SEC_PER_DAY = SEC_PER_HR * 24
# Seconds per (30-day) month
SEC_PER_MNTH = SEC_PER_DAY * 30
# Second per (30*12 = 360 day) year
SEC_PER_YR = SEC_PER_MNTH * 12
# Auxiliary method needed by Utils.secs_to_string
def Utils.secs_to_string_case(array, var, string, plural)
case var
when 1
array << "1 #{string}"
else
array << "#{var} #{plural}"
end
end
# Turn a number of seconds into a human readable string, e.g
# 2 days, 3 hours, 18 minutes and 10 seconds
def Utils.secs_to_string(secs)
ret = []
years, secs = secs.divmod SEC_PER_YR
secs_to_string_case(ret, years, _("year"), _("years")) if years > 0
months, secs = secs.divmod SEC_PER_MNTH
secs_to_string_case(ret, months, _("month"), _("months")) if months > 0
days, secs = secs.divmod SEC_PER_DAY
secs_to_string_case(ret, days, _("day"), _("days")) if days > 0
hours, secs = secs.divmod SEC_PER_HR
secs_to_string_case(ret, hours, _("hour"), _("hours")) if hours > 0
mins, secs = secs.divmod SEC_PER_MIN
secs_to_string_case(ret, mins, _("minute"), _("minutes")) if mins > 0
secs = secs.to_i
secs_to_string_case(ret, secs, _("second"), _("seconds")) if secs > 0 or ret.empty?
case ret.length
when 0
raise "Empty ret array!"
when 1
return ret.to_s
else
return [ret[0, ret.length-1].join(", ") , ret[-1]].join(_(" and "))
end
end
# Turn a number of seconds into a hours:minutes:seconds e.g.
# 3:18:10 or 5'12" or 7s
#
def Utils.secs_to_short(seconds)
secs = seconds.to_i # make sure it's an integer
mins, secs = secs.divmod 60
hours, mins = mins.divmod 60
if hours > 0
return ("%s:%s:%s" % [hours, mins, secs])
elsif mins > 0
return ("%s'%s\"" % [mins, secs])
else
return ("%ss" % [secs])
end
end
# Execute an external program, returning a String obtained by redirecting
# the program's standards errors and output
#
def Utils.safe_exec(command, *args)
IO.popen("-") { |p|
if p
return p.readlines.join("\n")
else
begin
$stderr.reopen($stdout)
exec(command, *args)
rescue Exception => e
puts "exec of #{command} led to exception: #{e.pretty_inspect}"
Kernel::exit! 0
end
puts "exec of #{command} failed"
Kernel::exit! 0
end
}
end
# Safely (atomically) save to _file_, by passing a tempfile to the block
# and then moving the tempfile to its final location when done.
#
# call-seq: Utils.safe_save(file, &block)
#
def Utils.safe_save(file)
raise 'No safe save directory defined!' if @@safe_save_dir.nil?
basename = File.basename(file)
temp = Tempfile.new(basename,@@safe_save_dir)
temp.binmode
yield temp if block_given?
temp.close
File.rename(temp.path, file)
end
# Decode HTML entities in the String _str_, using HTMLEntities if the
# package was found, or UNESCAPE_TABLE otherwise.
#
def Utils.decode_html_entities(str)
if defined? ::HTMLEntities
return HTMLEntities.decode_entities(str)
else
str.gsub(/(&(.+?);)/) {
symbol = $2
# remove the 0-paddng from unicode integers
if symbol =~ /^#(\d+)$/
symbol = $1.to_i.to_s
end
# output the symbol's irc-translated character, or a * if it's unknown
UNESCAPE_TABLE[symbol] || (symbol.match(/^\d+$/) ? [symbol.to_i].pack("U") : '*')
}
end
end
# Try to grab and IRCify the first HTML par (<p> tag) in the given string.
# If possible, grab the one after the first heading
#
# It is possible to pass some options to determine how the stripping
# occurs. Currently supported options are
# strip:: Regex or String to strip at the beginning of the obtained
# text
# min_spaces:: minimum number of spaces a paragraph should have
#
def Utils.ircify_first_html_par(xml_org, opts={})
if defined? ::Hpricot
Utils.ircify_first_html_par_wh(xml_org, opts)
else
Utils.ircify_first_html_par_woh(xml_org, opts)
end
end
# HTML first par grabber using hpricot
def Utils.ircify_first_html_par_wh(xml_org, opts={})
doc = Hpricot(xml_org)
# Strip styles and scripts
(doc/"style|script").remove
debug doc
strip = opts[:strip]
strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
min_spaces = opts[:min_spaces] || 8
min_spaces = 0 if min_spaces < 0
txt = String.new
pre_h = pars = by_span = nil
while true
debug "Minimum number of spaces: #{min_spaces}"
# Initial attempt: <p> that follows <h\d>
if pre_h.nil?
pre_h = Hpricot::Elements[]
found_h = false
doc.search("*") { |e|
next if e.bogusetag?
case e.pathname
when /^h\d/
found_h = true
when 'p'
pre_h << e if found_h
end
}
debug "Hx: found: #{pre_h.pretty_inspect}"
end
pre_h.each { |p|
debug p
txt = p.to_html.ircify_html
txt.sub!(strip, '') if strip
debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
break unless txt.empty? or txt.count(" ") < min_spaces
}
return txt unless txt.empty? or txt.count(" ") < min_spaces
# Second natural attempt: just get any <p>
pars = doc/"p" if pars.nil?
debug "par: found: #{pars.pretty_inspect}"
pars.each { |p|
debug p
txt = p.to_html.ircify_html
txt.sub!(strip, '') if strip
debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
break unless txt.empty? or txt.count(" ") < min_spaces
}
return txt unless txt.empty? or txt.count(" ") < min_spaces
# Nothing yet ... let's get drastic: we look for non-par elements too,
# but only for those that match something that we know is likely to
# contain text
# Some blogging and forum platforms use spans or divs with a 'body' or
# 'message' or 'text' in their class to mark actual text. Since we want
# the class match to be partial and case insensitive, we collect
# the common elements that may have this class and then filter out those
# we don't need. If no divs or spans are found, we'll accept additional
# elements too (td, tr, tbody, table).
if by_span.nil?
by_span = Hpricot::Elements[]
extra = Hpricot::Elements[]
doc.search("*") { |el|
next if el.bogusetag?
case el.pathname
when AFTER_PAR_PATH
by_span.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
when AFTER_PAR_EX
extra.push el if el[:class] =~ AFTER_PAR_CLASS or el[:id] =~ AFTER_PAR_CLASS
end
}
if by_span.empty? and not extra.empty?
by_span.concat extra
end
debug "other \#1: found: #{by_span.pretty_inspect}"
end
by_span.each { |p|
debug p
txt = p.to_html.ircify_html
txt.sub!(strip, '') if strip
debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
break unless txt.empty? or txt.count(" ") < min_spaces
}
return txt unless txt.empty? or txt.count(" ") < min_spaces
# At worst, we can try stuff which is comprised between two <br>
# TODO
debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
return txt unless txt.count(" ") < min_spaces
break if min_spaces == 0
min_spaces /= 2
end
end
# HTML first par grabber without hpricot
def Utils.ircify_first_html_par_woh(xml_org, opts={})
xml = xml_org.gsub(/<!--.*?-->/m, '').gsub(/<script(?:\s+[^>]*)?>.*?<\/script>/im, "").gsub(/<style(?:\s+[^>]*)?>.*?<\/style>/im, "")
strip = opts[:strip]
strip = Regexp.new(/^#{Regexp.escape(strip)}/) if strip.kind_of?(String)
min_spaces = opts[:min_spaces] || 8
min_spaces = 0 if min_spaces < 0
txt = String.new
while true
debug "Minimum number of spaces: #{min_spaces}"
header_found = xml.match(HX_REGEX)
if header_found
header_found = $'
while txt.empty? or txt.count(" ") < min_spaces
candidate = header_found[PAR_REGEX]
break unless candidate
txt = candidate.ircify_html
header_found = $'
txt.sub!(strip, '') if strip
debug "(Hx attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
end
end
return txt unless txt.empty? or txt.count(" ") < min_spaces
# If we haven't found a first par yet, try to get it from the whole
# document
header_found = xml
while txt.empty? or txt.count(" ") < min_spaces
candidate = header_found[PAR_REGEX]
break unless candidate
txt = candidate.ircify_html
header_found = $'
txt.sub!(strip, '') if strip
debug "(par attempt) #{txt.inspect} has #{txt.count(" ")} spaces"
end
return txt unless txt.empty? or txt.count(" ") < min_spaces
# Nothing yet ... let's get drastic: we look for non-par elements too,
# but only for those that match something that we know is likely to
# contain text
# Attempt #1
header_found = xml
while txt.empty? or txt.count(" ") < min_spaces
candidate = header_found[AFTER_PAR1_REGEX]
break unless candidate
txt = candidate.ircify_html
header_found = $'
txt.sub!(strip, '') if strip
debug "(other attempt \#1) #{txt.inspect} has #{txt.count(" ")} spaces"
end
return txt unless txt.empty? or txt.count(" ") < min_spaces
# Attempt #2
header_found = xml
while txt.empty? or txt.count(" ") < min_spaces
candidate = header_found[AFTER_PAR2_REGEX]
break unless candidate
txt = candidate.ircify_html
header_found = $'
txt.sub!(strip, '') if strip
debug "(other attempt \#2) #{txt.inspect} has #{txt.count(" ")} spaces"
end
debug "Last candidate #{txt.inspect} has #{txt.count(" ")} spaces"
return txt unless txt.count(" ") < min_spaces
break if min_spaces == 0
min_spaces /= 2
end
end
# This method extracts title, content (first par) and extra
# information from the given document _doc_.
#
# _doc_ can be an URI, a Net::HTTPResponse or a String.
#
# If _doc_ is a String, only title and content information
# are retrieved (if possible), using standard methods.
#
# If _doc_ is an URI or a Net::HTTPResponse, additional
# information is retrieved, and special title/summary
# extraction routines are used if possible.
#
def Utils.get_html_info(doc, opts={})
case doc
when String
Utils.get_string_html_info(doc, opts)
when Net::HTTPResponse
Utils.get_resp_html_info(doc, opts)
when URI
ret = DataStream.new
@@bot.httputil.get_response(doc) { |resp|
ret.replace Utils.get_resp_html_info(resp, opts)
}
return ret
else
raise
end
end
class ::UrlLinkError < RuntimeError
end
# This method extracts title, content (first par) and extra
# information from the given Net::HTTPResponse _resp_.
#
# Currently, the only accepted options (in _opts_) are
# uri_fragment:: the URI fragment of the original request
# full_body:: get the whole body instead of
# @@bot.config['http.info_bytes'] bytes only
#
# Returns a DataStream with the following keys:
# text:: the (partial) body
# title:: the title of the document (if any)
# content:: the first paragraph of the document (if any)
# headers::
# the headers of the Net::HTTPResponse. The value is
# a Hash whose keys are lowercase forms of the HTTP
# header fields, and whose values are Arrays.
#
def Utils.get_resp_html_info(resp, opts={})
case resp
when Net::HTTPSuccess
loc = URI.parse(resp['x-rbot-location'] || resp['location']) rescue nil
if loc and loc.fragment and not loc.fragment.empty?
opts[:uri_fragment] ||= loc.fragment
end
ret = DataStream.new(opts.dup)
ret[:headers] = resp.to_hash
ret[:text] = partial = opts[:full_body] ? resp.body : resp.partial_body(@@bot.config['http.info_bytes'])
filtered = Utils.try_htmlinfo_filters(ret)
if filtered
return filtered
elsif resp['content-type'] =~ /^text\/|(?:x|ht)ml/
ret.merge!(Utils.get_string_html_info(partial, opts))
end
return ret
else
raise UrlLinkError, "getting link (#{resp.code} - #{resp.message})"
end
end
# This method runs an appropriately-crafted DataStream _ds_ through the
# filters in the :htmlinfo filter group, in order. If one of the filters
# returns non-nil, its results are merged in _ds_ and returned. Otherwise
# nil is returned.
#
# The input DataStream shuold have the downloaded HTML as primary key
# (:text) and possibly a :headers key holding the resonse headers.
#
def Utils.try_htmlinfo_filters(ds)
filters = @@bot.filter_names(:htmlinfo)
return nil if filters.empty?
cur = nil
# TODO filter priority
filters.each { |n|
debug "testing filter #{n}"
cur = @@bot.filter(@@bot.global_filter_name(n, :htmlinfo), ds)
debug "returned #{cur.pretty_inspect}"
break if cur
}
return ds.merge(cur) if cur
end
# HTML info filters often need to check if the webpage location
# of a passed DataStream _ds_ matches a given Regexp.
def Utils.check_location(ds, rx)
debug ds[:headers]
if h = ds[:headers]
loc = [h['x-rbot-location'],h['location']].flatten.grep(rx)
end
loc ||= []
debug loc
return loc.empty? ? nil : loc
end
# This method extracts title and content (first par)
# from the given HTML or XML document _text_, using
# standard methods (String#ircify_html_title,
# Utils.ircify_first_html_par)
#
# Currently, the only accepted option (in _opts_) is
# uri_fragment:: the URI fragment of the original request
#
def Utils.get_string_html_info(text, opts={})
txt = text.dup
title = txt.ircify_html_title
if frag = opts[:uri_fragment] and not frag.empty?
fragreg = /.*?<a\s+[^>]*name=["']?#{frag}["']?.*?>/im
txt.sub!(fragreg,'')
end
c_opts = opts.dup
c_opts[:strip] ||= title
content = Utils.ircify_first_html_par(txt, c_opts)
content = nil if content.empty?
return {:title => title, :content => content}
end
# Get the first pars of the first _count_ _urls_.
# The pages are downloaded using the bot httputil service.
# Returns an array of the first paragraphs fetched.
# If (optional) _opts_ :message is specified, those paragraphs are
# echoed as replies to the IRC message passed as _opts_ :message
#
def Utils.get_first_pars(urls, count, opts={})
idx = 0
msg = opts[:message]
retval = Array.new
while count > 0 and urls.length > 0
url = urls.shift
idx += 1
begin
info = Utils.get_html_info(URI.parse(url), opts)
par = info[:content]
retval.push(par)
if par
msg.reply "[#{idx}] #{par}", :overlong => :truncate if msg
count -=1
end
rescue
debug "Unable to retrieve #{url}: #{$!}"
next
end
end
return retval
end
end
end
Irc::Utils.bot = Irc::Bot::Plugins.manager.bot
|