summaryrefslogtreecommitdiff
path: root/data/rbot/plugins/url.rb
blob: 8327dd945d169c0d1c3282e547fef01d08f1c32e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
#-- vim:sw=2:et
#++
#
# :title: Url plugin

define_structure :Url, :channel, :nick, :time, :url, :info

class UrlPlugin < Plugin
  LINK_INFO = "[Link Info]"
  OUR_UNSAFE = Regexp.new("[^#{URI::PATTERN::UNRESERVED}#{URI::PATTERN::RESERVED}%# ]", false, 'N')

  Config.register Config::IntegerValue.new('url.max_urls',
    :default => 100, :validate => Proc.new{|v| v > 0},
    :desc => "Maximum number of urls to store. New urls replace oldest ones.")
  Config.register Config::IntegerValue.new('url.display_link_info',
    :default => 0,
    :desc => "Get the title of links pasted to the channel and display it (also tells if the link is broken or the site is down). Do it for at most this many links per line (set to 0 to disable)")
  Config.register Config::BooleanValue.new('url.auto_shorten',
    :default => false,
    :desc => "Automatically spit out shortened URLs when they're seen. Check shortenurls for config options")
  Config.register Config::IntegerValue.new('url.auto_shorten_min_length',
    :default => 48,
    :desc => "Minimum length of URL to auto-shorten.  Only has an effect when url.auto_shorten is true.")
  Config.register Config::BooleanValue.new('url.titles_only',
    :default => false,
    :desc => "Only show info for links that have <title> tags (in other words, don't display info for jpegs, mpegs, etc.)")
  Config.register Config::BooleanValue.new('url.first_par',
    :default => false,
    :desc => "Also try to get the first paragraph of a web page")
  Config.register Config::BooleanValue.new('url.info_on_list',
    :default => false,
    :desc => "Show link info when listing/searching for urls")
  Config.register Config::ArrayValue.new('url.no_info_hosts',
    :default => ['localhost', '^192\.168\.', '^10\.', '^127\.', '^172\.(1[6-9]|2\d|31)\.'],
    :on_change => Proc.new { |bot, v| bot.plugins['url'].reset_no_info_hosts },
    :desc => "A list of regular expressions matching hosts for which no info should be provided")
  Config.register Config::ArrayValue.new('url.only_on_channels',
    :desc => "Show link info only on these channels",
    :default => [])
  Config.register Config::ArrayValue.new('url.ignore',
    :desc => "Don't show link info for urls from users represented as hostmasks on this list. Useful for ignoring other bots, for example.",
    :default => [])

  def initialize
    super
    @registry.set_default(Array.new)
    unless @bot.config['url.display_link_info'].kind_of?(Integer)
      @bot.config.items[:'url.display_link_info'].set_string(@bot.config['url.display_link_info'].to_s)
    end
    reset_no_info_hosts
    self.filter_group = :htmlinfo
    load_filters
  end

  def reset_no_info_hosts
    @no_info_hosts = Regexp.new(@bot.config['url.no_info_hosts'].join('|'), true)
    debug "no info hosts regexp set to #{@no_info_hosts}"
  end

  def help(plugin, topic="")
    "url info <url> => display link info for <url> (set url.display_link_info > 0 if you want the bot to do it automatically when someone writes an url), urls [<max>=4] => list <max> last urls mentioned in current channel, urls search [<max>=4] <regexp> => search for matching urls. In a private message, you must specify the channel to query, eg. urls <channel> [max], urls search <channel> [max] <regexp>"
  end

  def get_title_from_html(pagedata)
    return pagedata.ircify_html_title
  end

  def get_title_for_url(uri_str, opts = {})

    url = uri_str.kind_of?(URI) ? uri_str : URI.parse(uri_str)
    return if url.scheme !~ /https?/

    # also check the ip, the canonical name and the aliases
    begin
      checks = TCPSocket.gethostbyname(url.host)
      checks.delete_at(-2)
    rescue => e
      return "Unable to retrieve info for #{url.host}: #{e.message}"
    end

    checks << url.host
    checks.flatten!

    unless checks.grep(@no_info_hosts).empty?
      return ( opts[:always_reply] ? "Sorry, info retrieval for #{url.host} (#{checks.first}) is disabled" : false )
    end

    logopts = opts.dup

    title = nil
    extra = []

    begin
      debug "+ getting info for #{url.request_uri}"
      info = @bot.filter(:htmlinfo, url)
      debug info
      logopts[:htmlinfo] = info
      resp = info[:headers]

      logopts[:title] = title = info[:title]

      if info[:content]
        logopts[:extra] = info[:content]
        extra << "#{Bold}text#{Bold}: #{info[:content]}" if @bot.config['url.first_par']
      else
        logopts[:extra] = String.new
        logopts[:extra] << "Content Type: #{resp['content-type']}"
        extra << "#{Bold}type#{Bold}: #{resp['content-type']}" unless title
        if enc = resp['content-encoding']
          logopts[:extra] << ", encoding: #{enc}"
          extra << "#{Bold}encoding#{Bold}: #{enc}" if @bot.config['url.first_par'] or not title
        end

        size = resp['content-length'].first.gsub(/(\d)(?=\d{3}+(?:\.|$))(\d{3}\..*)?/,'\1,\2') rescue nil
        if size
          logopts[:extra] << ", size: #{size} bytes"
          extra << "#{Bold}size#{Bold}: #{size} bytes" if @bot.config['url.first_par'] or not title
        end
      end
    rescue Exception => e
      case e
      when UrlLinkError
        raise e
      else
        error e
        raise "connecting to site/processing information (#{e.message})"
      end
    end

    call_event(:url_added, url.to_s, logopts)
    if title
      extra.unshift("#{Bold}title#{Bold}: #{title}")
    end
    return extra.join(", ") if title or not @bot.config['url.titles_only']
  end

  def handle_urls(m, params={})
    opts = {
      :display_info => @bot.config['url.display_link_info'],
      :channels => @bot.config['url.only_on_channels'],
      :ignore => @bot.config['url.ignore']
    }.merge params
    urls = opts[:urls]
    display_info= opts[:display_info]
    channels = opts[:channels]
    ignore = opts[:ignore]

    unless channels.empty?
      return unless channels.map { |c| c.downcase }.include?(m.channel.downcase)
    end

    ignore.each { |u| return if m.source.matches?(u) }

    return if urls.empty?
    debug "found urls #{urls.inspect}"
    list = m.public? ? @registry[m.target] : nil
    debug "display link info: #{display_info}"
    urls_displayed = 0
    urls.each do |urlstr|
      debug "working on #{urlstr}"
      next unless urlstr =~ /^https?:\/\/./
      if @bot.config['url.auto_shorten'] == true and
         urlstr.length >= @bot.config['url.auto_shorten_min_length']
        m.reply(bot.plugins['shortenurls'].shorten(nil, {:url=>urlstr, :called=>true}))
        next
      end
      title = nil
      debug "Getting title for #{urlstr}..."
      reply = nil
      begin
        title = get_title_for_url(urlstr,
                                  :always_reply => m.address?,
                                  :nick => m.source.nick,
                                  :channel => m.channel,
                                  :ircline => m.message)
        debug "Title #{title ? '' : 'not '} found"
        reply = "#{LINK_INFO} #{title}" if title
      rescue => e
        debug e
        # we might get a 404 because of trailing punctuation, so we try again
        # with the last character stripped. this might generate invalid URIs
        # (e.g. because "some.url" gets chopped to some.url%2, so catch that too
        if e.message =~ /\(404 - Not Found\)/i or e.kind_of?(URI::InvalidURIError)
          # chop off last non-word character from the unescaped version of
          # the URL, and retry if we still have enough string to look like a
          # minimal URL
          unescaped = URI.unescape(urlstr)
          debug "Unescaped: #{unescaped}"
          if unescaped.sub!(/\W$/,'') and unescaped =~ /^https?:\/\/./
            urlstr.replace URI.escape(unescaped, OUR_UNSAFE)
            retry
          else
            debug "Not retrying #{unescaped}"
          end
        end
        reply = "Error #{e.message}"
      end

      if display_info > urls_displayed
        if reply
          m.reply reply, :overlong => :truncate, :to => :public,
            :nick => (m.address? ? :auto : false)
          urls_displayed += 1
        end
      end

      next unless list

      # check to see if this url is already listed
      next if list.find {|u| u.url == urlstr }

      url = Url.new(m.target, m.sourcenick, Time.new, urlstr, title)
      debug "#{list.length} urls so far"
      list.pop if list.length > @bot.config['url.max_urls']
      debug "storing url #{url.url}"
      list.unshift url
      debug "#{list.length} urls now"
    end
    @registry[m.target] = list
  end

  def info(m, params)
    escaped = URI.escape(params[:urls].to_s, OUR_UNSAFE)
    urls = URI.extract(escaped)
    Thread.new do
      handle_urls(m,
                  :urls => urls,
                  :display_info => params[:urls].length,
                  :channels => [])
    end
  end

  def message(m)
    return if m.address?

    escaped = URI.escape(m.message, OUR_UNSAFE)
    urls = URI.extract(escaped, ['http', 'https'])
    return if urls.empty?
    Thread.new { handle_urls(m, :urls => urls) }
  end

  def reply_urls(opts={})
    list = opts[:list]
    max = opts[:max]
    channel = opts[:channel]
    m = opts[:msg]
    return unless list and max and m
    list[0..(max-1)].each do |url|
      disp = "[#{url.time.strftime('%Y/%m/%d %H:%M:%S')}] <#{url.nick}> #{url.url}"
      if @bot.config['url.info_on_list']
        title = url.info ||
          get_title_for_url(url.url,
                            :nick => url.nick, :channel => channel) rescue nil
        # If the url info was missing and we now have some, try to upgrade it
        if channel and title and not url.info
          ll = @registry[channel]
          debug ll
          if el = ll.find { |u| u.url == url.url }
            el.info = title
            @registry[channel] = ll
          end
        end
        disp << " --> #{title}" if title
      end
      m.reply disp, :overlong => :truncate
    end
  end

  def urls(m, params)
    channel = params[:channel] ? params[:channel] : m.target
    max = params[:limit].to_i
    max = 10 if max > 10
    max = 1 if max < 1
    list = @registry[channel]
    if list.empty?
      m.reply "no urls seen yet for channel #{channel}"
    else
      reply_urls :msg => m, :channel => channel, :list => list, :max => max
    end
  end

  def search(m, params)
    channel = params[:channel] ? params[:channel] : m.target
    max = params[:limit].to_i
    string = params[:string]
    max = 10 if max > 10
    max = 1 if max < 1
    regex = Regexp.new(string, Regexp::IGNORECASE)
    list = @registry[channel].find_all {|url|
      regex.match(url.url) || regex.match(url.nick) ||
        (@bot.config['url.info_on_list'] && regex.match(url.info))
    }
    if list.empty?
      m.reply "no matches for channel #{channel}"
    else
      reply_urls :msg => m, :channel => channel, :list => list, :max => max
    end
  end
end

plugin = UrlPlugin.new
plugin.map 'urls info *urls', :action => 'info'
plugin.map 'url info *urls', :action => 'info'
plugin.map 'urls search :channel :limit :string', :action => 'search',
                          :defaults => {:limit => 4},
                          :requirements => {:limit => /^\d+$/},
                          :public => false
plugin.map 'urls search :limit :string', :action => 'search',
                          :defaults => {:limit => 4},
                          :requirements => {:limit => /^\d+$/},
                          :private => false
plugin.map 'urls :channel :limit', :defaults => {:limit => 4},
                          :requirements => {:limit => /^\d+$/},
                          :public => false
plugin.map 'urls :limit', :defaults => {:limit => 4},
                          :requirements => {:limit => /^\d+$/},
                          :private => false