# File lib/webrobots.rb, line 20
  def initialize(user_agent, options = nil)
    @user_agent = user_agent
    @parser = RobotsTxt::Parser.new(user_agent)
    @parser_mutex = Mutex.new

    options ||= {}
    @http_get = options[:http_get] || method(:http_get)

    @robotstxt = create_cache()
  end