class MogileFS::Mysql

Consider this deprecated, to be removed at some point…

read-only interface that can be a backend for MogileFS::MogileFS

This provides direct, read-only access to any slave MySQL database to provide better performance, scalability and eliminate mogilefsd as a point of failure

Constants

DEV_STATUS_READABLE
GET_DEVICES
GET_DOMAINS

Attributes

my[R]
query_method[R]

Public Class Methods

new(args = {}) click to toggle source

Creates a new MogileFS::Mysql instance. args must include a key :domain specifying the domain of this client and :mysql, specifying an already-initialized Mysql object.

The Mysql object can be either the standard Mysql driver or the Mysqlplus one supporting c_async_query.

# File lib/mogilefs/mysql.rb, line 20
def initialize(args = {})
  @my = args[:mysql]
  @query_method = @my.respond_to?(:c_async_query) ? :c_async_query : :query
  @last_update_device = @last_update_domain = 0
  @cache_domain = @cache_device = nil
end

Public Instance Methods

_get_paths(params = {}) click to toggle source

Get the paths for key.

# File lib/mogilefs/mysql.rb, line 82
def _get_paths(params = {})
  zone = params[:zone]
  # noverify = (params[:noverify] == 1) # TODO this is unused atm
  dmid = get_dmid(params[:domain])
  devices = refresh_device or raise MogileFS::Backend::NoDevicesError
  urls = []
  sql = <<-EOS
  SELECT fid FROM file
  WHERE dmid = #{dmid} AND dkey = '#{@my.quote(params[:key])}'
  LIMIT 1
  EOS

  res = query(sql).fetch_row
  res && res[0] or raise MogileFS::Backend::UnknownKeyError
  fid = res[0]
  sql = "SELECT devid FROM file_on WHERE fid = '#{@my.quote(fid)}'"
  query(sql).each do |devid,|
    unless devinfo = devices[devid.to_i]
      devices = refresh_device(true)
      devinfo = devices[devid.to_i] or next
    end
    devinfo[:readable] or next
    port = devinfo[:http_get_port]
    host = zone && zone == 'alt' ? devinfo[:altip] : devinfo[:hostip]
    nfid = '%010u' % fid
    b, mmm, ttt = /(\d)(\d{3})(\d{3})(?:\d{3})/.match(nfid)[1..3]
    uri = "/dev#{devid}/#{b}/#{mmm}/#{ttt}/#{nfid}.fid"
    urls << "http://#{host}:#{port}#{uri}"
  end
  urls
end
_list_keys(domain, prefix = '', after = '', limit = 1000) { |dkey, to_i, to_i| ... } click to toggle source

Lists keys starting with prefix follwing after up to limit. If after is nil the list starts at the beginning.

# File lib/mogilefs/mysql.rb, line 30
def _list_keys(domain, prefix = '', after = '', limit = 1000)
  # this code is based on server/lib/MogileFS/Worker/Query.pm
  dmid = get_dmid(domain)

  # don't modify passed arguments
  limit ||= 1000
  limit = limit.to_i
  limit = 1000 if limit > 1000 || limit <= 0
  after, prefix = "#{after}", "#{prefix}"

  if after.length > 0 && /^#{Regexp.quote(prefix)}/ !~ after
    raise MogileFS::Backend::AfterMismatchError
  end

  raise MogileFS::Backend::InvalidCharsError if /[%\]/ =~ prefix
  prefix.gsub!(/_/, '\_') # not sure why MogileFS::Worker::Query does this...

  sql = <<-EOS
  SELECT dkey,length,devcount FROM file
  WHERE dmid = #{dmid}
    AND dkey LIKE '#{@my.quote(prefix)}%'
    AND dkey > '#{@my.quote(after)}'
  ORDER BY dkey LIMIT #{limit}
  EOS

  keys = []
  query(sql).each do |dkey,length,devcount|
    yield(dkey, length.to_i, devcount.to_i) if block_given?
    keys << dkey
  end

  keys.empty? ? nil : [ keys, (keys.last || '') ]
end
_size(domain, key) click to toggle source

Returns the size of key.

# File lib/mogilefs/mysql.rb, line 66
def _size(domain, key)
  dmid = get_dmid(domain)

  sql = <<-EOS
  SELECT length FROM file
  WHERE dmid = #{dmid} AND dkey = '#{@my.quote(key)}'
  LIMIT 1
  EOS

  res = query(sql).fetch_row
  return res[0].to_i if res && res[0]
  raise MogileFS::Backend::UnknownKeyError
end
sleep(params) click to toggle source
# File lib/mogilefs/mysql.rb, line 114
def sleep(params); Kernel.sleep(params[:duration] || 10); {}; end

Private Instance Methods

get_dmid(domain) click to toggle source
# File lib/mogilefs/mysql.rb, line 173
def get_dmid(domain)
  refresh_domain[domain] || refresh_domain(true)[domain] or          raise MogileFS::Backend::DomainNotFoundError, domain
end
query(sql) click to toggle source
# File lib/mogilefs/mysql.rb, line 130
def query(sql)
  @my.send(@query_method, sql)
end
refresh_device(force = false) click to toggle source
# File lib/mogilefs/mysql.rb, line 140
def refresh_device(force = false)
  if ! force && ((MogileFS.now - @last_update_device) < 60)
    return @cache_device
  end
  tmp = {}
  res = query(GET_DEVICES)
  res.each do |devid, hostip, altip, http_port, http_get_port,
               dev_status, host_status|
    http_port = http_port ? http_port.to_i : 80
    tmp[devid.to_i] = {
      :hostip => hostip.freeze,
      :altip => (altip || hostip).freeze,
      :readable => (host_status == "alive" &&
                    DEV_STATUS_READABLE.include?(dev_status)),
      :http_port => http_port,
      :http_get_port => http_get_port ?  http_get_port.to_i : http_port,
    }.freeze
  end
  @last_update_device = MogileFS.now
  @cache_device = tmp.freeze
end
refresh_domain(force = false) click to toggle source
# File lib/mogilefs/mysql.rb, line 162
def refresh_domain(force = false)
  if ! force && ((MogileFS.now - @last_update_domain) < 5)
    return @cache_domain
  end
  tmp = {}
  res = query(GET_DOMAINS)
  res.each { |dmid,namespace| tmp[namespace] = dmid.to_i }
  @last_update_domain = MogileFS.now
  @cache_domain = tmp.freeze
end