module GuessHtmlEncoding

A small and simple library for guessing the encoding of HTML in Ruby 1.9.

Constants

VERSION

Public Class Methods

encode(html, headers = nil) click to toggle source

Force an HTML string into a guessed encoding.

# File lib/guess_html_encoding.rb, line 41
def self.encode(html, headers = nil)
  html_copy = html.to_s.dup
  encoding = guess(html_copy, headers)
  html_copy.force_encoding(encoding_loaded?(encoding) ? encoding : "UTF-8")
  if html_copy.valid_encoding?
    html_copy
  else
    html_copy.force_encoding('ASCII-8BIT').encode('UTF-8', :undef => :replace, :invalid => :replace)
  end
end
encoding_loaded?(encoding) click to toggle source

Is this encoding loaded?

# File lib/guess_html_encoding.rb, line 53
def self.encoding_loaded?(encoding)
  !!Encoding.find(encoding) rescue nil
end
guess(html, headers = nil) click to toggle source

Guess the encoding of an HTML string, using HTTP headers if provided. HTTP headers can be a string or a hash.

# File lib/guess_html_encoding.rb, line 6
def self.guess(html, headers = nil)
  html = html.to_s.dup.force_encoding("ASCII-8BIT")
  out = nil

  if headers
    headers = headers.map {|k, v| "#{k}: #{v}" }.join("\n") if headers.is_a?(Hash)
    headers = headers.dup.force_encoding("ASCII-8BIT")
    headers.gsub(/[\r\n]+/, "\n").split("\n").map {|i| i.split(":")}.each do |k,v|
      if k =~ /Content-Type/i && v =~ /charset=([\w\d-]+);?/i
        out = $1.upcase
        break
      end
    end
  end

  if out.nil? || out.empty? || !encoding_loaded?(out)

    out = HTMLScanner.new(html[0,2500]).encoding || out

    out.upcase! unless out.nil?
  end

  # Translate encodings with other names.
  if out
    out = "UTF-8" if %w[DEFAULT UTF8 UNICODE].include?(out)
    out = "CP1251" if out == "CP-1251"
    out = "ISO-8859-1" if %w[LATIN1 LATIN-1].include?(out)
    out = "WINDOWS-1250" if %w[WIN-1251 WIN1251].include?(out)
    out = "GB18030" if %w[GB2312 GB18030].include?(out) 
  end

  out
end