def self.filter_awstats(input, output = $stdout)
data = input.read
data.slice!(REGEXPS[:awstats_map])
referrers = data.slice!(REGEXPS[:awstats_pagerefs_extract], 1).strip
filtered = Array.new
referrers.each_line do |line|
@stats.lines += 1
unless line =~ REGEXPS[:awstats_url]
@stats.invalid += 1
filtered << line
next
end
if spam?($1)
@stats.spam += 1
else
@stats.ham += 1
filtered << line
end
end
if filtered.length > 0
referrers = filtered.join('').strip + "\n"
else
referrers = ''
end
output.puts data.gsub(REGEXPS[:awstats_pagerefs_replace],
"BEGIN_PAGEREFS #{filtered.length}\n#{referrers}END_PAGEREFS")
end