whatweb.rb 未完待续

#!/usr/bin/env ruby                #表示ruby的执行环境

=begin                             # ruby中用=begin来表示注释的开始

.$$$     $.                                   .$$$     $.

$$$$     $$. .$$$  $$$ .$$$$$$.  .$$$$$$$$$$. $$$$     $$. .$$$$$$$. .$$$$$$.

$ $$     $$$ $ $$  $$$ $ $$$$$$. $$$$$ $$$$$$ $ $$     $$$ $ $$   $$ $ $$$$$$.

$ `$     $$$ $ `$  $$$ $ `$  $$$ $$‘ $ `$ `$$ $ `$     $$$ $ `$      $ `$  $$$‘

$. $     $$$ $. $$$$$$ $. $$$$$$ `$  $. $  :‘ $. $     $$$ $. $$$$   $. $$$$$.

$::$  .  $$$ $::$  $$$ $::$  $$$     $::$     $::$  .  $$$ $::$      $::$  $$$$

$;;$ $$$ $$$ $;;$  $$$ $;;$  $$$     $;;$     $;;$ $$$ $$$ $;;$      $;;$  $$$$

$$$$$$ $$$$$ $$$$  $$$ $$$$  $$$     $$$$     $$$$$$ $$$$$ $$$$$$$$$ $$$$$$$$$‘
WhatWeb - Next generation web scanner.

Author: Andrew Horton aka urbanadventurer
Homepage: http://www.morningstarsecurity.com/research/whatweb
Copyright 2009-2012 Andrew Horton <andrew at morningstarsecurity dot com>
This file is part of WhatWeb.
WhatWeb is free software: you can redistribute it and/or modify

it under the terms of the GNU General Public License as published by

the Free Software Foundation, either version 2 of the License, or

(at your option) any later version.
WhatWeb is distributed in the hope that it will be useful,

but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

GNU General Public License for more details.
You should have received a copy of the GNU General Public License

along with WhatWeb.  If not, see <http://www.gnu.org/licenses/>.

=end                                 # ruby中用=end 表示注释的结束
#require ‘profile‘

require ‘getoptlong‘

require ‘pp‘

require ‘net/http‘

require ‘open-uri‘

require ‘cgi‘

require ‘thread‘

require ‘tempfile‘

require ‘rbconfig‘  # detect environment, e.g. windows or linux

require ‘resolv‘

require ‘resolv-replace‘ # asynchronous DNS
### ruby 1.9 changes

if RUBY_VERSION =~ /^1\.9/ || RUBY_VERSION =~ /^2\.0/                          #ruby 中使用=~ 运算符根据正则表达式测试一个字符串，这里用正则判断ruby的版本

        require ‘digest/md5‘

else

        require ‘md5‘

    require ‘net/https‘

end
### gem detection & loading

def gem_available?(gemname)

    #  gem_available_new_rubygems?(gemname) or gem_available_old_rubygems?(gemname)

    if defined?(Gem::Specification) and defined?(Gem::Specification.find_by_name)

        gem_available_new_rubygems?(gemname)

    else

        gem_available_old_rubygems?(gemname)

    end

end
# Needed by Ruby 1.8.7 (2010-08-16 patchlevel 302) used as stable in Debian in Aug2012.

def gem_available_old_rubygems?(gemname)

    Gem.available?(gemname)

end
def gem_available_new_rubygems?(gemname)

    begin

        true if Gem::Specification.find_by_name(gemname)

    rescue LoadError

        false

    end

end
gems = %w|json mongo rchardet |            ####  gems = ["json", "mongo", "rchardet"]
gems.each do |thisgem|                     ####  遍历gems，加载json,mongo,rchardet

    begin

        require ‘rubygems‘ # rubygems is optional

        if gem_available?(thisgem) #

            require thisgem

        else

end

    rescue LoadError

        # that failed.. no big deal

        raise if $WWDEBUG==true

    end

end
## set up load paths - must be before loading lib/ files       ####设置加载路径，必须在加载 lib 下的文件之前设置

# add the directory of the file currently being executed to the load path

$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless

    $:.include?(File.dirname(__FILE__)) || $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))

$LOAD_PATH << "/usr/share/whatweb/"
# if __FILE__ is a symlink then follow *every* symlink        ####如果文件是链接形式存在，要找到文件的真实路径

if File.symlink?(__FILE__)

  require ‘pathname‘

  $LOAD_PATH << File.dirname( Pathname.new(__FILE__).realpath )

end
require ‘lib/target.rb‘

require ‘lib/plugins.rb‘

require ‘lib/output.rb‘

require ‘lib/colour.rb‘

require ‘lib/tld.rb‘

require ‘lib/extend-http.rb‘

require ‘lib/version_class.rb‘
#### 加载插件目录

# look through LOAD_PATH for the following plugin directories. Could be in same dir as whatweb or /usr/share/whatweb, etc

PLUGIN_DIRS=[ "plugins", "my-plugins"].map {|x| $LOAD_PATH.map {|y| y+"/"+x if File.exists?(y+"/"+x) } }.flatten.compact
# nothing says pro-developer like using global variables

$VERSION="0.4.8-dev"                                         #### 程序版本

$WWDEBUG = false # raise exceptions in plugins, etc          #### 是否开启调试模式

$verbose=0 # $VERBOSE is reserved in ruby                    #### 冗长模式，在ruby中是预留的

$USE_EXAMPLE_URLS=false                                                #### 使用测试URL

$use_colour="auto"                                           #### 使用颜色

$USER_AGENT="WhatWeb/#{$VERSION}"                            #### HTTP 中的User-Agent

$MAX_THREADS=25                                              #### 最大线程数为25

$AGGRESSION=1                                                #### 攻击性

$FOLLOW_REDIRECT="always"                                    #### 跟踪重定向

$MAX_REDIRECTS=10                                            #### 重定向深度

$USE_PROXY=false                                             #### 使用代理

$PROXY_HOST=nil

$PROXY_PORT=8080

$PROXY_USER=nil

$PROXY_PASS=nil

$URL_PREFIX=""

$URL_SUFFIX=""

$URL_PATTERN=nil

$NO_THREADS=false

$HTTP_OPEN_TIMEOUT=15

$HTTP_READ_TIMEOUT=30

$WAIT=nil                                                    #### 延时

$OUTPUT_ERRORS=nil

$QUIET=false

$CUSTOM_HEADERS={}

$BASIC_AUTH_USER=nil

$BASIC_AUTH_PASS=nil

$PLUGIN_TIMES=Hash.new(0)                                    #### 创建hash表

$NO_ERRORS=false
### matching
# fuzzy matching ftw

def make_tag_pattern(b)

    # remove stuff between script and /script

    # don‘t bother with  !--, --> or noscript and /noscript

    inscript=false;
b.scan(/<([^\s>]*)/).flatten.map {|x| x.downcase!; r=nil;

            r=x if inscript==false

            inscript=true if x=="script"

            (inscript=false; r=x) if x=="/script"

            r

        }.compact.join(",")

end
def decode_html_entities(s)

    t = s.dup

    html_entities = { "&quot;"=>‘"‘, "&apos;"=>"‘", "&amp;"=>"&", "&lt;"=>"<", "&gt;"=>">" }

    html_entities.each_pair {|from,to| t.gsub!(from,to) }

    t

end
def certainty_to_words(p)

    case p

        when 0..49

            "maybe"

        when 50..99

            "probably"

        when 100

            "certain"

    end

end
# some plugins want a random string in URLs

def randstr

    rand(36**8).to_s(36)

end
def match_ghdb(ghdb, body, meta, status, base_uri)

    # this could be made faster by creating code to eval once for each plugin
pp "match_ghdb",ghdb if $verbose > 2
# take a GHDB string and turn it into code to be evaluated

    matches=[] # fill with true or false. succeeds if all true

    s = ghdb
# does it contain intitle?

    if s =~ /intitle:/i

        # extract either the next word or the following words enclosed in "s, it can‘t possibly be both

        intitle = (s.scan(/intitle:"([^"]*)"/i) + s.scan(/intitle:([^"]\w+)/i)).to_s

        matches << ((body =~ /<title>[^<]*#{Regexp.escape(intitle)}[^<]*<\/title>/i).nil? ? false : true)

        # strip out the intitle: part

        s=s.gsub(/intitle:"([^"]*)"/i,‘‘).gsub(/intitle:([^"]\w+)/i,‘‘)

    end
if s =~ /filetype:/i

        filetype = (s.scan(/filetype:"([^"]*)"/i) + s.scan(/filetype:([^"]\w+)/i)).to_s

        # lame method: check if the URL ends in the filetype

        unless base_uri.nil?

            unless base_uri.path.split("?")[0].nil?

                matches << ((base_uri.path.split("?")[0] =~ /#{Regexp.escape(filetype)}$/i).nil? ? false : true)

            end

        end

        s=s.gsub(/filetype:"([^"]*)"/i,‘‘).gsub(/filetype:([^"]\w+)/i,‘‘)

    end
if s =~ /inurl:/i

        inurl = (s.scan(/inurl:"([^"]*)"/i) + s.scan(/inurl:([^"]\w+)/i)).flatten

        # can occur multiple times.

        inurl.each {|x| matches << ((base_uri.to_s =~ /#{Regexp.escape(x)}/i).nil? ? false : true)  }

        # strip out the inurl: part

        s=s.gsub(/inurl:"([^"]*)"/i,‘‘).gsub(/inurl:([^"]\w+)/i,‘‘)

    end
# split the remaining words except those enclosed in quotes, remove the quotes and sort them
remaining_words = s.scan(/([^ "]+)|("[^"]+")/i).flatten.compact.each {|w| w.delete!(‘"‘)  }.sort.uniq
pp "Remaining GHDB words",     remaining_words if $verbose > 2
remaining_words.each do |w|

        # does it start with a - ?

        if w[0..0] == ‘-‘

            # reverse true/false if it begins with a -

            matches << ((body =~ /#{Regexp.escape(w[1..-1])}/i).nil? ? true : false)

        else

            w = w[1..-1] if w[0..0] == ‘+‘ # if it starts with +, ignore the 1st char

            matches << ((body =~ /#{Regexp.escape(w)}/i).nil? ? false : true)

        end

    end
pp matches if $verbose > 2
# if all matcbhes are true, then true

    if matches.uniq == [true]

        true

    else

        false

    end

end
### targets
def make_target_list(cmdline_args, inputfile=nil,pluginlist = nil)

    url_list = cmdline_args
# read each line as a url, skipping lines that begin with a #

    if !inputfile.nil? and File.exists?(inputfile)

        pp "loading input file: #{inputfile}" if $verbose > 2

        url_list += File.open(inputfile).readlines.each {|line| line.strip! }.delete_if {|line| line =~ /^#.*/ }.each {|line| line.delete!("\n") }

    end
# add example urls to url_list if required. plugins must be loaded already

    if $USE_EXAMPLE_URLS

        url_list += pluginlist.map {|name,plugin| plugin.examples unless plugin.examples.nil? }.compact.flatten

    end
genrange=url_list.map do |x|

      range=nil

      if x =~ /^[0-9\.\-*\/]+$/ and not x =~ /^[\d\.]+$/

        # check for nmap

        error "Target ranges require nmap to be in the path" if `which nmap` == ""

        range=`nmap -n -sL #{x} 2>&1 | egrep -o "([0-9]{1,3}\\.){3}[0-9]{1,3}"`

        range=range.split("\n")

      end

      range

    end.compact.flatten
url_list= url_list.select {|x| not x =~ /^[0-9\.\-*\/]+$/ or x =~ /^[\d\.]+$/ }

    url_list += genrange unless genrange.empty?
#make urls friendlier, test if it‘s a file, if test for not assume it‘s http://

    # http, https, ftp, etc

    url_list=url_list.map do |x|

        if File.exists?(x)

            x

        else

            # use url pattern

            if $URL_PATTERN

                x = $URL_PATTERN.gsub(‘%insert%‘,x)

            end

            # add prefix & suffix

            x=$URL_PREFIX + x + $URL_SUFFIX
if x =~ (/^[a-z]+:\/\//)

                x

            else

                x.sub(/^/,"http://")

            end

        end

    end
url_list=url_list.flatten #.sort.uniq

end
def next_target

    t=nil
puts "Target List:" + $targets.inspect if $verbose > 2

#

    while $recent_targets.include?(t) or t.nil?

        t=$targets.shift

        puts "# t at the end of the $targets list" if $verbose > 2

        # t at the end of the $targets list

        if t.nil?

            puts "t is nil" if $verbose > 2

            if $targets.empty?

                if Thread.list.size > 1

                    if $verbose > 2

                        puts "Thread list size: #{Thread.list.size}"

                        Thread.list.each do |thread|

                            puts "Thread: #{thread.inspect} is #{thread.status}"

                        end

                    end

                    #sleep 1

                    Thread.pass

                else

                    puts "breaking now" if $verbose > 2

                    break

                end

            end

        end

    end
puts "Recent Targets:" + $recent_targets.join(",") if $verbose > 2
$recent_targets.push t

    $recent_targets.pop if $recent_targets.size > 100 # we dont need to care about mroe than 100
t

end
# backwards compatible convenience method for plugins to use

def open_target(url)

    newt = Target.new(url)

    newt.open

    [newt.status,newt.uri,newt.ip,newt.body,newt.headers,newt.raw_headers]

end
### output
def error(s)

    return if $NO_ERRORS

    if defined?($semaphore)

        # We want the output mutex locked.

        # Has our current thread already locked the Mutex?

        begin

            $semaphore.lock

        rescue ThreadError

            # we‘re already locked. This was expected.

        end

    end

    if ($use_colour=="auto") or ($use_colour=="always")

        STDERR.puts red(s)

    else

        STDERR.puts s

    end

    STDERR.flush

    unless $OUTPUT_ERRORS.nil?

        $OUTPUT_ERRORS.out(s)

    end

    $semaphore.unlock if defined?($semaphore)

end
# takes a string and returns an array of lines. used by plugin_info

def word_wrap(s,width=10)

    ret=[]

    line=""

    s.split.map {|x|

        word=x

        if line.size + x.size + 1 <= width

            line += x + " "

        else

            if word.size > width

                ret << line

                line = ""

                w=word.clone

                while w.size > width

                    ret << w[0..(width-1)]

                    w=w[width.to_i..-1]

                end

                ret << w unless w.size == 0

            else

                ret << line

                line=x + " "

            end

        end

     }

    ret << line

    ret

end
### core
def run_plugins(target)

        results=[]

        $plugins_to_use.each do |name,plugin|

            begin

                while plugin.locked?

                    #sleep 0.1

                    puts "Waiting for plugin:#{name} to unlock" if $verbose > 2

                    Thread.pass

                end

                plugin.lock
plugin.init(target)
# eXecute the plugin

                #start_time = Time.now

                result=plugin.x

                #end_time = Time.now

                #$PLUGIN_TIMES[name] += end_time - start_time
plugin.unlock
rescue Exception => err

                error("ERROR: Plugin #{name} failed for #{target.to_s}. #{err}")

                plugin.unlock

                raise if $WWDEBUG==true

            end

            results << [name, result] unless result.nil? or result.empty?

        end

    results

end
def usage()

puts"

.$$$     $.                                   .$$$     $.

$$$$     $$. .$$$  $$$ .$$$$$$.  .$$$$$$$$$$. $$$$     $$. .$$$$$$$. .$$$$$$.

$ $$     $$$ $ $$  $$$ $ $$$$$$. $$$$$ $$$$$$ $ $$     $$$ $ $$   $$ $ $$$$$$.

$ `$     $$$ $ `$  $$$ $ `$  $$$ $$‘ $ `$ `$$ $ `$     $$$ $ `$      $ `$  $$$‘

$. $     $$$ $. $$$$$$ $. $$$$$$ `$  $. $  :‘ $. $     $$$ $. $$$$   $. $$$$$.

$::$  .  $$$ $::$  $$$ $::$  $$$     $::$     $::$  .  $$$ $::$      $::$  $$$$

$;;$ $$$ $$$ $;;$  $$$ $;;$  $$$     $;;$     $;;$ $$$ $$$ $;;$      $;;$  $$$$

$$$$$$ $$$$$ $$$$  $$$ $$$$  $$$     $$$$     $$$$$$ $$$$$ $$$$$$$$$ $$$$$$$$$‘
"
puts "WhatWeb - Next generation web scanner version #{$VERSION}.\nDeveloped by Andrew Horton aka urbanadventurer and Brendan Coles"

puts "Homepage: http://www.morningstarsecurity.com/research/whatweb"

puts

puts "Usage: whatweb [options] <URLs>"

puts "

TARGET SELECTION:

  <URLs>\t\tEnter URLs, filenames or nmap-format IP ranges.

\t\t\tUse /dev/stdin to pipe HTML directly

  --input-file=FILE, -i\tIdentify URLs found in FILE. You can pipe

\t\t\thostnames or URLs directly with -i /dev/stdin
TARGET MODIFICATION:

  --url-prefix\t\tAdd a prefix to target URLs

  --url-suffix\t\tAdd a suffix to target URLs

  --url-pattern\t\tInsert the targets into a URL. Requires --input-file,

\t\t\teg. www.example.com/%insert%/robots.txt
AGGRESSION:

  The aggression level controls the trade-off between speed/stealth and

  reliability.

  --aggression, -a=LEVEL Set the aggression level. Default: 1

  Aggression levels are:

  1. Stealthy\tMakes one HTTP request per target. Also follows redirects.

  2. Unused

  3. Aggressive\tCan make a handful of HTTP requests per target. This triggers

  \t\taggressive plugins for targets only when those plugins are

  \t\tidentified with a level 1 request first.

  4. Heavy\tMakes a lot of HTTP requests per target. Aggressive tests from

  \t\tall plugins are used for all URLs.
HTTP OPTIONS:

  --user-agent, -U=AGENT Identify as AGENT instead of WhatWeb/#{$VERSION}.

  --header, -H\t\tAdd an HTTP header. eg \"Foo:Bar\". Specifying a default

\t\t\theader will replace it. Specifying an empty value, eg.

\t\t\t\"User-Agent:\" will remove the header.

  --follow-redirect=WHEN Control when to follow redirects. WHEN may be `never‘,

\t\t\t`http-only‘, `meta-only‘, `same-site‘, `same-domain‘

\t\t\tor `always‘. Default: #{$FOLLOW_REDIRECT}

  --max-redirects=NUM\tMaximum number of contiguous redirects. Default: 10
AUTHENTICATION:

  --user, -u=<user:password> HTTP basic authentication

  --cookie, -c=COOKIES\tProvide cookies, e.g. ‘name=value; name2=value2  ‘
PROXY:

  --proxy\t\t<hostname[:port]> Set proxy hostname and port

\t\t\tDefault: #{$PROXY_PORT}

  --proxy-user\t\t<username:password> Set proxy user and password
PLUGINS:

  --list-plugins, -l\tList all plugins

  --plugins, -p=LIST\tSelect plugins. LIST is a comma delimited set of

\t\t\tselected plugins. Default is all.

\t\t\tEach element can be a directory, file or plugin name and

\t\t\tcan optionally have a modifier, eg. + or -

\t\t\tExamples: +/tmp/moo.rb,+/tmp/foo.rb

\t\t\ttitle,md5,+./plugins-disabled/

\t\t\t./plugins-disabled,-md5

\t\t\t-p + is a shortcut for -p +plugins-disabled

  --info-plugins, -I=PLUGINS\tDisplay detailed information for plugins.

\t\t\tOptionally search with keywords in a comma delimited

\t\t\tlist.

  --grep, -g=STRING\tSearch for STRING in HTTP responses. Reports with a

\t\t\tplugin named Grep

  --custom-plugin=DEFINITION\tDefine a custom plugin named Custom-Plugin,

\t\t\tExamples: \":text=>‘powered by abc‘\"

\t\t\t\":version=>/powered[ ]?by ab[0-9]/\"

\t\t\t\":ghdb=>‘intitle:abc \\\"powered by abc\\\"‘\"

\t\t\t\":md5=>‘8666257030b94d3bdb46e05945f60b42‘\"

\t\t\t\"{:text=>‘powered by abc‘},{:regexp=>/abc [ ]?1/i}\"

  --dorks=PLUGIN\tList google dorks for the selected plugin

  --example-urls, -e=PLUGIN Update the target list with example URLs from

\t\t\tthe selected plugins.
OUTPUT:

  --verbose, -v\t\tVerbose output includes plugin descriptions. Use twice

\t\t\tfor debugging.

  --colour,--color=WHEN\tcontrol whether colour is used. WHEN may be `never‘,

\t\t\t`always‘, or `auto‘

  --quiet, -q\t\tDo not display brief logging to STDOUT

  --no-errors\t\tSuppress error messages
LOGGING:

  --log-brief=FILE\tLog brief, one-line output

  --log-verbose=FILE\tLog verbose output

  --log-xml=FILE\tLog XML format

  --log-json=FILE\tLog JSON format

"+

#  --log-sql=FILE\tLog SQL INSERT statements

#  --log-sql-create=FILE\tCreate SQL database tables

"  --log-json-verbose=FILE Log JSON Verbose format

  --log-magictree=FILE\tLog MagicTree XML format

  --log-object=FILE\tLog Ruby object inspection format

  --log-mongo-database\tName of the MongoDB database

  --log-mongo-collection Name of the MongoDB collection. Default: whatweb

  --log-mongo-host\tMongoDB hostname or IP address. Default: 0.0.0.0

  --log-mongo-username\tMongoDB username. Default: nil

  --log-mongo-password\tMongoDB password. Default: nil

  --log-errors=FILE\tLog errors
PERFORMANCE & STABILITY:

  --max-threads, -t\tNumber of simultaneous threads. Default: #{$MAX_THREADS}.

  --open-timeout\tTime in seconds. Default: #{$HTTP_OPEN_TIMEOUT}

  --read-timeout\tTime in seconds. Default: #{$HTTP_READ_TIMEOUT}

  --wait=SECONDS\tWait SECONDS between connections

\t\t\tThis is useful when using a single thread.
HELP & MISCELLANEOUS:

  --help, -h\t\tThis help

  --debug\t\tRaise errors in plugins

  --version\t\tDisplay version information. (WhatWeb #{$VERSION})
EXAMPLE USAGE:

* Scan example.com

  whatweb example.com

* Scan reddit.com slashdot.org with verbose plugin descriptions

  whatweb -v reddit.com slashdot.org

* An aggressive scan of mashable.com detects the exact version of Wordpress

  whatweb -a 3 mashable.com

* Scan the local network quickly with 255 threads and suppress errors

  whatweb --no-errors -t 255 192.168.0.0/24\n"
suggestions=""

    suggestions << "To enable JSON logging install the json gem.\n" unless gem_available?(‘json‘)

    suggestions << "To enable MongoDB logging install the mongo gem.\n" unless gem_available?(‘mongo‘)

    suggestions << "To enable character set detection and MongoDB logging install the rchardet gem.\n" unless gem_available?(‘rchardet‘)
unless suggestions.empty?

        print "\nOPTIONAL DEPENDENCIES\n--------------------------------------------------------------------------------\n" + suggestions + "\n"

    end

if RUBY_VERSION =~ /^1\.9/

    puts "WARNING: Ruby 1.9 support is experimental. For stable usage use Ruby 1.8 instead. Please report bugs at https://github.com/urbanadventurer/WhatWeb/issue\n\n"

end

puts
end
if ARGV.size==0 # faster usage info

    usage

    exit

end
plugin_selection=nil

use_custom_plugin=false

use_custom_grep_plugin=false

input_file=nil

output_list = []

mongo={}

mongo[:use_mongo_log]=false
opts = GetoptLong.new(

      [ ‘--help‘, ‘-h‘, GetoptLong::NO_ARGUMENT ],

      [ ‘-v‘,‘--verbose‘, GetoptLong::NO_ARGUMENT ],

      [ ‘-l‘,‘--list-plugins‘, GetoptLong::NO_ARGUMENT ],

      [ ‘-p‘,‘--plugins‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘-I‘,‘--info-plugins‘, GetoptLong::OPTIONAL_ARGUMENT ],

      [ ‘--dorks‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘-e‘,‘--example-urls‘, GetoptLong::NO_ARGUMENT ],

      [ ‘--colour‘,‘--color‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-object‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-brief‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-xml‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-json‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-json-verbose‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-magictree‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-verbose‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-mongo-collection‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-mongo-host‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-mongo-database‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-mongo-username‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-mongo-password‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-sql‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-sql-create‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--log-errors‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--no-errors‘, GetoptLong::NO_ARGUMENT ],

      [ ‘-i‘,‘--input-file‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘-U‘,‘--user-agent‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘-a‘,‘--aggression‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘-t‘,‘--max-threads‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--follow-redirect‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--max-redirects‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--proxy‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--proxy-user‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--url-prefix‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--url-suffix‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--url-pattern‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--custom-plugin‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘-g‘,‘--grep‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--open-timeout‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--read-timeout‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--header‘,‘-H‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--cookie‘,‘-c‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--user‘,‘-u‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--wait‘, GetoptLong::REQUIRED_ARGUMENT ],

      [ ‘--debug‘, GetoptLong::NO_ARGUMENT ],

      [ ‘--version‘, GetoptLong::NO_ARGUMENT ],

      [ ‘-q‘,‘--quiet‘, GetoptLong::NO_ARGUMENT]

    )
begin

    opts.each do |opt, arg|

        case opt

            when ‘-i‘,‘--input-file‘

                input_file=arg

            when ‘-l‘,‘--list-plugins‘

                PluginSupport.load_plugins

                PluginSupport.plugin_list

                exit

            when ‘-p‘,‘--plugins‘

                plugin_selection=arg

            when ‘-I‘,‘--info-plugins‘

                PluginSupport.load_plugins

                PluginSupport.plugin_info(arg.split(","))

                exit

            when ‘--dorks‘

                PluginSupport.load_plugins

                PluginSupport.plugin_dorks(arg)

                exit

            when ‘-e‘,‘--example-urls‘

                $USE_EXAMPLE_URLS=true
when ‘--color‘,‘--colour‘

                case arg

                    when ‘auto‘

                        $use_colour="auto"

                    when ‘always‘

                        $use_colour="always"

                    when ‘never‘

                        $use_colour=false

                    else

                        raise("--colour argument not recognized")

                    end

            when ‘--log-object‘

                output_list << OutputObject.new(arg)

            when ‘--log-brief‘

                 output_list << OutputBrief.new(arg)

            when ‘--log-xml‘

                 output_list << OutputXML.new(arg)

            when ‘--log-magictree‘

                output_list << OutputMagicTreeXML.new(arg)

            when ‘--log-verbose‘

                output_list << OutputVerbose.new(arg)

            when ‘--log-sql‘

                 output_list << OutputSQL.new(arg)

            when ‘--log-sql-create‘

                PluginSupport.load_plugins("+")

                # delete the file if it already exists

                begin

                    File.delete(arg)

                rescue

                end

                 OutputSQL.new(arg).create_tables

                puts "SQL CREATE statements written to #{arg}"

                exit

            when ‘--log-json‘

                if defined?(JSON)

                     output_list << OutputJSON.new(arg)

                else

                    raise("Sorry. The JSON gem is required for JSON output")

                end

            when ‘--log-json-verbose‘

                if defined?(JSON)

                     output_list << OutputJSONVerbose.new(arg)

                else

                    raise("Sorry. The JSON gem is required for JSONVerbose output")

                end

            when ‘--log-mongo-collection‘

                if defined?(Mongo) and defined?(CharDet)

                    mongo[:collection]=arg

                    mongo[:use_mongo_log]=true

                else

                    raise("Sorry. The mongo and rchardet gems are required for Mongo output")

                end
when ‘--log-mongo-host‘

                if defined?(Mongo) and defined?(CharDet)

                     mongo[:host]=arg

                    mongo[:use_mongo_log]=true

                else

                    raise("Sorry. The mongo and rchardet gems are required for Mongo output")

                end
when ‘--log-mongo-database‘

                if defined?(Mongo) and defined?(CharDet)

                     mongo[:database]=arg

                    mongo[:use_mongo_log]=true

                else

                    raise("Sorry. The mongo and rchardet gems are required for Mongo output")

                end

            when ‘--log-mongo-username‘

                if defined?(Mongo) and defined?(CharDet)

                     mongo[:username]=arg

                    mongo[:use_mongo_log]=true

                else

                    raise("Sorry. The mongo and rchardet gems are required for Mongo output")

                end

            when ‘--log-mongo-password‘

                if defined?(Mongo) and defined?(CharDet)

                     mongo[:password]=arg

                    mongo[:use_mongo_log]=true

                else

                    raise("Sorry. The mongo and rchardet gems are required for Mongo output")

                end

            when ‘--log-errors‘

                 $OUTPUT_ERRORS = OutputErrors.new(arg)

            when ‘--no-errors‘

                 $NO_ERRORS = true

            when ‘-U‘,‘--user-agent‘

                $USER_AGENT=arg

            when ‘-t‘,‘--max-threads‘

                $MAX_THREADS=arg.to_i

            when ‘-a‘,‘--aggression‘

                $AGGRESSION=arg.to_i

            when ‘--proxy‘

                $USE_PROXY=true

                $PROXY_HOST = arg.to_s.split(":")[0]

                $PROXY_PORT = arg.to_s.split(":")[1].to_i if arg.to_s.include?(":")

            when ‘--proxy-user‘

                $PROXY_USER=arg.to_s.split(":")[0]

                $PROXY_PASS=arg.to_s.scan(/^[^:]*:(.+)/).to_s if arg =~ /^[^:]*:(.+)/

            when ‘-q‘,‘--quiet‘

                $QUIET=true

            when ‘--url-prefix‘

                $URL_PREFIX=arg

            when ‘--url-suffix‘

                $URL_SUFFIX=arg

            when ‘--url-pattern‘

                $URL_PATTERN=arg

            when ‘--custom-plugin‘

                use_custom_plugin=true if PluginSupport.custom_plugin(arg)

            when ‘--grep‘,‘-g‘

                use_custom_grep_plugin=true if PluginSupport.custom_plugin(arg,"grep")

            when ‘--follow-redirect‘

                if ["never","http-only","meta-only","same-site","same-domain","always"].include?(arg.downcase)

                    $FOLLOW_REDIRECT=arg.downcase

                else

                    raise("Invalid --follow-redirect parameter.")

                end

            when ‘--max-redirects‘

                $MAX_REDIRECTS=arg.to_i

            when ‘--open-timeout‘

                $HTTP_OPEN_TIMEOUT=arg.to_i

            when ‘--read-timeout‘

                $HTTP_READ_TIMEOUT=arg.to_i

            when ‘--wait‘

                $WAIT = arg.to_i

            when ‘-H‘,‘--header‘

                begin

                    x=arg.scan(/([^:]+):(.*)/).flatten

                    raise if x.empty?

                    $CUSTOM_HEADERS[x.first]=x.last

                rescue

                    raise("Invalid --header parameter.")

                end

            when ‘-c‘,‘--cookie‘

                begin

                    raise if arg.empty?

                    $CUSTOM_HEADERS["Cookie"]=arg

                rescue

                    raise("Cookie require a parameter, e.g. name=value; name2=value2")

                end

            when ‘-u‘,‘--user‘

                $BASIC_AUTH_USER=arg.split(":").first

                $BASIC_AUTH_PASS=arg.to_s.scan(/^[^:]*:(.+)/).to_s if arg =~ /^[^:]*:(.+)/

            when ‘--debug‘

                $WWDEBUG = true

            when ‘-h‘,‘--help‘

                usage

                exit

            when ‘-v‘,‘--verbose‘

                $verbose=$verbose+1

            when ‘--version‘

                puts "WhatWeb version #{$VERSION} ( http://www.morningstarsecurity.com/research/whatweb/ )"

                exit

        end

    end

rescue Errno::EPIPE

    exit

rescue StandardError, GetoptLong::Error => err

    puts

    usage

    puts err

    exit

end
# sanity check # Disable colours in Windows environments

if RbConfig::CONFIG[‘host_os‘] =~ /mswin|mingw/

    $use_colour = false

end
# example URLs are only allowed with selected plugins, not all plugins

if $USE_EXAMPLE_URLS and not plugin_selection

    error "You must select plugins with -p or --plugins to include examples in the target list"

    exit 1

end
### PLUGINS

plugin_selection += ",+Custom-Plugin"     if use_custom_plugin and plugin_selection

plugin_selection += ",+Grep"         if use_custom_grep_plugin and plugin_selection

$plugins_to_use = PluginSupport.load_plugins(plugin_selection)

# load all the plugins
# sanity check # no plugins?

if $plugins_to_use.size == 0

    error "No plugins selected, exiting."

    exit 1

end
# optimise plugins

PluginSupport.precompile_regular_expressions
### OUTPUT

output_list << OutputBrief.new unless $QUIET # by default output brief

output_list << OutputObject.new() if $verbose > 1 # full output if -vv

output_list << OutputVerbose.new() if $verbose > 0 # full output if -v
## output dependencies

if mongo[:use_mongo_log]

    if $plugins_to_use.map { |a,b| a }.include?("Charset")

        output_list << OutputMongo.new(mongo)

    else

        error("MongoDB logging requires the Charset plugin to be activated. The Charset plugin is the slowest whatweb plugin, it not included by default, and resides in the plugins-disabled folder. Use ./whatweb -p +./plugins-disabled/Charset.rb to enable it.")

        exit

    end

end
## Headers

$CUSTOM_HEADERS["User-Agent"]=$USER_AGENT unless $CUSTOM_HEADERS["User-Agent"]

$CUSTOM_HEADERS.delete_if {|k,v| v=="" }
### TARGETS

# clean up urls, add example urls if needed

$targets=make_target_list(ARGV, input_file, $plugins_to_use)

$recent_targets=[]
# fail & show usage if no targets.

if $targets.size <1

    error "No targets selected"

    usage

    exit 1

end
$semaphore=Mutex.new

Thread.abort_on_exception = true if $WWDEBUG
while t = next_target

        Thread.new(t) do |thistarget|

            begin

                target = Target.new(thistarget) # we set the target within the thread

            rescue => err

                error(err)

                next

            end
puts Thread.current.to_s + " started for " + target.to_s if $verbose>1

            sleep $WAIT unless $WAIT.nil? # wait
# follow redirects

            no_redirects =false

            num_redirects = 0

            while no_redirects == false do

                no_redirects=true if target.is_file?

                # if we redirect 10 times we give up

                if num_redirects == $MAX_REDIRECTS

                    error("ERROR Too many redirects: #{target.to_s}")

                    no_redirects=true

                    next

                end
begin

                    target.open

                rescue => err

                    error("ERROR Opening target: #{target.to_s} - #{err}")

                    no_redirects = true # without this we can get stuck in a loop

                    raise if $WWDEBUG

                    next

                end
if target.is_url? and target.status.nil?

                    # assume all HTTP sites return a status

                    no_redirects=true

                    next

                end
results = run_plugins(target)
# reporting

                # multiple output plugins simultaneously, some stdout, some files

                output_list.each do |o|

                    begin

                        o.out(target.to_s, target.status, results)

                    rescue => err

                        #srsly, logging failed

                        error("ERROR Logging failed: #{target.to_s} - #{err}")

                        raise if $WWDEBUG==true

                    end

                end
# REDIRECTION

                unless no_redirects

                    begin

                        if newtarget = target.get_redirection_target

                            num_redirects+=1

                            target=Target.new(newtarget)

                        else

                            no_redirects=true

                        end

                    rescue => err

                        error("ERROR Redirection broken: #{target.to_s} - #{err}")

                        no_redirects=true

                        raise if $WWDEBUG==true

                    end

                end

            end # while no_redirects

        end # Thread.new
while Thread.list.size>($MAX_THREADS+1)

        puts "Thread list full, passing control" if $verbose>1

        #sleep 0.5

        Thread.pass

    end

end # targets.each
# close output logs

output_list.each {|o|

    o.close

}
# shutdown plugins

Plugin.registered_plugins.map {|name,plugin| plugin.shutdown }
#pp $PLUGIN_TIMES.sort_by {|x,y|y }

whatweb.rb 未完待续,码迷,mamicode.com