Changeset 322

Show
Ignore:
Timestamp:
05/09/08 09:42:11 (3 months ago)
Author:
justin
Message:

applying patches from someone23

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • rubygems/tarantula/trunk/lib/relevance/tarantula.rb

    r287 r322  
    1010require 'active_support' 
    1111require 'action_controller' 
    12 xss_shield_path = File.join(TARANTULA_ROOT, %w{vendor xss-shield}) 
    13 $: << File.join(xss_shield_path, "lib") 
    14 require File.join(xss_shield_path, "init") 
     12#xss_shield_path = File.join(TARANTULA_ROOT, %w{vendor xss-shield}) 
     13#$: << File.join(xss_shield_path, "lib") 
     14#require File.join(xss_shield_path, "init") 
    1515 
    1616gem 'facets' 
    …… 
    4848require 'relevance/tarantula/html_reporter' 
    4949require 'relevance/tarantula/html_report_helper' 
     50require 'relevance/tarantula/io_reporter' 
    5051require 'relevance/tarantula/recording' 
    5152require 'relevance/tarantula/response' 
    …… 
    5758require 'relevance/tarantula/form' 
    5859require 'relevance/tarantula/form_submission' 
     60require 'relevance/tarantula/xss_form_submission' 
     61require 'relevance/tarantula/xss_document_checker_handler' 
    5962 
    6063require 'relevance/tarantula/tidy_handler' if ENV['TIDY_PATH'] 

  • rubygems/tarantula/trunk/lib/relevance/tarantula/crawler.rb

    r284 r322  
    88  attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber, 
    99                :reporters, :links_to_crawl, :links_queued, :forms_to_crawl, 
    10                 :form_signatures_queued, :max_url_length, :response_code_handler 
     10                :form_signatures_queued, :max_url_length, :response_code_handler, 
     11                :times_to_crawl, :fuzzers 
    1112  attr_reader   :transform_url_patterns, :referrers, :failures, :successes 
    1213    
    …… 
    2122    @forms_to_crawl = [] 
    2223    @referrers = {} 
    23     @skip_uri_patterns =[ 
     24    @skip_uri_patterns = [ 
    2425      /^javascript/, 
    2526      /^mailto/, 
    26       /^http/,                                       
     27      /^http/, 
    2728    ] 
    2829    self.transform_url_patterns = [ 
    2930      [/#.*$/, ''] 
    3031    ] 
    31     @reporters = [] 
     32    @reporters = [Relevance::Tarantula::IOReporter.new($stderr)] 
    3233    @decoder = HTMLEntities.new 
    33      
     34    @times_to_crawl = 1 
     35    @fuzzers = [Relevance::Tarantula::FormSubmission] 
    3436  end 
    3537   
    …… 
    4648   
    4749  def crawl(url = "/") 
    48     queue_link url 
    49     do_crawl 
     50    @times_to_crawl.times do |i| 
     51      queue_link url 
     52      do_crawl 
     53       
     54      puts "#{(i+1).ordinalize} crawl" if @times_to_crawl > 1 
     55       
     56      if i + 1 < @times_to_crawl 
     57        @links_queued = Set.new 
     58        @form_signatures_queued = Set.new 
     59        @referrers = {} 
     60      end 
     61    end 
    5062  rescue Interrupt 
    5163    $stderr.puts "CTRL-C" 
    …… 
    7587   
    7688  def save_result(result) 
    77     return if result.nil? 
    78     collection = result.success ? successes : failures 
    79     collection << result 
     89    reporters.each do |reporter| 
     90      reporter.report(result) 
     91    end 
    8092  end 
    8193   
    …… 
    166178   
    167179  def queue_form(form, referrer = nil) 
    168     fs = FormSubmission.new(Form.new(form)) 
    169     fs.action = transform_url(fs.action) 
    170     return if should_skip_form_submission?(fs) 
    171     @referrers[fs.action] = referrer if referrer 
    172     @forms_to_crawl << fs 
    173     @form_signatures_queued << fs.signature 
     180    fuzzers.each do |fuzzer| 
     181      fuzzer.mutate(Form.new(form)).each do |fs| 
     182        # fs = fuzzer.new(Form.new(form)) 
     183        fs.action = transform_url(fs.action) 
     184        return if should_skip_form_submission?(fs) 
     185        @referrers[fs.action] = referrer if referrer 
     186        @forms_to_crawl << fs 
     187        @form_signatures_queued << fs.signature 
     188      end 
     189    end 
    174190  end 
    175191   
    …… 
    179195 
    180196  def generate_reports 
    181     FileUtils.mkdir_p(report_dir) 
     197    errors = [] 
    182198    reporters.each do |reporter| 
    183       reporter.report(report_dir, self) 
    184     end 
    185   end 
    186    
    187   def report_to_console 
    188     unless (failures).empty? 
    189       $stderr.puts "****** FAILURES" 
    190       failures.each do |failure| 
    191         $stderr.puts "#{failure.code}: #{failure.url}" 
    192       end 
    193       raise "#{failures.size} failures" 
     199      begin 
     200        reporter.finish_report 
     201      rescue RuntimeError => e 
     202        errors << e 
     203      end 
     204    end 
     205    unless errors.empty? 
     206      raise errors.map(&:message).join("\n") 
    194207    end 
    195208  end 
    196209   
    197210  def report_results 
    198     puts "Writing results to #{report_dir}" 
    199211    generate_reports 
    200     report_to_console 
    201212  end 
    202213   

  • rubygems/tarantula/trunk/lib/relevance/tarantula/form.rb

    r132 r322  
    11class Relevance::Tarantula::Form 
    22  extend Forwardable 
    3   def_delegators("@tag", :find_all) 
     3  def_delegators("@tag", :search) 
    44   
    55  def initialize(tag) 
    …… 
    1616   
    1717  def rails_method_hack 
    18     (tag = @tag.find(:tag => 'input', :attributes => { :name => '_method'})) && tag["value"] 
     18    (tag = @tag.at('input[@name="_method"]')) && tag["value"] 
    1919  end 
    2020 

  • rubygems/tarantula/trunk/lib/relevance/tarantula/form_submission.rb

    r287 r322  
    55    @action = form.action 
    66    @data = mutate_selects(form).merge(mutate_text_areas(form)).merge(mutate_inputs(form)) 
     7  end 
     8   
     9  def self.mutate(form) 
     10    [self.new(form)] 
    711  end 
    812   
    …… 
    1822   
    1923  def create_random_data_for(form, tag_selector) 
    20     form.find_all(tag_selector).inject({}) do |form_args, input| 
     24    form.search(tag_selector).inject({}) do |form_args, input| 
    2125      # TODO: test 
    2226      form_args[input['name']] = random_data(input) if input['name'] 
    …… 
    2630 
    2731  def mutate_inputs(form) 
    28     create_random_data_for(form, :tag => 'input') 
     32    create_random_data_for(form, 'input') 
    2933  end 
    3034 
    3135  def mutate_text_areas(form) 
    32     create_random_data_for(form, :tag => 'textarea') 
     36    create_random_data_for(form, 'textarea') 
    3337  end 
    3438   
    3539  def mutate_selects(form) 
    36     form.find_all(:tag => 'select').inject({}) do |form_args, select| 
    37       options = select.find_all(:tag => 'option') 
     40    form.search('select').inject({}) do |form_args, select| 
     41      options = select.search('option') 
    3842      option = options.rand 
    3943      form_args[select['name']] = option['value']  
    …… 
    4751      when /_id$/           : random_whole_number 
    4852      when /uploaded_data/  : nil 
     53      when /^_method$/      : input['value'] 
    4954      when nil              : input['value'] 
    5055      else                    random_int 

  • rubygems/tarantula/trunk/lib/relevance/tarantula/html_document_handler.rb

    r233 r322  
     1require 'hpricot' 
     2 
    13class Relevance::Tarantula::HtmlDocumentHandler  
    24  extend Forwardable 
    …… 
    1113    body = nil 
    1214    Recording.stderr do 
    13       body = HTML::Document.new html 
     15      body = Hpricot html 
    1416    end        
    1517    body 
    …… 
    2022    return unless response.html? 
    2123    body = html_doc_without_stderr_noise(response.body) 
    22     body.find_all(:tag=>'a').each do |tag| 
     24    body.search('a').each do |tag| 
    2325      queue_link(tag['href'], url) 
    2426    end 
    25     body.find_all(:tag=>'link').each do |tag| 
     27    body.search('link').each do |tag| 
    2628      queue_link(tag['href'], url) 
    2729    end 
    28     body.find_all(:tag =>'form').each do |form| 
    29       form.attributes['action'] = url unless form.attributes['action'] 
     30    body.search('form').each do |form| 
     31      form['action'] = url unless form['action'] 
    3032      queue_form(form, url) 
    3133    end 

  • rubygems/tarantula/trunk/lib/relevance/tarantula/html_reporter.rb

    r296 r322  
    11class Relevance::Tarantula::HtmlReporter 
     2   
    23  include Relevance::Tarantula 
    3   attr_accessor :basedir, :results  
     4  attr_accessor :basedir, :results 
    45  delegate :successes, :failures, :to => :results 
    5   def self.report(basedir, results) 
    6     self.new(basedir, results) 
     6   
     7  HtmlResultOverview = Struct.new(:code, :url, :description, :method, :referrer, :file_name) 
     8   
     9  def initialize(basedir) 
     10    @basedir = basedir 
     11    @results = Struct.new(:successes, :failures).new([], []) 
     12    FileUtils.mkdir_p(@basedir) 
    713  end 
    8  
    9   def initialize(basedir, results) 
    10     @basedir = basedir 
    11     @results = results 
     14   
     15  def report(result) 
     16    return if result.nil? 
     17     
     18    create_detail_report(result) 
     19     
     20    collection = result.success ? results.successes : results.failures 
     21    collection << HtmlResultOverview.new( 
     22      result.code, result.url, result.description, result.method, result.referrer, result.file_name 
     23    ) 
     24  end 
     25   
     26  def finish_report 
     27    puts "Writing results to #{basedir}" 
    1228    copy_styles 
    1329    create_index 
    14     create_detail_reports 
    1530  end 
    16    
    17   def template(name) 
    18     File.read(File.join(File.dirname(__FILE__), name)) 
    19   end 
    20    
    21   def output(name, body) 
    22     File.open(File.join(basedir, name), "w") do |file| 
    23       file.write body 
    24     end 
    25   end       
    2631   
    2732  def copy_styles 
    …… 
    4550    output("index.html", template.result(binding)) 
    4651  end 
    47  
    48   def create_detail_reports 
     52   
     53  def template(name) 
     54    File.read(File.join(File.dirname(__FILE__), name)) 
     55  end 
     56   
     57  def output(name, body) 
     58    File.open(File.join(basedir, name), "w") do |file| 
     59      file.write body 
     60    end 
     61  end       
     62   
     63  def create_detail_report(result) 
    4964    template = ERB.new(template("detail.html.erb")) 
    50     results.successes.each do |result| 
    51       output(result.file_name, template.result(result.send(:binding))) 
    52     end 
    53     results.failures.each do |result| 
    54       output(result.file_name, template.result(result.send(:binding))) 
    55     end 
     65    output(result.file_name, template.result(result.send(:binding))) 
    5666  end  
    57  
     67   
    5868  # CSS class for HTML status codes 
    5969  def class_for_code(code) 
    6070    "r#{Integer(code)/100}"  
    6171  end 
     72   
     73   
    6274end 

  • rubygems/tarantula/trunk/lib/relevance/tarantula/html_report_helper.rb

    r296 r322  
     1require "erb" 
    12module Relevance::Tarantula::HtmlReportHelper  
     3  include ERB::Util 
    24  include Relevance::Tarantula 
    35  def wrap_in_line_number_table(text, &blk) 
    …… 
    4446  def wrap_stack_trace_line(text) 
    4547    if text =~ %r{^\s*(/[^:]+):(\d+):([^:]+)$} 
    46       file = $1.to_s_xss_protected 
     48      file = h($1) # .to_s_xss_protected 
    4749      line_number = $2 
    48       message = $3.to_s_xss_protected 
    49       "<a href='#{textmate_url(file, line_number)}'>#{file}:#{line_number}</a>:#{message}".mark_as_xss_protected 
     50      message = h($3) # .to_s_xss_protected 
     51      "<a href='#{textmate_url(file, line_number)}'>#{file}:#{line_number}</a>:#{message}" # .mark_as_xss_protected 
    5052    else 
    51       text.to_s_xss_protected 
     53      h(text) # .to_s_xss_protected 
    5254    end 
    5355  end 

  • rubygems/tarantula/trunk/lib/relevance/tarantula/rails_integration_proxy.rb

    r273 r322  
    1919      [/^http:\/\/#{integration_test.host}/, '']    # strip full path down to relative 
    2020    ] 
    21     t.reporters << Relevance::Tarantula::HtmlReporter 
     21    t.reporters << Relevance::Tarantula::HtmlReporter.new(t.report_dir) 
    2222    t 
    2323  end 

  • rubygems/tarantula/trunk/lib/relevance/tarantula/result.rb

    r296 r322  
    3131  end 
    3232  ALLOW_NNN_FOR = /^allow_(\d\d\d)_for$/ 
    33   class <<self 
     33  class << self 
    3434    attr_accessor :next_number 
    3535    def handle(result) 

  • rubygems/tarantula/trunk/test/relevance/core_extensions/test_case_test.rb

    r273 r322  
    55  it "can create the crawler" do  
    66    RailsIntegrationProxy.stubs(:rails_root).returns("STUB_RAILS_ROOT") 
     7    Crawler.any_instance.stubs(:rails_root).returns("STUB_RAILS_ROOT") 
    78    tarantula_crawler(stub_everything) 
    89  end 

  • rubygems/tarantula/trunk/test/relevance/tarantula/crawler_test.rb

    r284 r322  
    8383  it 'queues and remembers forms' do 
    8484    crawler = Crawler.new 
    85     form = HTML::Document.new('<form action="/action" method="post"/>').find(:tag =>'form') 
     85    form = Hpricot('<form action="/action" method="post"/>').at('form') 
    8686    signature = FormSubmission.new(Form.new(form)).signature 
    8787    crawler.queue_form(form) 
    …… 
    9999 
    100100describe 'Relevance::Tarantula::Crawler#report_results' do 
    101   it "delegates to generate_reports then report_to_console" do 
    102     crawler = Crawler.new 
    103     crawler.expects(:report_dir) 
    104     crawler.expects(:puts) 
     101  it "delegates to generate_reports" do 
     102    crawler = Crawler.new 
    105103    crawler.expects(:generate_reports) 
    106     crawler.expects(:report_to_console) 
    107104    crawler.report_results 
    108105  end 
    …… 
    188185  end 
    189186   
    190   it "reports errors to stderr and then raises" do 
    191     crawler = Crawler.new 
    192     crawler.failures << stub(:code => "404", :url => "/uh-oh") 
    193     $stderr.expects(:puts).with("****** FAILURES") 
    194     $stderr.expects(:puts).with("404: /uh-oh") 
    195     lambda {crawler.report_to_console}.should.raise RuntimeError 
    196   end 
    197    
    198187  it "asks each reporter to write its report in report_dir" do 
    199188    crawler = Crawler.new 
    200     crawler.failures << stub(:code => "404", :url => "/uh-oh") 
    201189    crawler.stubs(:report_dir).returns(test_output_dir) 
    202190    reporter = stub_everything 
    203191    reporter.expects(:report) 
     192    reporter.expects(:finish_report) 
    204193    crawler.reporters = [reporter] 
     194    crawler.save_result stub(:code => "404", :url => "/uh-oh") 
    205195    crawler.generate_reports 
    206196  end 

  • rubygems/tarantula/trunk/test/relevance/tarantula/form_submission_test.rb

    r141 r322  
    55  # TODO: add more from field types to this example form as needed 
    66  before do 
    7     @tag = HTML::Document.new(<<END) 
     7    @tag = Hpricot(<<END) 
    88<form action="/session" method="post"> 
    99  <input id="email" name="email" size="30" type="text" /> 
    …… 
    1717</form> 
    1818END 
    19     @form = Relevance::Tarantula::Form.new(@tag.find(:tag => 'form')) 
     19    @form = Relevance::Tarantula::Form.new(@tag.at('form')) 
    2020    @fs = Relevance::Tarantula::FormSubmission.new(@form) 
    2121  end 
    …… 
    2727   
    2828  it "can mutate selects" do 
    29     Array.any_instance.stubs(:rand).returns(stub(:[] => "2006-stub")) 
     29    Hpricot::Elements.any_instance.stubs(:rand).returns(stub(:[] => "2006-stub")) 
    3030    @fs.mutate_selects(@form).should == {"foo[opened_on(1i)]" => "2006-stub"} 
    3131  end 
    …… 
    5757describe "Relevance::Tarantula::FormSubmission for a crummy form" do 
    5858  before do 
    59     @tag = HTML::Document.new(<<END) 
     59    @tag = Hpricot(<<END) 
    6060<form action="/session" method="post"> 
    6161  <input value="no_name" /> 
    6262</form> 
    6363END 
    64     @form = Relevance::Tarantula::Form.new(@tag.find(:tag => 'form')) 
     64    @form = Relevance::Tarantula::Form.new(@tag.at('form')) 
    6565    @fs = Relevance::Tarantula::FormSubmission.new(@form) 
    6666  end 

  • rubygems/tarantula/trunk/test/relevance/tarantula/form_test.rb

    r132 r322  
    33describe "Relevance::Tarantula::Form large example" do 
    44  before do 
    5     @tag = HTML::Document.new(<<END) 
     5    @tag = Hpricot(<<END) 
    66<form action="/session" method="post"> 
    77  <input name="authenticity_token" type="hidden" value="1be0d07c6e13669a87b8f52a3c7e1d1ffa77708d" /> 
    …… 
    1212</form> 
    1313END 
    14     @form = Relevance::Tarantula::Form.new(@tag.find(:tag => 'form')) 
     14    @form = Relevance::Tarantula::Form.new(@tag.at('form')) 
    1515  end 
    1616   
    …… 
    2727describe "A Relevance::Tarantula::Form" do 
    2828  it "defaults method to 'get'" do 
    29     @tag = HTML::Document.new("<form/>") 
    30     @form = Relevance::Tarantula::Form.new(@tag.find(:tag => 'form')) 
     29    @tag = Hpricot("<form/>") 
     30    @form = Relevance::Tarantula::Form.new(@tag.at('form')) 
    3131    @form.method.should == 'get' 
    3232  end 
    …… 
    3535describe "A Relevance::Tarantula::Form with a hacked _method" do 
    3636  before do 
    37     @tag = HTML::Document.new(<<END) 
     37    @tag = Hpricot(<<END) 
    3838<form action="/foo"> 
    3939  <input name="authenticity_token" type="hidden" value="1be0d07c6e13669a87b8f52a3c7e1d1ffa77708d" /> 
    …… 
    4141</form> 
    4242END 
    43     @form = Relevance::Tarantula::Form.new(@tag.find(:tag => 'form')) 
     43    @form = Relevance::Tarantula::Form.new(@tag.at('form')) 
    4444  end 
    4545 

  • rubygems/tarantula/trunk/test/relevance/tarantula/html_document_handler_test.rb

    r233 r322  
    3131   
    3232  it "queues forms" do 
    33     @handler.expects(:queue_form).with{|tag,referrer| HTML::Tag === tag} 
     33    @handler.expects(:queue_form).with{|tag,referrer| Hpricot::Elem === tag} 
    3434    @handler.handle(Result.new(:response => stub(:html? => true, :body => '<form>stuff</form>'))) 
    3535  end 

  • rubygems/tarantula/trunk/test/relevance/tarantula/html_reporter_test.rb

    r296 r322  
    77    FileUtils.mkdir_p(test_output_dir) 
    88    Relevance::Tarantula::Result.next_number = 0 
    9     @results = (1..10).map do |index| 
     9    @success_results = (1..10).map do |index| 
    1010      Relevance::Tarantula::Result.new( 
    1111        :success => true,  
    …… 
    1313        :url => "/widgets/#{index}",  
    1414        :response => stub(:code => 200, :body => "<h1>header</h1>\n<p>text</p>"),  
     15        :referrer => "/random/#{rand(100)}",  
     16        :log => <<-END, 
     17Made-up stack trace: 
     18/some_module/some_class.rb:697:in `bad_method' 
     19/some_module/other_class.rb:12345677:in `long_method' 
     20this link should be <a href="#">escaped</a> 
     21blah blah blah 
     22END 
     23        :data => "{:param1 => :value, :param2 => :another_value}" 
     24      ) 
     25    end 
     26    @fail_results = (1..10).map do |index| 
     27      Relevance::Tarantula::Result.new( 
     28        :success => false,  
     29        :method => "get",  
     30        :url => "/widgets/#{index}",  
     31        :response => stub(:code => 500, :body => "<h1>header</h1>\n<p>text</p>"),  
    1532        :referrer => "/random/#{rand(100)}",  
    1633        :log => <<-END, 
    …… 
    3249  it "creates a report based on tarantula results" do     
    3350    Relevance::Tarantula::Result.any_instance.stubs(:rails_root).returns("STUB_ROOT") 
    34     results = stub(:successes => @results, :failures => @results) 
    35     Relevance::Tarantula::HtmlReporter.report(test_output_dir, results) 
     51    # results = stub(:successes => @results, :failures => @results) 
     52    reporter = Relevance::Tarantula::HtmlReporter.new(test_output_dir) 
     53    (@success_results + @fail_results).each {|r| reporter.report(r)} 
     54    reporter.finish_report 
    3655    File.should.exist @index 
    3756    File.should.exist @detail 

  • rubygems/tarantula/trunk/test/relevance/tarantula/rails_integration_proxy_test.rb

    r273 r322  
    55  before { 
    66    Crawler.any_instance.stubs(:crawl) 
     7    Crawler.any_instance.stubs(:rails_root).returns("STUB_RAILS_ROOT") 
    78    RailsIntegrationProxy.stubs(:rails_root).returns("STUB_RAILS_ROOT") 
    89  }