Ticket #12: tarantula_less_mem.patch

tarantula.orig/lib/relevance/tarantula/crawler.rb

@@ -20 +21 @@
-    @links_to_crawl = []
-    @forms_to_crawl = []
-    @referrers = {}
-
+ 
-      /^javascript/,
-      /^mailto/,
-                                      
+
-    ]
-    self.transform_url_patterns = [
-      [/#.*$/, '']
-    ]
-
+Relevance::Tarantula::IOReporter.new($stderr)
-    @decoder = HTMLEntities.new
-    
-  end
-  
-  def method_missing(meth, *args)
@@ -74 +87 @@
-  end  
-  
-  def save_result(result)
-turn if result.nil?
-collection = result.success ? successes : failures
-collection << result
+porters.each do |reporter|
+  reporter.report(result)
+end
-  end
-  
-  def handle_link_results(link, response)
@@ -178 +195 @@
-  end
-
-  def generate_reports
-FileUtils.mkdir_p(report_dir)
+errors = []
-    reporters.each do |reporter|
-
-
-
-
-
-
-
-
-
+
+
+
+
-      end
-
+
+
+
-    end
-  end
-  
-  def report_results
-    puts "Writing results to #{report_dir}"
-    generate_reports
-    report_to_console
-  end
-  
-  def total_links_count

tarantula.orig/lib/relevance/tarantula/html_reporter.rb

@@ -1 +1 @@
-class Relevance::Tarantula::HtmlReporter
+  
-  include Relevance::Tarantula
- 
+
-  delegate :successes, :failures, :to => :results
-
-
-
-
-
+
+
+
+
-    @basedir = basedir
-
-
-
-
+
+
-  end
-  
-
-
+
+
+
+
+
+
+
+
+
-  end
-  
-output(name, body)
-File.open(File.join(basedir, name), "w") do |file|
-  file.write body
-end
-      
+finish_report
+puts "Writing results to #{basedir}"
+copy_styles
+create_index
+
-  
-  def copy_styles
-    # not using cp_r because it picks up .svn crap
@@ -44 +49 @@
-    template = ERB.new(template("index.html.erb"))
-    output("index.html", template.result(binding))
-  end
-
-create_detail_reports
-template = ERB.new(template("detail.html.erb"
-  results.successes.each do |result|
-    output(result.file_name, template.result(result.send(:binding)))
-  end
-results.failures.each do |result
-output(result.file_name, template.result(result.send(:binding)))
+
+template(name)
+File.read(File.join(File.dirname(__FILE__), name
+end
+
+def output(name, body)
+File.open(File.join(basedir, name), "w") do |file
+file.write body
-    end
+  end      
+  
+  def create_detail_report(result)
+    template = ERB.new(template("detail.html.erb"))
+    output(result.file_name, template.result(result.send(:binding)))
-  end 
-
+
-  # CSS class for HTML status codes
-  def class_for_code(code)
-    "r#{Integer(code)/100}" 
-  end
+  
+  
-end

tarantula.orig/lib/relevance/tarantula/io_reporter.rb

@@ +1 @@
+class Relevance::Tarantula::IOReporter
+  
+  include Relevance::Tarantula
+  attr_accessor :io, :results
+  delegate :successes, :failures, :to => :results
+  
+  IOResultOverview = Struct.new(:code, :url)
+  
+  def initialize(io)
+    @io = io
+    @results = Struct.new(:successes, :failures).new([], [])
+  end
+  
+  def report(result)
+    return if result.nil?
+    
+    unless result.success # collection = result.success ? results.successes : results.failures
+      results.failures << IOResultOverview.new(
+        result.code, result.url
+      )
+    end
+  end
+  
+  def finish_report
+    unless (failures).empty?
+      io.puts "****** FAILURES"
+      failures.each do |failure|
+        io.puts "#{failure.code}: #{failure.url}"
+      end
+      raise "#{failures.size} failures"
+    end
+  end
+  
+end

tarantula.orig/lib/relevance/tarantula/rails_integration_proxy.rb

@@ -18 +18 @@
-      [/\?\d+$/, ''],                               # strip trailing numbers for assets
-      [/^http:\/\/#{integration_test.host}/, '']    # strip full path down to relative
-    ]
-
+.new(t.report_dir)
-    t
-  end
-

tarantula.orig/lib/relevance/tarantula/result.rb

@@ -30 +30 @@
-    "#{DEFAULT_LOCALHOST}#{url}"
-  end
-  ALLOW_NNN_FOR = /^allow_(\d\d\d)_for$/
-
+ 
-    attr_accessor :next_number
-    def handle(result)
-      retval = result.dup