Class: Arachnid2::Watir

Inherits:
Object
  • Object
show all
Includes:
Exoskeleton
Defined in:
lib/arachnid2/watir.rb

Constant Summary collapse

DEFAULT_AGENT = 
:desktop
DEFAULT_ORIENTATION = 
:landscape

Instance Method Summary collapse

Methods included from Exoskeleton

#bound_time, #bound_urls, #browser_type, #crawl_options, #extension_ignored?, #extract_hrefs, #in_docker?, #internal_link?, #make_absolute, #maximum_load_rate, #memory_danger?, #non_html_extensions, #preflight, #process, #proxy, #skip_link?, #timeout, #vacuum

Constructor Details

#initialize(url) ⇒ Watir

Returns a new instance of Watir.



8
9
10
11
 
# File 'lib/arachnid2/watir.rb', line 8

def initialize(url)
  @url = url
  @domain = Adomain[@url]
end

Instance Method Details

#crawl(opts) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/arachnid2/watir.rb', line 13

def crawl(opts)
  preflight(opts)
  watir_preflight
  @already_retried = false

  until @global_queue.empty?
    q = @global_queue.shift
    links = nil

    break if time_to_stop?

    @global_visited.insert(q)

    make_request(q, &Proc.new)
  end # until @global_queue.empty?
ensure
  @browser.close if @browser rescue nil
  @headless.destroy if @headless rescue nil
end