Still learning Ruby and trying to get a web scraper that is writing to a
csv file. I am able to get everything but the writing of the array to
the csv file. Getting the following error message when it runs.
search.rb:108:in block in find_att_value': undefined local variable or method
csv’ for main:Object (NameError)
from search.rb:71:in each' from search.rb:71:in
find_att_value’
from search.rb:120:in block in <main>' from /home/bob/.rvm/rubies/ruby-2.2.1/lib/ruby/2.2.0/csv.rb:1273:in
open’
from search.rb:17:in `’
Below is my code. How can I fix this?
Thanks in advance.
Bob
require ‘rubygems’
require ‘nokogiri’
require ‘open-uri’
require ‘mechanize’
require ‘csv’
BASEURL = “http://www.pbm-erson.com”
BASEURL1 =
“http://www.pbm-erson.com/Catalog/PBM/Camshafts_PBM/Performance_CPBM?&sortby=2&pageSize=5&page=”
HSH = Hash.new
RESULTS = []
BRY = Array.new #attribute name array from products
LAST_PAGE_NUMBER = 2 #last page to stop on
ARR1 = []
$bsv
#go through and get links for each product page
CSV.open(“results-test.csv”,“w”) do |csv|
#$bsv = csv
#def csvwrite
csv << $bsv
#end
def find_link
puts " Finding Links"
item_info_css = “.item-info”
product_item_row = “.item-row”
product_num = “span.item-num”
for current_page_number in 1..LAST_PAGE_NUMBER
url = "#{BASEURL1}#{current_page_number}"
puts url
doc = Nokogiri::HTML(open(url))
products = doc.css(item_info_css)
products.each do |product|
product_nodes = product.css(product_num)
links = product.css('a')
linkref = links[0]["href"]
raw_results = BASEURL + linkref
RESULTS << raw_results
sleep 1
end
end
puts RESULTS#finding links
end
def build_att_name_array
BRY.push(“ProductID:”)
BRY.push(“Name:”)
BRY.push(“Short Description:”)
BRY.push(“HTML Content:”)#building attribute name list
puts “Building Table Headers”
RESULTS.each do |link|
url = link
doc = Nokogiri::HTML(open(url))
doc.css(".attribute-list-item").each do |attrib|
att = attrib.at_css("span.attrib-name").text
BRY << att
end
sleep 1
end
end
#need to verify the lining up of the code from here down
def find_att_value
puts “Matching attributes to attribute name”
hry = BRY.uniq
RESULTS.each do |link|
url = link
doc = Nokogiri::HTML(open(url))
erp_num = doc.at_css("span.erp-num").text.strip #our part number
from Sxe
short_desc = doc.at_css(“.pd-header”).text.strip # short
description
long_desc = doc.at_css(“.product-cm”).text.strip #html content
id = doc.at_css(“input#CurrentProductId”)[‘value’]
hry.each do |ab|
if ab == "ProductID:"
HSH[ab]= id
elsif ab == "Name:"
HSH[ab]= erp_num
elsif ab == "Short Description:"
HSH[ab]= short_desc
elsif ab == "HTML Content:"
HSH[ab]= long_desc
else
begin
els = doc.search"[text()*='#{ab}']"
el = els.first
eln = el.next_element.text.strip
HSH[ab]= eln
rescue
HSH[ab]= ""
end
end
end
puts "-----------------------"
puts HSH
HSH.each_value do |val|
ARR1 << val
end
#$bsv << ARR1
#csvwrite
csv << ARR1 # <— HERE IS WHERE THE ERROR IS
ARR1.clear
HSH.clear
puts “------------------------”
end
end