3
\$\begingroup\$

I'm working on a lead generation bot that helps you find the emails of people you want to reach out to.

The bot grabs your spreadsheet from gdrive, logs into several email finding tools, and collect emails for every prospect in the spreadsheet from the tool.

My goal is for non-techies (i.e marketers) to be able to use the bot.

Here's the code

require "google_drive"
require "watir-webdriver"
#require "timeout"

#Your config - Give nimrod the details it needs
@your_config = {
    email_hunter_email: "",
    email_hunter_password: "",
    voila_norbert_email: "",
    voila_norbert_password: "",
    find_that_email_email: "",
    find_that_email_password: "",
    google_sheet_key: "",
}

#Email Hunter config
@email_hunter = {
    url: "https://hunter.io/users/sign_in",
    email_field: "user[email]",
    password_field: "user[password]",
    login_button: "Log in »",
  website_field: 'domain-field',
    button: 'search-btn',
    pattern_result_div: '/html/body/div[3]/div/div[2]/div[1]/div[1]/div/div/div[1]/div',
    email_result_div: '.search.index .search-results .result .email'
}

#Voila Norbert config
@voila_norbert = {
    url: "https://app.voilanorbert.com/#!/auth/login",
    email_field: "email",
    password_field: "password",
    login_button: "Let's do this, Norbert!",
    name_field: 'name',
  website_field: 'domain',
    button: 'Go ahead, Norbert!',
    email_result_div: '.contact-list'
}

#Find That Email config
@find_that_email = {
    url: "https://findthat.email/sign-in/",
    email_field: "email",
    password_field: "password",
    login_button: "t_sign_submit",
    first_name_field: 'first_name',
    last_name_field: 'last_name',
  website_field: 'company_domain',
    button: 't_home_sumbit',
    answer_div: '//*[@id="t_d_search_log_box"]/div/div[1]/div'
}



# Creates a session. This will prompt the credential via command line for the
# first time and save it to config.json file for later usages.
session = GoogleDrive::Session.from_config("google_drive_config.json")

# Go to spreadsheet
ws = session.spreadsheet_by_key(@your_config[:google_sheet_key]).worksheets[0]


# To do
#replace sleeps with wait timeouts
#https://ruby-doc.org/stdlib-2.4.1/libdoc/timeout/rdoc/Timeout.html

=begin
def get_proxy_address
    begin
        browser = Watir::Browser.new :phantomjs
        #maximize the browser
        browser.driver.manage.window.maximize
        browser.goto('http://proxylist.hidemyass.com')
        ip = ''
        browser.element(:xpath => '//*[@id="listable"]/tbody/tr[1]/td[2]/span').spans.select {|s| (s.visible?) ? ip << s.text : '';}
        port = browser.element(:xpath => '//*[@id="listable"]/tbody/tr[1]/td[3]').text
    end until IPAddress.valid? ip
    return "#{ip}" + ":" + "#{port}"
end
def with_timeout_handling
    begin
        Timeout::timeout(120) do
            yield
        end
    rescue Timeout::Error
        nil
    end
end
=end


def open_browser
    #open new browser
    browser = Watir::Browser.new :phantomjs

    #maximize the browser
    browser.driver.manage.window.maximize

    #return browser
    return browser
end




def email_hunter(browser, first_name, last_name, domain)
    return "Email Hunter login details missing." if @your_config[:email_hunter_email].nil? || @your_config[:email_hunter_password].nil?
  browser.goto(@email_hunter[:url])
    sleep 10
    browser.text_field(:name => @email_hunter[:email_field]).set @your_config[:email_hunter_email]
    browser.text_field(:name => @email_hunter[:password_field]).set @your_config[:email_hunter_password]
    browser.button(:value => @email_hunter[:login_button]).click
    #browser.button(:value => @email_hunter[:login_button]).click
    sleep 30 #wait for it to set the session
  browser.text_field(:id => @email_hunter[:website_field]).set domain
    browser.button(:id => @email_hunter[:button]).click
  sleep 20

  #Get email pattern
    #Replace the pattern with details you have
    if browser.div(:xpath => @email_hunter[:pattern_result_div]).exists?
        pattern = browser.div(:xpath => @email_hunter[:pattern_result_div]).text.split('Most common pattern: ')[-1]
        pattern_replacement = pattern.gsub('{first}', first_name.downcase).gsub('{f}', first_name[0].downcase).gsub('{last}', last_name.downcase).gsub('{l}', last_name[0].downcase)
    else
        pattern = 'Not found'
        pattern_replacement = 'Not found'
    end

    #Get email returned
    if browser.div(:css => @email_hunter[:email_result_div]).exists?
        email_gotten = browser.div(:css => @email_hunter[:email_result_div]).text
    else
        email_gotten = 'Not found'
    end

    #puts pattern + pattern_replacement + puts email_gotten
    return "Pattern: #{pattern}" + "\n" + "Pattern replacement: #{pattern_replacement}" + "\n" + "Email from source: #{email_gotten}"
end

def voila_norbert(browser, first_name, last_name, domain)
    return "Voila Norbert login details missing" if @your_config[:voila_norbert_email].nil? || @your_config[:voila_norbert_password].nil?
  browser.goto(@voila_norbert[:url])
    sleep 10
    browser.text_field(:name => @voila_norbert[:email_field]).set @your_config[:voila_norbert_email]
    browser.text_field(:name => @voila_norbert[:password_field]).set @your_config[:voila_norbert_password]
    browser.element(:css => "input[type=submit]").click
    sleep 30 #wait for it to set the session
  browser.text_field(:name => @voila_norbert[:name_field]).set "#{first_name}" + ' ' + "#{last_name}"
  browser.text_field(:name => @voila_norbert[:website_field]).set domain
    browser.button(:value => @voila_norbert[:button]).click
  sleep 40
    contact_list = browser.ul(:css => @voila_norbert[:email_result_div])
    contact_list.lis.each do |li|
        if li.text.include?("#{first_name}" + ' ' + "#{last_name}")
            return li.text
            break
        end
    end
    return "Not found"
end

def find_that_email(browser, first_name, last_name, domain)
    return "Find that email login details missing" if @your_config[:find_that_email_email].nil? || @your_config[:find_that_email_password].nil?
  browser.goto(@find_that_email[:url])
    browser.text_field(:name => @find_that_email[:email_field]).set @your_config[:find_that_email_email]
    browser.text_field(:name => @find_that_email[:password_field]).set @your_config[:find_that_email_password]
    browser.execute_script('$("#t_sign_submit").click()')
    sleep 30
    #browser.execute_script('$(".t_inputtext_home").click()')
  #browser.text_field(:name => @find_that_email[:name_field]).set "#{first_name}" + ' ' + "#{last_name}"

  browser.text_field(:name => @find_that_email[:first_name_field]).set first_name
  browser.text_field(:name => @find_that_email[:last_name_field]).set last_name
  browser.text_field(:name => @find_that_email[:website_field]).set domain #Can't use xpath because id changes
  #browser.div(:id => @find_that_email[:button]).click
    sleep 5
    browser.execute_script('$("#t_my_app_add_button").click()')
    #browser.form(:id =>'h_u_test').submit
    sleep 30
    if browser.div(:xpath=>@find_that_email[:answer_div]).exists?
        return browser.div(:xpath=>@find_that_email[:answer_div]).text.strip
    else
        return ''
    end

end

#browser.screenshot.save("screenshots/3.png")
#browser.execute_script('$("#get_domain_data").click();')

def whois(browser, first_name, last_name, domain)
    browser.goto('https://www.whoisxmlapi.com/?domainName=' + domain + '&outputFormat=xml')
    sleep 30
    content = browser.div(:id => 'wa-tab-content-whoislookup').text
    r = Regexp.new(/\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/)
    emails = content.scan(r).uniq
    unless emails.nil?
        return emails.first
    else
        return 'Not Found'
    end
end


def with_error_handling
    yield
rescue => e
    return e
end


#Main function
# Get all rows in the spreadsheet
(2..ws.num_rows).each do |row|
    #Next row in spreadsheet if first name, last name or domain is missing in the row
    next if ws[row, 1].nil? || ws[row, 2].nil? || ws[row, 3].nil?

    browser = open_browser

  #Step 1 email hunter
    email_hunter_result = with_error_handling {  email_hunter(browser, ws[row, 1], ws[row, 2], ws[row, 3]) }
    #Enter email hunter result into spreadsheet
    ws[row, 4] = email_hunter_result

    #Step 2 Voila Norbert
    voila_norbert_results = with_error_handling { voila_norbert(browser, ws[row, 1], ws[row, 2], ws[row, 3]) }
    ws[row, 5] = voila_norbert_results

  #Step 3 Find That Email
    find_that_email_result = with_error_handling { find_that_email(browser, ws[row, 1], ws[row, 2], ws[row, 3]) }
    ws[row, 6] = find_that_email_result


    #Step 4 whois
    whois_result = with_error_handling {  whois(browser, ws[row, 1], ws[row, 2], ws[row, 3]) }
    ws[row, 7] = whois_result

    ws.save
  browser.close
    sleep 10
end

Any feedback or suggestions on how I can make the code more readable / user friendly?

\$\endgroup\$

1 Answer 1

Reset to default
1
\$\begingroup\$

Congratulations on putting your code up for review for the first time here. It takes some courage and humility to ask invite criticism of one's code.

Indentation

Standard Ruby intentation is two spaces. Other indentations are jarring to the reader used to the standard.

Prefer lines shorter than 80 characters.

Long lines cause the code to be hard to read when someone is using an editor with a narrower window than you used, or when the code is printed, or when it is displayed on a stackexchange site (witness the horizontal scroll bars above). For that reason, prefer lines shorter than 80 characters.

Note: Unlike using two spaces for indentation, which is very standard, my prohibition on long lines is not an opinion not universally held by Ruby programmers. At the very least, there is some dissent on how long a line has to be in order to be "long."

Commented-out code

Commented-out code, especially without a comment explaining why, is a code smell. In general, at least leave a comment explaining why it's commented out. If possible, just delete it. If it is code that is sometimes used, then put it inside an if statement so that it can be enabled by configuration, argument, environment-variable, etc.

Use constants to explain "magic numbers"

In lines such as this:

next if ws[row, 1].nil? || ws[row, 2].nil? || ws[row, 3].nil?

The numbers are "magical." This is bad for two reasons: One is that they don't communicate their meaning. The other is that the same numbers, with the same meaning, appear elsewhere, but there is nothing in the code to tell the read that the "2" on this line has the same meaning as the "2" on this line (note: I've broken the line into several lines for ease of reading):

voila_norbert_results = with_error_handling { 
  voila_norbert(browser, ws[row, 1], ws[row, 2], ws[row, 3])
}

Instead, you can use constants to stand for these numbers and give them meaning. I like to put related constants in a module, like this:

module Columns
  FIRST_NAME = 1
  LAST_NAME = 2
  DOMAIN = 3
end

And then use them like this:

next if ws[row, Columns::FIRST_NAME].nil? || 
        ws[row, Columns::LAST_NAME].nil? || 
        ws[row, Columns::DOMAIN].nil?

Put long or complicated conditions in a separate method

This line: #Next row in spreadsheet if first name, last name or domain is missing in the row next if ws[row, 1].nil? || ws[row, 2].nil? || ws[row, 3].nil?

Would be nicer to read with the condition in a separate method. Doing this allows you to give the method a name that replaces the comment, essentially creating an executable comment:

def missing_required_column?(ws)
  ws[row, 1].nil? || ws[row, 2].nil? || ws[row, 3].nil?
end

...

next if missing_required_column?(ws)

Prefer implicit rather than explicit return

This code:

unless emails.nil?
    return emails.first
else
    return 'Not Found'
end

could be better expressed as:

unless emails.nil?
  emails.first
else
  'Not Found'
end

This works because the value of the last executed expression becomes the return value from the method. Also, an if or unless is itself an expression which has a value.

There's another way to express this code, though. I know I just told you to prefer explicit return, but implicit return communicates really well when you are handling an abnormal condition. So I'd do this instead:

return "Not Found" if emails.nil?
emails.first
\$\endgroup\$

You must log in to answer this question.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.