Thread safety issue with rails 4 about google-api-ads-ruby HOT 6 CLOSED

# purge_adgroups_without_keywords.rb
class PurgeAdgroupsWithoutKeywords
  include Sidekiq::Worker

  sidekiq_options retry: 2, backtrace: true

  def perform(adwords_campaign_id, test_run = false)
    adw = AdwordsCampaign.find adwords_campaign_id
    extraction = adw.extraction
    user = adw.user
    company = adw.company

    valid_for_sync = adw.adwords_registered? && extraction.present?

    return false if !valid_for_sync

    Rails.logger.info "Performing adgroup purge for campaign #{adw.id}"

    # Code below
    adgroups                  = adw.adwords.get_adgroups.map{|adgr| adgr[:id]}
    # Code below, everything fails here.
    adgroups_with_keywords    = adw.adwords.get_keywords(include_pending: true, ignore_index_limit: true).keys
    adgroups_without_keywords = adgroups - adgroups_with_keywords

    Rails.logger.info "Adgroups with keywords for campaign #{adw.id}: #{adgroups_with_keywords.join(' ')}"
    Rails.logger.info "Adgroups without keywords for campaign #{adw.id}: #{adgroups_without_keywords.join(' ')}"

    if !test_run
      # Code below
      adw.adwords.delete_adgroups_by_ids(adgroups_without_keywords)
      extraction.pages.where(adgroup_id: adgroups_without_keywords).update_all(adgroup_id: nil)

      Rails.logger.info "Removed #{adgroups_without_keywords.count} adgroups from AdWords"
    end
    adgroups_without_keywords
  end
end

  # adwords_helper.rb
  # Get all adgroups from a campaign
  def get_adgroups
    @api = get_adwords_api

    service = @api.service(:AdGroupService, API_VERSION)

    selector = {
      :fields => ['Id', 'Name'],
      :predicates => [
        {:field => 'CampaignId', :operator => 'IN', :values => [self.campaign_id]},
        {:field => 'Status', :operator => 'IN', :values => ['ENABLED']},
      ],
      :paging => {
        :start_index => 0,
        :number_results => 500
      }
    }

    offset, page = 0, {}
    result = []

    begin
      page = service.get(selector)

      if page[:entries]
        page[:entries].each do |adgroup|
          result.push({:id => adgroup[:id], :name => adgroup[:name]})
        end
        offset += 500
        selector[:paging][:start_index] = offset
      end

      sleep(@@RETRY_INTERVAL)
    end while page[:total_num_entries] > offset

    result
  end

  # Get all keywords of adgroups
  def get_keywords(limit_to_adgroups: nil, include_pending: false, ignore_index_limit: false)
    @api = get_adwords_api

    service = @api.service(:AdGroupCriterionService, API_VERSION)
    selector = {
      :fields => ['Id', 'AdGroupId', 'KeywordText', 'KeywordMatchType', 'CpcBid'],
      :predicates => [
        {:field => 'CampaignId', :operator => 'IN', :values => [self.campaign_id]},
        {:field => 'CriteriaType', :operator => 'IN', :values => ['KEYWORD']},
        {:field => 'Status', :operator => 'IN', :values => ['ENABLED']},
        {:field => 'ApprovalStatus', :operator => 'IN', :values => (include_pending ? ['APPROVED', 'PENDING_REVIEW', 'UNDER_REVIEW'] : ['APPROVED'])}
      ],
      :paging => {
        :start_index => 0,
        :number_results => 500
      }
    }

    if limit_to_adgroups
      selector[:predicates].push({:field => 'AdGroupId', :operator => 'IN', :values => limit_to_adgroups})
    elsif self.adgroup_id
      selector[:predicates].push({:field => 'AdGroupId', :operator => 'IN', :values => [self.adgroup_id]})
    end

    offset, page = 0, {}
    result = {}

    begin
      # Everything fails here.
      page = service.get(selector)

      if page[:entries]
        page[:entries].each do |keyword|
          result[keyword[:ad_group_id].to_i] ||= []
          result[keyword[:ad_group_id].to_i].push({:id => keyword[:criterion][:id], :text => keyword[:criterion][:text], :match_type => keyword[:criterion][:match_type], :bid => keyword[:bidding_strategy_configuration][:bids].detect{|bid| bid[:bids_type] == 'CpcBid'}[:bid]})
        end
        offset += 500
        selector[:paging][:start_index] = offset

        # Maximum start index in this call
        if offset > 100000 && !ignore_index_limit
          Rails.logger.info "Can't fetch more than 100k keywords for campaign #{self.campaign_id}"
          break
        end
      end

      sleep(@@RETRY_INTERVAL)
    end while page[:total_num_entries] > offset

    result
  end

  # Deletes specific adgroups
  def delete_adgroups_by_ids(ids)
    unless perform_dangerous_operations?
      return false
    end
    @api = get_adwords_api

    service = @api.service(:MutateJobService, API_VERSION)

    operations = ids.map{|id| {
      :xsi_type => 'AdGroupOperation',
      :operator => 'SET',
      :operand => {
        :id => id,
        :status => 'REMOVED'
      }
    }}

    operations.each_slice(1000) do |ops|
      response = service.mutate(ops, {:prerequisite_job_ids => []})
      job_id = response[:id]
      status = poll_job(service, job_id)

      if status != 'COMPLETED'
        Rails.logger.fatal "Could not complete deletion of adgroups"
      end

      sleep(@@RETRY_INTERVAL)
    end
  end

c:0048 p:---- s:0249 e:000248 CFUNC  :initialize
c:0047 p:---- s:0247 e:000246 CFUNC  :new
c:0046 p:0030 s:0244 e:000243 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/2.1.0/set.rb:81 [FINISH]
c:0045 p:---- s:0239 e:000238 CFUNC  :new
c:0044 p:2070 s:0236 e:000234 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/2.1.0/rexml/parsers/baseparser.rb:379
c:0043 p:0007 s:0204 e:000203 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/2.1.0/rexml/parsers/baseparser.rb:184
c:0042 p:0042 s:0201 e:000200 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/nori-1.1.5/lib/nori/parser/rexml.rb:16
c:0041 p:0027 s:0192 e:000191 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/nori-1.1.5/lib/nori/parser.rb:31
c:0040 p:0036 s:0186 e:000185 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/nori-1.1.5/lib/nori.rb:12
c:0039 p:0030 s:0181 e:000180 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/savon-1.2.0/lib/savon/soap/response.rb:85
c:0038 p:0007 s:0178 e:000177 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/savon-1.2.0/lib/savon/soap/response.rb:63
c:0037 p:0035 s:0175 e:000174 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/google-ads-common-0.9.5/lib/ads_common/results_extractor.rb:39
c:0036 p:0100 s:0167 e:000166 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/google-ads-common-0.9.5/lib/ads_common/savon_service.rb:85
c:0035 p:0013 s:0156 e:000155 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/google-adwords-api-0.13.2/lib/adwords_api/v201406/ad_group_criter
c:0034 p:0300 s:0151 e:000150 METHOD /home/deployer/apps/robot/lib/adwords_helper.rb:693
c:0033 p:0129 s:0139 e:000138 METHOD /home/deployer/apps/robot/app/workers/purge_adgroups_without_keywords.rb:19
c:0032 p:0010 s:0126 e:000125 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/bundler/gems/sidekiq-5047f9220b82/lib/sidekiq/processor.rb:75
c:0031 p:0020 s:0121 e:000120 BLOCK  /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/bundler/gems/sidekiq-5047f9220b82/lib/sidekiq/processor.rb:52 [FINISH]
c:0030 p:---- s:0119 e:000118 CFUNC  :call
c:0029 p:0016 s:0116 e:000115 LAMBDA /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/bundler/gems/sidekiq-5047f9220b82/lib/sidekiq/middleware/chain.rb:127
c:0028 p:0038 s:0114 e:000113 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/bundler/gems/sidetiq-f3668e03c72c/lib/sidetiq/middleware/history.rb:8
c:0027 p:0030 s:0107 e:000106 LAMBDA /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/bundler/gems/sidekiq-5047f9220b82/lib/sidekiq/middleware/chain.rb:129
c:0026 p:0014 s:0105 e:000104 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/bundler/gems/sidekiq-failures-1bddc5c12cea/lib/sidekiq/failures/middle
c:0025 p:0030 s:0097 e:000096 LAMBDA /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/bundler/gems/sidekiq-5047f9220b82/lib/sidekiq/middleware/chain.rb:129
c:0024 p:0024 s:0095 e:000094 METHOD /home/deployer/.rbenv/versions/2.1.5/lib/ruby/gems/2.1.0/gems/sidekiq-pro-1.9.1/lib/sidekiq/batch/middleware.rb:26

  @@config = 'adwords_api.yml'

  def initialize(args = {})
    self.adwords_id = AdwordsConfig.config[:master_account]

    return unless args.is_a?(Hash)
    args.each do |k,v|
      instance_variable_set("@#{k}", v) unless v.nil?
    end
  end

  def get_adwords_api
    @api ||= create_adwords_api
    return @api
  end

  # Creates an instance of AdWords API class. Uses a configuration file and
  # Rails config directory.
  def create_adwords_api
    config_filename = File.join(Rails.root, 'config', @@config)
    config = YAML::load_file(config_filename)

    if self.adwords_id
      config[:authentication][:client_customer_id] = self.adwords_id
    end

    config[:authentication][:oauth2_token].merge!(self.credentials) if self.credentials

    @api = AdwordsApi::Api.new(config)
    @api.logger = logger if logger

    return @api
  end