serpapi / google-search-results-python Goto Github PK

# stop if backend miss to return serpapi_pagination
if not 'serpapi_pagination' in result:
  raise StopIteration

# stop if no next page
if not 'next' in result['serpapi_pagination']:
    raise StopIteration

SSLCertVerificationError Traceback (most recent call last)
/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
698 # Make the request on the httplib connection object.
--> 699 httplib_response = self._make_request(
700 conn,

SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1125)

from serpapi import GoogleSearch
search = GoogleSearch({
    "q": "coffee", 
    "location": "Austin,Texas",
    "api_key": "<your secret api key>"
  })
result = search.get_dict()

Collecting google-search-results
  Using cached https://files.pythonhosted.org/packages/08/eb/38646304d98db83d85f57599d2ccc8caf325961e8792100a1014950197a6/google_search_results-1.5.2.tar.gz
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "/private/var/folders/3m/91gj9l890y71886_7sfndl3r0000gn/T/pip-install-YVqFKL/google-search-results/setup.py", line 7, in <module>
        with open(path.join(here, 'SHORT_README.rst'), encoding='utf-8') as f:
      File "/usr/local/Cellar/python@2/2.7.15_3/Frameworks/Python.framework/Versions/2.7/lib/python2.7/codecs.py", line 898, in open
        file = __builtin__.open(filename, mode, buffering)
    IOError: [Errno 2] No such file or directory: '/private/var/folders/3m/91gj9l890y71886_7sfndl3r0000gn/T/pip-install-YVqFKL/google-search-results/SHORT_README.rst'

    ----------------------------------------
Command "python setup.py egg_info" failed with error code 1 in /private/var/folders/3m/91gj9l890y71886_7sfndl3r0000gn/T/pip-install-YVqFKL/google-search-results/

~ ❯❯❯ pip --version
pip 19.0.2 from /usr/local/lib/python2.7/site-packages/pip (python 2.7)
~ ❯❯❯ python --version
Python 2.7.15

params = {
    'q': 'Aspen Pumps Ltd',
    'engine': 'google',
    'api_key': <api_key>,
    'num': 100
  }

result_set = GoogleSearchResults(params).get_dict()

print(result_set.keys())

dict_keys(['search_metadata', 'search_parameters', 'search_information', 'ads', 'shopping_results', 'organic_results', 'related_searches', 'pagination', 'serpapi_pagination'])

---------------------------------------------------------------------------                                                                                                                                   JSONDecodeError                           Traceback (most recent call last)
<ipython-input-8-b6d24cb47bf7> in <module>
----> 1 GoogleSearchResults({}).get_search_archive(search_id='5df0db57ab3f5837994cd5a1', format='html')

C:\ProgramData\Anaconda3\lib\site-packages\serpapi\serp_api_client.py in get_search_archive(self, search_id, format)
78             dict|string: search result from the archive
79         """
---> 80         return json.loads(self.get_results("/searches/{0}.{1}".format(search_id, format)))
81
82     def get_account(self):

C:\ProgramData\Anaconda3\lib\json\__init__.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
352             parse_int is None and parse_float is None and
353             parse_constant is None and object_pairs_hook is None and not kw):
--> 354         return _default_decoder.decode(s)
355     if cls is None:
356         cls = JSONDecoder

C:\ProgramData\Anaconda3\lib\json\decoder.py in decode(self, s, _w)
337
338         """
--> 339         obj, end = self.raw_decode(s, idx=_w(s, 0).end())
340         end = _w(s, end).end()
341         if end != len(s):

C:\ProgramData\Anaconda3\lib\json\decoder.py in raw_decode(self, s, idx)
355             obj, end = self.scan_once(s, idx)
356         except StopIteration as err:
--> 357             raise JSONDecodeError("Expecting value", s, err.value) from None
358         return obj, end

JSONDecodeError: Expecting value: line 1 column 1 (char 0)

from langchain import SerpAPIWrapper
from langchain.requests import RequestsWrapper
import requests
#verfy=False
search = SerpAPIWrapper(serpapi_api_key="api")
results = search.run("how to make a cake")`
```

https://serpapi.com/search
---------------------------------------------------------------------------
ConnectionResetError                      Traceback (most recent call last)
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    676                 headers=headers,
--> 677                 chunked=chunked,
    678             )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    380         try:
--> 381             self._validate_conn(conn)
    382         except (SocketTimeout, BaseSSLError) as e:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
    977         if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
--> 978             conn.connect()
    979 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connection.py in connect(self)
    370             server_hostname=server_hostname,
--> 371             ssl_context=context,
    372         )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/util/ssl_.py in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data)
    385         if HAS_SNI and server_hostname is not None:
--> 386             return context.wrap_socket(sock, server_hostname=server_hostname)
    387 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
    406                          server_hostname=server_hostname,
--> 407                          _context=self, _session=session)
    408 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in __init__(self, sock, keyfile, certfile, server_side, cert_reqs, ssl_version, ca_certs, do_handshake_on_connect, family, type, proto, fileno, suppress_ragged_eofs, npn_protocols, ciphers, server_hostname, _context, _session)
    816                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
--> 817                     self.do_handshake()
    818 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self, block)
   1076                 self.settimeout(None)
-> 1077             self._sslobj.do_handshake()
   1078         finally:

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self)
    688         """Start the SSL/TLS handshake."""
--> 689         self._sslobj.do_handshake()
    690         if self.context.check_hostname:

ConnectionResetError: [Errno 104] Connection reset by peer

During handling of the above exception, another exception occurred:

ProtocolError                             Traceback (most recent call last)
/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    448                     retries=self.max_retries,
--> 449                     timeout=timeout
    450                 )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    726             retries = retries.increment(
--> 727                 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
    728             )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    409             if read is False or not self._is_method_retryable(method):
--> 410                 raise six.reraise(type(error), error, _stacktrace)
    411             elif read is not None:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
    733             if value.__traceback__ is not tb:
--> 734                 raise value.with_traceback(tb)
    735             raise value

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    676                 headers=headers,
--> 677                 chunked=chunked,
    678             )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    380         try:
--> 381             self._validate_conn(conn)
    382         except (SocketTimeout, BaseSSLError) as e:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
    977         if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
--> 978             conn.connect()
    979 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/connection.py in connect(self)
    370             server_hostname=server_hostname,
--> 371             ssl_context=context,
    372         )

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/urllib3/util/ssl_.py in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data)
    385         if HAS_SNI and server_hostname is not None:
--> 386             return context.wrap_socket(sock, server_hostname=server_hostname)
    387 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
    406                          server_hostname=server_hostname,
--> 407                          _context=self, _session=session)
    408 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in __init__(self, sock, keyfile, certfile, server_side, cert_reqs, ssl_version, ca_certs, do_handshake_on_connect, family, type, proto, fileno, suppress_ragged_eofs, npn_protocols, ciphers, server_hostname, _context, _session)
    816                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
--> 817                     self.do_handshake()
    818 

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self, block)
   1076                 self.settimeout(None)
-> 1077             self._sslobj.do_handshake()
   1078         finally:

/anaconda/envs/azureml_py36/lib/python3.6/ssl.py in do_handshake(self)
    688         """Start the SSL/TLS handshake."""
--> 689         self._sslobj.do_handshake()
    690         if self.context.check_hostname:

ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)
<ipython-input-26-45ac328ca8f8> in <module>
      1 question = 'where to get best coffee'
----> 2 results = performSearch(question)

<ipython-input-25-5bc778bad4e2> in performSearch(question)
     12 
     13     search = GoogleSearch(params)
---> 14     results = search.get_dict()
     15     return results

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_dict(self)
    101             (alias for get_dictionary)
    102         """
--> 103         return self.get_dictionary()
    104 
    105     def get_object(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_dictionary(self)
     94             Dict with the formatted response content
     95         """
---> 96         return dict(self.get_json())
     97 
     98     def get_dict(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_json(self)
     81         """
     82         self.params_dict["output"] = "json"
---> 83         return json.loads(self.get_results())
     84 
     85     def get_raw_json(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_results(self, path)
     68             Response text field
     69         """
---> 70         return self.get_response(path).text
     71 
     72     def get_html(self):

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/serpapi/serp_api_client.py in get_response(self, path)
     57             url, parameter = self.construct_url(path)
     58             print(url)
---> 59             response = requests.get(url, parameter, timeout=self.timeout)
     60             return response
     61         except requests.HTTPError as e:

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/api.py in get(url, params, **kwargs)
     73     """
     74 
---> 75     return request('get', url, params=params, **kwargs)
     76 
     77 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/api.py in request(method, url, **kwargs)
     59     # cases, and look like a memory leak in others.
     60     with sessions.Session() as session:
---> 61         return session.request(method=method, url=url, **kwargs)
     62 
     63 

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    540         }
    541         send_kwargs.update(settings)
--> 542         resp = self.send(prep, **send_kwargs)
    543 
    544         return resp

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/sessions.py in send(self, request, **kwargs)
    653 
    654         # Send the request
--> 655         r = adapter.send(request, **kwargs)
    656 
    657         # Total elapsed time of the request (approximately)

/anaconda/envs/azureml_py36/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    496 
    497         except (ProtocolError, socket.error) as err:
--> 498             raise ConnectionError(err, request=request)
    499 
    500         except MaxRetryError as e:

ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer'))

Python 3.9.10 (main, Mar  1 2022, 21:02:54) 
[GCC 8.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import requests
>>> requests.get('https://serpapi.com', {"api_key":VALID_API_KEY, "engine": "google_jobs", "q": "Barista"})
Fatal Python error: Segmentation fault

Current thread 0x0000ffff8e999010 (most recent call first):
  File "/usr/local/lib/python3.9/ssl.py", line 1173 in send
  File "/usr/local/lib/python3.9/ssl.py", line 1204 in sendall
  File "/usr/local/lib/python3.9/http/client.py", line 1001 in send
  File "/usr/local/lib/python3.9/http/client.py", line 1040 in _send_output
  File "/usr/local/lib/python3.9/http/client.py", line 1280 in endheaders
  File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 395 in request
  File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 496 in _make_request
  File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790 in urlopen
  File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 486 in send
  File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703 in send
  File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589 in request
  File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 59 in request
  File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 73 in get
  File "<stdin>", line 1 in <module>
Segmentation fault

FROM python:3.9.10-slim-buster

ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1

# OLD: RUN apt-get update && apt-get upgrade -y && apt-get install gcc -y && apt-get install apt-utils -y

# Install build-essential for celery worker otherwise it says gcc not found
RUN apt-get update \
  # dependencies for building Python packages
  && apt-get install -y build-essential \
  # psycopg2 dependencies
  && apt-get install -y libpq-dev \
  # Additional dependencies
  && apt-get install -y telnet netcat \
  # cleaning up unused files
  && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
  && rm -rf /var/lib/apt/lists/*

WORKDIR /app

COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt

COPY ./compose/local/flask/start /start
RUN sed -i 's/\r$//g' /start
RUN chmod +x /start

# COPY ./compose/local/flask/celery/worker/start /start-celeryworker
# RUN sed -i 's/\r$//g' /start-celeryworker
# RUN chmod +x /start-celeryworker

# COPY ./compose/local/flask/celery/beat/start /start-celerybeat
# RUN sed -i 's/\r$//g' /start-celerybeat
# RUN chmod +x /start-celerybeat

# COPY ./compose/local/flask/celery/flower/start /start-flower
# RUN sed -i 's/\r$//g' /start-flower
# RUN chmod +x /start-flower

COPY . .

# COPY entrypoint.sh /usr/local/bin/
# ENTRYPOINT ["entrypoint.sh"]

version: "3.9"

services:
  flask_app:
    restart: always
    container_name: flask_app
    image: meder/flask_live_app:1.0.0
    command: /start
    build: .
    ports:
      - "4000:4000"
    volumes:
      - .:/app
    env_file:
      - local.env
    environment:
      - FLASK_ENV=development
      - FLASK_APP=app.py
    depends_on:
      - db
  db:
    container_name: flask_db
    image: postgres:16.1-alpine
    ports:
      - "5432:5432"
    environment:
      - POSTGRES_USER=USER
      - POSTGRES_PASSWORD=PW
      - POSTGRES_DB=DB
    volumes: 
      - postgres_data:/var/lib/postgresql/data
  redis:
    container_name: redis
    image: redis:7.2-alpine
    ports:
      - "6379:6379"
volumes:
  postgres_data: {}

flask==3.0.0
psycopg2-binary==2.9.9
google-search-results==2.4.2

mkdir deps
python3 -m pip install --target deps "google-search-results==2.4.2"
ls -1 deps/tests

__init__.py
__pycache__
test_account_api.py
(etc)

from serpapi import GoogleSearch
ImportError: cannot import name 'GoogleSearch' from 'serpapi' (C:\Users\XXXX\AppData\Local\Programs\Python\Python310\lib\site-packages\serpapi\__init__.py)

from serpapi import GoogleSearch

params = {
  "q": "What's the definition of transparent?",
  "hl": "en",
  "gl": "us",
  "api_key": ""
}

search = GoogleSearch(params)
results = search.get_dict()
answer_box = results['answer_box']

from serpapi import GoogleSearch
import pandas as pd

PARAMS = {'engine': 'google_trends',
          'data_type' : 'RELATED_QUERIES',
          'q' : "health insurance",
          'geo': "IE",
          'date' : "2022-01-01 2022-12-31",
          'hl' : 'en-GB',
          'csv' : True,
          'api_key' : '[Key]'}

search = GoogleSearch(PARAMS) 
results = search.get_dict() 

rel = results['related_queries']['top']
df = pd.DataFrame(rel)

df[["query", "value"]]

df.to_csv("serpapi_results.csv")