from camoufox.sync_api import Camoufox from bs4 import BeautifulSoup from urllib.parse import urlsplit,parse_qsl from http.server import BaseHTTPRequestHandler, HTTPServer import time import os hostName = os.getenv("LISTEN_HOSTNAME","localhost") serverPort = os.getenv("LISTEN_PORT", 9090) class MyServer(BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() if self.getsearchstring() == None: with open("nosearch.html") as file: html = file.read() self.wfile.write(bytes(html, "utf-8")) else: #debug swap these over, not good for production with Camoufox(headless="virtual") as browser: #with Camoufox(headless=False) as browser: page = browser.new_page() page.goto("https://google.com/search?udm=14&q="+self.getsearchstring()) try: page.locator('xpath=//*[@id="W0wltc"]').click() except: suppressedError = true if self.getpage() != 1: if self.getpage() > 10: page.get_by_text("10", exact=True).click() if self.getpage() > 19: page.get_by_text("19", exact=True).click() page.get_by_text(self.getpagestr(), exact=True).click() time.sleep(4) page.wait_for_url("**search**") html = page.content() with open("/tmp/ghost.html", "w", encoding="utf-8") as f: f.write(html) self.wfile.write(bytes(googlepage_to_cleanhtml(html,self.getsearchstring(),self.getpage()), "utf-8")) def getsearchstring(self): spl = urlsplit(self.path) params = parse_qsl(spl.query) for item in params: if item[0] == "q": return item[1] return None def getpage(self): spl = urlsplit(self.path) params = parse_qsl(spl.query) for item in params: if item[0] == "page": return int(item[1]) return 1 def getpagestr(self): spl = urlsplit(self.path) params = parse_qsl(spl.query) for item in params: if item[0] == "page": return item[1] return "1" def geturlfromdata(data): actual_url = parse_qsl(data) for item in actual_url: if item[0] == "url": return item[1] return None def googlepage_to_cleanhtml(googlehtml,searchstring="",currentpage=1): soup = BeautifulSoup(googlehtml, "lxml") searchresults = soup.find_all("div", attrs={"id":"search"})[0] preamble = """Ghostgle

Ghostgle

Web results(only kind ghostgle does)

""" postamble = "" results = "" for h3 in searchresults.select("h3"): preview = str(h3.parent.parent.parent.parent.parent.next_sibling.div) heading = h3.string cite = h3.parent.cite del cite["class"] if len(list(cite.children)) > 1: del cite.span["class"] if h3.parent.img: imagetag = "" else: imagetag = '' source = h3.next_sibling.next_sibling.div source.span.extract() del source.span["class"] source = source.span try: url = geturlfromdata(h3.parent['data-sb']) except: url = h3.parent['href'] results = results+imagetag+"
"+str(source)+"
"+str(cite)+"

"+heading+"

"+preview+"

" navigation = soup.find("td", attrs={"role":"heading"}).parent pagetds = navigation.find_all("td") del pagetds[0] pages = '
' for page in pagetds: if page.a != None: try: # we check if it's an int because "Next" and international versions of "Next" aren't. page_num_str = int(page.get_text()) page_num_str = page.get_text() except: page_num_str = str(currentpage+1) pages = pages + "
"+page.get_text()+"  
" else: pages = pages + "
"+page.get_text()+"  
" pages = pages + "
" return preamble+results+pages+postamble #debug swap these over, not good for production #with Camoufox(headless=False) as browser: with Camoufox(headless="virtual") as browser: page = browser.new_page() page.goto("https://google.com?udm=14") if __name__ == "__main__": webServer = HTTPServer((hostName, serverPort), MyServer) print("Server started http://%s:%s" % (hostName, serverPort)) try: webServer.serve_forever() except KeyboardInterrupt: pass webServer.server_close() print("Server stopped.")