153 lines
No EOL
6.3 KiB
Python
153 lines
No EOL
6.3 KiB
Python
from camoufox.sync_api import Camoufox
|
|
from bs4 import BeautifulSoup
|
|
from urllib.parse import urlsplit,parse_qsl
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
import time
|
|
import os
|
|
|
|
hostName = os.getenv("LISTEN_HOSTNAME","localhost")
|
|
serverPort = os.getenv("LISTEN_PORT", 9090)
|
|
|
|
class MyServer(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
self.send_response(200)
|
|
self.send_header("Content-type", "text/html")
|
|
self.end_headers()
|
|
if self.getsearchstring() == None:
|
|
with open("nosearch.html") as file:
|
|
html = file.read()
|
|
self.wfile.write(bytes(html, "utf-8"))
|
|
else:
|
|
#debug swap these over, not good for production
|
|
with Camoufox(headless="virtual") as browser:
|
|
#with Camoufox(headless=False) as browser:
|
|
page = browser.new_page()
|
|
page.goto("https://google.com/search?udm=14&q="+self.getsearchstring())
|
|
try:
|
|
page.locator('xpath=//*[@id="W0wltc"]').click()
|
|
except:
|
|
suppressedError = true
|
|
if self.getpage() != 1:
|
|
if self.getpage() > 10:
|
|
page.get_by_text("10", exact=True).click()
|
|
if self.getpage() > 19:
|
|
page.get_by_text("19", exact=True).click()
|
|
page.get_by_text(self.getpagestr(), exact=True).click()
|
|
time.sleep(4)
|
|
page.wait_for_url("**search**")
|
|
html = page.content()
|
|
with open("/tmp/ghost.html", "w", encoding="utf-8") as f:
|
|
f.write(html)
|
|
self.wfile.write(bytes(googlepage_to_cleanhtml(html,self.getsearchstring(),self.getpage()), "utf-8"))
|
|
|
|
def getsearchstring(self):
|
|
spl = urlsplit(self.path)
|
|
params = parse_qsl(spl.query)
|
|
for item in params:
|
|
if item[0] == "q":
|
|
return item[1]
|
|
return None
|
|
|
|
def getpage(self):
|
|
spl = urlsplit(self.path)
|
|
params = parse_qsl(spl.query)
|
|
for item in params:
|
|
if item[0] == "page":
|
|
return int(item[1])
|
|
return 1
|
|
|
|
def getpagestr(self):
|
|
spl = urlsplit(self.path)
|
|
params = parse_qsl(spl.query)
|
|
for item in params:
|
|
if item[0] == "page":
|
|
return item[1]
|
|
return "1"
|
|
|
|
|
|
def geturlfromdata(data):
|
|
actual_url = parse_qsl(data)
|
|
for item in actual_url:
|
|
if item[0] == "url":
|
|
return item[1]
|
|
return None
|
|
|
|
def googlepage_to_cleanhtml(googlehtml,searchstring="",currentpage=1):
|
|
soup = BeautifulSoup(googlehtml, "lxml")
|
|
searchresults = soup.find_all("div", attrs={"id":"search"})[0]
|
|
preamble = """<html><head><title>Ghostgle</title></head><body><h1>Ghostgle</h1><form id="search-form" action="search" method="get">
|
|
<div class="search-fields" style="display:flex; flex=1">
|
|
<div class="autocomplete">
|
|
|
|
<input
|
|
type="text"
|
|
name="q"
|
|
id="search-bar"
|
|
class="home-search"
|
|
autofocus="autofocus"
|
|
autocapitalize="none"
|
|
spellcheck="false"
|
|
autocorrect="off"
|
|
autocomplete="off"
|
|
dir="auto">
|
|
</div>
|
|
<input type="submit" id="search-submit" value="Search">
|
|
</div>
|
|
</form><h2>Web results(only kind ghostgle does)</h2>"""
|
|
postamble = "</body></html>"
|
|
results = ""
|
|
for h3 in searchresults.select("h3"):
|
|
preview = str(h3.parent.parent.parent.parent.parent.next_sibling.div)
|
|
heading = h3.string
|
|
cite = h3.parent.cite
|
|
del cite["class"]
|
|
if len(list(cite.children)) > 1:
|
|
del cite.span["class"]
|
|
if h3.parent.img:
|
|
imagetag = "<img src=\""+h3.parent.img["src"]+"\" style=\"height:26px;width:26px;float:left;max-width:26px;\"/>"
|
|
else:
|
|
imagetag = '<svg focusable="false" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" style="max-width:26px;float:left;"><path d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm-1 17.93c-3.95-.49-7-3.85-7-7.93 0-.62.08-1.21.21-1.79L9 15v1c0 1.1.9 2 2 2v1.93zm6.9-2.54c-.26-.81-1-1.39-1.9-1.39h-1v-3c0-.55-.45-1-1-1H8v-2h2c.55 0 1-.45 1-1V7h2c1.1 0 2-.9 2-2v-.41c2.93 1.19 5 4.06 5 7.41 0 2.08-.8 3.97-2.1 5.39z"></path></svg>'
|
|
source = h3.next_sibling.next_sibling.div
|
|
source.span.extract()
|
|
del source.span["class"]
|
|
source = source.span
|
|
try:
|
|
url = geturlfromdata(h3.parent['data-sb'])
|
|
except:
|
|
url = h3.parent['href']
|
|
results = results+imagetag+"<div class=\"source\">"+str(source)+"<br/>"+str(cite)+"</div><h3><a href=\""+url+"\">"+heading+"</a></h3><p>"+preview+"</p>"
|
|
navigation = soup.find("td", attrs={"role":"heading"}).parent
|
|
pagetds = navigation.find_all("td")
|
|
del pagetds[0]
|
|
pages = '<div id="pages" style="display:flex; flex=1">'
|
|
for page in pagetds:
|
|
if page.a != None:
|
|
try:
|
|
# we check if it's an int because "Next" and international versions of "Next" aren't.
|
|
page_num_str = int(page.get_text())
|
|
page_num_str = page.get_text()
|
|
except:
|
|
page_num_str = str(currentpage+1)
|
|
pages = pages + "<div style=\"minwidth=14px\"><a href=\"?q="+searchstring+"&page="+page_num_str+"\">"+page.get_text()+"</a> </div>"
|
|
else:
|
|
pages = pages + "<div style=\"minwidth=14px\">"+page.get_text()+" </div>"
|
|
pages = pages + "</div>"
|
|
return preamble+results+pages+postamble
|
|
|
|
#debug swap these over, not good for production
|
|
#with Camoufox(headless=False) as browser:
|
|
with Camoufox(headless="virtual") as browser:
|
|
page = browser.new_page()
|
|
page.goto("https://google.com?udm=14")
|
|
|
|
if __name__ == "__main__":
|
|
webServer = HTTPServer((hostName, serverPort), MyServer)
|
|
print("Server started http://%s:%s" % (hostName, serverPort))
|
|
|
|
try:
|
|
webServer.serve_forever()
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
webServer.server_close()
|
|
print("Server stopped.") |