It only bloody works!
This commit is contained in:
parent
09d32a1e39
commit
43dae28a36
4 changed files with 223 additions and 0 deletions
30
Dockerfile
Normal file
30
Dockerfile
Normal file
|
|
@ -0,0 +1,30 @@
|
||||||
|
# temp stage
|
||||||
|
FROM python:3.12.2-slim AS builder
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1
|
||||||
|
ENV PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y --no-install-recommends gcc
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip wheel --no-cache-dir --no-deps --wheel-dir /app/wheels -r requirements.txt
|
||||||
|
|
||||||
|
|
||||||
|
# final stage
|
||||||
|
FROM python:3.12.2-slim
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# mostly install firefox to ensure all the libs are there
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y --no-install-recommends firefox-esr xvfb
|
||||||
|
|
||||||
|
COPY --from=builder /app/wheels /wheels
|
||||||
|
COPY --from=builder /app/requirements.txt .
|
||||||
|
RUN pip install --no-cache /wheels/*
|
||||||
|
RUN python -m camoufox fetch
|
||||||
|
COPY nosearch.html /app/nosearch.html
|
||||||
|
COPY main.py /app/main.py
|
||||||
153
main.py
Normal file
153
main.py
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
from camoufox.sync_api import Camoufox
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from urllib.parse import urlsplit,parse_qsl
|
||||||
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||||
|
import time
|
||||||
|
import os
|
||||||
|
|
||||||
|
hostName = os.getenv("LISTEN_HOSTNAME","localhost")
|
||||||
|
serverPort = os.getenv("LISTEN_PORT", 9090)
|
||||||
|
|
||||||
|
class MyServer(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self):
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-type", "text/html")
|
||||||
|
self.end_headers()
|
||||||
|
if self.getsearchstring() == None:
|
||||||
|
with open("nosearch.html") as file:
|
||||||
|
html = file.read()
|
||||||
|
self.wfile.write(bytes(html, "utf-8"))
|
||||||
|
else:
|
||||||
|
#debug swap these over, not good for production
|
||||||
|
with Camoufox(headless="virtual") as browser:
|
||||||
|
#with Camoufox(headless=False) as browser:
|
||||||
|
page = browser.new_page()
|
||||||
|
page.goto("https://google.com/search?udm=14&q="+self.getsearchstring())
|
||||||
|
try:
|
||||||
|
page.locator('xpath=//*[@id="W0wltc"]').click()
|
||||||
|
except:
|
||||||
|
suppressedError = true
|
||||||
|
if self.getpage() != 1:
|
||||||
|
if self.getpage() > 10:
|
||||||
|
page.get_by_text("10", exact=True).click()
|
||||||
|
if self.getpage() > 19:
|
||||||
|
page.get_by_text("19", exact=True).click()
|
||||||
|
page.get_by_text(self.getpagestr(), exact=True).click()
|
||||||
|
time.sleep(4)
|
||||||
|
page.wait_for_url("**search**")
|
||||||
|
html = page.content()
|
||||||
|
with open("/tmp/ghost.html", "w", encoding="utf-8") as f:
|
||||||
|
f.write(html)
|
||||||
|
self.wfile.write(bytes(googlepage_to_cleanhtml(html,self.getsearchstring(),self.getpage()), "utf-8"))
|
||||||
|
|
||||||
|
def getsearchstring(self):
|
||||||
|
spl = urlsplit(self.path)
|
||||||
|
params = parse_qsl(spl.query)
|
||||||
|
for item in params:
|
||||||
|
if item[0] == "q":
|
||||||
|
return item[1]
|
||||||
|
return None
|
||||||
|
|
||||||
|
def getpage(self):
|
||||||
|
spl = urlsplit(self.path)
|
||||||
|
params = parse_qsl(spl.query)
|
||||||
|
for item in params:
|
||||||
|
if item[0] == "page":
|
||||||
|
return int(item[1])
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def getpagestr(self):
|
||||||
|
spl = urlsplit(self.path)
|
||||||
|
params = parse_qsl(spl.query)
|
||||||
|
for item in params:
|
||||||
|
if item[0] == "page":
|
||||||
|
return item[1]
|
||||||
|
return "1"
|
||||||
|
|
||||||
|
|
||||||
|
def geturlfromdata(data):
|
||||||
|
actual_url = parse_qsl(data)
|
||||||
|
for item in actual_url:
|
||||||
|
if item[0] == "url":
|
||||||
|
return item[1]
|
||||||
|
return None
|
||||||
|
|
||||||
|
def googlepage_to_cleanhtml(googlehtml,searchstring="",currentpage=1):
|
||||||
|
soup = BeautifulSoup(googlehtml, "lxml")
|
||||||
|
searchresults = soup.find_all("div", attrs={"id":"search"})[0]
|
||||||
|
preamble = """<html><head><title>Ghostgle</title></head><body><h1>Ghostgle</h1><form id="search-form" action="search" method="get">
|
||||||
|
<div class="search-fields" style="display:flex; flex=1">
|
||||||
|
<div class="autocomplete">
|
||||||
|
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
name="q"
|
||||||
|
id="search-bar"
|
||||||
|
class="home-search"
|
||||||
|
autofocus="autofocus"
|
||||||
|
autocapitalize="none"
|
||||||
|
spellcheck="false"
|
||||||
|
autocorrect="off"
|
||||||
|
autocomplete="off"
|
||||||
|
dir="auto">
|
||||||
|
</div>
|
||||||
|
<input type="submit" id="search-submit" value="Search">
|
||||||
|
</div>
|
||||||
|
</form><h2>Web results(only kind ghostgle does)</h2>"""
|
||||||
|
postamble = "</body></html>"
|
||||||
|
results = ""
|
||||||
|
for h3 in searchresults.select("h3"):
|
||||||
|
preview = str(h3.parent.parent.parent.parent.parent.next_sibling.div)
|
||||||
|
heading = h3.string
|
||||||
|
cite = h3.parent.cite
|
||||||
|
del cite["class"]
|
||||||
|
if len(list(cite.children)) > 1:
|
||||||
|
del cite.span["class"]
|
||||||
|
if h3.parent.img:
|
||||||
|
imagetag = "<img src=\""+h3.parent.img["src"]+"\" style=\"height:26px;width:26px;float:left;max-width:26px;\"/>"
|
||||||
|
else:
|
||||||
|
imagetag = '<svg focusable="false" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" style="max-width:26px;float:left;"><path d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm-1 17.93c-3.95-.49-7-3.85-7-7.93 0-.62.08-1.21.21-1.79L9 15v1c0 1.1.9 2 2 2v1.93zm6.9-2.54c-.26-.81-1-1.39-1.9-1.39h-1v-3c0-.55-.45-1-1-1H8v-2h2c.55 0 1-.45 1-1V7h2c1.1 0 2-.9 2-2v-.41c2.93 1.19 5 4.06 5 7.41 0 2.08-.8 3.97-2.1 5.39z"></path></svg>'
|
||||||
|
source = h3.next_sibling.next_sibling.div
|
||||||
|
source.span.extract()
|
||||||
|
del source.span["class"]
|
||||||
|
source = source.span
|
||||||
|
try:
|
||||||
|
url = geturlfromdata(h3.parent['data-sb'])
|
||||||
|
except:
|
||||||
|
url = h3.parent['href']
|
||||||
|
results = results+imagetag+"<div class=\"source\">"+str(source)+"<br/>"+str(cite)+"</div><h3><a href=\""+url+"\">"+heading+"</a></h3><p>"+preview+"</p>"
|
||||||
|
navigation = soup.find("td", attrs={"role":"heading"}).parent
|
||||||
|
pagetds = navigation.find_all("td")
|
||||||
|
del pagetds[0]
|
||||||
|
pages = '<div id="pages" style="display:flex; flex=1">'
|
||||||
|
for page in pagetds:
|
||||||
|
if page.a != None:
|
||||||
|
try:
|
||||||
|
# we check if it's an int because "Next" and international versions of "Next" aren't.
|
||||||
|
page_num_str = int(page.get_text())
|
||||||
|
page_num_str = page.get_text()
|
||||||
|
except:
|
||||||
|
page_num_str = str(currentpage+1)
|
||||||
|
pages = pages + "<div style=\"minwidth=14px\"><a href=\"?q="+searchstring+"&page="+page_num_str+"\">"+page.get_text()+"</a> </div>"
|
||||||
|
else:
|
||||||
|
pages = pages + "<div style=\"minwidth=14px\">"+page.get_text()+" </div>"
|
||||||
|
pages = pages + "</div>"
|
||||||
|
return preamble+results+pages+postamble
|
||||||
|
|
||||||
|
#debug swap these over, not good for production
|
||||||
|
#with Camoufox(headless=False) as browser:
|
||||||
|
with Camoufox(headless="virtual") as browser:
|
||||||
|
page = browser.new_page()
|
||||||
|
page.goto("https://google.com?udm=14")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
webServer = HTTPServer((hostName, serverPort), MyServer)
|
||||||
|
print("Server started http://%s:%s" % (hostName, serverPort))
|
||||||
|
|
||||||
|
try:
|
||||||
|
webServer.serve_forever()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
pass
|
||||||
|
|
||||||
|
webServer.server_close()
|
||||||
|
print("Server stopped.")
|
||||||
37
nosearch.html
Normal file
37
nosearch.html
Normal file
|
|
@ -0,0 +1,37 @@
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta name="referrer" content="no-referrer">
|
||||||
|
<title>Ghostgle Search</title>
|
||||||
|
</head>
|
||||||
|
<body id="main">
|
||||||
|
<div class="search-container">
|
||||||
|
<h1>Ghostgle</h1>
|
||||||
|
<form id="search-form" action="search" method="get">
|
||||||
|
<div class="search-fields" style="display:flex; flex=1">
|
||||||
|
<div class="autocomplete">
|
||||||
|
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
name="q"
|
||||||
|
id="search-bar"
|
||||||
|
class="home-search"
|
||||||
|
autofocus="autofocus"
|
||||||
|
autocapitalize="none"
|
||||||
|
spellcheck="false"
|
||||||
|
autocorrect="off"
|
||||||
|
autocomplete="off"
|
||||||
|
dir="auto">
|
||||||
|
</div>
|
||||||
|
<input type="submit" id="search-submit" value="Search">
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<footer>
|
||||||
|
<p class="footer">
|
||||||
|
Ghostgle Search v0.0.1 ||
|
||||||
|
<a class="link" href="https://example.com">Don't View on GitHub</a>
|
||||||
|
</p>
|
||||||
|
</footer>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
3
requirements.txt
Normal file
3
requirements.txt
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
cloverlabs-camoufox==0.6.0
|
||||||
|
bs4==0.0.2
|
||||||
|
playwright==1.60.0
|
||||||
Loading…
Add table
Reference in a new issue