It only bloody works!
This commit is contained in:
parent
09d32a1e39
commit
43dae28a36
4 changed files with 223 additions and 0 deletions
30
Dockerfile
Normal file
30
Dockerfile
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# temp stage
|
||||
FROM python:3.12.2-slim AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends gcc
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip wheel --no-cache-dir --no-deps --wheel-dir /app/wheels -r requirements.txt
|
||||
|
||||
|
||||
# final stage
|
||||
FROM python:3.12.2-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# mostly install firefox to ensure all the libs are there
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends firefox-esr xvfb
|
||||
|
||||
COPY --from=builder /app/wheels /wheels
|
||||
COPY --from=builder /app/requirements.txt .
|
||||
RUN pip install --no-cache /wheels/*
|
||||
RUN python -m camoufox fetch
|
||||
COPY nosearch.html /app/nosearch.html
|
||||
COPY main.py /app/main.py
|
||||
153
main.py
Normal file
153
main.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
from camoufox.sync_api import Camoufox
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.parse import urlsplit,parse_qsl
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
import time
|
||||
import os
|
||||
|
||||
hostName = os.getenv("LISTEN_HOSTNAME","localhost")
|
||||
serverPort = os.getenv("LISTEN_PORT", 9090)
|
||||
|
||||
class MyServer(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "text/html")
|
||||
self.end_headers()
|
||||
if self.getsearchstring() == None:
|
||||
with open("nosearch.html") as file:
|
||||
html = file.read()
|
||||
self.wfile.write(bytes(html, "utf-8"))
|
||||
else:
|
||||
#debug swap these over, not good for production
|
||||
with Camoufox(headless="virtual") as browser:
|
||||
#with Camoufox(headless=False) as browser:
|
||||
page = browser.new_page()
|
||||
page.goto("https://google.com/search?udm=14&q="+self.getsearchstring())
|
||||
try:
|
||||
page.locator('xpath=//*[@id="W0wltc"]').click()
|
||||
except:
|
||||
suppressedError = true
|
||||
if self.getpage() != 1:
|
||||
if self.getpage() > 10:
|
||||
page.get_by_text("10", exact=True).click()
|
||||
if self.getpage() > 19:
|
||||
page.get_by_text("19", exact=True).click()
|
||||
page.get_by_text(self.getpagestr(), exact=True).click()
|
||||
time.sleep(4)
|
||||
page.wait_for_url("**search**")
|
||||
html = page.content()
|
||||
with open("/tmp/ghost.html", "w", encoding="utf-8") as f:
|
||||
f.write(html)
|
||||
self.wfile.write(bytes(googlepage_to_cleanhtml(html,self.getsearchstring(),self.getpage()), "utf-8"))
|
||||
|
||||
def getsearchstring(self):
|
||||
spl = urlsplit(self.path)
|
||||
params = parse_qsl(spl.query)
|
||||
for item in params:
|
||||
if item[0] == "q":
|
||||
return item[1]
|
||||
return None
|
||||
|
||||
def getpage(self):
|
||||
spl = urlsplit(self.path)
|
||||
params = parse_qsl(spl.query)
|
||||
for item in params:
|
||||
if item[0] == "page":
|
||||
return int(item[1])
|
||||
return 1
|
||||
|
||||
def getpagestr(self):
|
||||
spl = urlsplit(self.path)
|
||||
params = parse_qsl(spl.query)
|
||||
for item in params:
|
||||
if item[0] == "page":
|
||||
return item[1]
|
||||
return "1"
|
||||
|
||||
|
||||
def geturlfromdata(data):
|
||||
actual_url = parse_qsl(data)
|
||||
for item in actual_url:
|
||||
if item[0] == "url":
|
||||
return item[1]
|
||||
return None
|
||||
|
||||
def googlepage_to_cleanhtml(googlehtml,searchstring="",currentpage=1):
|
||||
soup = BeautifulSoup(googlehtml, "lxml")
|
||||
searchresults = soup.find_all("div", attrs={"id":"search"})[0]
|
||||
preamble = """<html><head><title>Ghostgle</title></head><body><h1>Ghostgle</h1><form id="search-form" action="search" method="get">
|
||||
<div class="search-fields" style="display:flex; flex=1">
|
||||
<div class="autocomplete">
|
||||
|
||||
<input
|
||||
type="text"
|
||||
name="q"
|
||||
id="search-bar"
|
||||
class="home-search"
|
||||
autofocus="autofocus"
|
||||
autocapitalize="none"
|
||||
spellcheck="false"
|
||||
autocorrect="off"
|
||||
autocomplete="off"
|
||||
dir="auto">
|
||||
</div>
|
||||
<input type="submit" id="search-submit" value="Search">
|
||||
</div>
|
||||
</form><h2>Web results(only kind ghostgle does)</h2>"""
|
||||
postamble = "</body></html>"
|
||||
results = ""
|
||||
for h3 in searchresults.select("h3"):
|
||||
preview = str(h3.parent.parent.parent.parent.parent.next_sibling.div)
|
||||
heading = h3.string
|
||||
cite = h3.parent.cite
|
||||
del cite["class"]
|
||||
if len(list(cite.children)) > 1:
|
||||
del cite.span["class"]
|
||||
if h3.parent.img:
|
||||
imagetag = "<img src=\""+h3.parent.img["src"]+"\" style=\"height:26px;width:26px;float:left;max-width:26px;\"/>"
|
||||
else:
|
||||
imagetag = '<svg focusable="false" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" style="max-width:26px;float:left;"><path d="M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2zm-1 17.93c-3.95-.49-7-3.85-7-7.93 0-.62.08-1.21.21-1.79L9 15v1c0 1.1.9 2 2 2v1.93zm6.9-2.54c-.26-.81-1-1.39-1.9-1.39h-1v-3c0-.55-.45-1-1-1H8v-2h2c.55 0 1-.45 1-1V7h2c1.1 0 2-.9 2-2v-.41c2.93 1.19 5 4.06 5 7.41 0 2.08-.8 3.97-2.1 5.39z"></path></svg>'
|
||||
source = h3.next_sibling.next_sibling.div
|
||||
source.span.extract()
|
||||
del source.span["class"]
|
||||
source = source.span
|
||||
try:
|
||||
url = geturlfromdata(h3.parent['data-sb'])
|
||||
except:
|
||||
url = h3.parent['href']
|
||||
results = results+imagetag+"<div class=\"source\">"+str(source)+"<br/>"+str(cite)+"</div><h3><a href=\""+url+"\">"+heading+"</a></h3><p>"+preview+"</p>"
|
||||
navigation = soup.find("td", attrs={"role":"heading"}).parent
|
||||
pagetds = navigation.find_all("td")
|
||||
del pagetds[0]
|
||||
pages = '<div id="pages" style="display:flex; flex=1">'
|
||||
for page in pagetds:
|
||||
if page.a != None:
|
||||
try:
|
||||
# we check if it's an int because "Next" and international versions of "Next" aren't.
|
||||
page_num_str = int(page.get_text())
|
||||
page_num_str = page.get_text()
|
||||
except:
|
||||
page_num_str = str(currentpage+1)
|
||||
pages = pages + "<div style=\"minwidth=14px\"><a href=\"?q="+searchstring+"&page="+page_num_str+"\">"+page.get_text()+"</a> </div>"
|
||||
else:
|
||||
pages = pages + "<div style=\"minwidth=14px\">"+page.get_text()+" </div>"
|
||||
pages = pages + "</div>"
|
||||
return preamble+results+pages+postamble
|
||||
|
||||
#debug swap these over, not good for production
|
||||
#with Camoufox(headless=False) as browser:
|
||||
with Camoufox(headless="virtual") as browser:
|
||||
page = browser.new_page()
|
||||
page.goto("https://google.com?udm=14")
|
||||
|
||||
if __name__ == "__main__":
|
||||
webServer = HTTPServer((hostName, serverPort), MyServer)
|
||||
print("Server started http://%s:%s" % (hostName, serverPort))
|
||||
|
||||
try:
|
||||
webServer.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
webServer.server_close()
|
||||
print("Server stopped.")
|
||||
37
nosearch.html
Normal file
37
nosearch.html
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
<html>
|
||||
<head>
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<title>Ghostgle Search</title>
|
||||
</head>
|
||||
<body id="main">
|
||||
<div class="search-container">
|
||||
<h1>Ghostgle</h1>
|
||||
<form id="search-form" action="search" method="get">
|
||||
<div class="search-fields" style="display:flex; flex=1">
|
||||
<div class="autocomplete">
|
||||
|
||||
<input
|
||||
type="text"
|
||||
name="q"
|
||||
id="search-bar"
|
||||
class="home-search"
|
||||
autofocus="autofocus"
|
||||
autocapitalize="none"
|
||||
spellcheck="false"
|
||||
autocorrect="off"
|
||||
autocomplete="off"
|
||||
dir="auto">
|
||||
</div>
|
||||
<input type="submit" id="search-submit" value="Search">
|
||||
</div>
|
||||
</form>
|
||||
|
||||
</div>
|
||||
<footer>
|
||||
<p class="footer">
|
||||
Ghostgle Search v0.0.1 ||
|
||||
<a class="link" href="https://example.com">Don't View on GitHub</a>
|
||||
</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
3
requirements.txt
Normal file
3
requirements.txt
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
cloverlabs-camoufox==0.6.0
|
||||
bs4==0.0.2
|
||||
playwright==1.60.0
|
||||
Loading…
Add table
Reference in a new issue