diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c0bb6ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# temp stage +FROM python:3.12.2-slim AS builder + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends gcc + +COPY requirements.txt . +RUN pip wheel --no-cache-dir --no-deps --wheel-dir /app/wheels -r requirements.txt + + +# final stage +FROM python:3.12.2-slim + +WORKDIR /app + +# mostly install firefox to ensure all the libs are there +RUN apt-get update && \ + apt-get install -y --no-install-recommends firefox-esr xvfb + +COPY --from=builder /app/wheels /wheels +COPY --from=builder /app/requirements.txt . +RUN pip install --no-cache /wheels/* +RUN python -m camoufox fetch +COPY nosearch.html /app/nosearch.html +COPY main.py /app/main.py \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..513bbb7 --- /dev/null +++ b/main.py @@ -0,0 +1,153 @@ +from camoufox.sync_api import Camoufox +from bs4 import BeautifulSoup +from urllib.parse import urlsplit,parse_qsl +from http.server import BaseHTTPRequestHandler, HTTPServer +import time +import os + +hostName = os.getenv("LISTEN_HOSTNAME","localhost") +serverPort = os.getenv("LISTEN_PORT", 9090) + +class MyServer(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + if self.getsearchstring() == None: + with open("nosearch.html") as file: + html = file.read() + self.wfile.write(bytes(html, "utf-8")) + else: + #debug swap these over, not good for production + with Camoufox(headless="virtual") as browser: + #with Camoufox(headless=False) as browser: + page = browser.new_page() + page.goto("https://google.com/search?udm=14&q="+self.getsearchstring()) + try: + page.locator('xpath=//*[@id="W0wltc"]').click() + except: + suppressedError = true + if self.getpage() != 1: + if self.getpage() > 10: + page.get_by_text("10", exact=True).click() + if self.getpage() > 19: + page.get_by_text("19", exact=True).click() + page.get_by_text(self.getpagestr(), exact=True).click() + time.sleep(4) + page.wait_for_url("**search**") + html = page.content() + with open("/tmp/ghost.html", "w", encoding="utf-8") as f: + f.write(html) + self.wfile.write(bytes(googlepage_to_cleanhtml(html,self.getsearchstring(),self.getpage()), "utf-8")) + + def getsearchstring(self): + spl = urlsplit(self.path) + params = parse_qsl(spl.query) + for item in params: + if item[0] == "q": + return item[1] + return None + + def getpage(self): + spl = urlsplit(self.path) + params = parse_qsl(spl.query) + for item in params: + if item[0] == "page": + return int(item[1]) + return 1 + + def getpagestr(self): + spl = urlsplit(self.path) + params = parse_qsl(spl.query) + for item in params: + if item[0] == "page": + return item[1] + return "1" + + +def geturlfromdata(data): + actual_url = parse_qsl(data) + for item in actual_url: + if item[0] == "url": + return item[1] + return None + +def googlepage_to_cleanhtml(googlehtml,searchstring="",currentpage=1): + soup = BeautifulSoup(googlehtml, "lxml") + searchresults = soup.find_all("div", attrs={"id":"search"})[0] + preamble = """Ghostgle

Ghostgle

+
+
+ + +
+ +
+

Web results(only kind ghostgle does)

""" + postamble = "" + results = "" + for h3 in searchresults.select("h3"): + preview = str(h3.parent.parent.parent.parent.parent.next_sibling.div) + heading = h3.string + cite = h3.parent.cite + del cite["class"] + if len(list(cite.children)) > 1: + del cite.span["class"] + if h3.parent.img: + imagetag = "" + else: + imagetag = '' + source = h3.next_sibling.next_sibling.div + source.span.extract() + del source.span["class"] + source = source.span + try: + url = geturlfromdata(h3.parent['data-sb']) + except: + url = h3.parent['href'] + results = results+imagetag+"
"+str(source)+"
"+str(cite)+"

"+heading+"

"+preview+"

" + navigation = soup.find("td", attrs={"role":"heading"}).parent + pagetds = navigation.find_all("td") + del pagetds[0] + pages = '
' + for page in pagetds: + if page.a != None: + try: + # we check if it's an int because "Next" and international versions of "Next" aren't. + page_num_str = int(page.get_text()) + page_num_str = page.get_text() + except: + page_num_str = str(currentpage+1) + pages = pages + "
"+page.get_text()+"  
" + else: + pages = pages + "
"+page.get_text()+"  
" + pages = pages + "
" + return preamble+results+pages+postamble + +#debug swap these over, not good for production +#with Camoufox(headless=False) as browser: +with Camoufox(headless="virtual") as browser: + page = browser.new_page() + page.goto("https://google.com?udm=14") + +if __name__ == "__main__": + webServer = HTTPServer((hostName, serverPort), MyServer) + print("Server started http://%s:%s" % (hostName, serverPort)) + + try: + webServer.serve_forever() + except KeyboardInterrupt: + pass + + webServer.server_close() + print("Server stopped.") \ No newline at end of file diff --git a/nosearch.html b/nosearch.html new file mode 100644 index 0000000..f25e169 --- /dev/null +++ b/nosearch.html @@ -0,0 +1,37 @@ + + + + Ghostgle Search + + +
+

Ghostgle

+
+
+
+ + +
+ +
+
+ +
+ + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3748c79 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +cloverlabs-camoufox==0.6.0 +bs4==0.0.2 +playwright==1.60.0 \ No newline at end of file