aboutsummaryrefslogtreecommitdiffstats
path: root/main.py
diff options
context:
space:
mode:
Diffstat (limited to 'main.py')
-rwxr-xr-xmain.py111
1 files changed, 102 insertions, 9 deletions
diff --git a/main.py b/main.py
index ef89ca1..8176293 100755
--- a/main.py
+++ b/main.py
@@ -3,6 +3,12 @@
import argparse
import logging
+import subprocess
+import sys
+import tika
+import docker
+import os
+import nltk
from newspaper import Article, build, Config
from bs4 import BeautifulSoup
from contextlib import closing
@@ -17,15 +23,31 @@ from datetime import datetime as time
class Argparser(object):
def __init__(self):
parser = argparse.ArgumentParser()
- parser.add_argument(
- "--source",
- type=str, help="the url where the urls to be extracted reside")
- parser.add_argument("--bool", action="store_true",
- help="bool", default=False)
+ parser.add_argument("--source",
+ type=str, help="the url where the \
+ urls to be extracted reside")
+ parser.add_argument("--out", type=str,
+ help="the output file", default="")
+ parser.add_argument("--singlelink", action="store_true",
+ help="whether the app should work in single-link \
+ meaning only one page's contents will be used \
+ mode", default=False)
+ parser.add_argument("--multilink", action="store_true",
+ help="whether the app should work in multi-link \
+ mode meaning the srouce contians a list of links \
+ rather than being the actual source itself",
+ default=False)
+ parser.add_argument("--sourcetype", type=str,
+ help="determines the type of the \
+ source.html,text,...")
+ parser.add_argument("--pdftomp3", action="store_true",
+ default=False, help="convert pdf to mp3. \
+ source should be the path to a pdf file and\
+ out should be the path to the mp3 output file")
self.args = parser.parse_args()
-# TODO-maybe actually really do some logging
+# FIXME-maybe actually really do some logging
def logError(err: RequestException) -> None:
"""logs the errors"""
logging.exception(err)
@@ -35,7 +57,7 @@ def isAGoodResponse(resp: Response) -> bool:
"""checks whether the get we sent got a 200 response"""
content_type = resp.headers['Content-Type'].lower()
return (resp.status_code == 200 and
- content_type is not None and content_type.find("html") > -1)
+ content_type is not None)
def simpleGet(url: str) -> bytes:
@@ -81,8 +103,67 @@ def configNews(config: Config) -> None:
config.browser_user_agent = "Chrome/91.0.4464.5"
-def main() -> None:
- argparser = Argparser()
+def call_from_shell_list(command_list):
+ # should probably deprecate this at some point
+ if sys.version_info < (3, 7):
+ return subprocess.run(command_list, stdout=subprocess.PIPE)
+ else:
+ return subprocess.run(command_list, capture_output=True)
+
+
+def pdfToVoice(argparser: Argparser) -> None:
+ """main function for converting a pdf to an mp3"""
+ TIKA_SERVER_ENDPOINT = "127.0.0.1:9977"
+ os.environ["TIKA_SERVER_ENDPOINT"] = TIKA_SERVER_ENDPOINT
+ dockerClient = docker.from_env()
+ container = dockerClient.containers.run("apache/tika:2.0.0", detach=True,
+ ports={TIKA_SERVER_ENDPOINT:
+ "9998"})
+ while True:
+ resp = get("http://127.0.0.1:9977")
+ if resp.status_code == 200:
+ break
+ time.sleep(.5)
+ rawText = tika.parser.from_file()
+ tts = gTTS(rawText['content'])
+ tts.save(argparser.args.out)
+ container.stop()
+ dockerClient.close()
+
+
+def extractRequirements(textBody: str) -> list:
+ result = []
+ REQ_KEYWORDS = ["shall", "should", "must", "may", "can", "could"]
+ nltk.download("punkt")
+ sentences = nltk.sent_tokenize(textBody)
+ for sentence in sentences:
+ for keyword in REQ_KEYWORDS:
+ if sentence.find(keyword) >= 0:
+ result.append(sentence)
+ return result
+
+
+def singleLinkMode(argparser: Argparser) -> dict:
+ """runs the single-link main function"""
+ if argparser.args.sourcetype == "html":
+ parser = build(argparser.args.source)
+ for article in parser.articles:
+ a = Article(article.url)
+ try:
+ a.download()
+ a.parse()
+ doc = Document(a.html)
+ print(doc.summary())
+ extractRequirements(doc.summary())
+ except Exception as e:
+ logging.exception(e)
+ elif argparser.args.sourcetype == "text":
+ bytesText = simpleGet(argparser.args.source)
+ extractRequirements(bytesText.decode("utf-8"))
+
+
+def multiLinkMode(argparser: Argparser) -> None:
+ """run the multi-link main function"""
config = Config()
configNews(config)
urls = getURLS(argparser.args.source)
@@ -102,5 +183,17 @@ def main() -> None:
logging.exception(e)
+def main() -> None:
+ argparser = Argparser()
+ if argparser.args.singlelink:
+ singleLinkMode(argparser)
+ elif argparser.args.multilink:
+ multiLinkMode(argparser)
+ elif argparser.args.pdftomp3:
+ pdfToVoice(argparser)
+ else:
+ pass
+
+
if __name__ == "__main__":
main()