From 43d266146a18466fb842e0637f3351a2eaef38c7 Mon Sep 17 00:00:00 2001 From: terminaldweller Date: Tue, 14 Jun 2022 21:27:25 +0430 Subject: added keyword extraction feature, some pylint cleanup --- devourer/devourer.py | 128 ++++++++++++++++++++++++++++-------------------- devourer/poetry.lock | 30 ++++++------ devourer/pyproject.toml | 1 + devourer/tests.sh | 4 ++ 4 files changed, 97 insertions(+), 66 deletions(-) diff --git a/devourer/devourer.py b/devourer/devourer.py index 29d4506..7ee2fb1 100644 --- a/devourer/devourer.py +++ b/devourer/devourer.py @@ -15,6 +15,7 @@ import fastapi import gtts # type:ignore import newspaper # type:ignore import nltk # type:ignore +import rake_nltk # type:ignore import readability # type:ignore import refextract # type:ignore import requests @@ -24,42 +25,42 @@ from tika import parser as tparser # FIXME-maybe actually really do some logging -def logError(err: str) -> None: +def log_error(err: str) -> None: """Logs the errors.""" logging.exception(err) -def isAGoodResponse(resp: requests.Response) -> bool: +def is_a_good_response(resp: requests.Response) -> bool: """Checks whether the get we sent got a 200 response.""" content_type = resp.headers["Content-Type"].lower() return resp.status_code == 200 and content_type is not None -def simpleGet(url: str) -> bytes: +def simple_get(url: str) -> bytes: """Issues a simple get request.""" content = bytes() try: with contextlib.closing(requests.get(url, stream=True)) as resp: - if isAGoodResponse(resp): + if is_a_good_response(resp): content = resp.content except requests.exceptions.RequestException as e: - logError("Error during requests to {0} : {1}".format(url, str(e))) + log_error("Error during requests to {0} : {1}".format(url, str(e))) finally: return content -def getWithParams(url: str, params: dict) -> typing.Optional[dict]: +def get_with_params(url: str, params: dict) -> typing.Optional[dict]: """Issues a get request with params.""" try: with contextlib.closing( requests.get(url, params=params, stream=True) ) as resp: - if isAGoodResponse(resp): + if is_a_good_response(resp): return resp.json() else: return None except requests.exceptions.RequestException as e: - logError("Error during requests to {0} : {1}".format(url, str(e))) + log_error("Error during requests to {0} : {1}".format(url, str(e))) return None @@ -70,8 +71,8 @@ def getRandStr(n): def getURLS(source: str, summary: str) -> dict: """Extracts the urls from a website.""" - result = dict() - raw_ml = simpleGet(source) + result = {} + raw_ml = simple_get(source) ml = bs4.BeautifulSoup(raw_ml, "lxml") rand_tmp = "/tmp/" + getRandStr(20) @@ -94,7 +95,7 @@ def getURLS(source: str, summary: str) -> dict: return result -def configNews(config: newspaper.Config) -> None: +def config_news(config: newspaper.Config) -> None: """Configures newspaper.""" config.fetch_images = False config.keep_article_html = True @@ -102,7 +103,7 @@ def configNews(config: newspaper.Config) -> None: config.browser_user_agent = "Chrome/91.0.4464.5" -def sanitizeText(text: str) -> str: +def sanitize_text(text: str) -> str: """Sanitize the strings.""" text = text.replace("\n", "") text = text.replace("\n\r", "") @@ -111,7 +112,7 @@ def sanitizeText(text: str) -> str: # FIXME-have to decide whether to use files or urls -def pdfToVoice() -> str: +def pdf_to_voice() -> str: """Main function for converting a pdf to an mp3.""" outfile = str() try: @@ -145,13 +146,13 @@ def extractRequirements(textBody: str) -> list: for sentence in sentences: for keyword in REQ_KEYWORDS: if sentence.casefold().find(keyword) >= 0: - result.append(sanitizeText(sentence)) + result.append(sanitize_text(sentence)) return result -def extractRefs(url: str) -> list: +def extract_refs(url: str) -> list: """Extract the references from an article.""" - refs = list() + refs = [] try: refs = refextract.extract_references_from_url(url) return refs @@ -161,12 +162,12 @@ def extractRefs(url: str) -> list: return refs -def pdfToText(url: str) -> str: +def pdf_to_text(url: str) -> str: """Convert the PDF file to a string.""" - tikaResult = dict() + tikaResult = {} try: with tempfile.NamedTemporaryFile(mode="w+b", delete=True) as tmpFile: - content = simpleGet(url) + content = simple_get(url) if content is not None: tmpFile.write(content) tikaResult = tparser.from_file( @@ -179,13 +180,13 @@ def pdfToText(url: str) -> str: logging.exception(e) finally: if "content" in tikaResult: - return sanitizeText(tikaResult["content"]) + return sanitize_text(tikaResult["content"]) else: return "" -# FIXME doesnt work for long texts -def summarizeText(text: str) -> str: +# TODO-very performance-intensive +def summarize_text(text: str) -> str: """Summarize the given text using bart.""" result = str() # TODO move me later @@ -226,7 +227,7 @@ def summarizeText(text: str) -> str: return result -def summarizeText_v2(text: str) -> str: +def summarize_text_v2(text: str) -> str: """Text summarization using nltk.""" stop_words = set(nltk.corpus.stopwords.words("english")) words = nltk.tokenize.word_tokenize(text) @@ -267,7 +268,7 @@ def summarizeText_v2(text: str) -> str: return summary -def textToAudio(text: str) -> str: +def text_to_audio(text: str) -> str: """Transform the given text into audio.""" path = str() try: @@ -284,7 +285,7 @@ def textToAudio(text: str) -> str: def getRequirements(url: str, sourcetype: str) -> list: """Runs the single-link main function.""" result = str() - results = list() + results = [] try: if sourcetype == "html": parser = newspaper.build(url) @@ -299,7 +300,7 @@ def getRequirements(url: str, sourcetype: str) -> list: # results = extractRequirements(doc.summary()) results = extractRequirements(doc) elif sourcetype == "text": - bytesText = simpleGet(url) + bytesText = simple_get(url) results = extractRequirements(bytesText.decode("utf-8")) except Exception as e: logging.exception(e) @@ -328,7 +329,7 @@ def summarizeLinkToAudio(url: str, summary: str) -> str: else: print("invalid option for summary type.") if result != "": - result = sanitizeText(result) + result = sanitize_text(result) except Exception as e: logging.exception(e) finally: @@ -338,11 +339,11 @@ def summarizeLinkToAudio(url: str, summary: str) -> str: # FIXME-change my name def summarizeLinksToAudio(url: str, summary: str) -> str: """Summarize a list of urls into audio files.""" - results = list() + results = [] result = str() try: config = newspaper.Config() - configNews(config) + config_news(config) urls = getURLS(url, summary) for url in urls: results.append(summarizeLinkToAudio(url, summary)) @@ -366,19 +367,19 @@ def searchWikipedia(search_term: str, summary: str) -> str: "limit": "10", "format": "json", } - res = getWithParams(os.environ["WIKI_SEARCH_URL"], searchParmas) + res = get_with_params(os.environ["WIKI_SEARCH_URL"], searchParmas) # FIXME-handle wiki redirects/disambiguations if res is not None: source = res[3][0] result = summarizeLinkToAudio(source, summary) - result = sanitizeText(result) + result = sanitize_text(result) except Exception as e: logging.exception(e) finally: return result -def getAudioFromFile(audio_path: str) -> bytes: +def get_audio_from_file(audio_path: str) -> bytes: """Returns the contents of a file in binary format.""" with open(audio_path, "rb") as audio: return audio.read() @@ -397,15 +398,23 @@ def getSentiments(detailed: bool) -> list: """ +def get_keywords_from_text(text: str) -> typing.List[str]: + """Extract keywords out of text.""" + rake_nltk_var = rake_nltk.Rake() + rake_nltk_var.extract_keywords_from_text(text) + return rake_nltk_var.get_ranked_phrases() + + app = fastapi.FastAPI() nltk.download("punkt") nltk.download("stopwords") +nltk.download("wordnet") # https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html @app.middleware("http") -async def addSecureHeaders( +async def add_secure_headers( request: fastapi.Request, call_next ) -> fastapi.Response: """Adds security headers proposed by OWASP.""" @@ -425,35 +434,50 @@ def pdf_ep( ): """The pdf manupulation endpoint.""" if feat == "": - text = pdfToText(url) + text = pdf_to_text(url) if summarize: - text = summarizeText_v2(text) + text = summarize_text_v2(text) if audio: - audio_path = textToAudio(text) + audio_path = text_to_audio(text) return fastapi.Response( - getAudioFromFile(audio_path) if audio_path != "" else "", + get_audio_from_file(audio_path) if audio_path != "" else "", media_type="audio/mpeg", ) return { "Content-Type": "application/json", - "isOk": True if text != "" else False, + "isOk": bool(text), "result": text, } elif feat == "refs": - refs = extractRefs(url) + refs = extract_refs(url) return { "Content-Type": "application/json", - "isOk": True if refs is not None else False, + "isOk": bool(refs), "result": refs, } + elif feat == "keyword": + text = pdf_to_text(url) + keywords = get_keywords_from_text(text) + return { + "Content-Type": "application/json", + "isOk": bool(keywords), + "result": keywords, + } + else: + return { + "Content-Type": "application/json", + "isOk": False, + "result": "unknown feature requested", + } +# TODO- currently not working @app.get("/mila/tika") def pdf_to_audio_ep(url: str): """Turns a pdf into an audiofile.""" - audio_path = pdfToVoice() + audio_path = pdf_to_voice() return fastapi.Response( - getAudioFromFile(audio_path) if audio_path != "" else "", + get_audio_from_file(audio_path) if audio_path != "" else "", media_type="audio/mpeg", ) @@ -464,7 +488,7 @@ def extract_reqs_ep(url: str, sourcetype: str = "html"): result = getRequirements(url, sourcetype) return { "Content-Type": "application/json", - "isOK": True if result is not None else False, + "isOK": bool(result), "reqs": result, } @@ -474,15 +498,15 @@ def wiki_search_ep(term: str, summary: str = "none", audio: bool = False): """Search and summarizes from wikipedia.""" text = searchWikipedia(term, summary) if audio: - audio_path = textToAudio(text) + audio_path = text_to_audio(text) return fastapi.Response( - getAudioFromFile(audio_path) if audio_path != "" else "", + get_audio_from_file(audio_path) if audio_path != "" else "", media_type="audio/mpeg", ) else: return { "Content-Type": "application/json", - "isOK": True if text != "" else False, + "isOK": bool(text), "audio": "", "text": text, } @@ -493,16 +517,16 @@ def summarize_ep(url: str, summary: str = "none", audio: bool = False): """Summarize and turn the summary into audio.""" text = summarizeLinkToAudio(url, summary) if audio: - audio_path = textToAudio(text) + audio_path = text_to_audio(text) print(audio_path) return fastapi.Response( - getAudioFromFile(audio_path) if audio_path != "" else "", + get_audio_from_file(audio_path) if audio_path != "" else "", media_type="audio/mpeg", ) else: return { "Content-Type": "application/json", - "isOK": True if text != "" else False, + "isOK": bool(text), # "audio": "", "text": text, } @@ -513,16 +537,16 @@ def mila_ep(url: str, summary: str = "newspaper", audio: bool = False): """Extract all the urls and then summarize and turn into audio.""" text = summarizeLinksToAudio(url, summary) if audio: - audio_path = textToAudio(text) + audio_path = text_to_audio(text) print(audio_path) return fastapi.Response( - getAudioFromFile(audio_path) if audio_path != "" else "", + get_audio_from_file(audio_path) if audio_path != "" else "", media_type="audio/mpeg", ) else: return { "Content-Type": "application/json", - "isOK": True if text != "" else False, + "isOK": bool(text), "audio": "", "text": text, } diff --git a/devourer/poetry.lock b/devourer/poetry.lock index ce2a8e6..be1bb56 100644 --- a/devourer/poetry.lock +++ b/devourer/poetry.lock @@ -662,14 +662,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -[[package]] -name = "python-magic-bin" -version = "0.4.14" -description = "File type identification using libmagic binary package" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "pytz" version = "2022.1" @@ -686,6 +678,17 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "rake-nltk" +version = "1.0.6" +description = "RAKE short for Rapid Automatic Keyword Extraction algorithm, is a domain independent keyword extraction algorithm which tries to determine key phrases in a body of text by analyzing the frequency of word appearance and its co-occurance with other words in the text." +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +nltk = ">=3.6.2,<4.0.0" + [[package]] name = "readability-lxml" version = "0.8.1" @@ -1157,7 +1160,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "a740bd5805861994b28f7a187d06be052f26dd920355f6624955134e62cb6059" +content-hash = "24d72bba62f852cba715eb9f954a00454da5f045138088d365fc27d8de2a035f" [metadata.files] absl-py = [ @@ -1642,11 +1645,6 @@ python-magic = [ {file = "python-magic-0.4.25.tar.gz", hash = "sha256:21f5f542aa0330f5c8a64442528542f6215c8e18d2466b399b0d9d39356d83fc"}, {file = "python_magic-0.4.25-py2.py3-none-any.whl", hash = "sha256:1a2c81e8f395c744536369790bd75094665e9644110a6623bcc3bbea30f03973"}, ] -python-magic-bin = [ - {file = "python_magic_bin-0.4.14-py2.py3-none-macosx_10_6_intel.whl", hash = "sha256:7b1743b3dbf16601d6eedf4e7c2c9a637901b0faaf24ad4df4d4527e7d8f66a4"}, - {file = "python_magic_bin-0.4.14-py2.py3-none-win32.whl", hash = "sha256:34a788c03adde7608028203e2dbb208f1f62225ad91518787ae26d603ae68892"}, - {file = "python_magic_bin-0.4.14-py2.py3-none-win_amd64.whl", hash = "sha256:90be6206ad31071a36065a2fc169c5afb5e0355cbe6030e87641c6c62edc2b69"}, -] pytz = [ {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, @@ -1686,6 +1684,10 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +rake-nltk = [ + {file = "rake-nltk-1.0.6.tar.gz", hash = "sha256:7813d680b2ce77b51cdac1757f801a87ff47682c9dbd2982aea3b66730346122"}, + {file = "rake_nltk-1.0.6-py3-none-any.whl", hash = "sha256:1c1ffdb64cae8cb99d169d53a5ffa4635f1c4abd3a02c6e22d5d083136bdc5c1"}, +] readability-lxml = [ {file = "readability-lxml-0.8.1.tar.gz", hash = "sha256:e51fea56b5909aaf886d307d48e79e096293255afa567b7d08bca94d25b1a4e1"}, {file = "readability_lxml-0.8.1-py3-none-any.whl", hash = "sha256:e0d366a21b1bd6cca17de71a4e6ea16fcfaa8b0a5b4004e39e2c7eff884e6305"}, diff --git a/devourer/pyproject.toml b/devourer/pyproject.toml index 2f79961..52b6b8c 100644 --- a/devourer/pyproject.toml +++ b/devourer/pyproject.toml @@ -19,6 +19,7 @@ transformers = "^4.11.2" fastapi = "^0.70.0" uvicorn = "^0.15.0" refextract = "^1.1.4" +rake-nltk = "^1.0.6" [tool.poetry.dev-dependencies] diff --git a/devourer/tests.sh b/devourer/tests.sh index e673acc..4ba4ea0 100755 --- a/devourer/tests.sh +++ b/devourer/tests.sh @@ -1,8 +1,12 @@ #!/usr/bin/env sh curl -k -X GET "https://localhost:19019/mila/summ?url=https://dilipkumar.medium.com/standalone-mongodb-on-kubernetes-cluster-19e7b5896b27&summary=newspaper&audio=true" + curl -k -X GET "https://localhost:19019/mila/wiki?term=iommu&summary=none&audio=false" + curl -k -X GET "https://localhost:19019/mila/reqs?url=https://www.ietf.org/rfc/rfc2865.txt&sourcetype=text" + curl -k -X GET "https://localhost:19019/mila/pdf?feat=&url=https://www.rroij.com/open-access/mutation-testing-a-review-33-36.pdf" curl -k -X GET "https://localhost:19019/mila/pdf?feat=refs&url=https://www.rroij.com/open-access/mutation-testing-a-review-33-36.pdf" +curl -k -X GET "https://localhost:19019/mila/pdf?feat=keyword&url=https://www.rroij.com/open-access/mutation-testing-a-review-33-36.pdf&summarize=true" curl -k -X GET "https://localhost:19019/mila/pdf?feat=&url=https://www.rroij.com/open-access/mutation-testing-a-review-33-36.pdf&summarize=true" -- cgit v1.2.3