aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorterminaldweller <thabogre@gmail.com>2022-12-02 06:56:35 +0000
committerterminaldweller <thabogre@gmail.com>2022-12-02 06:56:35 +0000
commitd1e30be24d01152cd6357055ef8a8271c56d1ae1 (patch)
tree59cbed0ecfcbbb7614b753e92136e2a7551362d1
parenta crawler (diff)
downloadkaminokumo-master.tar.gz
kaminokumo-master.zip
fixed a bug with a bad output. increased max workers to 32.HEADmaster
-rwxr-xr-xkaminokumo11
1 files changed, 6 insertions, 5 deletions
diff --git a/kaminokumo b/kaminokumo
index 2b75181..0c9c101 100755
--- a/kaminokumo
+++ b/kaminokumo
@@ -4,11 +4,11 @@ import argparse
import json
import sys
import os
-import requests
import re
import concurrent.futures
-import bs4
import pathlib
+import requests
+import bs4
class bcolors:
@@ -74,7 +74,7 @@ def single_get(url: str) -> requests.Response:
def multi_get(urls: list) -> list:
- with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
+ with concurrent.futures.ThreadPoolExecutor(max_workers=32) as pool:
response_list = list(pool.map(single_get, urls))
return response_list
@@ -84,7 +84,7 @@ def single_get_tag(input: list) -> (requests.Response, str):
def multi_get_tag(urls: list) -> list:
- with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
+ with concurrent.futures.ThreadPoolExecutor(max_workers=32) as pool:
response_list = list(pool.map(single_get_tag, urls))
return response_list
@@ -139,7 +139,8 @@ def manga_scrape() -> None:
try:
result += re_res[0] + "\n"
except IndexError:
- result += thing["title"] + "--> nothing\n"
+ pass
+ # result += thing["title"] + "--> nothing\n"
print(result, end="")