diff options
author | terminaldweller <thabogre@gmail.com> | 2022-12-02 06:56:35 +0000 |
---|---|---|
committer | terminaldweller <thabogre@gmail.com> | 2022-12-02 06:56:35 +0000 |
commit | d1e30be24d01152cd6357055ef8a8271c56d1ae1 (patch) | |
tree | 59cbed0ecfcbbb7614b753e92136e2a7551362d1 | |
parent | a crawler (diff) | |
download | kaminokumo-master.tar.gz kaminokumo-master.zip |
-rwxr-xr-x | kaminokumo | 11 |
1 files changed, 6 insertions, 5 deletions
@@ -4,11 +4,11 @@ import argparse import json import sys import os -import requests import re import concurrent.futures -import bs4 import pathlib +import requests +import bs4 class bcolors: @@ -74,7 +74,7 @@ def single_get(url: str) -> requests.Response: def multi_get(urls: list) -> list: - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as pool: response_list = list(pool.map(single_get, urls)) return response_list @@ -84,7 +84,7 @@ def single_get_tag(input: list) -> (requests.Response, str): def multi_get_tag(urls: list) -> list: - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as pool: response_list = list(pool.map(single_get_tag, urls)) return response_list @@ -139,7 +139,8 @@ def manga_scrape() -> None: try: result += re_res[0] + "\n" except IndexError: - result += thing["title"] + "--> nothing\n" + pass + # result += thing["title"] + "--> nothing\n" print(result, end="") |