path: root/digester.py

#!/usr/bin/python3
# _*_ coding=utf-8 _*_
# original source:https://github.com/polyrabbit/hacker-news-digest/blob/master/%5Btutorial%5D%20How-to-extract-main-content-from-web-pages-using-Machine-Learning.ipynb

import argparse
import code
import readline
import signal
import sys
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.svm import SVC


def SigHandler_SIGINT(signum, frame):
    print()
    sys.exit(0)


class Argparser(object):
    def __init__(self):
        parser = argparse.ArgumentParser()
        parser.add_argument("--string", type=str, help="string")
        parser.add_argument(
            "--bool", action="store_true", help="bool", default=False
        )
        parser.add_argument(
            "--dbg", action="store_true", help="debug", default=False
        )
        self.args = parser.parse_args()


# write code here
def premain(argparser):
    signal.signal(signal.SIGINT, SigHandler_SIGINT)
    # here
    dataframe = pd.read_csv("/tmp/features.csv")
    dataframe.head()
    y = dataframe.target
    X = dataframe.drop(["target"], axis=1)

    corpus = X["attr"]
    vc = CountVectorizer()
    vc.fit(corpus)

    numeric_features = pd.concat(
        [
            X.drop(["attr"], axis=1),
            pd.DataFrame(
                vc.transform(corpus).toarray(), columns=vc.vocabulary_
            ),
        ],
        axis=1,
    )
    numeric_features.head()
    plt.scatter(dataframe.index, dataframe.target, color="red", label="target")
    plt.scatter(
        numeric_features.index,
        numeric_features.depth,
        color="green",
        label="depth",
    )
    plt.scatter(
        numeric_features.index,
        numeric_features.text_ratio,
        color="blue",
        label="text_ratio",
    )
    plt.scatter(
        numeric_features.index,
        numeric_features.alink_text_ratio,
        color="skyblue",
        label="alink_text_ratio",
    )
    plt.legend(loc=(1, 0))
    plt.show()
    scaler = preprocessing.StandardScaler()
    scaler.fit(numeric_features)
    scaled_X = scaler.transform(numeric_features)

    # clf = MultinomialNB()
    # clf = RandomForestClassifier()
    clf = SVC(C=1, kernel="poly", probability=True)
    clf.fit(scaled_X, y)
    predicted_index = clf.predict(scaled_X).tolist().index(True)

    scaled_X = scaler.transform(numeric_features)
    pred_y = clf.predict(scaled_X)

    print(pd.DataFrame(clf.predict_log_proba(scaled_X), columns=clf.classes_))
    print(
        "Number of mispredicted out of %d is %d (%.2f%%)"
        % (
            y.shape[0],
            (y != pred_y).sum(),
            (y != pred_y).sum() * 100.0 / y.shape[0],
        )
    )
    print()
    print("Predicted rows:")
    print(
        dataframe[pred_y]
        .drop(["text_ratio", "alink_text_ratio", "contain_title"], axis=1)
        .merge(
            pd.DataFrame(
                clf.predict_log_proba(scaled_X)[pred_y],
                columns=clf.classes_,
                index=dataframe[pred_y].index,
            ),
            left_index=True,
            right_index=True,
        )
    )
    print()

    # print 'Acutual rows:'
    # print dataframe[dataframe.target]


def main():
    argparser = Argparser()
    if argparser.args.dbg:
        try:
            premain(argparser)
        except Exception as e:
            print(e.__doc__)
            if e.message:
                print(e.message)
            variables = globals().copy()
            variables.update(locals())
            shell = code.InteractiveConsole(variables)
            shell.interact(banner="DEBUG REPL")
    else:
        premain(argparser)


if __name__ == "__main__":
    main()