Release koco-v0.1.1

Author: inmoonlight

MANIFEST.in

@@ -0,0 +1 @@
+include requirements.txt

koco/__init__.py

@@ -0,0 +1,4 @@
+from koco.about import __version__, __description__  # noqa: F401
+from koco.load import list_datasets, load_dataset  # noqa: F401
+
+from koco import korean_hate_speech  # noqa: F401

koco/korean_hate_speech/__init__.py

@@ -0,0 +1 @@
+from .korean_hate_speech import load  # noqa: F401

koco/korean_hate_speech/korean_hate_speech.py

@@ -0,0 +1,131 @@
+"""Load kocohub/korean-hate-speech dataset
+
+The dataset contains
+    - labeled train, dev set
+    - label-removed test set
+    - unlabeled data
+    - news title for each train, dev, test, and unlabeled corpus
+
+For more information, see https://github.com/kocohub/korean-hate-speech
+"""
+
+import pandas as pd
+
+from ..utils import DOWNLOAD_DIR, read_lines
+
+dataset = 'korean-hate-speech'
+datadir = f'{DOWNLOAD_DIR}/{dataset}-master'
+
+
+def _load_labeled():
+    """Load labeled train, dev set
+
+    Returns:
+        labeled_dataset (dict):
+            {
+                'train': [
+                             {
+                                  'comments': str,
+                                  'contain_gender_bias: bool,
+                                  'bias': str,
+                                  'hate': str,
+                                  'news_title': str,
+                             },
+                             ...
+                         ]
+
+                'dev': [
+                           {
+                                'comments': str,
+                                'contain_gender_bias: bool,
+                                'bias': str,
+                                'hate': str,
+                                'news_title': str,
+                           },
+                           ...
+                       ]
+            }
+    """
+    train = pd.read_csv(f'{datadir}/labeled/train.tsv', sep='\t')
+    dev = pd.read_csv(f'{datadir}/labeled/dev.tsv', sep='\t')
+    train_news_title = read_lines(f'{datadir}/news_title/train.news_title.txt')
+    dev_news_title = read_lines(f'{datadir}/news_title/dev.news_title.txt')
+    assert train.shape[0] == len(train_news_title)
+    assert dev.shape[0] == len(dev_news_title)
+
+    train['news_title'] = train_news_title
+    dev['news_title'] = dev_news_title
+
+    labeled_dataset = dict()
+    labeled_dataset['train'] = train.to_dict('records')
+    labeled_dataset['dev'] = dev.to_dict('records')
+    return labeled_dataset
+
+
+def _load_unlabeled():
+    """Load unlabeled corpus
+
+    Returns:
+        unlabeled_dataset (list of dict):
+            [
+                {
+                    'comments': str,
+                    'news_title': str,
+                }, ...
+            ]
+    """
+    unlabeled_comments = []
+    unlabeled_news_titles = []
+    for i in range(5):
+        unlabeled_comments_tmp = read_lines(f'{datadir}/unlabeled/unlabeled_comments_{i}.txt')
+        unlabeled_comments.extend(unlabeled_comments_tmp)
+        unlabeled_news_title_tmp = read_lines(f'{datadir}/news_title/unlabeled_comments.news_title_{i}.txt')
+        unlabeled_news_titles.extend(unlabeled_news_title_tmp)
+    assert len(unlabeled_comments) == len(unlabeled_news_titles)
+
+    # TODO: multi-processing
+    unlabeled_dataset = []
+    for c, nt in zip(unlabeled_comments, unlabeled_news_titles):
+        d = {'comments': c, 'news_title': nt}
+        unlabeled_dataset.append(d)
+    return unlabeled_dataset
+
+
+def _load_testset():
+    """Load testset
+
+    Note that testset doesn't contain any labels
+
+    Returns:
+        testset (list of dict):
+            [
+                {
+                    'comments': str,
+                    'news_title': str,
+                }, ...
+            ]
+    """
+    test = pd.read_csv(f'{datadir}/test.no_label.tsv', sep='\t')
+    test_news_title = read_lines(f'{datadir}/news_title/test.news_title.txt')
+    assert test.shape[0] == len(test_news_title)
+
+    test['news_title'] = test_news_title
+    return test.to_dict('records')
+
+
+AVAILABLE_MODE = {
+    'labeled': _load_labeled,
+    'unlabeled': _load_unlabeled,
+    'testset': _load_testset
+}
+
+
+def load(mode):
+    """Load korean-hate-speech dataset
+
+    Args:
+        mode (str): Either labeled, unlabeld, or testset
+    """
+    if mode not in AVAILABLE_MODE:
+        raise ValueError(f'Invalid mode. Try one of {AVAILABLE_MODE.keys()}')
+    return AVAILABLE_MODE[mode]()

koco/load.py

@@ -0,0 +1,52 @@
+import logging
+import requests
+
+from .patch import download_dataset
+from .utils import DOWNLOAD_DIR, exist_dataset
+from .korean_hate_speech import load as khs_loader
+
+
+KOCOHUB = 'https://api.github.com/orgs/kocohub/repos'
+
+logger = logging.getLogger(__name__)
+
+
+def list_datasets():
+    """List datasets in kocohub
+    """
+    success = False
+    while not success:
+        r = requests.get(KOCOHUB, params={'per_page': '500'})
+        success = r.ok
+    return [info['name'] for info in r.json()]
+
+
+def is_valid_dataset(dataset):
+    all_datasets = list_datasets()
+    if dataset in all_datasets:
+        return True
+    else:
+        return False
+
+
+def patch_dataset(dataset, verbose=True):
+    """Download and unzip dataset from kocohub
+
+    Args:
+        dataset (str): dataset name (e.g., korean-hate-speech)
+        verbose (bool): whether to show dataset installation path
+    """
+    if exist_dataset(dataset):
+        if verbose:
+            logger.info(f'{dataset} is already installed in {DOWNLOAD_DIR}.')
+    else:
+        if not is_valid_dataset(dataset):
+            raise ValueError(f'{dataset} is not in {list_datasets()}')
+        download_dataset(dataset, verbose=verbose)
+
+
+def load_dataset(dataset, mode, verbose=True):
+    patch_dataset(dataset, verbose)
+
+    if dataset == 'korean-hate-speech':
+        return khs_loader(mode)

koco/patch.py

@@ -0,0 +1,24 @@
+import logging
+import zipfile
+
+import wget
+
+from .utils import DOWNLOAD_DIR, make_dirs
+
+baseurl = 'https://codeload.github.com/kocohub/{}/zip/master'
+
+logger = logging.getLogger(__name__)
+
+
+def download_dataset(dataset, verbose=True):
+    make_dirs(DOWNLOAD_DIR)
+    url = baseurl.format(dataset)
+    wget.download(url, f'{DOWNLOAD_DIR}/{dataset}.zip')
+    unzip(f'{DOWNLOAD_DIR}/{dataset}.zip')
+    if verbose:
+        logger.info(f'Dataset {dataset} downloaded to {DOWNLOAD_DIR}.')
+
+
+def unzip(zippath):
+    with zipfile.ZipFile(zippath) as z:
+        z.extractall(DOWNLOAD_DIR)

koco/utils.py

@@ -0,0 +1,20 @@
+import os
+
+DOWNLOAD_DIR = f'{os.path.expanduser("~")}/.kocohub'
+
+
+def exist_dataset(dataset):
+    return os.path.exists(f'{DOWNLOAD_DIR}/{dataset}-master')
+
+
+def exist_dir(dirpath):
+    return os.path.exists(dirpath)
+
+
+def make_dirs(dirpath):
+    if not exist_dir(dirpath):
+        os.makedirs(dirpath)
+
+
+def read_lines(path):
+    return [line.rstrip('\n') for line in open(path)]

requirements.txt

@@ -0,0 +1,3 @@
+pandas>=1.0.3
+requests>=2.23.0
+wget>=3.2

setup.py

@@ -0,0 +1,29 @@
+import os
+from setuptools import find_packages, setup
+
+REQ_FILE = 'requirements.txt'
+VERSION = '0.1.1'
+
+
+def get_requires():
+    thisdir = os.path.dirname(__file__)
+    reqpath = os.path.join(thisdir, REQ_FILE)
+    return [line.rstrip('\n') for line in open(reqpath)]
+
+
+setup(
+    name='koco',
+    version=VERSION,
+    description='A library to easily access kocohub datasets',
+    author='Jihyung Moon',
+    author_email='[email protected]',
+    url='https://github.com/inmoonlight/koco',
+    license='MIT',
+    packages=find_packages(),
+    install_requires=get_requires(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "Intended Audience :: Science/Research",
+    ],
+    keywords='korean nlp datasets',
+)