examples/example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# gokart task run\n",
    "---\n",
    "running sample task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TASK_WORKSPACE_DIRECTORY'] = './resources'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# define tasks\n",
      "import gokart\n",
      "import luigi\n",
      "from luigi.util import requires\n",
      "from logging import getLogger\n",
      "\n",
      "logger = getLogger(__name__)\n",
      "\n",
      "\n",
      "class SampleTask(gokart.TaskOnKart):\n",
      "    task_namespace = 'sample'\n",
      "    name = luigi.Parameter()\n",
      "    number = luigi.IntParameter()\n",
      "    \n",
      "    def run(self):\n",
      "        self.dump(f'this is sample output. model number: {self.number}')\n",
      "\n",
      "        \n",
      "@requires(SampleTask)\n",
      "class SecondTask(gokart.TaskOnKart):\n",
      "    task_namespace = 'sample'\n",
      "    param = luigi.Parameter()\n",
      "\n",
      "    def run(self):\n",
      "        sample = self.load()\n",
      "        self.dump(sample + f'add task: {self.param}')\n",
      "        \n",
      "gokart.run()"
     ]
    }
   ],
   "source": [
    "!cat ./task.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample task run\n",
    "!rm -rf ./resources\n",
    "!python task.py sample.SampleTask --name='EXAMPLE' --number=1 --local-scheduler 2> /dev/null\n",
    "!python task.py sample.SampleTask --name='EXAMPLE' --number=2 --local-scheduler 2> /dev/null\n",
    "!python task.py sample.SampleTask --name='EXAMPLE' --number=3 --local-scheduler 2> /dev/null\n",
    "!python task.py sample.SampleTask --name='TEMP' --number=1 --local-scheduler 2> /dev/null\n",
    "!python task.py sample.SampleTask --name='TEMP' --number=2 --local-scheduler 2> /dev/null\n",
    "!python task.py sample.SecondTask --name='TEMP' --number=2 --param='RUN' --local-scheduler --local-temporary-directory='./resource' 2> /dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m./resources/\u001b[00m\n",
      "├── \u001b[34m__main__\u001b[00m\n",
      "│   ├── SampleTask_84b0b9c5a39bce072271599c9f730660.pkl\n",
      "│   ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
      "│   ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
      "│   ├── SampleTask_d57cff8074e2560896974850e5d3174d.pkl\n",
      "│   ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
      "│   └── SecondTask_ea1806322904199b2455d6e115c525ea.pkl\n",
      "└── \u001b[34mlog\u001b[00m\n",
      "    ├── \u001b[34mmodule_versions\u001b[00m\n",
      "    │   ├── SampleTask_84b0b9c5a39bce072271599c9f730660.txt\n",
      "    │   ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.txt\n",
      "    │   ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.txt\n",
      "    │   ├── SampleTask_d57cff8074e2560896974850e5d3174d.txt\n",
      "    │   ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.txt\n",
      "    │   └── SecondTask_ea1806322904199b2455d6e115c525ea.txt\n",
      "    ├── \u001b[34mprocessing_time\u001b[00m\n",
      "    │   ├── SampleTask_84b0b9c5a39bce072271599c9f730660.pkl\n",
      "    │   ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
      "    │   ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
      "    │   ├── SampleTask_d57cff8074e2560896974850e5d3174d.pkl\n",
      "    │   ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
      "    │   └── SecondTask_ea1806322904199b2455d6e115c525ea.pkl\n",
      "    ├── \u001b[34mtask_log\u001b[00m\n",
      "    │   ├── SampleTask_84b0b9c5a39bce072271599c9f730660.pkl\n",
      "    │   ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
      "    │   ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
      "    │   ├── SampleTask_d57cff8074e2560896974850e5d3174d.pkl\n",
      "    │   ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
      "    │   └── SecondTask_ea1806322904199b2455d6e115c525ea.pkl\n",
      "    └── \u001b[34mtask_params\u001b[00m\n",
      "        ├── SampleTask_84b0b9c5a39bce072271599c9f730660.pkl\n",
      "        ├── SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl\n",
      "        ├── SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl\n",
      "        ├── SampleTask_d57cff8074e2560896974850e5d3174d.pkl\n",
      "        ├── SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl\n",
      "        └── SecondTask_ea1806322904199b2455d6e115c525ea.pkl\n",
      "\n",
      "6 directories, 30 files\n"
     ]
    }
   ],
   "source": [
    "!tree ./resources/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Init Thunderbolt\n",
    "---\n",
    "using thunderbolt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from thunderbolt import Thunderbolt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1st arg is gokart's output directory (default: $TASK_WORKSPACE_DIRECTORY)\n",
    "# 2nd arg is `task_filters` (for fast)\n",
    "\n",
    "tb = Thunderbolt() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check tasks param\n",
    "checking thunderbolt's task_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = tb.get_task_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task_id</th>\n",
       "      <th>task_name</th>\n",
       "      <th>last_modified</th>\n",
       "      <th>task_params</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:13.704206</td>\n",
       "      <td>{'name': 'TEMP', 'number': '1'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:15.728998</td>\n",
       "      <td>{'name': 'TEMP', 'number': '2'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>SecondTask</td>\n",
       "      <td>2020-02-13 14:17:17.735793</td>\n",
       "      <td>{'name': 'TEMP', 'number': '2', 'param': 'RUN'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:07.787346</td>\n",
       "      <td>{'name': 'EXAMPLE', 'number': '1'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:09.776602</td>\n",
       "      <td>{'name': 'EXAMPLE', 'number': '2'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:11.699651</td>\n",
       "      <td>{'name': 'EXAMPLE', 'number': '3'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   task_id   task_name              last_modified  \\\n",
       "0        0  SampleTask 2020-02-13 14:17:13.704206   \n",
       "1        1  SampleTask 2020-02-13 14:17:15.728998   \n",
       "2        2  SecondTask 2020-02-13 14:17:17.735793   \n",
       "3        3  SampleTask 2020-02-13 14:17:07.787346   \n",
       "4        4  SampleTask 2020-02-13 14:17:09.776602   \n",
       "5        5  SampleTask 2020-02-13 14:17:11.699651   \n",
       "\n",
       "                                       task_params  \n",
       "0                  {'name': 'TEMP', 'number': '1'}  \n",
       "1                  {'name': 'TEMP', 'number': '2'}  \n",
       "2  {'name': 'TEMP', 'number': '2', 'param': 'RUN'}  \n",
       "3               {'name': 'EXAMPLE', 'number': '1'}  \n",
       "4               {'name': 'EXAMPLE', 'number': '2'}  \n",
       "5               {'name': 'EXAMPLE', 'number': '3'}  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.set_option(\"display.max_colwidth\", 200)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## thunderbolt filter\n",
    "2nd arg 'task_filters' is str or list. So fast.\n",
    "\n",
    "task_filters is partial match word for example: \n",
    " - 'Tag' -> HogeTag, NormalizeHogeTag, TagTask, ...\n",
    " - ['Train', 'Tag'] -> TrainModel, TrainData, HogeTag, NormalizeHogeTag, TagTask, ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tb = Thunderbolt(task_filters='Sample') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task_id</th>\n",
       "      <th>task_name</th>\n",
       "      <th>last_modified</th>\n",
       "      <th>task_params</th>\n",
       "      <th>task_hash</th>\n",
       "      <th>task_log</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:13.704206</td>\n",
       "      <td>{'name': 'TEMP', 'number': '1'}</td>\n",
       "      <td>d57cff8074e2560896974850e5d3174d</td>\n",
       "      <td>{'file_path': ['./resources/__main__/SampleTask_d57cff8074e2560896974850e5d3174d.pkl']}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:15.728998</td>\n",
       "      <td>{'name': 'TEMP', 'number': '2'}</td>\n",
       "      <td>d05a2ab961781d3d8eca3e2e5f0d608b</td>\n",
       "      <td>{'file_path': ['./resources/__main__/SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl']}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:07.787346</td>\n",
       "      <td>{'name': 'EXAMPLE', 'number': '1'}</td>\n",
       "      <td>e883bcfad65f5fb68259d1cd4691f384</td>\n",
       "      <td>{'file_path': ['./resources/__main__/SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl']}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:09.776602</td>\n",
       "      <td>{'name': 'EXAMPLE', 'number': '2'}</td>\n",
       "      <td>84b0b9c5a39bce072271599c9f730660</td>\n",
       "      <td>{'file_path': ['./resources/__main__/SampleTask_84b0b9c5a39bce072271599c9f730660.pkl']}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>SampleTask</td>\n",
       "      <td>2020-02-13 14:17:11.699651</td>\n",
       "      <td>{'name': 'EXAMPLE', 'number': '3'}</td>\n",
       "      <td>944fc52ef5011b71b5839f035f4d7e48</td>\n",
       "      <td>{'file_path': ['./resources/__main__/SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl']}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   task_id   task_name              last_modified  \\\n",
       "0        0  SampleTask 2020-02-13 14:17:13.704206   \n",
       "1        1  SampleTask 2020-02-13 14:17:15.728998   \n",
       "2        3  SampleTask 2020-02-13 14:17:07.787346   \n",
       "3        4  SampleTask 2020-02-13 14:17:09.776602   \n",
       "4        5  SampleTask 2020-02-13 14:17:11.699651   \n",
       "\n",
       "                          task_params                         task_hash  \\\n",
       "0     {'name': 'TEMP', 'number': '1'}  d57cff8074e2560896974850e5d3174d   \n",
       "1     {'name': 'TEMP', 'number': '2'}  d05a2ab961781d3d8eca3e2e5f0d608b   \n",
       "2  {'name': 'EXAMPLE', 'number': '1'}  e883bcfad65f5fb68259d1cd4691f384   \n",
       "3  {'name': 'EXAMPLE', 'number': '2'}  84b0b9c5a39bce072271599c9f730660   \n",
       "4  {'name': 'EXAMPLE', 'number': '3'}  944fc52ef5011b71b5839f035f4d7e48   \n",
       "\n",
       "                                                                                  task_log  \n",
       "0  {'file_path': ['./resources/__main__/SampleTask_d57cff8074e2560896974850e5d3174d.pkl']}  \n",
       "1  {'file_path': ['./resources/__main__/SampleTask_d05a2ab961781d3d8eca3e2e5f0d608b.pkl']}  \n",
       "2  {'file_path': ['./resources/__main__/SampleTask_e883bcfad65f5fb68259d1cd4691f384.pkl']}  \n",
       "3  {'file_path': ['./resources/__main__/SampleTask_84b0b9c5a39bce072271599c9f730660.pkl']}  \n",
       "4  {'file_path': ['./resources/__main__/SampleTask_944fc52ef5011b71b5839f035f4d7e48.pkl']}  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# example: all_data=True\n",
    "\n",
    "tb.get_task_df(all_data=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Data Load\n",
    "---\n",
    "using load method\n",
    "- arg: thunderbolt's task_id\n",
    "- return : data list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is sample output. model number: 1\n"
     ]
    }
   ],
   "source": [
    "x = tb.load(task_id=3)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### newest data load example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'this is sample output. model number: 2'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tb.get_data('SampleTask')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}