fj_embedding_composite.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fj_embedding_composite.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNt76jHk4zu3EaOLTB2Mw/r",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/r-dube/fakejobs/blob/main/fj_embedding_composite.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFWTUL3GCHsG"
      },
      "source": [
        "# Load the modules used\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "import string\n",
        "import nltk\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, roc_auc_score\n",
        "from keras.models import Model\n",
        "from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional, GlobalMaxPool1D, Conv1D, MaxPool1D, Flatten, RepeatVector, Input, Embedding, Concatenate\n",
        "from keras.optimizers import Adam\n",
        "from keras import metrics\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.preprocessing.text import Tokenizer"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9wN7spE3GCrZ"
      },
      "source": [
        "# For reproducible results, set seeds\n",
        "import random as rn\n",
        "import os\n",
        "import tensorflow as tf\n",
        "os.environ['PYTHONHASHSEED'] = '42'\n",
        "os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
        "np.random.seed(42)\n",
        "rn.seed(42)\n",
        "tf.random.set_seed(42)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rrnTamUHDzxI"
      },
      "source": [
        "# Set data_url, the location of the data\n",
        "# Data is not loaded from a local file\n",
        "# data_url=\"https://raw.githubusercontent.com/r-dube/fakejobs/main/data/fj_small.csv\"\n",
        "# data_url=\"https://raw.githubusercontent.com/r-dube/fakejobs/main/data/fj_medium.csv\"\n",
        "data_url=\"https://raw.githubusercontent.com/r-dube/fakejobs/main/data/fake_job_postings.csv\""
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "usYPD_l1Bimz"
      },
      "source": [
        "def fj_load_df_from_url():\n",
        "    \"\"\"\n",
        "    Load dataframe from csv file\n",
        "    Input:\n",
        "        None\n",
        "    Returns:\n",
        "        dataframe\n",
        "    \"\"\"\n",
        "\n",
        "    df = pd.read_csv(data_url)\n",
        "\n",
        "    print ('Loaded dataframe shape', df.shape)\n",
        "\n",
        "    counts = fj_label_stats(df)\n",
        "    print ('Not fraudulent', counts[0], 'Fraudulent', counts[1])\n",
        "\n",
        "    print(df.describe())\n",
        "\n",
        "    print ('NAs/NANs in data =>')\n",
        "    print(df.isna().sum())\n",
        "\n",
        "    return df\n",
        "\n",
        "def fj_label_stats(df):\n",
        "    \"\"\"\n",
        "    Very basic label statistics\n",
        "    Input: \n",
        "        Dataframe\n",
        "    Returns:\n",
        "        Number of samples with 0, 1 as the label\n",
        "    \"\"\"\n",
        "    counts = np.bincount(df['fraudulent'])\n",
        "    return counts\n",
        "\n",
        "def fj_txt_only(df):\n",
        "    \"\"\"\n",
        "    Combine all the text fields, discard everything else except for the label\n",
        "    Input: \n",
        "        Dataframe\n",
        "    Returns:\n",
        "        Processed dataframe\n",
        "    \"\"\"\n",
        "    \n",
        "    df.fillna(\" \", inplace = True)\n",
        "\n",
        "    df['text'] = df['title'] + ' ' + df['location'] + ' ' + df['department'] + \\\n",
        "    ' ' + df['company_profile'] + ' ' + df['description'] + ' ' + \\\n",
        "    df['requirements'] + ' ' + df['benefits'] + ' ' + df['employment_type'] + \\\n",
        "    ' ' + df['required_education'] + ' ' + df['industry'] + ' ' + df['function'] \n",
        "\n",
        "    del df['title']\n",
        "    del df['location']\n",
        "    del df['department']\n",
        "    del df['company_profile']\n",
        "    del df['description']\n",
        "    del df['requirements']\n",
        "    del df['benefits']\n",
        "    del df['employment_type']\n",
        "    del df['required_experience']\n",
        "    del df['required_education']\n",
        "    del df['industry']\n",
        "    del df['function']  \n",
        "    \n",
        "    del df['salary_range']\n",
        "    del df['job_id']\n",
        "    del df['telecommuting']\n",
        "    del df['has_company_logo']\n",
        "    del df['has_questions']\n",
        "\n",
        "    return df"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lks9Mm0Tc1l2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9d57d03b-61bb-43fb-cb6d-7cf5b9ff52a1"
      },
      "source": [
        "df = fj_load_df_from_url()\n",
        "df = fj_txt_only(df)\n",
        "print('Maximum text length', df['text'].str.len().max())"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded dataframe shape (17880, 18)\n",
            "Not fraudulent 17014 Fraudulent 866\n",
            "             job_id  telecommuting  ...  has_questions    fraudulent\n",
            "count  17880.000000   17880.000000  ...   17880.000000  17880.000000\n",
            "mean    8940.500000       0.042897  ...       0.491723      0.048434\n",
            "std     5161.655742       0.202631  ...       0.499945      0.214688\n",
            "min        1.000000       0.000000  ...       0.000000      0.000000\n",
            "25%     4470.750000       0.000000  ...       0.000000      0.000000\n",
            "50%     8940.500000       0.000000  ...       0.000000      0.000000\n",
            "75%    13410.250000       0.000000  ...       1.000000      0.000000\n",
            "max    17880.000000       1.000000  ...       1.000000      1.000000\n",
            "\n",
            "[8 rows x 5 columns]\n",
            "NAs/NANs in data =>\n",
            "job_id                     0\n",
            "title                      0\n",
            "location                 346\n",
            "department             11547\n",
            "salary_range           15012\n",
            "company_profile         3308\n",
            "description                1\n",
            "requirements            2695\n",
            "benefits                7210\n",
            "telecommuting              0\n",
            "has_company_logo           0\n",
            "has_questions              0\n",
            "employment_type         3471\n",
            "required_experience     7050\n",
            "required_education      8105\n",
            "industry                4903\n",
            "function                6455\n",
            "fraudulent                 0\n",
            "dtype: int64\n",
            "Maximum text length 14991\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxtcnlHBpPro"
      },
      "source": [
        "# Utilities to clean text\n",
        "\n",
        "def remove_URL(text):\n",
        "    url = re.compile(r\"https?://\\S+|www\\.\\S+\")\n",
        "    return url.sub(r\"\", text)\n",
        "\n",
        "def remove_html(text):\n",
        "    html = re.compile(r\"<.*?>\")\n",
        "    return html.sub(r\"\", text)\n",
        "\n",
        "def remove_emoji(string):\n",
        "    emoji_pattern = re.compile(\n",
        "        \"[\"\n",
        "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "        u\"\\U00002702-\\U000027B0\"\n",
        "        u\"\\U000024C2-\\U0001F251\"\n",
        "        \"]+\",\n",
        "        flags=re.UNICODE,\n",
        "    )\n",
        "    return emoji_pattern.sub(r\"\", string)\n",
        "\n",
        "def remove_punct(text):\n",
        "    table = str.maketrans(\"\", \"\", string.punctuation)\n",
        "    return text.translate(table)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z5ESVXOyxzK9",
        "outputId": "ac2d0fe0-4e90-431d-8ddf-62003221b557"
      },
      "source": [
        "# more text cleaning - remove stopwords using NLTK\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "stop = set(stopwords.words(\"english\"))\n",
        "\n",
        "def remove_stopwords(text):\n",
        "    text = [word.lower() for word in text.split() if word.lower() not in stop]\n",
        "\n",
        "    return \" \".join(text)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TrqwWWQ2uruM"
      },
      "source": [
        "# Actually clean the text\n",
        "df['text'] = df['text'].map(lambda x: remove_URL(x))\n",
        "df['text'] = df['text'].map(lambda x: remove_html(x))\n",
        "df['text'] = df['text'].map(lambda x: remove_emoji(x))\n",
        "df['text'] = df['text'].map(lambda x: remove_punct(x))\n",
        "df['text'] = df[\"text\"].map(remove_stopwords)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EUCnYLkBpnNu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b5e77774-3634-4d39-a050-7b33c959f930"
      },
      "source": [
        "# train-test split\n",
        "train_text, test_text, train_labels , test_labels = train_test_split(df['text'], df['fraudulent'] , test_size = 0.15)\n",
        "print(train_text.shape)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(15198,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P1QfnRxKUxlV"
      },
      "source": [
        "# Token model: Configuration parameters\n",
        "max_num_words = 50000 # maximum allowed size of vocabulary\n",
        "max_length = 250 # maximum allowed number of words in a job description\n",
        "embed_dim = 32 # number of dimensions for learned embedding\n",
        "# settimg embedding_dim to 50 as min globe embedding size available is 50"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kW2n8ilFghmh",
        "outputId": "0c80892e-8002-45ea-a5e8-311b4c07c4aa"
      },
      "source": [
        "# Token model: Prepare the token based train and test input\n",
        "\n",
        "# max_num_words variable in case we want to clip the number of words\n",
        "tokenizer = Tokenizer(num_words=max_num_words)\n",
        "tokenizer.fit_on_texts(train_text)\n",
        "\n",
        "num_tokens = len(tokenizer.word_index)\n",
        "print(\"Found %s unique tokens.\" % num_tokens)\n",
        "\n",
        "# Fix dictionary for later use when embeddings are loaded\n",
        "# While the vocabulary size is restricted to max_num_words above\n",
        "#   the dictionary returned by the Tokenizer has values outside of max_num_words\n",
        "#   the embedding loading code needs all values to be in range\n",
        "tokenizer.word_index = {e:i for e,i in tokenizer.word_index.items() if i < max_num_words}\n",
        "\n",
        "# handle the case where the number of tokens < max_num_words, the max size of dictionary\n",
        "max_num_words = min(num_tokens + 1, max_num_words)\n",
        "print(\"Set %s max_num_words.\" % max_num_words)\n",
        "\n",
        "word_index = tokenizer.word_index\n",
        "\n",
        "train_sequences = tokenizer.texts_to_sequences(train_text)\n",
        "train_padded = pad_sequences(\n",
        "    train_sequences, maxlen=max_length, padding=\"post\", truncating=\"post\"\n",
        ")\n",
        "\n",
        "test_sequences = tokenizer.texts_to_sequences(test_text)\n",
        "test_padded = pad_sequences(\n",
        "    test_sequences, maxlen=max_length, padding=\"post\", truncating=\"post\"\n",
        ")\n",
        "\n",
        "print(f\"Shape of train {train_padded.shape}\")\n",
        "print(f\"Shape of test {test_padded.shape}\")"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 157210 unique tokens.\n",
            "Set 50000 max_num_words.\n",
            "Shape of train (15198, 250)\n",
            "Shape of test (2682, 250)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "EtPnDS0s8lgb",
        "outputId": "0e2adffa-8a51-4072-f604-1f0722b123fe"
      },
      "source": [
        "\"\"\"\n",
        "# test to ensure that there are no out of index values in dictionary\n",
        "print(word_index[\"cma\"])\n",
        "\"\"\""
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\n# test to ensure that there are no out of index values in dictionary\\nprint(word_index[\"cma\"])\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 103
        },
        "id": "LMgPGUA7r-kn",
        "outputId": "427cdb14-bd64-45ec-e072-34e150a997c7"
      },
      "source": [
        "\"\"\"\n",
        "# Token model: but with FASTTEXT (use either FASTTEXT or word2vec but not both)\n",
        "# Token model: Assumes that FASTTEXT data is available on locally mounted drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "fasttext_file='/content/drive/My Drive/Data/fakejobs/gensim_fasttext.mod'\n",
        "\n",
        "# Token model: load in pre-trained FASTTEXT word vectors\n",
        "print('Loading word vectors...')\n",
        "from gensim.models.fasttext import FastText as FT_gensim\n",
        "loaded_model = FT_gensim.load(fasttext_file)\n",
        "\n",
        "print('Filling pre-trained embeddings...')\n",
        "embedding_matrix = np.zeros((max_num_words, embed_dim))\n",
        "for word, i in word_index.items():\n",
        "      # XXX not sure why gensim uses the quotes below\n",
        "      if (\"word\" in loaded_model):\n",
        "          embedding_matrix[i] = loaded_model[\"word\"]\n",
        "          # word's vector not found in embedding index will be all zeros.\n",
        "\n",
        "word=\"salary\"\n",
        "print(word)\n",
        "if (\"word\" in loaded_model):\n",
        "  print (loaded_model[\"word\"])\n",
        "else:\n",
        "  print (\"no vector\")\n",
        "\"\"\""
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\n# Token model: but with FASTTEXT (use either FASTTEXT or word2vec but not both)\\n# Token model: Assumes that FASTTEXT data is available on locally mounted drive\\nfrom google.colab import drive\\ndrive.mount(\\'/content/drive\\')\\nfasttext_file=\\'/content/drive/My Drive/Data/fakejobs/gensim_fasttext.mod\\'\\n\\n# Token model: load in pre-trained FASTTEXT word vectors\\nprint(\\'Loading word vectors...\\')\\nfrom gensim.models.fasttext import FastText as FT_gensim\\nloaded_model = FT_gensim.load(fasttext_file)\\n\\nprint(\\'Filling pre-trained embeddings...\\')\\nembedding_matrix = np.zeros((max_num_words, embed_dim))\\nfor word, i in word_index.items():\\n      # XXX not sure why gensim uses the quotes below\\n      if (\"word\" in loaded_model):\\n          embedding_matrix[i] = loaded_model[\"word\"]\\n          # word\\'s vector not found in embedding index will be all zeros.\\n\\nword=\"salary\"\\nprint(word)\\nif (\"word\" in loaded_model):\\n  print (loaded_model[\"word\"])\\nelse:\\n  print (\"no vector\")\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "evHj8J2MjwMS",
        "outputId": "64689e0e-0397-41c9-c6b5-1b8d76283e5c"
      },
      "source": [
        "# \"\"\"\n",
        "# Token model: but with word2vec (use either FASTTEXT or word2vec but not both)\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "import gensim.models\n",
        "word2vec_file='/content/drive/My Drive/Data/fakejobs/gensim_word2vec.mod'\n",
        "\n",
        "print('Loading word vectors...')\n",
        "loaded_model = gensim.models.Word2Vec.load(word2vec_file)\n",
        "\n",
        "# Token model: prepare embedding matrix\n",
        "print('Filling pre-trained embeddings...')\n",
        "embedding_matrix = np.zeros((max_num_words, embed_dim))\n",
        "for word, i in word_index.items():\n",
        "      if (word in loaded_model.wv.vocab):\n",
        "          embedding_matrix[i] = loaded_model[word]\n",
        "          # word's vector not found in embedding index will be all zeros.\n",
        "\n",
        "word=\"salary\"\n",
        "print(word)\n",
        "if (word in loaded_model.wv.vocab):\n",
        "  print (loaded_model[word])\n",
        "else:\n",
        "  print (\"no vector\")\n",
        "#\"\"\""
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n",
            "Loading word vectors...\n",
            "Filling pre-trained embeddings...\n",
            "salary\n",
            "[-0.29170936 -0.61309755 -0.46258762  0.07202664 -0.09073978  0.077778\n",
            " -0.12148292  0.37506315 -1.0836064  -0.00529616  0.35912657  0.8394321\n",
            "  0.10721996 -0.40348828  0.12436266  0.68627995 -0.13415784  0.81693923\n",
            "  0.4995241   0.333784    0.31136537 -0.05352516  0.3026262   0.41333514\n",
            "  0.53737676  0.16790484 -0.31380147 -0.24655591  0.47631833  0.7520056\n",
            "  0.59845304  0.10776368]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:17: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:23: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_g3Yeivx0r7"
      },
      "source": [
        "# Token model: load pre-trained word embeddings into an Embedding layer\n",
        "# We set trainable = True if want to allow the embedding layer to be trained further\n",
        "\n",
        "embedding_layer = Embedding(\n",
        "  max_num_words,\n",
        "  embed_dim,\n",
        "  weights=[embedding_matrix],\n",
        "  input_length=max_length,\n",
        "  trainable=False\n",
        ")"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PHNBeD6dslhd",
        "outputId": "9ee6fafe-03dc-4509-bf28-396a99c41fab"
      },
      "source": [
        "# Char model: Configuration parameters\n",
        "\n",
        "# Assuming that the text has been processed for word tokenization\n",
        "# alphabet=\"abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\\\"/\\\\|_@#$%^&*~`+-=<>()[]{}\"\n",
        "alphabet=\"abcdefghijklmnopqrstuvwxyz0123456789\"\n",
        "encoding_size = len(alphabet)\n",
        "char_input_size=1000\n",
        "\n",
        "# Create a dictionary for encoding characters\n",
        "dict = {}  # Maps each character to an integer\n",
        "for idx, char in enumerate(alphabet):\n",
        "  dict[char] = idx + 1\n",
        "\n",
        "print(encoding_size)\n",
        "print (dict)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "36\n",
            "{'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, '0': 27, '1': 28, '2': 29, '3': 30, '4': 31, '5': 32, '6': 33, '7': 34, '8': 35, '9': 36}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OxU9Y3fL1jeU"
      },
      "source": [
        "# Char model: Utility function(s)\n",
        "\n",
        "# Return one-hot-vector character encoding for string\n",
        "# Memory is allocated outside this routine\n",
        "def str_to_ohv(s, ohv):\n",
        "  max_length = min(len(s), char_input_size)\n",
        "  for i in range(0, max_length):\n",
        "    c = s[i]\n",
        "    if c in dict:\n",
        "      ohv[i, dict[c]-1] = 1\n",
        "  return ohv"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hwyr--LJ_luP"
      },
      "source": [
        "# Char model: create train input\n",
        "num_jobs = train_text.shape[0]\n",
        "train_t = np.zeros((num_jobs, char_input_size, encoding_size), dtype=np.int8)\n",
        "i = 0\n",
        "for _, val in train_text.iteritems():\n",
        "  str_to_ohv(val, train_t[i])\n",
        "  i=i+1"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UlXCalahJlar"
      },
      "source": [
        "# Char model: create test input\n",
        "num_jobs = test_text.shape[0]\n",
        "test_t = np.zeros((num_jobs, char_input_size, encoding_size), dtype=np.int8)\n",
        "i = 0\n",
        "for _, val in test_text.iteritems():\n",
        "  str_to_ohv(val, test_t[i])\n",
        "  i=i+1"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AX_1JKmA6BZy"
      },
      "source": [
        "# The Composite model: Token + Char\n",
        "\n",
        "# Not specificying input dtype as Keras internally assumes float32\n",
        "\n",
        "# Token (word) model\n",
        "token_input = Input(shape=(max_length,), name=\"token_input\")\n",
        "token_embedding = embedding_layer(token_input)\n",
        "# uncomment line below (and comment line above) to switch to inline-only embedding\n",
        "# token_embedding = Embedding(max_num_words, embed_dim, input_length=max_length, name=\"token_embedding\")(token_input)\n",
        "token_conv = Conv1D(64, kernel_size=3, strides=1, padding=\"valid\", activation=\"relu\", name=\"token_conv\")(token_embedding)\n",
        "token_pool = MaxPool1D(pool_size=3, strides=3, name=\"token_pool\")(token_conv)\n",
        "token_drop = Dropout(.5, name=\"token_drop\")(token_pool)\n",
        "\n",
        "# Char model\n",
        "char_input = Input(shape=(char_input_size, encoding_size), name=\"char_input\")\n",
        "char_conv = Conv1D(64, kernel_size=3, strides=1, padding=\"valid\",activation=\"relu\", name=\"char_conv\")(char_input)\n",
        "char_pool = GlobalMaxPool1D(name=\"char_pool\")(char_conv)\n",
        "char_drop = Dropout(.5, name=\"char_drop\")(char_pool)\n",
        "char_repeated = RepeatVector(token_drop.get_shape()[1], name=\"char_repeated\")(char_drop)\n",
        "\n",
        "# Merge\n",
        "merged = Concatenate(axis=2, name=\"concat\")([token_drop, char_repeated])\n",
        "lstm = Bidirectional(LSTM(32, dropout=0.3, recurrent_dropout=0.01, name=\"lstm\"), name=\"bidir\")(merged)\n",
        "output = Dense(1, activation=\"sigmoid\", name=\"output\")(lstm)\n",
        "\n",
        "# define a model with a list of two inputs\n",
        "model = Model(inputs=[token_input, char_input], outputs=output)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZUzcT443YL0U",
        "outputId": "57173de9-382d-4af4-a432-19f5ac88df3e"
      },
      "source": [
        "model.compile(optimizer = 'adam' , loss = 'binary_crossentropy' , metrics = ['accuracy', tf.keras.metrics.FalsePositives(), tf.keras.metrics.FalseNegatives()])\n",
        "model.summary()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "token_input (InputLayer)        [(None, 250)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "char_input (InputLayer)         [(None, 1000, 36)]   0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding (Embedding)           (None, 250, 32)      1600000     token_input[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "char_conv (Conv1D)              (None, 998, 64)      6976        char_input[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "token_conv (Conv1D)             (None, 248, 64)      6208        embedding[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "char_pool (GlobalMaxPooling1D)  (None, 64)           0           char_conv[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "token_pool (MaxPooling1D)       (None, 82, 64)       0           token_conv[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "char_drop (Dropout)             (None, 64)           0           char_pool[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "token_drop (Dropout)            (None, 82, 64)       0           token_pool[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "char_repeated (RepeatVector)    (None, 82, 64)       0           char_drop[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concat (Concatenate)            (None, 82, 128)      0           token_drop[0][0]                 \n",
            "                                                                 char_repeated[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "bidir (Bidirectional)           (None, 64)           41216       concat[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "output (Dense)                  (None, 1)            65          bidir[0][0]                      \n",
            "==================================================================================================\n",
            "Total params: 1,654,465\n",
            "Trainable params: 54,465\n",
            "Non-trainable params: 1,600,000\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "KdX53khljBIf",
        "outputId": "0f093325-6f58-4e61-bcfd-6c1962684388"
      },
      "source": [
        "\"\"\"\n",
        "# Uncomment to save image of model architecture\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "dot_img_file = '/content/drive/My Drive/Results/fj_embedding_composite.png'\n",
        "tf.keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True, show_dtype=True)\n",
        "\"\"\""
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"\\n# Uncomment to save image of model architecture\\nfrom google.colab import drive\\ndrive.mount('/content/drive')\\n\\ndot_img_file = '/content/drive/My Drive/Results/fj_embedding_composite.png'\\ntf.keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True, show_dtype=True)\\n\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7872Kjn0I6Gc",
        "outputId": "8c09128c-de4e-4a5a-818d-bc52d141afeb"
      },
      "source": [
        "model.fit([train_padded, train_t], train_labels, epochs = 15)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/15\n",
            "475/475 [==============================] - 120s 239ms/step - loss: 0.1970 - accuracy: 0.9458 - false_positives: 8.4811 - false_negatives: 367.2311\n",
            "Epoch 2/15\n",
            "475/475 [==============================] - 114s 239ms/step - loss: 0.1251 - accuracy: 0.9555 - false_positives: 44.4664 - false_negatives: 284.7080\n",
            "Epoch 3/15\n",
            "475/475 [==============================] - 112s 236ms/step - loss: 0.1064 - accuracy: 0.9631 - false_positives: 61.7647 - false_negatives: 218.0105\n",
            "Epoch 4/15\n",
            "475/475 [==============================] - 112s 237ms/step - loss: 0.0911 - accuracy: 0.9696 - false_positives: 53.2353 - false_negatives: 177.3445\n",
            "Epoch 5/15\n",
            "475/475 [==============================] - 112s 236ms/step - loss: 0.0813 - accuracy: 0.9708 - false_positives: 54.4790 - false_negatives: 158.2605\n",
            "Epoch 6/15\n",
            "475/475 [==============================] - 112s 236ms/step - loss: 0.0680 - accuracy: 0.9778 - false_positives: 35.8151 - false_negatives: 136.7059\n",
            "Epoch 7/15\n",
            "475/475 [==============================] - 112s 236ms/step - loss: 0.0612 - accuracy: 0.9796 - false_positives: 42.2584 - false_negatives: 114.3382\n",
            "Epoch 8/15\n",
            "475/475 [==============================] - 112s 237ms/step - loss: 0.0553 - accuracy: 0.9802 - false_positives: 41.1597 - false_negatives: 106.4748\n",
            "Epoch 9/15\n",
            "475/475 [==============================] - 112s 237ms/step - loss: 0.0500 - accuracy: 0.9820 - false_positives: 35.2605 - false_negatives: 103.6681\n",
            "Epoch 10/15\n",
            "475/475 [==============================] - 113s 238ms/step - loss: 0.0420 - accuracy: 0.9854 - false_positives: 34.4223 - false_negatives: 82.9391\n",
            "Epoch 11/15\n",
            "475/475 [==============================] - 112s 236ms/step - loss: 0.0373 - accuracy: 0.9876 - false_positives: 27.3235 - false_negatives: 72.2059\n",
            "Epoch 12/15\n",
            "475/475 [==============================] - 112s 235ms/step - loss: 0.0487 - accuracy: 0.9840 - false_positives: 35.3046 - false_negatives: 78.0903\n",
            "Epoch 13/15\n",
            "475/475 [==============================] - 113s 238ms/step - loss: 0.0403 - accuracy: 0.9863 - false_positives: 30.9769 - false_negatives: 69.8782\n",
            "Epoch 14/15\n",
            "475/475 [==============================] - 113s 238ms/step - loss: 0.0387 - accuracy: 0.9868 - false_positives: 32.8866 - false_negatives: 68.2122\n",
            "Epoch 15/15\n",
            "475/475 [==============================] - 112s 236ms/step - loss: 0.0347 - accuracy: 0.9876 - false_positives: 28.3424 - false_negatives: 65.8004\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f36ca78f780>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F4B3OYymJPHW"
      },
      "source": [
        "pred_soft = model.predict([test_padded, test_t])"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ci0Z6vXdJP2p",
        "outputId": "0c564a0c-07e7-4465-cf92-ad7089f3a9bb"
      },
      "source": [
        "# pred = np.around(pred_soft, decimals = 0)\n",
        "pred = np.where(pred_soft > 0.15, 1, 0)\n",
        "\n",
        "acc = accuracy_score(pred, test_labels)\n",
        "f1 = f1_score(pred, test_labels)\n",
        "\n",
        "cm = confusion_matrix(test_labels, pred)\n",
        "tn = cm[0][0]\n",
        "fn = cm[1][0]\n",
        "tp = cm[1][1]\n",
        "fp = cm[0][1]\n",
        "\n",
        "print('Accuracy score: {:.4f}'.format(acc), 'F1 score: {:.4f}'.format(f1))\n",
        "print('False Positives: {:.0f}'.format(fp), 'False Negatives: {:.0f}'.format(fn))\n",
        "print('Confusion matrix:\\n', cm)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy score: 0.9836 F1 score: 0.8417\n",
            "False Positives: 17 False Negatives: 27\n",
            "Confusion matrix:\n",
            " [[2521   17]\n",
            " [  27  117]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xm1MoM4U4aQa",
        "outputId": "c4c9410b-0f6d-4e02-f88f-a73fe08c24bb"
      },
      "source": [
        "auc = roc_auc_score(test_labels, pred_soft)\n",
        "print('AUC score: {:.4f}'.format(auc))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AUC score: 0.9872\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0BBcKCVqOKCy",
        "outputId": "86c62c1a-ba50-4834-8a28-b0ee41c1c9c6"
      },
      "source": [
        "\"\"\"\"\n",
        "# Uncomment to save results on drive to a csv file\n",
        "df_results1 = pd.DataFrame(data=test_labels)\n",
        "df_results1.reset_index(drop=True, inplace=True)\n",
        "df_results2 = pd.DataFrame(data=pred_soft, columns=['embedding_composite'])\n",
        "df_results = pd.concat([df_results1, df_results2], axis=1)\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "results_file='/content/drive/My Drive/Results/fj_embedding_composite.csv'\n",
        "\n",
        "df_results.to_csv(results_file)\n",
        "\"\"\""
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}