NLP/Models+K-Means/main/csv_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "676be61a-bd65-4510-8357-94859f596330",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "94f312ae-87d2-4d5e-ae75-9fc85a2a980c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_json('../data/SIR_test_set.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7d949b04-4929-4921-a818-4e8cbb57826b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             CVE_ID                                      Issue_Url_old  \\\n",
      "0    CVE-2021-45822      https://github.com/btiteam/xbtit-3.1/issues/7   \n",
      "1    CVE-2021-45769  https://github.com/mz-automation/libiec61850/i...   \n",
      "2    CVE-2021-45773  https://github.com/mz-automation/lib60870/issu...   \n",
      "3    CVE-2022-25014      https://github.com/gamonoid/icehrm/issues/283   \n",
      "4    CVE-2022-25013      https://github.com/gamonoid/icehrm/issues/284   \n",
      "..              ...                                                ...   \n",
      "705  CVE-2022-32417          https://github.com/Snakinya/Vuln/issues/1   \n",
      "706  CVE-2021-34485  https://github.com/github/advisory-database/is...   \n",
      "707  CVE-2021-44906   https://github.com/minimistjs/minimist/issues/11   \n",
      "708   CVE-2020-8927  https://github.com/github/advisory-database/is...   \n",
      "709  CVE-2021-31402            https://github.com/cfug/dio/issues/1752   \n",
      "\n",
      "                                         Issue_Url_new  \\\n",
      "0        https://github.com/btiteam/xbtit-3.1/issues/7   \n",
      "1    https://github.com/mz-automation/libiec61850/i...   \n",
      "2    https://github.com/mz-automation/lib60870/issu...   \n",
      "3        https://github.com/gamonoid/icehrm/issues/283   \n",
      "4        https://github.com/gamonoid/icehrm/issues/284   \n",
      "..                                                 ...   \n",
      "705          https://github.com/snakinya/vuln/issues/1   \n",
      "706  https://github.com/github/advisory-database/is...   \n",
      "707   https://github.com/minimistjs/minimist/issues/11   \n",
      "708  https://github.com/github/advisory-database/is...   \n",
      "709            https://github.com/cfug/dio/issues/1752   \n",
      "\n",
      "                      Repo_new          Issue_Created_At  \\\n",
      "0            btiteam/xbtit-3.1 2021-12-22 20:25:58+00:00   \n",
      "1    mz-automation/libiec61850 2021-12-23 00:53:55+00:00   \n",
      "2       mz-automation/lib60870 2021-12-23 06:01:26+00:00   \n",
      "3              gamonoid/icehrm 2021-12-23 08:09:18+00:00   \n",
      "4              gamonoid/icehrm 2021-12-23 08:13:20+00:00   \n",
      "..                         ...                       ...   \n",
      "705              Snakinya/Vuln 2022-08-04 10:38:48+00:00   \n",
      "706   github/advisory-database 2022-10-12 20:44:32+00:00   \n",
      "707        minimistjs/minimist 2022-10-19 14:23:14+00:00   \n",
      "708   github/advisory-database 2022-10-31 20:04:11+00:00   \n",
      "709                   cfug/dio 2023-03-21 16:54:52+00:00   \n",
      "\n",
      "                                           description  \\\n",
      "0    Stored & Reflected XSS affecting Xbtit NUMBERT...   \n",
      "1    NULL Pointer Dereference in APITAG NULL Pointe...   \n",
      "2    NULL Pointer Dereference in APITAG NULL Pointe...   \n",
      "3    Reflected XSS vulnerability NUMBERTAG in icehr...   \n",
      "4    Reflected XSS vulnerabilities NUMBERTAG in ice...   \n",
      "..                                                 ...   \n",
      "705  pboot cms NUMBERTAG RCE. 漏洞详情： URLTAG 声明 APITA...   \n",
      "706  .NET CVE backfill round NUMBERTAG Hello, Pleas...   \n",
      "707  Backport of NUMBERTAG fixes to NUMBERTAG Thank...   \n",
      "708  Update impacted packages for CVETAG . Hi, This...   \n",
      "709  CVE Dio NUMBERTAG Google OVS Scanner. Package ...   \n",
      "\n",
      "                                     vectorString  severity  baseScore  \\\n",
      "0    CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:C/C:L/I:L/A:N    MEDIUM        6.1   \n",
      "1    CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H      HIGH        7.5   \n",
      "2    CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H      HIGH        7.5   \n",
      "3    CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:C/C:L/I:L/A:N    MEDIUM        6.1   \n",
      "4    CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:C/C:L/I:L/A:N    MEDIUM        6.1   \n",
      "..                                            ...       ...        ...   \n",
      "705  CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H  CRITICAL        9.8   \n",
      "706  CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N    MEDIUM        5.5   \n",
      "707  CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H  CRITICAL        9.8   \n",
      "708  CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:L/A:L    MEDIUM        6.5   \n",
      "709  CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N      HIGH        7.5   \n",
      "\n",
      "     impactScore  exploitabilityScore  \n",
      "0            2.7                  2.8  \n",
      "1            3.6                  3.9  \n",
      "2            3.6                  3.9  \n",
      "3            2.7                  2.8  \n",
      "4            2.7                  2.8  \n",
      "..           ...                  ...  \n",
      "705          5.9                  3.9  \n",
      "706          3.6                  1.8  \n",
      "707          5.9                  3.9  \n",
      "708          2.5                  3.9  \n",
      "709          3.6                  3.9  \n",
      "\n",
      "[710 rows x 11 columns]\n"
     ]
    }
   ],
   "source": [
    "train_data_temp = pd.DataFrame()\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d4272d23-2c40-416a-aa83-40b09817ea0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_temp['description'] = data['description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "101f87d6-38d7-4562-a572-49ab74eec58d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0      False\n",
      "1      False\n",
      "2      False\n",
      "3      False\n",
      "4      False\n",
      "       ...  \n",
      "705    False\n",
      "706    False\n",
      "707    False\n",
      "708    False\n",
      "709    False\n",
      "Name: description, Length: 710, dtype: bool\n"
     ]
    }
   ],
   "source": [
    "print(train_data_temp['description'].isna())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "85b0cd35-3862-43fb-b4ab-d88c0ceae6da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Empty DataFrame\n",
      "Columns: [description]\n",
      "Index: []\n"
     ]
    }
   ],
   "source": [
    "# 获取 NaN 值的行索引\n",
    "nan_rows = train_data_temp[train_data_temp['description'].isna()]\n",
    "print(nan_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8eaf202e-b96b-4f79-8b3b-89e4757add04",
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorString = data['vectorString']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "49331613-93f9-4d86-9e43-384c16ff8813",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    AV AC PR UI  S  C  I  A\n",
      "0    N  L  N  R  C  L  L  N\n",
      "1    N  L  N  N  U  N  N  H\n",
      "2    N  L  N  N  U  N  N  H\n",
      "3    N  L  N  R  C  L  L  N\n",
      "4    N  L  N  R  C  L  L  N\n",
      "..  .. .. .. .. .. .. .. ..\n",
      "705  N  L  N  N  U  H  H  H\n",
      "706  L  L  L  N  U  H  N  N\n",
      "707  N  L  N  N  U  H  H  H\n",
      "708  N  L  N  N  U  N  L  L\n",
      "709  N  L  N  N  U  H  N  N\n",
      "\n",
      "[710 rows x 8 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lx\\AppData\\Local\\Temp\\ipykernel_38864\\3052899741.py:14: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
      "  train_data = train_data.applymap(transform_value)\n"
     ]
    }
   ],
   "source": [
    "#转换数据\n",
    "def transform_value(val):\n",
    "    return val.split(':')[1]\n",
    "    \n",
    "columns = ['AV', 'AC', 'PR', 'UI', 'S', 'C', 'I', 'A']\n",
    "\n",
    "temp = []\n",
    "\n",
    "for i in range(vectorString.size):\n",
    "    part =  vectorString[i].split('/')\n",
    "    list_items = part[1::]\n",
    "    temp.append(list_items)\n",
    "train_data = pd.DataFrame(temp, columns=columns)\n",
    "train_data = train_data.applymap(transform_value)\n",
    "print(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "79a6f3ee-0517-4a4f-b26a-6f2dabf9d3b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_cvss_score(params):\n",
    "    # 字典映射分值\n",
    "    AV = {'N': 0.85, 'A': 0.62, 'L': 0.55, 'P': 0.2}\n",
    "    AC = {'L': 0.77, 'H': 0.44}\n",
    "    PR = {'N': 0.85, 'L': 0.68, 'H': 0.5}\n",
    "    UI = {'N': 0.85, 'R': 0.62}\n",
    "    S = {'U': 1, 'C': 1.08}\n",
    "    C = {'N': 0, 'L': 0.22, 'H': 0.56}\n",
    "    I = {'N': 0, 'L': 0.22, 'H': 0.56}\n",
    "    A = {'N': 0, 'L': 0.22, 'H': 0.56}\n",
    "\n",
    "    # 获取参数值\n",
    "    av = AV[params['AV']]\n",
    "    ac = AC[params['AC']]\n",
    "    pr = PR[params['PR']]\n",
    "    ui = UI[params['UI']]\n",
    "    s = S[params['S']]\n",
    "    c = C[params['C']]\n",
    "    i = I[params['I']]\n",
    "    a = A[params['A']]\n",
    "\n",
    "    # 计算临时分数\n",
    "    impact = 1 - (1 - c) * (1 - i) * (1 - a)\n",
    "    exploitability = 8.22 * av * ac * pr * ui\n",
    "\n",
    "    if impact == 0:\n",
    "        base_score = 0\n",
    "    else:\n",
    "        if s == 1:  # 未改变\n",
    "            base_score = round(min(1.176 * (exploitability + impact), 10), 1)\n",
    "        else:  # 改变\n",
    "            base_score = round(min(1.08 * (exploitability + impact), 10), 1)\n",
    "\n",
    "    return base_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "622cf1dd-082c-4d2a-a880-34d22e96d053",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    AV AC PR UI  S  C  I  A  score\n",
      "0    N  L  N  R  C  L  L  N    3.5\n",
      "1    N  L  N  N  U  N  N  H    5.2\n",
      "2    N  L  N  N  U  N  N  H    5.2\n",
      "3    N  L  N  R  C  L  L  N    3.5\n",
      "4    N  L  N  R  C  L  L  N    3.5\n",
      "..  .. .. .. .. .. .. .. ..    ...\n",
      "705  N  L  N  N  U  H  H  H    5.6\n",
      "706  L  L  L  N  U  H  N  N    3.0\n",
      "707  N  L  N  N  U  H  H  H    5.6\n",
      "708  N  L  N  N  U  N  L  L    5.0\n",
      "709  N  L  N  N  U  H  N  N    5.2\n",
      "\n",
      "[710 rows x 9 columns]\n"
     ]
    }
   ],
   "source": [
    "# 为每一行创建字典\n",
    "train_dicts = train_data.apply(lambda row: {col: row[col][0] for col in train_data.columns}, axis=1)\n",
    "train_score = train_dicts.apply(calculate_cvss_score)\n",
    "train_data['score'] = train_score\n",
    "print(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f767e3c9-634b-4c0d-9145-eb4c013e1a6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict = {\n",
    "    'AV': {\n",
    "        'N': 'NETWORK',\n",
    "        'A': 'ADJACENT',\n",
    "        'L': 'LOCAL',\n",
    "        'P': 'PHYSICAL'\n",
    "    },\n",
    "    'AC': {\n",
    "        'L': 'LOW',\n",
    "        'H': 'HIGH'\n",
    "    }, \n",
    "    'PR': {\n",
    "        'N': 'NONE',\n",
    "        'L': 'LOW',\n",
    "        'H': 'HIGH'\n",
    "    }, \n",
    "    'UI': {\n",
    "        'N': 'NONE',\n",
    "        'R': 'REQUIRED'\n",
    "    },\n",
    "    'S': {\n",
    "        'U': 'UNCHANGED',\n",
    "        'C': 'CHANGED'\n",
    "    },\n",
    "    'C': {\n",
    "        'N': 'NONE',\n",
    "        'L': 'LOW',\n",
    "        'H': 'HIGH'\n",
    "    },\n",
    "    'I': {\n",
    "        'N': 'NONE',\n",
    "        'L': 'LOW',\n",
    "        'H': 'HIGH'\n",
    "    },\n",
    "    'A': {\n",
    "        'N': 'NONE', \n",
    "        'L': 'LOW',\n",
    "        'H': 'HIGH'\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d42106a3-9eb5-4580-9143-d7ae061b6d4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "          AV   AC    PR        UI          S     C     I     A  score\n0    NETWORK  LOW  NONE  REQUIRED    CHANGED   LOW   LOW  NONE    3.5\n1    NETWORK  LOW  NONE      NONE  UNCHANGED  NONE  NONE  HIGH    5.2\n2    NETWORK  LOW  NONE      NONE  UNCHANGED  NONE  NONE  HIGH    5.2\n3    NETWORK  LOW  NONE  REQUIRED    CHANGED   LOW   LOW  NONE    3.5\n4    NETWORK  LOW  NONE  REQUIRED    CHANGED   LOW   LOW  NONE    3.5\n..       ...  ...   ...       ...        ...   ...   ...   ...    ...\n705  NETWORK  LOW  NONE      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6\n706    LOCAL  LOW   LOW      NONE  UNCHANGED  HIGH  NONE  NONE    3.0\n707  NETWORK  LOW  NONE      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6\n708  NETWORK  LOW  NONE      NONE  UNCHANGED  NONE   LOW   LOW    5.0\n709  NETWORK  LOW  NONE      NONE  UNCHANGED  HIGH  NONE  NONE    5.2\n\n[710 rows x 9 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>AV</th>\n      <th>AC</th>\n      <th>PR</th>\n      <th>UI</th>\n      <th>S</th>\n      <th>C</th>\n      <th>I</th>\n      <th>A</th>\n      <th>score</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>REQUIRED</td>\n      <td>CHANGED</td>\n      <td>LOW</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>3.5</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>UNCHANGED</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>HIGH</td>\n      <td>5.2</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>UNCHANGED</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>HIGH</td>\n      <td>5.2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>REQUIRED</td>\n      <td>CHANGED</td>\n      <td>LOW</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>3.5</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>REQUIRED</td>\n      <td>CHANGED</td>\n      <td>LOW</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>3.5</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>705</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>UNCHANGED</td>\n      <td>HIGH</td>\n      <td>HIGH</td>\n      <td>HIGH</td>\n      <td>5.6</td>\n    </tr>\n    <tr>\n      <th>706</th>\n      <td>LOCAL</td>\n      <td>LOW</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>UNCHANGED</td>\n      <td>HIGH</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>707</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>UNCHANGED</td>\n      <td>HIGH</td>\n      <td>HIGH</td>\n      <td>HIGH</td>\n      <td>5.6</td>\n    </tr>\n    <tr>\n      <th>708</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>UNCHANGED</td>\n      <td>NONE</td>\n      <td>LOW</td>\n      <td>LOW</td>\n      <td>5.0</td>\n    </tr>\n    <tr>\n      <th>709</th>\n      <td>NETWORK</td>\n      <td>LOW</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>UNCHANGED</td>\n      <td>HIGH</td>\n      <td>NONE</td>\n      <td>NONE</td>\n      <td>5.2</td>\n    </tr>\n  </tbody>\n</table>\n<p>710 rows × 9 columns</p>\n</div>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 替换 DataFrame 中的值\n",
    "train_data.replace(dict, inplace=True)\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f07b2d92-8271-46c8-ab90-a19570dd2566",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.insert(0, 'description', train_data_temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5ca97546-e120-4e80-b7dc-67a00c1bbf45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           description       AV   AC    PR  \\\n",
      "0    Stored & Reflected XSS affecting Xbtit NUMBERT...  NETWORK  LOW  NONE   \n",
      "1    NULL Pointer Dereference in APITAG NULL Pointe...  NETWORK  LOW  NONE   \n",
      "2    NULL Pointer Dereference in APITAG NULL Pointe...  NETWORK  LOW  NONE   \n",
      "3    Reflected XSS vulnerability NUMBERTAG in icehr...  NETWORK  LOW  NONE   \n",
      "4    Reflected XSS vulnerabilities NUMBERTAG in ice...  NETWORK  LOW  NONE   \n",
      "..                                                 ...      ...  ...   ...   \n",
      "705  pboot cms NUMBERTAG RCE. 漏洞详情： URLTAG 声明 APITA...  NETWORK  LOW  NONE   \n",
      "706  .NET CVE backfill round NUMBERTAG Hello, Pleas...    LOCAL  LOW   LOW   \n",
      "707  Backport of NUMBERTAG fixes to NUMBERTAG Thank...  NETWORK  LOW  NONE   \n",
      "708  Update impacted packages for CVETAG . Hi, This...  NETWORK  LOW  NONE   \n",
      "709  CVE Dio NUMBERTAG Google OVS Scanner. Package ...  NETWORK  LOW  NONE   \n",
      "\n",
      "           UI          S     C     I     A  score  \n",
      "0    REQUIRED    CHANGED   LOW   LOW  NONE    3.5  \n",
      "1        NONE  UNCHANGED  NONE  NONE  HIGH    5.2  \n",
      "2        NONE  UNCHANGED  NONE  NONE  HIGH    5.2  \n",
      "3    REQUIRED    CHANGED   LOW   LOW  NONE    3.5  \n",
      "4    REQUIRED    CHANGED   LOW   LOW  NONE    3.5  \n",
      "..        ...        ...   ...   ...   ...    ...  \n",
      "705      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "706      NONE  UNCHANGED  HIGH  NONE  NONE    3.0  \n",
      "707      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "708      NONE  UNCHANGED  NONE   LOW   LOW    5.0  \n",
      "709      NONE  UNCHANGED  HIGH  NONE  NONE    5.2  \n",
      "\n",
      "[710 rows x 10 columns]\n"
     ]
    }
   ],
   "source": [
    "print(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3e5e14c5-8f88-43d3-945c-1505e11a2490",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           description       AV   AC    PR  \\\n",
      "0    Stored & Reflected XSS affecting Xbtit NUMBERT...  NETWORK  LOW  NONE   \n",
      "10   illegal memcpy during njs_vmcode_typeof in PAT...  NETWORK  LOW  NONE   \n",
      "11   Heap UAF in njs_await_fulfilled. Env CODETAG P...  NETWORK  LOW  NONE   \n",
      "48   Add nonce to the logout link. The logout link ...  NETWORK  LOW  NONE   \n",
      "50   Divide By Zero in H5T__complete_copy () at PAT...  NETWORK  LOW  NONE   \n",
      "..                                                 ...      ...  ...   ...   \n",
      "705  pboot cms NUMBERTAG RCE. 漏洞详情： URLTAG 声明 APITA...  NETWORK  LOW  NONE   \n",
      "706  .NET CVE backfill round NUMBERTAG Hello, Pleas...    LOCAL  LOW   LOW   \n",
      "707  Backport of NUMBERTAG fixes to NUMBERTAG Thank...  NETWORK  LOW  NONE   \n",
      "708  Update impacted packages for CVETAG . Hi, This...  NETWORK  LOW  NONE   \n",
      "709  CVE Dio NUMBERTAG Google OVS Scanner. Package ...  NETWORK  LOW  NONE   \n",
      "\n",
      "           UI          S     C     I     A  score      prefix  \n",
      "0    REQUIRED    CHANGED   LOW   LOW  NONE    3.5  Stored & R  \n",
      "10       NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  illegal me  \n",
      "11       NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  Heap UAF i  \n",
      "48   REQUIRED    CHANGED  NONE  HIGH  NONE    3.7  Add nonce   \n",
      "50   REQUIRED  UNCHANGED  NONE  NONE  HIGH    4.0  Divide By   \n",
      "..        ...        ...   ...   ...   ...    ...         ...  \n",
      "705      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  pboot cms   \n",
      "706      NONE  UNCHANGED  HIGH  NONE  NONE    3.0  .NET CVE b  \n",
      "707      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  Backport o  \n",
      "708      NONE  UNCHANGED  NONE   LOW   LOW    5.0  Update imp  \n",
      "709      NONE  UNCHANGED  HIGH  NONE  NONE    5.2  CVE Dio NU  \n",
      "\n",
      "[264 rows x 11 columns]\n"
     ]
    }
   ],
   "source": [
    "# 提取前20个字符\n",
    "train_data['prefix'] = train_data['description'].str[:10]\n",
    "\n",
    "# 计算每个前20个字符的出现次数\n",
    "prefix_counts = train_data['prefix'].value_counts()\n",
    "\n",
    "# 只保留那些前20个字符出现次数为1的描述\n",
    "unique_prefixes = prefix_counts[prefix_counts == 1].index\n",
    "unique_descriptions = train_data[train_data['prefix'].isin(unique_prefixes)]\n",
    "print(unique_descriptions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d660a495-fdab-41fc-932c-6e593babc88e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           description       AV   AC    PR  \\\n",
      "0    Stored & Reflected XSS affecting Xbtit NUMBERT...  NETWORK  LOW  NONE   \n",
      "10   illegal memcpy during njs_vmcode_typeof in PAT...  NETWORK  LOW  NONE   \n",
      "11   Heap UAF in njs_await_fulfilled. Env CODETAG P...  NETWORK  LOW  NONE   \n",
      "48   Add nonce to the logout link. The logout link ...  NETWORK  LOW  NONE   \n",
      "50   Divide By Zero in H5T__complete_copy () at PAT...  NETWORK  LOW  NONE   \n",
      "..                                                 ...      ...  ...   ...   \n",
      "698  A NUMBERTAG specific heap buffer overflow with...  NETWORK  LOW  NONE   \n",
      "699  Mitigation for CVETAG . Hi there. It appears a...  NETWORK  LOW  NONE   \n",
      "703  Contact APITAG Product Security Team and ask t...  NETWORK  LOW  NONE   \n",
      "707  Backport of NUMBERTAG fixes to NUMBERTAG Thank...  NETWORK  LOW  NONE   \n",
      "709  CVE Dio NUMBERTAG Google OVS Scanner. Package ...  NETWORK  LOW  NONE   \n",
      "\n",
      "           UI          S     C     I     A  score      prefix  \n",
      "0    REQUIRED    CHANGED   LOW   LOW  NONE    3.5  Stored & R  \n",
      "10       NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  illegal me  \n",
      "11       NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  Heap UAF i  \n",
      "48   REQUIRED    CHANGED  NONE  HIGH  NONE    3.7  Add nonce   \n",
      "50   REQUIRED  UNCHANGED  NONE  NONE  HIGH    4.0  Divide By   \n",
      "..        ...        ...   ...   ...   ...    ...         ...  \n",
      "698      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  A NUMBERTA  \n",
      "699      NONE  UNCHANGED  NONE  NONE  HIGH    5.2  Mitigation  \n",
      "703      NONE  UNCHANGED   LOW   LOW   LOW    5.2  Contact AP  \n",
      "707      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  Backport o  \n",
      "709      NONE  UNCHANGED  HIGH  NONE  NONE    5.2  CVE Dio NU  \n",
      "\n",
      "[197 rows x 11 columns]\n"
     ]
    }
   ],
   "source": [
    "# 删除描述长度大于1000的行\n",
    "temp = pd.DataFrame()\n",
    "filtered_train_data = pd.DataFrame()\n",
    "temp = unique_descriptions[unique_descriptions['description'].str.len() <= 1000]\n",
    "filtered_train_data = temp[temp['description'].str.len() > 100]\n",
    "print(filtered_train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "eaa6e29e-7fc1-45b9-809b-a4d43686362c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lx\\AppData\\Local\\Temp\\ipykernel_38864\\3885197511.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  filtered_train_data.sort_values(by='prefix', inplace=True)\n"
     ]
    }
   ],
   "source": [
    "filtered_train_data.sort_values(by='prefix', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "db7adb10-83a9-438f-9681-ac02293cba3e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                           description       AV   AC    PR  \\\n",
      "698  A NUMBERTAG specific heap buffer overflow with...  NETWORK  LOW  NONE   \n",
      "488  A Remote Code Execution (RCE) vulnerability ex...  NETWORK  LOW  HIGH   \n",
      "51   A heap use after free in in H5AC_unpin_entry. ...  NETWORK  LOW  NONE   \n",
      "408  A package should never try to do unrelated thi...  NETWORK  LOW  NONE   \n",
      "236  A security vulnerability which will lead to co...  NETWORK  LOW  NONE   \n",
      "..                                                 ...      ...  ...   ...   \n",
      "480  一个后台存储型xss漏洞. When adding movie names, malicio...  NETWORK  LOW   LOW   \n",
      "639  关于 CVETAG 漏洞，不要在发 issues 了！！！. APITAG 如果你从前端传递...  NETWORK  LOW  NONE   \n",
      "606  后台服务器组中存在XSS漏洞. 进入后台，点击视频 >服务器组 >添加， 在名称框插入pay...  NETWORK  LOW   LOW   \n",
      "509  固定的cookie NUMBERTAG APITAG FILETAG NUMBERTAG H...  NETWORK  LOW  NONE   \n",
      "52   默认的 APITAG 为什么选择 APITAG 呢？. 版本情况 JDK版本： corret...  NETWORK  LOW  NONE   \n",
      "\n",
      "           UI          S     C     I     A  score  \n",
      "698      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "488      NONE  UNCHANGED  HIGH  HIGH  HIGH    3.8  \n",
      "51   REQUIRED  UNCHANGED  HIGH  HIGH  HIGH    4.4  \n",
      "408      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "236      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "..        ...        ...   ...   ...   ...    ...  \n",
      "480  REQUIRED    CHANGED   LOW   LOW  NONE    2.9  \n",
      "639      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "606  REQUIRED    CHANGED   LOW   LOW  NONE    2.9  \n",
      "509      NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "52       NONE  UNCHANGED  HIGH  HIGH  HIGH    5.6  \n",
      "\n",
      "[197 rows x 10 columns]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lx\\AppData\\Local\\Temp\\ipykernel_38864\\3846312463.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  filtered_train_data.drop('prefix', axis=1, inplace=True)\n"
     ]
    }
   ],
   "source": [
    "filtered_train_data.drop('prefix', axis=1, inplace=True)\n",
    "print(filtered_train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9cc98bb5-1ae4-4024-b9cc-d3db88996221",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "2.0"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_train_data['score'].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0c2e00f4-cf7b-4b15-8901-537abb4524e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_train_data.to_csv(r\"../dataset/filtered_test_dataset.csv\",header=None,index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "080b3051-8eb8-4c4f-81ad-d3a215c2f693",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}