From bd91363afac6e85e7c5d32a3ce9b6b63b7c2d469 Mon Sep 17 00:00:00 2001 From: Eli <43382407+eli64s@users.noreply.github.com> Date: Tue, 11 Jul 2023 19:08:37 -0500 Subject: [PATCH] Enhancements and Refactoring of Python Code Extraction Methods (#467) * Refactor and enhance code extraction methods. * Use f-strings to print filepaths, improving readability. --- examples/Code_search.ipynb | 190 +++++++++++++++++++++---------------- 1 file changed, 108 insertions(+), 82 deletions(-) diff --git a/examples/Code_search.ipynb b/examples/Code_search.ipynb index 90e3936..02779ee 100644 --- a/examples/Code_search.ipynb +++ b/examples/Code_search.ipynb @@ -7,86 +7,110 @@ "source": [ "## Code search\n", "\n", - "We index our own [openai-python code repository](https://github.com/openai/openai-python), and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files." + "We index our own [openai-python code repository](https://github.com/openai/openai-python), and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files.\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Total number of py files: 51\n", - "Total number of functions extracted: 97\n" + "Total number of .py files: 57\n", + "Total number of functions extracted: 118\n" ] } ], "source": [ - "import os\n", - "from glob import glob\n", "import pandas as pd\n", + "from pathlib import Path\n", + "\n", + "DEF_PREFIXES = ['def ', 'async def ']\n", + "NEWLINE = '\\n'\n", + "\n", "\n", "def get_function_name(code):\n", " \"\"\"\n", - " Extract function name from a line beginning with \"def \"\n", + " Extract function name from a line beginning with 'def' or 'async def'.\n", " \"\"\"\n", - " assert code.startswith(\"def \")\n", - " return code[len(\"def \"): code.index(\"(\")]\n", + " for prefix in DEF_PREFIXES:\n", + " if code.startswith(prefix):\n", + " return code[len(prefix): code.index('(')]\n", "\n", - "def get_until_no_space(all_lines, i) -> str:\n", + "\n", + "def get_until_no_space(all_lines, i):\n", " \"\"\"\n", " Get all lines until a line outside the function definition is found.\n", " \"\"\"\n", " ret = [all_lines[i]]\n", - " for j in range(i + 1, i + 10000):\n", - " if j < len(all_lines):\n", - " if len(all_lines[j]) == 0 or all_lines[j][0] in [\" \", \"\\t\", \")\"]:\n", - " ret.append(all_lines[j])\n", - " else:\n", - " break\n", - " return \"\\n\".join(ret)\n", + " for j in range(i + 1, len(all_lines)):\n", + " if len(all_lines[j]) == 0 or all_lines[j][0] in [' ', '\\t', ')']:\n", + " ret.append(all_lines[j])\n", + " else:\n", + " break\n", + " return NEWLINE.join(ret)\n", + "\n", "\n", "def get_functions(filepath):\n", " \"\"\"\n", " Get all functions in a Python file.\n", " \"\"\"\n", - " whole_code = open(filepath).read().replace(\"\\r\", \"\\n\")\n", - " all_lines = whole_code.split(\"\\n\")\n", - " for i, l in enumerate(all_lines):\n", - " if l.startswith(\"def \"):\n", - " code = get_until_no_space(all_lines, i)\n", - " function_name = get_function_name(code)\n", - " yield {\"code\": code, \"function_name\": function_name, \"filepath\": filepath}\n", + " with open(filepath, 'r') as file:\n", + " all_lines = file.read().replace('\\r', NEWLINE).split(NEWLINE)\n", + " for i, l in enumerate(all_lines):\n", + " for prefix in DEF_PREFIXES:\n", + " if l.startswith(prefix):\n", + " code = get_until_no_space(all_lines, i)\n", + " function_name = get_function_name(code)\n", + " yield {\n", + " 'code': code,\n", + " 'function_name': function_name,\n", + " 'filepath': filepath,\n", + " }\n", + " break\n", "\n", "\n", - "# get user root directory\n", - "root_dir = os.path.expanduser(\"~\")\n", - "# note: for this code to work, the openai-python repo must be downloaded and placed in your root directory\n", + "def extract_functions_from_repo(code_root):\n", + " \"\"\"\n", + " Extract all .py functions from the repository.\n", + " \"\"\"\n", + " code_files = list(code_root.glob('**/*.py'))\n", "\n", - "# path to code repository directory\n", - "code_root = root_dir + \"/openai-python\"\n", + " num_files = len(code_files)\n", + " print(f'Total number of .py files: {num_files}')\n", "\n", - "code_files = [y for x in os.walk(code_root) for y in glob(os.path.join(x[0], '*.py'))]\n", - "print(\"Total number of py files:\", len(code_files))\n", + " if num_files == 0:\n", + " print('Verify openai-python repo exists and code_root is set correctly.')\n", + " return None\n", "\n", - "if len(code_files) == 0:\n", - " print(\"Double check that you have downloaded the openai-python repo and set the code_root variable correctly.\")\n", + " all_funcs = [\n", + " func\n", + " for code_file in code_files\n", + " for func in get_functions(str(code_file))\n", + " ]\n", "\n", - "all_funcs = []\n", - "for code_file in code_files:\n", - " funcs = list(get_functions(code_file))\n", - " for func in funcs:\n", - " all_funcs.append(func)\n", + " num_funcs = len(all_funcs)\n", + " print(f'Total number of functions extracted: {num_funcs}')\n", "\n", - "print(\"Total number of functions extracted:\", len(all_funcs))" + " return all_funcs\n", + "\n", + "\n", + "# Set user root directory to the 'openai-python' repository\n", + "root_dir = Path.home()\n", + "\n", + "# Assumes the 'openai-python' repository exists in the user's root directory\n", + "code_root = root_dir / 'openai-python'\n", + "\n", + "# Extract all functions from the repository\n", + "all_funcs = extract_functions_from_repo(code_root)" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -121,36 +145,36 @@ " 0\n", " def _console_log_level():\\n if openai.log i...\n", " _console_log_level\n", - " /openai/util.py\n", - " [0.03389773145318031, -0.004390408284962177, 0...\n", + " openai/util.py\n", + " [0.033906757831573486, -0.00418944051489234, 0...\n", " \n", " \n", " 1\n", " def log_debug(message, **params):\\n msg = l...\n", " log_debug\n", - " /openai/util.py\n", - " [-0.004034275189042091, 0.004895383026450872, ...\n", + " openai/util.py\n", + " [-0.004059609025716782, 0.004895503632724285, ...\n", " \n", " \n", " 2\n", " def log_info(message, **params):\\n msg = lo...\n", " log_info\n", - " /openai/util.py\n", - " [0.004882764536887407, 0.0033515947870910168, ...\n", + " openai/util.py\n", + " [0.0048639848828315735, 0.0033139237202703953,...\n", " \n", " \n", " 3\n", " def log_warn(message, **params):\\n msg = lo...\n", " log_warn\n", - " /openai/util.py\n", - " [0.002535992069169879, -0.010829543694853783, ...\n", + " openai/util.py\n", + " [0.0024026145692914724, -0.010721310041844845,...\n", " \n", " \n", " 4\n", " def logfmt(props):\\n def fmt(key, val):\\n ...\n", " logfmt\n", - " /openai/util.py\n", - " [0.016732551157474518, 0.017367802560329437, 0...\n", + " openai/util.py\n", + " [0.01664826273918152, 0.01730910874903202, 0.0...\n", " \n", " \n", "\n", @@ -164,15 +188,15 @@ "3 def log_warn(message, **params):\\n msg = lo... log_warn \n", "4 def logfmt(props):\\n def fmt(key, val):\\n ... logfmt \n", "\n", - " filepath code_embedding \n", - "0 /openai/util.py [0.03389773145318031, -0.004390408284962177, 0... \n", - "1 /openai/util.py [-0.004034275189042091, 0.004895383026450872, ... \n", - "2 /openai/util.py [0.004882764536887407, 0.0033515947870910168, ... \n", - "3 /openai/util.py [0.002535992069169879, -0.010829543694853783, ... \n", - "4 /openai/util.py [0.016732551157474518, 0.017367802560329437, 0... " + " filepath code_embedding \n", + "0 openai/util.py [0.033906757831573486, -0.00418944051489234, 0... \n", + "1 openai/util.py [-0.004059609025716782, 0.004895503632724285, ... \n", + "2 openai/util.py [0.0048639848828315735, 0.0033139237202703953,... \n", + "3 openai/util.py [0.0024026145692914724, -0.010721310041844845,... \n", + "4 openai/util.py [0.01664826273918152, 0.01730910874903202, 0.0... " ] }, - "execution_count": 2, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -182,41 +206,41 @@ "\n", "df = pd.DataFrame(all_funcs)\n", "df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, engine='text-embedding-ada-002'))\n", - "df['filepath'] = df['filepath'].apply(lambda x: x.replace(code_root, \"\"))\n", + "df['filepath'] = df['filepath'].map(lambda x: Path(x).relative_to(code_root))\n", "df.to_csv(\"data/code_search_openai-python.csv\", index=False)\n", "df.head()" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "/openai/tests/test_endpoints.py:test_completions score=0.826\n", + "openai/tests/test_endpoints.py:test_completions score=0.826\n", "def test_completions():\n", " result = openai.Completion.create(prompt=\"This was a test\", n=5, engine=\"ada\")\n", " assert len(result.choices) == 5\n", "\n", "\n", "----------------------------------------------------------------------\n", - "/openai/tests/test_endpoints.py:test_completions_model score=0.811\n", - "def test_completions_model():\n", - " result = openai.Completion.create(prompt=\"This was a test\", n=5, model=\"ada\")\n", + "openai/tests/asyncio/test_endpoints.py:test_completions score=0.824\n", + "async def test_completions():\n", + " result = await openai.Completion.acreate(\n", + " prompt=\"This was a test\", n=5, engine=\"ada\"\n", + " )\n", " assert len(result.choices) == 5\n", - " assert result.model.startswith(\"ada\")\n", "\n", "\n", "----------------------------------------------------------------------\n", - "/openai/tests/test_endpoints.py:test_completions_multiple_prompts score=0.808\n", - "def test_completions_multiple_prompts():\n", - " result = openai.Completion.create(\n", - " prompt=[\"This was a test\", \"This was another test\"], n=5, engine=\"ada\"\n", - " )\n", - " assert len(result.choices) == 10\n", + "openai/tests/asyncio/test_endpoints.py:test_completions_model score=0.82\n", + "async def test_completions_model():\n", + " result = await openai.Completion.acreate(prompt=\"This was a test\", n=5, model=\"ada\")\n", + " assert len(result.choices) == 5\n", + " assert result.model.startswith(\"ada\")\n", "\n", "\n", "----------------------------------------------------------------------\n" @@ -231,11 +255,13 @@ " df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n", "\n", " res = df.sort_values('similarities', ascending=False).head(n)\n", + "\n", " if pprint:\n", " for r in res.iterrows():\n", - " print(r[1].filepath+\":\"+r[1].function_name + \" score=\" + str(round(r[1].similarities, 3)))\n", + " print(f\"{r[1].filepath}:{r[1].function_name} score={round(r[1].similarities, 3)}\")\n", " print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n", - " print('-'*70)\n", + " print('-' * 70)\n", + "\n", " return res\n", "\n", "res = search_functions(df, 'Completions API tests', n=3)" @@ -243,14 +269,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "/openai/validators.py:format_inferrer_validator score=0.751\n", + "openai/validators.py:format_inferrer_validator score=0.751\n", "def format_inferrer_validator(df):\n", " \"\"\"\n", " This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n", @@ -259,7 +285,7 @@ " ft_type = infer_task_type(df)\n", " immediate_msg = None\n", "----------------------------------------------------------------------\n", - "/openai/validators.py:get_validators score=0.748\n", + "openai/validators.py:get_validators score=0.748\n", "def get_validators():\n", " return [\n", " num_examples_validator,\n", @@ -268,7 +294,7 @@ " additional_column_validator,\n", " non_empty_field_validator,\n", "----------------------------------------------------------------------\n", - "/openai/validators.py:infer_task_type score=0.738\n", + "openai/validators.py:infer_task_type score=0.739\n", "def infer_task_type(df):\n", " \"\"\"\n", " Infer the likely fine-tuning task type from the data\n", @@ -286,14 +312,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "/openai/validators.py:get_common_xfix score=0.793\n", + "openai/validators.py:get_common_xfix score=0.794\n", "def get_common_xfix(series, xfix=\"suffix\"):\n", " \"\"\"\n", " Finds the longest common suffix or prefix of all the values in a series\n", @@ -305,7 +331,7 @@ " if xfix == \"suffix\"\n", " else series.str[: len(common_xfix) + 1]\n", "----------------------------------------------------------------------\n", - "/openai/validators.py:common_completion_suffix_validator score=0.778\n", + "openai/validators.py:common_completion_suffix_validator score=0.778\n", "def common_completion_suffix_validator(df):\n", " \"\"\"\n", " This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n", @@ -326,14 +352,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "/openai/cli.py:tools_register score=0.773\n", + "openai/cli.py:tools_register score=0.78\n", "def tools_register(parser):\n", " subparsers = parser.add_subparsers(\n", " title=\"Tools\", help=\"Convenience client side tools\"\n", @@ -382,7 +408,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.9.16" }, "orig_nbformat": 4 },