1{
2 "cells": [
3 {
4 "cell_type": "code",
5 "execution_count": null,
6 "metadata": {},
7 "outputs": [],
8 "source": [
9 "\"\"\"\n",
10 "We show here how to use the retrieval completion function to add context from documents to any OpenAI Evals task\n",
11 "The toy example here will be to augment our Born-First task with a dataset of presidential birthdays\n",
12 "\"\"\"\n",
13 "\n",
14 "# Download the dataset manually, or use curl\n",
15 "!curl -O https://people.math.sc.edu/Burkardt/datasets/presidents/president_birthdays.csv"
16 ]
17 },
18 {
19 "cell_type": "code",
20 "execution_count": null,
21 "metadata": {},
22 "outputs": [],
23 "source": [
24 "import os\n",
25 "from openai import OpenAI\n",
26 "import pandas as pd\n",
27 "\n",
28 "df = pd.read_csv(\"president_birthdays.csv\").rename(columns={\" \\\"Name\\\"\": \"Name\", \" \\\"Month\\\"\": \"Month\", \" \\\"Day\\\"\": \"Day\", \" \\\"Year\\\"\": \"Year\"}).set_index(\"Index\")\n",
29 "df[\"text\"] = df.apply(lambda r: f\"{r['Name']} was born on {r['Month']}/{r['Day']}/{r['Year']}\", axis=1)\n",
30 "display(df.head())\n",
31 "\n",
32 "def embed(text):\n",
33 " return OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\")).embeddings.create(\n",
34 " model=\"text-embedding-ada-002\",\n",
35 " input=text\n",
36 " ).data[0].embedding\n",
37 "\n",
38 "df[\"embedding\"] = df['text'].apply(embed)\n",
39 "df[[\"text\", \"embedding\"]].to_csv(\"presidents_embeddings.csv\")"
40 ]
41 },
42 {
43 "cell_type": "code",
44 "execution_count": null,
45 "metadata": {},
46 "outputs": [],
47 "source": [
48 "\"\"\"\n",
49 "We create a registry entry here in code. Notice we set number of retrieved documents k=2.\n",
50 "\"\"\"\n",
51 "\n",
52 "registry_yaml = f\"\"\"\n",
53 "retrieval/presidents/gpt-3.5-turbo:\n",
54 " class: evals.completion_fns.retrieval:RetrievalCompletionFn\n",
55 " args:\n",
56 " completion_fn: gpt-3.5-turbo\n",
57 " embeddings_and_text_path: {os.path.abspath('presidents_embeddings.csv')}\n",
58 " k: 2\n",
59 "\n",
60 "retrieval/presidents/cot/gpt-3.5-turbo:\n",
61 " class: evals.completion_fns.retrieval:RetrievalCompletionFn\n",
62 " args:\n",
63 " completion_fn: cot/gpt-3.5-turbo\n",
64 " embeddings_and_text_path: {os.path.abspath('presidents_embeddings.csv')}\n",
65 " k: 2\n",
66 "\"\"\".strip()\n",
67 "\n",
68 "# Replace with path to your registry\n",
69 "os.makedirs(\"completion_fns\", exist_ok=True)\n",
70 "with open(\"completion_fns/retrieval.yaml\", \"w\") as f:\n",
71 " f.write(registry_yaml)\n",
72 "\n",
73 "# GPT-3.5-turbo base: accuracy 0.7\n",
74 "!oaieval gpt-3.5-turbo born-first --max_samples 10 --registry_path .\n",
75 "\n",
76 "# GPT-3.5-turbo with retrieval: accuracy 0.9 -> The failure mode here is the retrieved president is incorrect: Andrew Johnson vs Andrew Jackson\n",
77 "!oaieval retrieval/presidents/gpt-3.5-turbo born-first --max_samples 10 --registry_path .\n",
78 "\n",
79 "# GPT-3.5-turbo with retrieval and chain-of-thought: accuracy 1.0\n",
80 "!oaieval retrieval/presidents/cot/gpt-3.5-turbo born-first --max_samples 10 --registry_path ."
81 ]
82 },
83 {
84 "cell_type": "code",
85 "execution_count": null,
86 "metadata": {},
87 "outputs": [],
88 "source": []
89 }
90 ],
91 "metadata": {
92 "kernelspec": {
93 "display_name": "base",
94 "language": "python",
95 "name": "python3"
96 },
97 "language_info": {
98 "codemirror_mode": {
99 "name": "ipython",
100 "version": 3
101 },
102 "file_extension": ".py",
103 "mimetype": "text/x-python",
104 "name": "python",
105 "nbconvert_exporter": "python",
106 "pygments_lexer": "ipython3",
107 "version": "3.10.9"
108 },
109 "orig_nbformat": 4
110 },
111 "nbformat": 4,
112 "nbformat_minor": 2
113}
114
1{
2 "cells": [
3 {
4 "cell_type": "code",
5 "execution_count": null,
6 "metadata": {},
7 "outputs": [],
8 "source": [
9 "\"\"\"\n",
10 "We show here how to use the retrieval completion function to add context from documents to any OpenAI Evals task\n",
11 "The toy example here will be to augment our Born-First task with a dataset of presidential birthdays\n",
12 "\"\"\"\n",
13 "\n",
14 "# Download the dataset manually, or use curl\n",
15 "!curl -O https://people.math.sc.edu/Burkardt/datasets/presidents/president_birthdays.csv"
16 ]
17 },
18 {
19 "cell_type": "code",
20 "execution_count": null,
21 "metadata": {},
22 "outputs": [],
23 "source": [
24 "import os\n",
25 "from openai import OpenAI\n",
26 "import pandas as pd\n",
27 "\n",
28 "df = pd.read_csv(\"president_birthdays.csv\").rename(columns={\" \\\"Name\\\"\": \"Name\", \" \\\"Month\\\"\": \"Month\", \" \\\"Day\\\"\": \"Day\", \" \\\"Year\\\"\": \"Year\"}).set_index(\"Index\")\n",
29 "df[\"text\"] = df.apply(lambda r: f\"{r['Name']} was born on {r['Month']}/{r['Day']}/{r['Year']}\", axis=1)\n",
30 "display(df.head())\n",
31 "\n",
32 "def embed(text):\n",
33 " return OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\")).embeddings.create(\n",
34 " model=\"text-embedding-ada-002\",\n",
35 " input=text\n",
36 " ).data[0].embedding\n",
37 "\n",
38 "df[\"embedding\"] = df['text'].apply(embed)\n",
39 "df[[\"text\", \"embedding\"]].to_csv(\"presidents_embeddings.csv\")"
40 ]
41 },
42 {
43 "cell_type": "code",
44 "execution_count": null,
45 "metadata": {},
46 "outputs": [],
47 "source": [
48 "\"\"\"\n",
49 "We create a registry entry here in code. Notice we set number of retrieved documents k=2.\n",
50 "\"\"\"\n",
51 "\n",
52 "registry_yaml = f\"\"\"\n",
53 "retrieval/presidents/gpt-3.5-turbo:\n",
54 " class: evals.completion_fns.retrieval:RetrievalCompletionFn\n",
55 " args:\n",
56 " completion_fn: gpt-3.5-turbo\n",
57 " embeddings_and_text_path: {os.path.abspath('presidents_embeddings.csv')}\n",
58 " k: 2\n",
59 "\n",
60 "retrieval/presidents/cot/gpt-3.5-turbo:\n",
61 " class: evals.completion_fns.retrieval:RetrievalCompletionFn\n",
62 " args:\n",
63 " completion_fn: cot/gpt-3.5-turbo\n",
64 " embeddings_and_text_path: {os.path.abspath('presidents_embeddings.csv')}\n",
65 " k: 2\n",
66 "\"\"\".strip()\n",
67 "\n",
68 "# Replace with path to your registry\n",
69 "os.makedirs(\"completion_fns\", exist_ok=True)\n",
70 "with open(\"completion_fns/retrieval.yaml\", \"w\") as f:\n",
71 " f.write(registry_yaml)\n",
72 "\n",
73 "# GPT-3.5-turbo base: accuracy 0.7\n",
74 "!oaieval gpt-3.5-turbo born-first --max_samples 10 --registry_path .\n",
75 "\n",
76 "# GPT-3.5-turbo with retrieval: accuracy 0.9 -> The failure mode here is the retrieved president is incorrect: Andrew Johnson vs Andrew Jackson\n",
77 "!oaieval retrieval/presidents/gpt-3.5-turbo born-first --max_samples 10 --registry_path .\n",
78 "\n",
79 "# GPT-3.5-turbo with retrieval and chain-of-thought: accuracy 1.0\n",
80 "!oaieval retrieval/presidents/cot/gpt-3.5-turbo born-first --max_samples 10 --registry_path ."
81 ]
82 },
83 {
84 "cell_type": "code",
85 "execution_count": null,
86 "metadata": {},
87 "outputs": [],
88 "source": []
89 }
90 ],
91 "metadata": {
92 "kernelspec": {
93 "display_name": "base",
94 "language": "python",
95 "name": "python3"
96 },
97 "language_info": {
98 "codemirror_mode": {
99 "name": "ipython",
100 "version": 3
101 },
102 "file_extension": ".py",
103 "mimetype": "text/x-python",
104 "name": "python",
105 "nbconvert_exporter": "python",
106 "pygments_lexer": "ipython3",
107 "version": "3.10.9"
108 },
109 "orig_nbformat": 4
110 },
111 "nbformat": 4,
112 "nbformat_minor": 2
113}
114