Skip to content
Project: Organizing Medical Transcriptions with the OpenAI API
Medical professionals often summarize patient encounters in transcripts written in natural language, which include details about symptoms, diagnosis, and treatments. These transcripts can be used for other medical documentation, such as for insurance purposes, but as they are densely packed with medical information, extracting the key data accurately can be challenging.
You and your team at Lakeside Healthcare Network have decided to leverage the OpenAI API to automatically extract medical information from these transcripts and automate the matching with the appropriate ICD-10 codes. ICD-10 codes are a standardized system used worldwide for diagnosing and billing purposes, such as insurance claims processing.
The Data
The dataset contains anonymized medical transcriptions categorized by specialty.
transcriptions.csv
| Column | Description |
|---|---|
"medical_specialty" | The medical specialty associated with each transcription. |
"transcription" | Detailed medical transcription texts, with insights into the medical case. |
# Import the necessary libraries
import pandas as pd
from openai import OpenAI
import json# Load the data
df = pd.read_csv("data/transcriptions.csv")
df.head()import re
import json
import pandas as pd
from openai import OpenAI
# Initialize the OpenAI client (expects OPENAI_API_KEY in env, or pass api_key="...")
client = OpenAI()
model = "gpt-4o-mini" # or your preferred model
# --- Define the function-calling schema ---
function_definition = [
{
"type": "function",
"function": {
"name": "extract_patient_info",
"description": "Extract key patient info from a clinical transcription.",
"parameters": {
"type": "object",
"properties": {
"age": {
"type": ["integer", "null"],
"description": "Patient age in years if mentioned or inferable; otherwise null."
},
"medical_specialty": {
"type": "string",
"description": "Clinical specialty responsible for the case (e.g., Orthopedic, Cardiology). Keep it short."
},
"recommended_treatment_or_procedure": {
"type": "string",
"description": "The main recommended treatment or procedure, concise phrase."
},
"icd10_code": {
"type": "string",
"description": "Best-fit ICD-10-CM code for the condition addressed by the recommendation."
},
"icd10_description": {
"type": "string",
"description": "Short description of that ICD-10 code."
},
"confidence": {
"type": "number",
"description": "Model confidence from 0 to 1.",
"minimum": 0,
"maximum": 1
},
"rationale": {
"type": "string",
"description": "1–2 sentence explanation for traceability."
}
},
"required": [
"age",
"medical_specialty",
"recommended_treatment_or_procedure",
"icd10_code",
"icd10_description",
"confidence",
"rationale"
],
"additionalProperties": False
}
}
}
]
system_prompt = (
"You are a clinical coding assistant. Extract structured data from transcripts. "
"If age is explicitly given, use it. If not, infer cautiously from context (e.g., '23-year-old'). "
"Return the single main recommended treatment/procedure. "
"Provide the most appropriate ICD-10-CM code and a short description. "
"Be concise and consistent."
)
def regex_age_fallback(text: str):
# Basic patterns like "23-year-old", "23 year old", "age 23", "23 y/o"
patterns = [
r"\b(\d{1,3})\s*-\s*year\s*-\s*old\b",
r"\b(\d{1,3})\s*year\s*old\b",
r"\bage[:\s]+(\d{1,3})\b",
r"\b(\d{1,3})\s*y\/o\b",
]
for p in patterns:
m = re.search(p, text, flags=re.IGNORECASE)
if m:
try:
val = int(m.group(1))
if 0 < val < 120:
return val
except:
pass
return None
# Ensure df already exists with columns ["transcription", "medical_specialty"]
# Example:
# df = pd.read_csv("data/transcriptions.csv")
records = []
for i, row in df.iterrows():
transcript = str(row.get("transcription", "") or "")
specialty_from_df = str(row.get("medical_specialty", "") or "").strip()
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": (
f"Medical specialty (from metadata): {specialty_from_df}\n\n"
f"Transcript:\n{transcript}\n\n"
"Extract the fields using the function. If uncertain, choose the best single ICD-10-CM code."
)}
]
resp = client.chat.completions.create(
model=model,
messages=messages,
tools=function_definition,
tool_choice={"type": "function", "function": {"name": "extract_patient_info"}},
temperature=0.2,
)
# Read tool call args (correctly)
tool_calls = resp.choices[0].message.tool_calls or []
if not tool_calls:
# Rare: retry once if the tool wasn't called
resp = client.chat.completions.create(
model=model,
messages=messages,
tools=function_definition,
tool_choice={"type": "function", "function": {"name": "extract_patient_info"}},
temperature=0.2,
)
tool_calls = resp.choices[0].message.tool_calls or []
args = {}
if tool_calls:
tc = tool_calls[0]
raw_args = getattr(tc.function, "arguments", {}) or {}
if isinstance(raw_args, str):
try:
args = json.loads(raw_args)
except json.JSONDecodeError:
args = {}
elif isinstance(raw_args, dict):
args = raw_args
# Fallback for age if the model left it null but it’s obvious in text
if args.get("age") is None:
fallback_age = regex_age_fallback(transcript)
if fallback_age is not None:
args["age"] = fallback_age
# Compose final row
out = {
"index": i,
"age": args.get("age"),
"recommended_treatment_or_procedure": args.get("recommended_treatment_or_procedure", ""),
"medical_specialty": args.get("medical_specialty") or specialty_from_df,
"icd10_code": args.get("icd10_code", ""),
"icd10_description": args.get("icd10_description", ""),
"confidence": args.get("confidence", None),
"rationale": args.get("rationale", "")
}
records.append(out)
df_structured = pd.DataFrame.from_records(records).set_index("index")
df_structured.head()