Optimization in Python

import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import minimize

# Function to minimize: f(x) = (x - 3)^2
def function(x):
    return (x - 3) ** 2

# Derivative of the function: f'(x) = 2 * (x - 3)
def derivative(x):
    return 2 * (x - 3)

# Gradient Descent function
def gradient_descent(starting_point, learning_rate, n_iterations):
    x = starting_point
    points = []  # To store points at each iteration

    for _ in range(n_iterations):
        points.append(x)
        gradient = derivative(x)
        x -= learning_rate * gradient  # Update x using gradient descent

    return points

# Parameters
starting_point = -7  # Starting point of the algorithm
learning_rate = 0.1  # Learning rate
n_iterations = 25  # Number of iterations

# Get the points from gradient descent
points = gradient_descent(starting_point, learning_rate, n_iterations)

# Create an array of x values for plotting the function
x_values = np.linspace(-8, 8, 400)
y_values = function(x_values)

# Plot the function and gradient descent steps
plt.figure(figsize=(10, 6))
plt.plot(x_values, y_values, label='f(x) = (x - 3)^2', color='blue')

# Plot gradient descent points
gd_x = np.array(points)
gd_y = function(gd_x)
plt.scatter(gd_x, gd_y, color='red', zorder=5)
plt.plot(gd_x, gd_y, color='red', linestyle='--', marker='o', zorder=4, label='Gradient Descent Steps')

# Highlight the minimum
plt.scatter([3], [0], color='green', zorder=6, marker='x', s=100, label='Minimum')

# Labels and title
plt.title('Gradient Descent on f(x) = (x - 3)^2')
plt.xlabel('x')
plt.ylabel('f(x)')
plt.legend()

# Show the plot
plt.show()

# Function to minimize: f(x) = (x - 3)^2
def function(x):
    return (x - 3) ** 2

# First derivative of the function: f'(x) = 2 * (x - 3)
def first_derivative(x):
    return 2 * (x - 3)

# Second derivative of the function: f''(x) = 2
def second_derivative(x):
    return 2

# Newton's Method function
def newtons_method(starting_point, tolerance=1e-6, max_iterations=25):
    x = starting_point
    points = []  # To store points at each iteration

    for _ in range(max_iterations):
        points.append(x)
        gradient = first_derivative(x)
        hessian = second_derivative(x)
        if hessian == 0:
            break  # Avoid division by zero
        x -= gradient / hessian  # Update x using Newton's Method

        # Check for convergence
        if abs(gradient) < tolerance:
            break

    return points

# Parameters
starting_point = -7  # Starting point of the algorithm

# Get the points from Newton's method
points = newtons_method(starting_point)

# Create an array of x values for plotting the function
x_values = np.linspace(-8, 8, 400)
y_values = function(x_values)

# Plot the function and Newton's method steps
plt.figure(figsize=(10, 6))
plt.plot(x_values, y_values, label='f(x) = (x - 3)^2', color='blue')

# Plot Newton's method points
nm_x = np.array(points)
nm_y = function(nm_x)
plt.scatter(nm_x, nm_y, color='red', zorder=5)
plt.plot(nm_x, nm_y, color='red', linestyle='--', marker='o', zorder=4, label='Newton\'s Method Steps')

# Highlight the minimum
plt.scatter([3], [0], color='green', zorder=6, marker='x', s=100, label='Minimum')

# Labels and title
plt.title('Newton\'s Method on f(x) = (x - 3)^2')
plt.xlabel('x')
plt.ylabel('f(x)')
plt.legend()

# Show the plot
plt.show()

# Function to minimize: f(x) = x^2 + 2x + 2
def function(x):
    return x**2 + 2*x + 2

# Gradient of the function: f'(x) = 2*x + 2
def gradient(x):
    return 2*x + 2

# Conjugate Gradient Method using Scipy
def conjugate_gradient_method(starting_point):
    # Optimization result
    result = minimize(
        fun=function,             # Objective function
        x0=np.array([starting_point]),  # Initial guess
        jac=gradient,             # Gradient function
        method='CG',              # Optimization method (Conjugate Gradient)
        options={'disp': True, 'return_all': True}  # Display and return all points
    )
    return result

# Parameters
starting_point = -4  # Starting point of the algorithm

# Perform Conjugate Gradient optimization
result = conjugate_gradient_method(starting_point)

# Extract optimization path
points = result.allvecs  # All points visited by CG
points = np.array(points).flatten()  # Flatten to 1D array for plotting

# Create an array of x values for plotting the function
x_values = np.linspace(-6, 2, 400)
y_values = function(x_values)

# Plot the function and Conjugate Gradient method steps
plt.figure(figsize=(10, 6))

# Plot function curve
plt.plot(x_values, y_values, label='f(x) = x^2 + 2x + 2', color='blue', linewidth=2)

# Plot Conjugate Gradient method points
plt.scatter(points, function(points), color='red', zorder=5)
plt.plot(points, function(points), color='red', linestyle='--', marker='o', zorder=4, label='CG Steps')

# Highlight the minimum
plt.scatter([-1], [function(-1)], color='green', zorder=6, marker='x', s=100, label='Minimum')

# Labels and title
plt.title('Conjugate Gradient Method on f(x) = x^2 + 2x + 2')
plt.xlabel('x')
plt.ylabel('f(x)')
plt.legend()

# Ensure grid and plot show
plt.ylim(bottom=min(y_values) - 1, top=max(y_values) + 1)  # Set y-axis limits for better visibility
plt.show()

# Function to minimize: f(x) = x^2 + 2x + 2
def function(x):
    return x**2 + 2*x + 2

# Gradient of the function: f'(x) = 2*x + 2
def gradient(x):
    return 2*x + 2

# BFGS Method using Scipy
def bfgs_method(starting_point):
    # Optimization result
    result = minimize(
        fun=function,             # Objective function
        x0=np.array([starting_point]),  # Initial guess
        jac=gradient,             # Gradient function
        method='BFGS',            # Optimization method
        options={'disp': True, 'return_all': True}  # Display and return all points
    )
    return result

# Parameters
starting_point = -4  # Starting point of the algorithm

# Perform BFGS optimization
result = bfgs_method(starting_point)

# Extract optimization path
points = result.allvecs  # All points visited by BFGS
points = np.array(points).flatten()  # Flatten to 1D array for plotting

# Create an array of x values for plotting the function
x_values = np.linspace(-6, 2, 400)
y_values = function(x_values)

# Plot the function and BFGS method steps
plt.figure(figsize=(10, 6))

# Plot function curve
plt.plot(x_values, y_values, label='f(x) = x^2 + 2x + 2', color='blue', linewidth=2)

# Plot BFGS method points
plt.scatter(points, function(points), color='red', zorder=5)
plt.plot(points, function(points), color='red', linestyle='--', marker='o', zorder=4, label='BFGS Steps')

# Highlight the minimum
plt.scatter([-1], [function(-1)], color='green', zorder=6, marker='x', s=100, label='Minimum')

# Labels and title
plt.title('BFGS Method on f(x) = x^2 + 2x + 2')
plt.xlabel('x')
plt.ylabel('f(x)')
plt.legend()

# Ensure grid and plot show
plt.ylim(bottom=min(y_values) - 1, top=max(y_values) + 1)  # Set y-axis limits for better visibility
plt.show()

# Linear function to minimize: f(x) = x
def objective_function(x):
    return x

# Constraints: x >= 0
def constraint(x):
    return x >= 0

# Simplex Method for Linear Programming (Manual Implementation)
def simplex_method():
    # Define the initial feasible solution
    x_values = np.linspace(0, 2, 100)  # Range for x
    y_values = objective_function(x_values)  # Compute objective function values

    # Initial solution
    x_start = 0
    optimal_x = x_start
    optimal_value = objective_function(optimal_x)

    # Optimization loop (simple manual iteration for demonstration)
    for x in x_values:
        if constraint(x):
            value = objective_function(x)
            if value < optimal_value:
                optimal_value = value
                optimal_x = x

    return optimal_x, optimal_value, x_values, y_values

# Perform Simplex optimization
optimal_x, optimal_value, x_values, y_values = simplex_method()

# Plotting
plt.figure(figsize=(10, 6))

# Plot the linear function
plt.plot(x_values, y_values, label='f(x) = x', color='blue', linewidth=2)

# Plot Simplex method result
plt.scatter(optimal_x, optimal_value, color='red', zorder=5, label='Optimal Solution')
plt.plot([optimal_x], [optimal_value], 'ro', linestyle='--', zorder=4)  # Mark the optimal point

# Labels and title
plt.title('Simplex Method on Linear Function f(x) = x')
plt.xlabel('x')
plt.ylabel('f(x)')
plt.legend()

# Plot show
plt.show()

# Example of the minimize function 
from scipy.optimize import minimize
def objective_function(x):
    return x[0]**2 + x[1]**2

result = minimize(objective_function, [1, 1], method='BFGS')
print(result.x)  

# Example of the root function 
from scipy.optimize import root

def equations(vars):
    x, y = vars
    return [x + 2*y - 3, x - y - 1]

result = root(equations, [0, 0])
print(result.x)

# Example of the curve_fit
from scipy.optimize import curve_fit
import numpy as np

def model(x, a, b):
    return a * np.exp(b * x)

x_data = np.array([1, 2, 3])
y_data = np.array([2.7, 7.4, 20.1])
params, covariance = curve_fit(model, x_data, y_data)
print(params)