ISYE 6669 Homework 4

By: Mark Pearl

Sept. 17, 2020

import numpy as np
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

##Q1
func_1 = lambda x: abs(x[0]**2-3*x[2]+4)+(x[1]-2)**2

sol1 = minimize(f1, x0=[0,0,0])
print(sol1)

      fun: 0.02183880722146802
 hess_inv: array([[ 1.00000000e+00, -1.78970444e-07,  1.49529676e-09],
       [-1.78970444e-07,  2.75764507e-01,  1.91203618e-03],
       [ 1.49529676e-09,  1.91203618e-03,  1.12731876e-02]])
      jac: array([5.96046448e-08, 2.95559156e-01, 2.23586226e+00])
  message: 'Desired error not necessarily achieved due to precision loss.'
     nfev: 340
      nit: 5
     njev: 82
   status: 2
  success: False
        x: array([1.36503469e-08, 2.14777957e+00, 1.33333333e+00])

From the above plot we can see that the following solution has found an optimal solution at x: array([0. , 2.99999999, 1.33333333])

##Q2
func_2 = lambda x: (x-1)*(x-2)*(x-3)+x**6

sol2 = minimize(func_2,x0=[0], bounds=[(-10,10)])
print(sol2)

      fun: array([-28.38478066])
 hess_inv: <1x1 LbfgsInvHessProduct with dtype=float64>
      jac: array([-7.1054274e-07])
  message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'
     nfev: 20
      nit: 3
     njev: 10
   status: 0
  success: True
        x: array([-1.41450924])

From the above plot we can see that the following solution has found an optimal solution at x: array([-1.41450924])

##Q3
A = np.matrix([[2,-3],[4,1]])
b = np.array([2,1])
func_3 = lambda x: np.linalg.norm(A.dot(x)-b)

sol3 = minimize(func_3, x0=[0,0])
print(sol3)

      fun: 2.1026487223951748e-08
 hess_inv: array([[1.91706262e-09, 1.25195345e-09],
       [1.25195345e-09, 1.51383874e-08]])
      jac: array([2.44043117, 0.91918012])
  message: 'Desired error not necessarily achieved due to precision loss.'
     nfev: 195
      nit: 27
     njev: 61
   status: 2
  success: False
        x: array([ 0.35714285, -0.42857143])

From the above plot we can see that the following solution has found an optimal solution at x: array([ 0.35714285, -0.42857143])

##Q4
func_4  = lambda x: (1.25 - x[0] + x[0]*x[1])**2 + (2.5 - x[0] + x[0]*x[1]**2)**2 + (3 - x[0] + x[0]*x[1]**3)**2

x0 = np.array([0,0])
xs = [x0]

bnds = ((-5, None), (None, 5))
sol4 = minimize(func_4,x0,bounds=bnds)

x = np.linspace(-5, 5, 50)
y = np.linspace(-5, 5, 50)
X,Y = np.meshgrid(x,y) # meshgrid
XY = np.vstack([X.ravel(), Y.ravel()]) # 2D to 1D vertically
Z = func_4(XY).reshape(50,50) 

xs = np.array(xs)
plt.figure()
plt.contour(X, Y, Z, np.linspace(0,25,50))
plt.plot(xs[:, 0], xs[:, 1], '-o')
plt.gca().set_aspect('equal', adjustable='box')
plt.show()

print(sol4)

      fun: 0.03940195167848258
 hess_inv: <2x2 LbfgsInvHessProduct with dtype=float64>
      jac: array([-3.46944697e-09, -1.94289028e-07])
  message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'
     nfev: 60
      nit: 18
     njev: 20
   status: 0
  success: True
        x: array([4.87469383, 0.71930641])

We can see from our 2D result that we have a boxed constraint which is causing the effect of our objective function to look like a diamond since both the upper right and bottom left quadrant our bounded to the constraint. We will have multiple local minimums in this situation as well in both the upper left and bottom right quadrants.

In our case we can see that based on the starting point, our equation is converging to the local optimum position in the bottom right quadrant at x: array([4.87469383, 0.71930641])

#surf = axes.plot_surface(x, y, z, cmap='coolwarm',linewidth=0, antialiased=True)
fig = plt.figure(figsize=(10,10))
axes = fig.gca(projection='3d')

surf = axes.contour3D(X, Y, Z, 50,  cmap='coolwarm')

axes.set_xlabel('X Label')
axes.set_ylabel('Y Label')
axes.set_zlabel('Z Label')

plt.show()

#Q5 
func_5 = lambda x: (x[0]**4-25*x[0]**2+4*x[0])+(x[1]**4-25*x[1]**2+4*x[1])+(x[2]**4-25*x[2]**2+4*x[2])+(x[3]**4-25*x[3]**2+4*x[3])

b = (-4, 4)
bnds = (b,b,b,b)
sol5 = minimize(func_5, x0=[0,0,0,0],bounds=bnds)
print(sol5) 

      fun: -681.8850016761318
 hess_inv: <4x4 LbfgsInvHessProduct with dtype=float64>
      jac: array([ 0.00000000e+00,  0.00000000e+00, -1.13686838e-05, -1.13686838e-05])
  message: b'CONVERGENCE: REL_REDUCTION_OF_F_<=_FACTR*EPSMCH'
     nfev: 50
      nit: 4
     njev: 10
   status: 0
  success: True
        x: array([-3.57487491, -3.57487491, -3.57487491, -3.57487491])

You can see with our result that each variable is converging to the same respective point. Since nothing will be different with the objective function in our case, since we're only incrementing the variable name. Nothing changes to the overall result. Therefore, this solution converged at the point: x: array([-3.57487491, -3.57487491, -3.57487491, -3.57487491])

##Q6 Attempt with starting point of [0,0]
func_6_1  = lambda x: (x[0]**2 + 2*x[1]-4)**2 + (x[0] + x[1]**2-8)**2

x0 = np.array([0,0])
xs = [x0]

bnds = ((-5, None), (None, 5))
sol_6_1 = minimize(func_6_1,x0,bounds=bnds)

x = np.linspace(-5, 5, 50)
y = np.linspace(-5, 5, 50)
X,Y = np.meshgrid(x,y) # meshgrid
XY = np.vstack([X.ravel(), Y.ravel()]) # 2D to 1D vertically
Z = func_6_1(XY).reshape(50,50) 

xs = np.array(xs)
plt.figure()
plt.contour(X, Y, Z, np.linspace(0,25,50))
plt.plot(xs[:, 0], xs[:, 1], '-o')
plt.gca().set_aspect('equal', adjustable='box')
plt.show()

#surf = axes.plot_surface(x, y, z, cmap='coolwarm',linewidth=0, antialiased=True)
fig = plt.figure(figsize=(10,10))
axes = fig.gca(projection='3d')

surf = axes.contour3D(X, Y, Z, 50,  cmap='coolwarm')

axes.set_xlabel('X Label')
axes.set_ylabel('Y Label')
axes.set_zlabel('Z Label')

plt.show()

print(sol_6_1)

      fun: 2.3325913012338164
 hess_inv: <2x2 LbfgsInvHessProduct with dtype=float64>
      jac: array([-8.88178420e-08, -3.10862449e-07])
  message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'
     nfev: 57
      nit: 12
     njev: 19
   status: 0
  success: True
        x: array([0.1852601 , 2.69890794])

From our result of the first model we can see that we have 3 local minimums for our solution. When choosing starting points that are closer to each of the 3 minima, we can see the resulting solution will change, as alluded to in the wording of the question. In our case, our first iteration started at x0 = [0,0]. Which means that it will converge to the minimum at the top of our 2D plot, therefore out solution for this first iteration is x: array([0.1852601 , 2.69890794])

##Q6 Attempt with starting point of [5,-2]
func_6_2  = lambda x: (x[0]**2 + 2*x[1]-4)**2 + (x[0] + x[1]**2-8)**2

x0 = np.array([5,-2])
xs = [x0]

bnds = ((-5, None), (None, 5))
sol_6_2 = minimize(func_6_2,x0,bounds=bnds)

x = np.linspace(-5, 5, 50)
y = np.linspace(-5, 5, 50)
X,Y = np.meshgrid(x,y) # meshgrid
XY = np.vstack([X.ravel(), Y.ravel()]) # 2D to 1D vertically
Z = func_6_2(XY).reshape(50,50) 

xs = np.array(xs)
plt.figure()
plt.contour(X, Y, Z, np.linspace(0,25,50))
plt.plot(xs[:, 0], xs[:, 1], '-o')
plt.gca().set_aspect('equal', adjustable='box')
plt.show()

#surf = axes.plot_surface(x, y, z, cmap='coolwarm',linewidth=0, antialiased=True)
fig = plt.figure(figsize=(10,10))
axes = fig.gca(projection='3d')

surf = axes.contour3D(X, Y, Z, 50,  cmap='coolwarm')

axes.set_xlabel('X Label')
axes.set_ylabel('Y Label')
axes.set_zlabel('Z Label')

plt.show()

print(sol_6_2)

      fun: 6.577393938445569e-13
 hess_inv: <2x2 LbfgsInvHessProduct with dtype=float64>
      jac: array([-8.88360458e-06,  4.80730083e-07])
  message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'
     nfev: 36
      nit: 11
     njev: 12
   status: 0
  success: True
        x: array([ 2.91703377, -2.25454339])

Now when choosing a starting point closer to the minima at the bottom right. We can see the impact to our solution, as the optimal solution is now positioned at: x: array([ 2.91703377, -2.25454339])

##Q6 Attempt with starting point of [-4,-4]
func_6_3  = lambda x: (x[0]**2 + 2*x[1]-4)**2 + (x[0] + x[1]**2-8)**2

x0 = np.array([-4,-4])
xs = [x0]

bnds = ((-5, None), (None, 5))
sol_6_3 = minimize(func_6_3,x0,bounds=bnds)

x = np.linspace(-5, 5, 50)
y = np.linspace(-5, 5, 50)
X,Y = np.meshgrid(x,y) # meshgrid
XY = np.vstack([X.ravel(), Y.ravel()]) # 2D to 1D vertically
Z = func_6_3(XY).reshape(50,50) 

xs = np.array(xs)
plt.figure()
plt.contour(X, Y, Z, np.linspace(0,25,50))
plt.plot(xs[:, 0], xs[:, 1], '-o')
plt.gca().set_aspect('equal', adjustable='box')
plt.show()

#surf = axes.plot_surface(x, y, z, cmap='coolwarm',linewidth=0, antialiased=True)
fig = plt.figure(figsize=(10,10))
axes = fig.gca(projection='3d')

surf = axes.contour3D(X, Y, Z, 50,  cmap='coolwarm')

axes.set_xlabel('X Label')
axes.set_ylabel('Y Label')
axes.set_zlabel('Z Label')

plt.show()

print(sol_6_3)

      fun: 5.039055124106524e-14
 hess_inv: <2x2 LbfgsInvHessProduct with dtype=float64>
      jac: array([1.50161290e-06, 2.66751814e-06])
  message: b'CONVERGENCE: NORM_OF_PROJECTED_GRADIENT_<=_PGTOL'
     nfev: 21
      nit: 6
     njev: 7
   status: 0
  success: True
        x: array([-3.27340466, -3.35758908])

Now when choosing a starting point closer to the minima at the bottom left. We can see the impact to our solution, as the optimal solution is now positioned at: x: array([-3.27340466, -3.35758908])!!

This goes to show based on the lectures, that the optimization problem will not find the global minimum. Which in our case is sol_6_1 based on an objective value of fun: 2.3325913012338164. Based on the starting point, it will find the optimal based on the closest local minima!