Linear regression, LARS - Least angle regression, Lasso regression

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

import sklearn
from sklearn.linear_model import LinearRegression,LogisticRegression,Ridge, Lars
from sklearn.linear_model import enet_path, lars_path, lasso_path

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import Normalizer,StandardScaler

Regression, Continuous target

Prostate dataset https://hastie.su.domains/ElemStatLearn/

Hastie,Tibshirani, Friedman: The Elements of Statistical Learning

data=pd.read_csv('data/prostate.csv',sep='\t')
data.columns=['id']+data.columns[1:].tolist()
data.set_index(data.id,inplace=True)
data.drop(columns=['id'],inplace=True)
data.describe()

	lcavol	lweight	age	lbph	svi	lcp	gleason	pgg45	lpsa
count	97.000000	97.000000	97.000000	97.000000	97.000000	97.000000	97.000000	97.000000	97.000000
mean	1.350010	3.628943	63.865979	0.100356	0.216495	-0.179366	6.752577	24.381443	2.478387
std	1.178625	0.428411	7.445117	1.450807	0.413995	1.398250	0.722134	28.204035	1.154329
min	-1.347074	2.374906	41.000000	-1.386294	0.000000	-1.386294	6.000000	0.000000	-0.430783
25%	0.512824	3.375880	60.000000	-1.386294	0.000000	-1.386294	6.000000	0.000000	1.731656
50%	1.446919	3.623007	65.000000	0.300105	0.000000	-0.798508	7.000000	15.000000	2.591516
75%	2.127041	3.876396	68.000000	1.558145	0.000000	1.178655	7.000000	40.000000	3.056357
max	3.821004	4.780383	79.000000	2.326302	1.000000	2.904165	9.000000	100.000000	5.582932

Train test split

train=data[data.train=='T'].drop(columns=['train'])
test=data[data.train=='F'].drop(columns=['train'])

columns=train.columns
y=train[columns[-1]].values
y_test=test[columns[-1]]
len(columns)

9

ct = ColumnTransformer(
    transformers=[
#        ('', 'passthrough', [2]),
        ('s',StandardScaler(), list(range(len(columns)-1)))
    ],
    remainder='passthrough'
)
X = ct.fit_transform(train[columns[:-1]])
#X_test=ct.transform(test[columns[1:-1]])
#ct.get_feature_names_out(input_features=train.columns)

lr=LinearRegression()
lr.fit(X,y)

lars_reg = Lars(n_nonzero_coefs=5)
lars_reg.fit(X, y)

Lars(n_nonzero_coefs=5)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

eps = 5e-3  # the smaller it is the longer is the path
l1_ratio=0.1
fig = plt.figure(figsize=(7,3))
#alphas_lasso, coefs_lasso, _ = lasso_path(X, y)

alphas_lars, _, coefs_lars = lars_path(X, y, method="lasso")

alphas_enet, coefs_enet, _ = enet_path(X, y, l1_ratio=l1_ratio)


colors = ["r", "g", "c", "k",'magenta','orange','cyan','yellow']
for coef_enet, coef_lars, c,col in zip(coefs_enet, coefs_lars, colors,columns[:-1]):
    l1 = plt.semilogx(alphas_enet*l1_ratio, coef_enet,linestyle="--", c=c)
    l2 = plt.semilogx(alphas_lars, coef_lars,  c=c, label=col)

plt.xlabel("alpha")
plt.ylabel("coefficients")
plt.title("LARS and Elastic Net(0.1) Paths")
plt.legend(loc="best", title='Lars')
plt.axis("tight")

(np.float64(0.0006221996838294731),
 np.float64(1.2414515815304998),
 np.float64(-0.3367348162062802),
 np.float64(0.7609346593258187))

print(f"{alphas_enet[-17]=}")
pd.DataFrame({'linreg':lr.coef_,'Lars':lars_reg.coef_,'lars_path':coefs_lars[:,-4],'enet_path':coefs_enet[:,-17]} ,index=columns[:-1])

alphas_enet[-17]=np.float64(0.026839737928660783)

	linreg	Lars	lars_path	enet_path
lcavol	0.711041	0.575349	0.575349	0.663364
lweight	0.290450	0.243782	0.243782	0.287505
age	-0.141482	0.000000	0.000000	-0.124893
lbph	0.210420	0.142426	0.142426	0.204212
svi	0.307300	0.198799	0.198799	0.293244
lcp	-0.286841	0.000000	0.000000	-0.221260
gleason	-0.020757	0.000000	0.000000	-0.000000
pgg45	0.275268	0.089406	0.089406	0.233048