<reponame>linyuehzzz/5526_neural_networks<filename>Lab2/rbf.py
'''
Lab 2
This code implements an RBF network.

<NAME> (lin.3326 at osu.edu)
Created: 10/3/2020
'''

import numpy as np
import matplotlib.pyplot as plt


def kmeans(X, k): 
    # randomly select initial clusters from input data
    clusters = np.random.choice(np.squeeze(X), size=k)
    prevClusters = clusters.copy()
    stds = np.zeros(k)
    converged = False
 
    while not converged:
        distances = np.squeeze(np.abs(X[:, np.newaxis] - clusters[np.newaxis, :]))
 
        # find the cluster that's closest to each point
        closestCluster = np.argmin(distances, axis=1)
 
        # update clusters by taking the mean of all of the points assigned to that cluster
        for i in range(k):
            pointsForCluster = X[closestCluster == i]
            if len(pointsForCluster) > 0:
                clusters[i] = np.mean(pointsForCluster, axis=0)
 
        # converge if clusters haven't moved
        converged = np.linalg.norm(clusters - prevClusters) < 1e-6
        prevClusters = clusters.copy()
 
    distances = np.squeeze(np.abs(X[:, np.newaxis] - clusters[np.newaxis, :]))
    closestCluster = np.argmin(distances, axis=1)
 
    clustersWithNoPoints = []
    for i in range(k):
        pointsForCluster = X[closestCluster == i]
        if len(pointsForCluster) < 2:
            # keep track of clusters with no points or 1 point
            clustersWithNoPoints.append(i)
            continue
        else:
            stds[i] = np.std(X[closestCluster == i])
 
    # if there are clusters with 0 or 1 points, take the mean std of the other clusters
    if len(clustersWithNoPoints) > 0:
        pointsToAverage = []
        for i in range(k):
            if i not in clustersWithNoPoints:
                pointsToAverage.append(X[closestCluster == i])
        pointsToAverage = np.concatenate(pointsToAverage).ravel()
        stds[clustersWithNoPoints] = np.mean(np.std(pointsToAverage))
 
    return clusters, stds



def rbf(x, c, s):
    return np.exp(-1 / (2 * s**2) * (x-c)**2)



class RBFNet(object):
    """Implementation of a Radial Basis Function Network"""
    def __init__(self, k=2, lr=0.01, epochs=10, inferStds=True, rbf=rbf):
        self.k = k
        self.lr = lr
        self.epochs = epochs
        self.rbf = rbf
        self.inferStds = inferStds
 
        self.w = np.random.randn(k)
        self.b = np.random.randn(1)

    def fit(self, X, y):
        if self.inferStds:
            # compute stds from data
            self.centers, self.stds = kmeans(X, self.k)
        else:
            # use a fixed std 
            self.centers, _ = kmeans(X, self.k)
            dMax = max([np.abs(c1 - c2) for c1 in self.centers for c2 in self.centers])
            self.stds = np.repeat(dMax / np.sqrt(2*self.k), self.k)
    
        # training
        for epoch in range(self.epochs):
            y_pred = []
            for i in range(X.shape[0]):
                # forward pass
                a = np.array([self.rbf(X[i], c, s) for c, s, in zip(self.centers, self.stds)])
                F = a.T.dot(self.w) + self.b
                y_pred.append(F)
    
                loss = (y[i] - F).flatten() ** 2
    
                # backward pass
                error = -(y[i] - F).flatten()
    
                # online update
                self.w = self.w - self.lr * a * error
                self.b = self.b - self.lr * error
        return np.array(y_pred)



# sample inputs and add noise
NUM_SAMPLES = 75
X = np.random.uniform(0., 1., NUM_SAMPLES)
X = np.sort(X, axis=0)
noise = np.random.uniform(-0.1, 0.1, NUM_SAMPLES)
y = 0.4 * np.sin(2 * np.pi * X) + 0.5 + noise
y_true = 0.4 * np.sin(2 * np.pi * X) + 0.5
 
rbfnet = RBFNet(k=2, lr=0.01, epochs=100, inferStds=True)
y_pred = rbfnet.fit(X, y)

plt.title('base=2, lr=0.01')
plt.plot(X, y_true, '-o', label='True h(x)')
plt.plot(X, y_pred, '-o', label='RBF-Net')
plt.legend()
 
plt.tight_layout()
plt.show()

'''
Lab 2
This code implements an RBF network.

<NAME> (lin.3326 at osu.edu)
Created: 10/3/2020
'''

import numpy as np
import matplotlib.pyplot as plt


def kmeans(X, k): 
    # randomly select initial clusters from input data
    clusters = np.random.choice(np.squeeze(X), size=k)
    prevClusters = clusters.copy()
    stds = np.zeros(k)
    converged = False
 
    while not converged:
        distances = np.squeeze(np.abs(X[:, np.newaxis] - clusters[np.newaxis, :]))
 
        # find the cluster that's closest to each point
        closestCluster = np.argmin(distances, axis=1)
 
        # update clusters by taking the mean of all of the points assigned to that cluster
        for i in range(k):
            pointsForCluster = X[closestCluster == i]
            if len(pointsForCluster) > 0:
                clusters[i] = np.mean(pointsForCluster, axis=0)
 
        # converge if clusters haven't moved
        converged = np.linalg.norm(clusters - prevClusters) < 1e-6
        prevClusters = clusters.copy()
 
    distances = np.squeeze(np.abs(X[:, np.newaxis] - clusters[np.newaxis, :]))
    closestCluster = np.argmin(distances, axis=1)
 
    clustersWithNoPoints = []
    for i in range(k):
        pointsForCluster = X[closestCluster == i]
        if len(pointsForCluster) < 2:
            # keep track of clusters with no points or 1 point
            clustersWithNoPoints.append(i)
            continue
        else:
            stds[i] = np.std(X[closestCluster == i])
 
    # if there are clusters with 0 or 1 points, take the mean std of the other clusters
    if len(clustersWithNoPoints) > 0:
        pointsToAverage = []
        for i in range(k):
            if i not in clustersWithNoPoints:
                pointsToAverage.append(X[closestCluster == i])
        pointsToAverage = np.concatenate(pointsToAverage).ravel()
        stds[clustersWithNoPoints] = np.mean(np.std(pointsToAverage))
 
    return clusters, stds



def rbf(x, c, s):
    return np.exp(-1 / (2 * s**2) * (x-c)**2)



class RBFNet(object):
    """Implementation of a Radial Basis Function Network"""
    def __init__(self, k=2, lr=0.01, epochs=10, inferStds=True, rbf=rbf):
        self.k = k
        self.lr = lr
        self.epochs = epochs
        self.rbf = rbf
        self.inferStds = inferStds
 
        self.w = np.random.randn(k)
        self.b = np.random.randn(1)

    def fit(self, X, y):
        if self.inferStds:
            # compute stds from data
            self.centers, self.stds = kmeans(X, self.k)
        else:
            # use a fixed std 
            self.centers, _ = kmeans(X, self.k)
            dMax = max([np.abs(c1 - c2) for c1 in self.centers for c2 in self.centers])
            self.stds = np.repeat(dMax / np.sqrt(2*self.k), self.k)
    
        # training
        for epoch in range(self.epochs):
            y_pred = []
            for i in range(X.shape[0]):
                # forward pass
                a = np.array([self.rbf(X[i], c, s) for c, s, in zip(self.centers, self.stds)])
                F = a.T.dot(self.w) + self.b
                y_pred.append(F)
    
                loss = (y[i] - F).flatten() ** 2
    
                # backward pass
                error = -(y[i] - F).flatten()
    
                # online update
                self.w = self.w - self.lr * a * error
                self.b = self.b - self.lr * error
        return np.array(y_pred)



# sample inputs and add noise
NUM_SAMPLES = 75
X = np.random.uniform(0., 1., NUM_SAMPLES)
X = np.sort(X, axis=0)
noise = np.random.uniform(-0.1, 0.1, NUM_SAMPLES)
y = 0.4 * np.sin(2 * np.pi * X) + 0.5 + noise
y_true = 0.4 * np.sin(2 * np.pi * X) + 0.5
 
rbfnet = RBFNet(k=2, lr=0.01, epochs=100, inferStds=True)
y_pred = rbfnet.fit(X, y)

plt.title('base=2, lr=0.01')
plt.plot(X, y_true, '-o', label='True h(x)')
plt.plot(X, y_pred, '-o', label='RBF-Net')
plt.legend()
 
plt.tight_layout()
plt.show()
Lab 2 This code implements an RBF network.  <NAME> (lin.3326 at osu.edu) Created: 10/3/2020 # randomly select initial clusters from input data # find the cluster that's closest to each point # update clusters by taking the mean of all of the points assigned to that cluster # converge if clusters haven't moved # keep track of clusters with no points or 1 point # if there are clusters with 0 or 1 points, take the mean std of the other clusters Implementation of a Radial Basis Function Network # compute stds from data # use a fixed std # training # forward pass # backward pass # online update # sample inputs and add noise