import torch
# dimension
D = 3
K = 5
# specify the parameters of the neural network
## the first layer
W1 = torch.randn((K, D), requires_grad=True)
b1 = torch.randn((K, 1), requires_grad=True)
print(W1)
tensor([[-0.3395, -1.0059, 0.6492],
[-0.0455, -0.3685, -0.9871],
[-0.4889, -0.9101, 0.8806],
[ 1.8505, -2.0295, -0.5187],
[-0.3543, 0.0921, 0.5322]], requires_grad=True)
print(b1)
tensor([[-0.4550],
[ 0.1326],
[ 0.7193],
[ 0.9713],
[-1.2404]], requires_grad=True)
## the second layer
W2 = torch.randn((K, K), requires_grad=True)
b2 = torch.randn((K, 1), requires_grad=True)
## the third layer
W3 = torch.randn((K, K), requires_grad=True)
b3 = torch.randn((K, 1), requires_grad=True)
## the last layer
W4 = torch.randn((1, K), requires_grad=True)
b4 = torch.randn((1,), requires_grad=True)
def sigmoid(x):
return 1 / (1 + torch.exp(-x))
## Sigmoid introduces nonlinearity and maps every input to a value between zero and one.
## Without nonlinear activations, stacking several linear layers would still behave like one linear transformation
# input
x = torch.randn((D, 1), requires_grad=True)
# ground truth
y_ground = torch.randn((1,))
# forward
## the first layer
z1 = torch.matmul(W1, x) + b1
a1 = sigmoid(z1)
print(a1)
tensor([[0.8187],
[0.3972],
[0.9320],
[0.9967],
[0.2329]], grad_fn=<MulBackward0>)
## the second layer
z2 = torch.matmul(W2, a1) + b2
a2 = sigmoid(z2)
print(a2)
tensor([[0.8757],
[0.6685],
[0.3927],
[0.4712],
[0.3323]], grad_fn=<MulBackward0>)
## the third layer
z3 = torch.matmul(W3, a2) + b3
a3 = sigmoid(z3)
print(a3)
tensor([[0.5654],
[0.8512],
[0.7149],
[0.6051],
[0.6423]], grad_fn=<MulBackward0>)
## the last layer
z4 = torch.matmul(W4, a3) + b4
y_pred = sigmoid(z4)
print(y_pred)
tensor([[0.7558]], grad_fn=<MulBackward0>)
# calculate the loss
J = (y_pred - y_ground) ** 2
print(J)
tensor([[4.3802]], grad_fn=<PowBackward0>)
print('the gradient of W2:')
print(W2.grad)
print('the gradient of b2:')
print(b2.grad)
the gradient of W2: None the gradient of b2: None
# Compute gradients
J.backward()
# Displays the gradients created by J.backward().
print('the gradient of W2:')
print(W2.grad)
print('the gradient of b2:')
print(b2.grad)
the gradient of W2:
tensor([[ 0.0226, 0.0109, 0.0257, 0.0275, 0.0064],
[ 0.0302, 0.0147, 0.0344, 0.0368, 0.0086],
[-0.0194, -0.0094, -0.0221, -0.0236, -0.0055],
[-0.1068, -0.0518, -0.1216, -0.1300, -0.0304],
[ 0.0422, 0.0205, 0.0480, 0.0513, 0.0120]])
the gradient of b2:
tensor([[ 0.0276],
[ 0.0369],
[-0.0237],
[-0.1304],
[ 0.0515]])
W <- W - lr * W_gradient
lr = 0.1
print('the weight of W2 before updation:')
print(W2)
the weight of W2 before updation:
tensor([[ 2.0902, -0.8325, 1.9016, -1.3918, -0.8954],
[ 1.4625, 0.6328, -0.3945, -0.5695, -1.3394],
[-1.0338, 0.6264, 0.6308, -0.7689, 1.6072],
[ 0.0658, 0.6266, -0.8462, -0.9436, 1.1116],
[ 1.3930, 0.5118, -1.6905, -0.4347, -0.0496]], requires_grad=True)
expected_new_W2 = W2 - lr * W2.grad
print(expected_new_W2)
tensor([[ 2.0879, -0.8336, 1.8991, -1.3945, -0.8960],
[ 1.4595, 0.6313, -0.3980, -0.5732, -1.3402],
[-1.0318, 0.6273, 0.6330, -0.7665, 1.6078],
[ 0.0765, 0.6318, -0.8341, -0.9306, 1.1147],
[ 1.3887, 0.5097, -1.6953, -0.4398, -0.0508]], grad_fn=<SubBackward0>)
# specify an optimizer
optimizer = torch.optim.SGD([W1, b1, W2, b2, W3, b3, W4, b4], lr=lr)
# Update parameters
optimizer.step()
print('the weight of W2 after updation:')
print(W2)
the weight of W2 after updation:
tensor([[ 2.0879, -0.8336, 1.8991, -1.3945, -0.8960],
[ 1.4595, 0.6313, -0.3980, -0.5732, -1.3402],
[-1.0318, 0.6273, 0.6330, -0.7665, 1.6078],
[ 0.0765, 0.6318, -0.8341, -0.9306, 1.1147],
[ 1.3887, 0.5097, -1.6953, -0.4398, -0.0508]], requires_grad=True)
# check if two tensors, W2 and expected_new_W2, are element-wise equal
torch.eq(W2, expected_new_W2).all()
tensor(True)
W1 = torch.randn((K, D), requires_grad=True)
b1 = torch.randn((K, 1), requires_grad=True)
def sigmoid(x):
...# define
layer1 = torch.nn.Linear(in_features=D, out_features=K, bias=True)
# input
x = torch.randn((D,))
z1 = torch.matmul(W1, x) + b1
a1 = sigmoid(z1)# forward
z1 = layer1(x)
a1 = torch.nn.functional.sigmoid(z1)
class neural_network(torch.nn.Module):
def __init__(self, D, K):
super(neural_network, self).__init__()
self.layer1 = torch.nn.Linear(in_features=D, out_features=K, bias=True)
self.layer2 = torch.nn.Linear(in_features=K, out_features=K, bias=True)
self.layer3 = torch.nn.Linear(in_features=K, out_features=K, bias=True)
self.layer4 = torch.nn.Linear(in_features=K, out_features=1, bias=True)
def forward(self, x):
# the first layer
z1 = self.layer1(x)
a1 = torch.nn.functional.sigmoid(z1)
# the second layer
z2 = self.layer2(a1)
a2 = torch.nn.functional.sigmoid(z2)
# the third layer
z3 = self.layer3(a2)
a3 = torch.nn.functional.sigmoid(z3)
# the fourth layer
z4 = self.layer4(a3)
a4 = torch.nn.functional.sigmoid(z4)
return a4
# specify the network and the optimizer
network = neural_network(D, K)
optimizer = torch.optim.SGD(network.parameters(), lr=lr)
# input
x = torch.randn((D,))
# ground truth
y_ground = torch.randn((1,))
# forward
y_pred = network(x)
# calculate the loss
J = (y_pred - y_ground) ** 2
print(J)
tensor([1.1442], grad_fn=<PowBackward0>)
# backward
J.backward()
# update the parameters
optimizer.step()
[x.grad for x in network.parameters()]
[tensor([[-1.4293e-03, 5.3471e-04, -1.8739e-04],
[ 8.6041e-05, -3.2187e-05, 1.1280e-05],
[ 6.2966e-04, -2.3555e-04, 8.2552e-05],
[-2.5285e-04, 9.4591e-05, -3.3150e-05],
[-1.3750e-04, 5.1439e-05, -1.8027e-05]]),
tensor([-5.9831e-04, 3.6016e-05, 2.6357e-04, -1.0584e-04, -5.7557e-05]),
tensor([[-2.6130e-03, -2.2001e-03, -9.7722e-04, -1.4353e-03, -2.4829e-03],
[ 2.1089e-03, 1.7756e-03, 7.8870e-04, 1.1584e-03, 2.0039e-03],
[-9.7956e-04, -8.2476e-04, -3.6634e-04, -5.3806e-04, -9.3079e-04],
[-2.5521e-05, -2.1488e-05, -9.5444e-06, -1.4018e-05, -2.4250e-05],
[ 1.7465e-03, 1.4705e-03, 6.5317e-04, 9.5934e-04, 1.6596e-03]]),
tensor([-3.7203e-03, 3.0026e-03, -1.3947e-03, -3.6336e-05, 2.4866e-03]),
tensor([[-0.0187, -0.0113, -0.0118, -0.0135, -0.0108],
[-0.0032, -0.0019, -0.0020, -0.0023, -0.0018],
[ 0.0161, 0.0098, 0.0102, 0.0116, 0.0093],
[-0.0309, -0.0187, -0.0195, -0.0223, -0.0179],
[ 0.0100, 0.0060, 0.0063, 0.0072, 0.0058]]),
tensor([-0.0297, -0.0051, 0.0256, -0.0492, 0.0158]),
tensor([[-0.2554, -0.2706, -0.2890, -0.2924, -0.2608]]),
tensor([-0.5310])]
optimizer.zero_grad()
[x.grad for x in network.parameters()]
[None, None, None, None, None, None, None, None]
# Create a numpy array.
import numpy as np
x = np.array([[1, 2], [3, 4]])
# Convert the numpy array to a torch tensor.
y = torch.from_numpy(x)
print(y)
tensor([[1, 2],
[3, 4]])
# Convert the torch tensor to a numpy array.
z = y.numpy()
print(z)
[[1 2] [3 4]]
# Download and construct CIFAR-10 dataset.
import torchvision
import torchvision.transforms as transforms
train_dataset = torchvision.datasets.CIFAR10(root='../../data/',
train=True,
transform=transforms.ToTensor(),
download=True)
100%|██████████| 170M/170M [19:30<00:00, 146kB/s]
# Fetch one data pair (read data from disk).
image, label = train_dataset[0]
print (image.size())
print (label)
torch.Size([3, 32, 32]) 6
# Data loader (this provides queues and threads in a very simple way).
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=64,
shuffle=True)
# When iteration starts, queue and thread start to load data from files.
data_iter = iter(train_loader)
# Get a mini-batch of images and labels
images, labels = next(data_iter)
for images, labels in train_loader:
# Training code should be written here.
pass
# You can build your custom dataset as below.
class CustomDataset(torch.utils.data.Dataset):
def __init__(self):
# TODO
# 1. Initialize file paths or a list of file names.
# pass
# Create 100 samples, each containing 3 features
self.inputs = torch.randn(100, 3)
# Create 100 binary labels: either 0 or 1
self.labels = torch.randint(0, 2, (100,))
def __getitem__(self, index):
# TODO
# 1. Read one data from file (e.g. using numpy.fromfile, PIL.Image.open).
# 2. Preprocess the data (e.g. torchvision.Transform).
# 3. Return a data pair (e.g. image and label).
# pass
return self.inputs[index], self.labels[index]
def __len__(self):
# You should change 0 to the total size of your dataset.
# return 0
return len(self.inputs)
# You can then use the prebuilt data loader.
custom_dataset = CustomDataset()
train_loader = torch.utils.data.DataLoader(dataset=custom_dataset,
batch_size=64,
shuffle=True)
# Test that it works by retrieving one batch
inputs, labels = next(iter(train_loader))
print("Dataset size:", len(custom_dataset))
print("Input batch shape:", inputs.shape)
print("Label batch shape:", labels.shape)
Dataset size: 100 Input batch shape: torch.Size([64, 3]) Label batch shape: torch.Size([64])