<reponame>bzcheeseman/snapshot-ensemble<gh_stars>0
#
# Created by <NAME> on 7/5/17.
#
# snapshot_ensemble
# Copyright (c) 2017 <NAME>
# Full license at snapshot_ensemble/LICENSE.txt
#

# Network is copied directly from pytorch tutorial to see if we can be more accurate with no changes

##################### Starting Copy ######################

import torch
import torchvision
import torchvision.transforms as transforms

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=16,
                                          shuffle=True, num_workers=4)

testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=1,
                                         shuffle=False, num_workers=4)

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x


net = Net()
net.cuda()
num_epochs = 150

import torch.optim as optim

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

# for epoch in range(num_epochs):  # loop over the dataset multiple times
#
#     running_loss = 0.0
#     for i, data in enumerate(trainloader, 0):
#         # get the inputs
#         inputs, labels = data
#
#         # wrap them in Variable
#         inputs, labels = Variable(inputs).cuda(), Variable(labels).cuda()
#
#         # zero the parameter gradients
#         optimizer.zero_grad()
#
#         # forward + backward + optimize
#         outputs = net(inputs)
#         loss = criterion(outputs, labels)
#         loss.backward()
#         optimizer.step()
#
#         # print statistics
#         running_loss += loss.data[0]
#         if i % 2000 == 1999:    # print every 2000 mini-batches
#             print('[%d, %5d] loss: %.3f' %
#                   (epoch + 1, i + 1, running_loss / 2000))
#             running_loss = 0.0
#
# print('Finished Training')
#
# correct = 0
# total = 0
# for data in testloader:
#     images, labels = data
#     outputs = net(Variable(images).cuda())
#     _, predicted = torch.max(outputs.data, 1)
#     total += labels.size(0)
#     correct += (predicted == labels.cuda()).sum()
#
# print('Accuracy of the network on the 10000 test images: %.2f %%' % (
#     100 * float(correct / total)))

##################### Starting Modified Code ######################

from module import SnapshotEnsemble

ensemble = SnapshotEnsemble(net, criterion, restart_lr=0.1, epochs=num_epochs, batch_size=16,
                            num_snapshots=6, train_dataset=trainset, test_dataset=testset)

# ensemble.train(closure=ensemble.default_closure, print_steps=2500)


def check(output, target):
    _, predicted = torch.max(output.data, 1)
    c = (predicted == target.data).squeeze()
    return bool(c[0])


ensemble.load(6)
# call this to optimize voting - I recommend using the full ensemble for this,
# also must be >= ensemble_size for validation/testing
# ensemble.optimize_ensemble_weights(forward=ensemble.default_forward, n_iters=1, ensemble_size=2)
# ensemble.save()
ensemble.validate(forward=ensemble.default_forward, ensemble_size=2, check_correctness=check)
ensemble.validate(forward=ensemble.default_forward, ensemble_size=6, check_correctness=check, use_weights=False)

# 50 epochs training, 6 snapshots
# 6 has accuracy = 63.70%
# 3 has accuracy = 62.08%
# 5 has accuracy = 63.22%
# 1 has accuracy = 60.83%



#
# Created by <NAME> on 7/5/17.
#
# snapshot_ensemble
# Copyright (c) 2017 <NAME>
# Full license at snapshot_ensemble/LICENSE.txt
#

# Network is copied directly from pytorch tutorial to see if we can be more accurate with no changes

##################### Starting Copy ######################

import torch
import torchvision
import torchvision.transforms as transforms

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=16,
                                          shuffle=True, num_workers=4)

testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=1,
                                         shuffle=False, num_workers=4)

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x


net = Net()
net.cuda()
num_epochs = 150

import torch.optim as optim

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

# for epoch in range(num_epochs):  # loop over the dataset multiple times
#
#     running_loss = 0.0
#     for i, data in enumerate(trainloader, 0):
#         # get the inputs
#         inputs, labels = data
#
#         # wrap them in Variable
#         inputs, labels = Variable(inputs).cuda(), Variable(labels).cuda()
#
#         # zero the parameter gradients
#         optimizer.zero_grad()
#
#         # forward + backward + optimize
#         outputs = net(inputs)
#         loss = criterion(outputs, labels)
#         loss.backward()
#         optimizer.step()
#
#         # print statistics
#         running_loss += loss.data[0]
#         if i % 2000 == 1999:    # print every 2000 mini-batches
#             print('[%d, %5d] loss: %.3f' %
#                   (epoch + 1, i + 1, running_loss / 2000))
#             running_loss = 0.0
#
# print('Finished Training')
#
# correct = 0
# total = 0
# for data in testloader:
#     images, labels = data
#     outputs = net(Variable(images).cuda())
#     _, predicted = torch.max(outputs.data, 1)
#     total += labels.size(0)
#     correct += (predicted == labels.cuda()).sum()
#
# print('Accuracy of the network on the 10000 test images: %.2f %%' % (
#     100 * float(correct / total)))

##################### Starting Modified Code ######################

from module import SnapshotEnsemble

ensemble = SnapshotEnsemble(net, criterion, restart_lr=0.1, epochs=num_epochs, batch_size=16,
                            num_snapshots=6, train_dataset=trainset, test_dataset=testset)

# ensemble.train(closure=ensemble.default_closure, print_steps=2500)


def check(output, target):
    _, predicted = torch.max(output.data, 1)
    c = (predicted == target.data).squeeze()
    return bool(c[0])


ensemble.load(6)
# call this to optimize voting - I recommend using the full ensemble for this,
# also must be >= ensemble_size for validation/testing
# ensemble.optimize_ensemble_weights(forward=ensemble.default_forward, n_iters=1, ensemble_size=2)
# ensemble.save()
ensemble.validate(forward=ensemble.default_forward, ensemble_size=2, check_correctness=check)
ensemble.validate(forward=ensemble.default_forward, ensemble_size=6, check_correctness=check, use_weights=False)

# 50 epochs training, 6 snapshots
# 6 has accuracy = 63.70%
# 3 has accuracy = 62.08%
# 5 has accuracy = 63.22%
# 1 has accuracy = 60.83%
# # Created by <NAME> on 7/5/17. # # snapshot_ensemble # Copyright (c) 2017 <NAME> # Full license at snapshot_ensemble/LICENSE.txt # # Network is copied directly from pytorch tutorial to see if we can be more accurate with no changes ##################### Starting Copy ###################### # for epoch in range(num_epochs):  # loop over the dataset multiple times # #     running_loss = 0.0 #     for i, data in enumerate(trainloader, 0): #         # get the inputs #         inputs, labels = data # #         # wrap them in Variable #         inputs, labels = Variable(inputs).cuda(), Variable(labels).cuda() # #         # zero the parameter gradients #         optimizer.zero_grad() # #         # forward + backward + optimize #         outputs = net(inputs) #         loss = criterion(outputs, labels) #         loss.backward() #         optimizer.step() # #         # print statistics #         running_loss += loss.data[0] #         if i % 2000 == 1999:    # print every 2000 mini-batches #             print('[%d, %5d] loss: %.3f' % #                   (epoch + 1, i + 1, running_loss / 2000)) #             running_loss = 0.0 # # print('Finished Training') # # correct = 0 # total = 0 # for data in testloader: #     images, labels = data #     outputs = net(Variable(images).cuda()) #     _, predicted = torch.max(outputs.data, 1) #     total += labels.size(0) #     correct += (predicted == labels.cuda()).sum() # # print('Accuracy of the network on the 10000 test images: %.2f %%' % ( #     100 * float(correct / total))) ##################### Starting Modified Code ###################### # ensemble.train(closure=ensemble.default_closure, print_steps=2500) # call this to optimize voting - I recommend using the full ensemble for this, # also must be >= ensemble_size for validation/testing # ensemble.optimize_ensemble_weights(forward=ensemble.default_forward, n_iters=1, ensemble_size=2) # ensemble.save() # 50 epochs training, 6 snapshots # 6 has accuracy = 63.70% # 3 has accuracy = 62.08% # 5 has accuracy = 63.22% # 1 has accuracy = 60.83%