LearningDL

SvAm Softmax Linear

#SV-AM-Softmax
class SVAMLinear(torch.nn.Module):
    def __init__(self,
                 in_channels,
                 num_class,
                 t = 1.2,
                 m = 0.35,
                 scale = 30):
        super(SVLinear,self).__init__()
        self.in_channels = in_channels
        self.num_class = num_class
        self.t = t
        self.m = m
        self.scale = scale
        self.weights = Parameter(torch.Tensor(in_channels, num_class))
        self.weights.data.uniform_(-1,1).renorm(2,1,1e-5).mul(1e5)
    def forward(self,input,target):
        norm_weights = F.normalize(self.weights,dim=0)
        cos_theta = torch.mm(input,norm_weights)#test
        batch_size = target.size(0)
        gtScore = cos_theta[torch.arange(0,batch_size),target].view(-1,1)
        mask = cos_theta > (gtScore - self.m)
        finalScore = torch.where(gtScore > self.m, gtScore - self.m,gtScore)
        hardEx = cos_theta[mask]
        cos_theta[mask] = self.t * hardEx + self.t - 1.0
        cos_theta.scatter_(1, target.data.view(-1,1),finalScore)
        cos_theta *= self.scale
        return cos_theta

Full Connected NN

import math

def sigmoid(x):
    return 1 / (1 + math.exp(-x))

    class Node(object):
        def __init__(self,layer_index,node_index):
            self.layer_index = layer_index
            self.node_index = node_index
            self.upstream = []
            self.downstream = []
            self.output = 0
            self.delta = 0
        
        def set_output(self,output):
            self.output = output
        
        def set_delta(self,delta):
            self.delta = delta

        def downstream_append(self,conn):
            self.downstream.append(conn)

        def upstream_append(self,conn):
            self.upstream.append(conn)
        
        def calc_output(self):
            output = reduce(lambda ret,conn:ret + conn.upstream_node.output * conn.weight,self.upstream,0)
            self.output = sigmoid(output)

        def calc_hidden_layer_delta(self):#hidden layer
            downstream_delta = reduce(lambda ret,conn:ret + conn.downstream_node.delta * conn.weight,self.downstream,0.0)
            self.delta = self.output * (1 - self.output) * downstream_delta

        def calc_output_layer_delta(self,label):
            self.delta = self.output * (1 - self.output) * (label - self.output)

        def __str__(self):
            node_str = "%u-%u:output:%f delta:%f" %(self.layer_index,self.node_index,self.output,self.delta)
            downstream_str = reduce(lambda ret,conn:ret + '\n\t' + str(conn),self.downstream,'')
            upstream_str = reduce(lambda ret,conn:ret + '\n\t' + str(conn),self.upstream,'')
            return node_str + '\n\tdownstream:' + downstream_str + '\n\tupstream:' + upstream_str

    class ConstNode(object):
        def __init__(self,layer_index,node_index):
            self.layer_index = layer_index
            self.node_index = node_index
            self.downstream = []
            self.output = 1

        def downstream_append(self,conn):
            self.downstream.append(conn)
        
        def set_delta(self):
            downstream_delta = reduce(lambda ret,conn:ret + conn.downstream_node.delta * conn.weight,downstream_delta,0.0)
            self.delta = self.output * (1 - self.output) * downstream_delta
        
        def __str__(self):
            node_str = '%u-%u:output:1 delta:%f' % (self.layer_index,self.node_index,self.delta)
            downstream_str = reduce(lambda ret,conn:ret + '\n\t' + str(node),self.downstream,'')
            return node_str + '\n\tdownstream:' + downstream_str

    class Layer(object):
        def __init__(self,layer_index,node_count):
            self.layer_index = layer_index
            self.node_count = node_count
            self.nodes = []
            for i in range(self.node_count):
                self.nodes.append(Node(layer_index,i))
            self.nodes.append(ConstNode(layer_index,self.node_count))

        def set_output(self,data):#input layer x1,x2,x3...
            for i in range(len(data)):
                self.nodes[i].set_output(data[i])
        
        def calc_output(self):
            for node in self.nodes[:-1]:
                node.calc_output()

        def dump(self):
            for node in self.nodes
                print(node)

    class Connection(object):
        def __init__(self,upstream_node,downstream_node):
            self.upstream_node = upstream_node
            self.downstream_node = downstream_node
            self.weight = random.uniform(-0.1,0.1)
            self.gradient = 0.0

        def calc_gradient(self):
            self.gradient = self.downstream_node.delta * self.upstream_node.output

        def get_gradient(self):
            return self.gradient

        def update_weight(self,rate):
            calc_gradient()
            self.weight += self.gradient * rate

        def __str__(self):
            return '(%u-%u) -> (%u-%u) = %f' % (
                self.upstream_node.layer_index,
                self.upstream_node.node_index,
                self.downstream_node.layer_index,
                self.downstream_node.node_index,
                self.weight
            )

    class Connections(object):
        def __init__(self):
            self.connections = []
        
        def connection_append(self,conn):
            self.connections.append(conn)

        def dump(self):
            for conn in self.connections:
                print(conn)

    class Network(object):
        def __init__(self,layers):#layers[i] <----> # of nodes in layer i
            self.connections = Connections()
            self.layers = []
            layer_count = len(layers)
            node_count = 0
            #how many layers?
            for i in range(layer_count):
                self.layers.append(Layer(i,layers[i]))   
            for layer in range(layer_count - 1):
                connections = [Connection(upstream_node,downstream_node)
                               for upstream_node in self.layers[layer].nodes
                               for downstream_node in self.layers[layer+1].nodes[:-1]]
                                    #cant get it?
                                    #temp = [Node(a,b)
                                    #for a in [1,2,3,4,5]
                                    #for b in [2,4,6,8]]  
                for conn in connections:
                    self.connections.add_connection(conn)
                    conn.downstream_node.upstream_append(conn)
                    conn.upstream_node.downstream_append(conn)