Pytorch Auto Derivative Functions Detailed Flow and Comparison with TensorFlow to Build Networks

import torch
class myrelu():#Customized subclasses
    # Customized autograd functions via created subclasses, complete with forward and backward propagation of tensor
    @staticmethod
    def forward(ctx, x):
        # In forward propagation, a context object and a tensor containing the inputs are accepted, a tensor containing the outputs must be returned, and the context object can be used to cache the objects for use in backpropagation
        ctx.save_for_backward(x)
        return (min=0)
    @staticmethod
    def backward(ctx, grad_output):
        """
        In backpropagation, we receive the context object and a tensor that
        which contains the gradient of the loss relative to the output produced during forward propagation.
        We can retrieve the cached data from the context object
        and must compute and return the gradient of the loss with respect to the input of the forward propagation.
        """
        x, = ctx.saved_tensors
        grad_x = grad_output.clone()
        grad_x[x < 0] = 0
        return grad_x

#%%
device=('cuda' if .is_available() else 'cpu')
# n is the batch size and d_in is the input dimension
# h is the hidden dimension and d_out is the output dimension
n,d_in,h,d_out=64,1000,100,10
# Create randomized input and output data, requires_grad is set to False by default, indicating that no post-differential operation is required.
x=(n,d_in,device=device)
y=(n,d_out,device=device)
# Randomly initialize the weights, requires_grad is set to True by default, indicating that you want to compute their differentials.
w1=(d_in,h,device=device,requires_grad=True)
w2=(h,d_out,device=device,requires_grad=True)

learning_rate=1e-6
for i in range(500):
    #Forward propagation, using operations on the tensor to compute the prediction y
  # Call customized functions
    y_pred=((w1)).mm(w2)
       
    
    #Calculate the loss value using the operation in tensor, () to get the value corresponding to the tensor of loss
    loss=(y_pred-y).pow(2).sum()
    print(i,())
    
    #Calculate the backpropagation using autograd, this call will calculate the tensor gradient of the loss for all requisites_grad=True.
    # After the call, and will be the gradient tensor of loss for w1 and w2, respectively
    ()
    #Using gradient descent to update weights, only want to make in-place changes to w1 and w2 values: don't want to update build computation graphs
    #so use torch.no_grad() to prevent pytorch from updating the build computation graphs
    with torch.no_grad():
        w1-=learning_rate*
        w2-=learning_rate*
        # Manually set the gradient to zero after backpropagation
        .zero_()
        .zero_() 

#%% Using TensorFlow
import .v1 as tf   #Whatever it takes to use placeholder
tf.disable_v2_behavior()
import numpy as np
#%%
# Create calculation maps

# n is the batch size and d_in is the input dimension
# h is the hidden dimension and d_out is the output dimension
n,d_in,h,d_out=64,1000,100,10

# Create placeholders for input and target data, which will be populated with real data when the computational graph is executed
x=(tf.float32,shape=(None,d_in))
y=(tf.float32,shape=(None,d_out))

# Create variables for the weights and initialize them with random data, TensorFlow's variables don't change when executing the computed graphs
w1 = (tf.random_normal((d_in,h)))
w2=(tf.random_normal((h,d_out)))
# Forward propagation: use TensorFlow's tensor operations to compute the prediction y (this code does not perform any numerical operations, it just builds a graph of the computations to be performed later)
h=(x,w1)
h_relu=(h,(1))
y_pred=(h_relu,w2)
# tensor computing lossloss with TensorFlow
loss=tf.reduce_sum((y-y_pred)**2.0)
# Calculate the derivative of loss with respect to the weights w1 and w2
grad_w1,grad_w2=(loss,[w1,w2])
# Use gradient descent to update the weights, to actually update the weights we need to compute new_w1 and new_w2 when performing the computation of the graphs
# Note: In TensorFlow, the act of updating weight values is part of the computational graph, but in Pytorch it occurs outside of the computational graph
learning_rate=1e-6
new_w1=(w1-learning_rate*grad_w1)
new_w2=(w2-learning_rate*grad_w2)

# Now that the computational graph is built, start a TensorFlow talkback to execute the computational graph
with () as sess:
    # Operate a computational graph to account for variables w1 and w2 #
    (tf.global_variables_initializer())
    # Create numpy arrays to store the actual data for input x and target y
    x_value=(n,d_in)
    y_value=(n,d_out)
    for i in range(500):
        # Multiple runs to compute the graph, with the feed_dict parameter on each execution.
        # Bind x_value to x and y_value to y. Calculate the loss each time the computational graph is executed.
        # new_w1 and new_w2, the values of these tensors are returned as numpy arrays
        loss_value,i,i=([loss,new_w1,new_w2],
                                feed_dict={x:x_value,y:y_value})
        print(loss_value)