PyTorch Notes

import torch

# Initialize a tensor with values ranging from -10 to 10
data = torch.tensor([-10.0, -5.0, 0.5, 5.0, 10.0])

print(f"Original: {data}")

# 1. Clamp between a min and max range [-1, 1]
# Values < -1 become -1; Values > 1 become 1
clamped_both = data.clamp(min=-1.0, max=1.0)
print(f"Range [-1, 1]: {clamped_both}")

# 2. Clamp with only a lower bound (min=-2)
# Values < -2 become -2; No upper limit
clamped_min = data.clamp(min=-2.0)
print(f"Min -2 only: {clamped_min}")

# 3. Clamp with only an upper bound (max=3)
# Values > 3 become 3; No lower limit
clamped_max = data.clamp(max=3.0)
print(f"Max 3 only:  {clamped_max}")

# x 是一个 3x4 的矩阵
x = torch.tensor([
    [1, 1, 1, 1], # row 0
    [2, 2, 2, 2], # row 1
    [3, 3, 3, 3]  # row 2
])

# y 是想取的行号
y = torch.tensor([0, 2])

print(x[y])
# 意思：取出第0行和第2行，组成新张量
# Output:
# tensor([[1, 1, 1, 1],
#         [3, 3, 3, 3]])

import torch

# x: 词表，有5个词，每个词向量长度为4
x = torch.randn(5, 4) 

# y: 两个句子，每个句子3个词的索引
y = torch.tensor([
    [1, 2, 4],  # Sentence 1: word 1, word 2, word 4
    [0, 0, 3]   # Sentence 2: word 0, word 0, word 3
])

out = x[y]

# 结果形状解析：
# y.shape = [2, 3]
# x.shape = [5, 4] (我们在第0维索引，剩下了第1维)
# Result  = [2, 3] + [4] = [2, 3, 4]

print("x shape:", x.shape)
print("y shape:", y.shape)
print("Result shape:", out.shape) 
# torch.Size([2, 3, 4]) 
# 解释：2个句子，每个句子3个词，每个词是4维向量。

import torch

x = torch.tensor([
    [1, 6, 3],
    [8, 2, 9]
])

# 生成一个布尔张量 y (Mask)
# y 会变成 [[False, True, False], [True, False, True]]
y = x > 5 

print("Mask:\n", y)

# 使用掩码索引
result = x[y]

print("Result:", result)
# Output: tensor([6, 8, 9])
# 注意：结果变成 1D 了！因为原来矩阵里大于5的位置并不构成一个矩形。

import torch

# 1. 创建数据 x (3D)
# 假设是 2个样本，每个样本3个像素，每个像素有4个特征(RGBA)
# Shape: (2, 3, 4)
x = torch.tensor([
    # Sample 0
    [[1, 1, 1, 1], [2, 2, 2, 2], [3, 3, 3, 3]],
    # Sample 1
    [[4, 4, 4, 4], [5, 5, 5, 5], [6, 6, 6, 6]]
])

# 2. 创建掩码 y (2D)
# Shape: (2, 3) - 必须匹配 x 的前两维
y = torch.tensor([
    [True,  False, True ], # Sample 0: 选第1个和第3个像素
    [False, True,  False]  # Sample 1: 选第2个像素
])

# 3. 执行索引
result = x[y]

print("x shape:", x.shape)       # [2, 3, 4]
print("y shape:", y.shape)       # [2, 3]
print("True count in y:", y.sum()) # 3 个 True

print("\nResult shape:", result.shape) 
# Expect: [3, 4] -> (True的总数, 剩余的最后一维)

print("Result data:\n", result)
# Output:
# tensor([[1, 1, 1, 1],  <- 来自 (0,0)
#         [3, 3, 3, 3],  <- 来自 (0,2)
#         [5, 5, 5, 5]]) <- 来自 (1,1)

import torch

# A small 2x2 matrix
x = torch.tensor([
    [1, 2],
    [3, 4]
])

# Repeat 2 times vertically, 3 times horizontally
y = x.repeat(2, 3)

print("Original Shape:", x.shape) # [2, 2]
print("New Shape     :", y.shape) # [2*2, 2*3] = [4, 6]
print("Result:\n", y)

# Output:
# tensor([[1, 2, 1, 2, 1, 2],  <-- Top-Left block repeated 3 times across
#         [3, 4, 3, 4, 3, 4],
#         [1, 2, 1, 2, 1, 2],  <-- The whole row block repeated 2 times down
#         [3, 4, 3, 4, 3, 4]])

import torch

x = torch.tensor([1, 2, 3]) # Shape: [3]

# unsqueeze to (1, 3) first

# Repeat 4 times along a NEW dimension (dim 0), 
# and 1 time along the original dimension (dim 1)
y = x.repeat(4, 1)

print("Shape:", y.shape)
print("Result:\n", y)

# Output:
# Shape: torch.Size([4, 3])
# Result:
#  tensor([[1, 2, 3],
#          [1, 2, 3],
#          [1, 2, 3],
#          [1, 2, 3]])
# Explanation: It effectively stacked 4 copies of the vector.

import torch

# 1. 原始数据：1D 向量
x = torch.tensor([10, 20])
# Shape: [2]
# 样子: [10, 20]

# 2. 调用 repeat(3, 1)
# 参数有两个 (3, 1)，原数据只有一维。
# 于是 PyTorch 自动把它变成了 [[10, 20]] (Shape: [1, 2])

y = x.repeat(3, 1)

# 3. 执行复制
# 第一维重复 3 次：意味着有 3 行。
# 第二维重复 1 次：意味着列数不变。

print(y)
# Output:
# tensor([[10, 20],
#         [10, 20],
#         [10, 20]])
# Shape: [3, 2]

import torch

A = torch.randn(3, 5) # i=3, k=5
B = torch.randn(5, 4) # k=5, j=4

# 传统写法
res_traditional = torch.matmul(A, B)

# Einsum 写法
# i: 3, k: 5, j: 4
res_einsum = torch.einsum('ik, kj -> ij', A, B)

print(torch.allclose(res_traditional, res_einsum)) # True


batch_size = 10
A = torch.randn(batch_size, 3, 5) # b i k
B = torch.randn(batch_size, 5, 4) # b k j

res_traditional = torch.bmm(A, B)

# Einsum 写法 (非常直观：b保留, k消掉)
res_einsum = torch.einsum('bik, bkj -> bij', A, B)

# self-attention
# b: batch, h: heads, i: query_len, j: key_len, d: head_dim
# Q: b h i d
# K: b h j d
# 我们想把 d 消掉 (点积)，保留 b h i j
attention_scores = torch.einsum('bhid, bhjd -> bhij', Q, K)

from einops import rearrange

out = rearrange(x, 'batch channel height width -> batch height width channel')

import torch

# Batch size = 3, Classes = 4
logits = torch.tensor([
    [0.1, 0.8, 0.05, 0.05], # Sample 0: Class 1 is largest
    [0.9, 0.0, 0.1,  0.0 ], # Sample 1: Class 0 is largest
    [0.2, 0.2, 0.5,  0.1 ]  # Sample 2: Class 2 is largest
])

# Find the class index with the highest score along the class dimension (dim=1)
pred_labels = torch.argmax(logits, dim=1)

print("Predicted Labels:", pred_labels)
# Output: tensor([1, 0, 2])

import torch

# Batch size = 3, Number of classes = 5
preds = torch.tensor([
    [0.1, 0.9, 0.0, 0.0, 0.0],  # Sample 0: Class 1 is highest
    [0.2, 0.1, 0.4, 0.1, 0.2],  # Sample 1: Class 2 is highest, then 0 or 4
    [0.0, 0.0, 0.1, 0.8, 0.1]   # Sample 2: Class 3 is highest
])

# Find top 2 predictions for each sample (along dim=1, which is the class dimension)
k = 2
top_vals, top_inds = torch.topk(preds, k=k, dim=1)

print("Top 2 Probabilities:\n", top_vals)
print("Top 2 Class Indices:\n", top_inds)

# Output:
# Top 2 Probabilities:
#  tensor([[0.9000, 0.1000],
#          [0.4000, 0.2000],
#          [0.8000, 0.1000]])
#
# Top 2 Class Indices (The predicted labels):
#  tensor([[1, 0],   -> Sample 0 predicts class 1, then class 0
#          [2, 0],   -> Sample 1 predicts class 2, then class 0
#          [3, 2]])  -> Sample 2 predicts class 3, then class 2

import torch

# Target: 3x5 matrix of zeros
target = torch.zeros(3, 5)

# Source: 3x2 matrix (values to fill)
src = torch.tensor([
    [1.0, 2.0],
    [3.0, 4.0],
    [5.0, 6.0]
])

# Indices: Where to put the values in each row?
# Row 0: put src[0][0] at col 1, src[0][1] at col 3
# Row 1: put src[1][0] at col 0, src[1][1] at col 2
# Row 2: put src[2][0] at col 4, src[2][1] at col 1
index = torch.tensor([
    [1, 3], 
    [0, 2], 
    [4, 1]
])

target.scatter_(dim=1, index=index, src=src)

print(target)
# Output explanation:
# Row 0: indices are 1 and 3 -> values 1.0 and 2.0 go there.
# Row 1: indices are 0 and 2 -> values 3.0 and 4.0 go there.
# Row 2: indices are 4 and 1 -> values 5.0 and 6.0 go there.

# Result:
# tensor([[0., 1., 0., 2., 0.],
#         [3., 0., 4., 0., 0.],
#         [0., 6., 0., 0., 5.]])

import torch

# Source: 3x3 Matrix
# [[ 1,  2,  3],
#  [ 4,  5,  6],
#  [ 7,  8,  9]]
input_tensor = torch.tensor([
    [1, 2, 3],
    [4, 5, 6],
    [7, 8, 9]
])

# Indices: 3x2 Matrix
# We want to gather 2 values for each of the 3 rows.
# Row 0: take element at index 0 and 2
# Row 1: take element at index 1 and 1
# Row 2: take element at index 2 and 0
index = torch.tensor([
    [0, 2],
    [1, 1],
    [2, 0]
])

output = torch.gather(input_tensor, dim=1, index=index)

print(output)
# Logic breakdown:
# Row 0: input[0][0]=1, input[0][2]=3  -> [1, 3]
# Row 1: input[1][1]=5, input[1][1]=5  -> [5, 5]
# Row 2: input[2][2]=9, input[2][0]=7  -> [9, 7]

# Result:
# tensor([[1, 3],
#         [5, 5],
#         [9, 7]])

a = torch.tensor([[1,2,3],[4,5,6]])
print(a.shape) # torch.Size([2, 3])
print(a.sum(0)) # tensor([5, 7, 9])
print(a.sum(1)) # tensor([ 6, 15])

import torch

x = torch.zeros(2, 3, 4, 5)
print(x.unsqueeze(0).shape)  # torch.Size([1, 2, 3, 4, 5])
print(x.unsqueeze(1).shape)  # torch.Size([2, 1, 3, 4, 5])


y = torch.zeros(1, 2, 1, 3, 4, 5)
print(y.squeeze(0).shape)  # torch.Size([2, 1, 3, 4, 5])
print(y.squeeze(2).shape)  # torch.Size([1, 2, 3, 4, 5])

>>> a = (1,2,3,4)
>>> a[:1]
(1,)
>>> a[:0]
()
>>> a[:-1]
(1, 2, 3)
>>> a[:3]
(1, 2, 3)
>>> a[:4]
(1, 2, 3, 4)

import torch

a = torch.tensor([[1, 2, 3], [4, 5, 6]])
print(a[:, :-1])
# tensor([[1, 2],
#        [4, 5]])
print(a[:-1, :])
# tensor([[1, 2, 3]])

# Simulating a third-party library class
class DataProcessor:
    def fetch_data(self):
        # Pretend this connects to a slow external server
        print("Connecting to external server... (Slow)")
        return {"data": "original"}

# 1. Standard usage
processor = DataProcessor()
processor.fetch_data()
print("-" * 20)

# 2. Define the patch function
# It must accept 'self' if it replaces an instance method
def mock_fetch_data(self):
    print("Using patched method (Fast)")
    return {"data": "patched_mock"}

# 3. Apply Monkey Patching
# We dynamically replace the method on the Class itself
DataProcessor.fetch_data = mock_fetch_data

# 4. Verification
# Now, any instance calls the new function
processor.fetch_data()

# Even new instances are affected
new_processor = DataProcessor()
new_processor.fetch_data()

import torch
import torch.nn.functional as F

# Setup
learning_rate = 0.1
x = torch.randn(1, 5)          # Input data
y_true = torch.tensor([[1.0]])   # True label

# Model parameters initialized manually
# requires_grad=True tells PyTorch to calculate gradients for them
w = torch.randn(5, 1, requires_grad=True)
b = torch.randn(1, requires_grad=True)

print(f"Initial weight:\n{w.data}\n")

# 1. Forward Pass
# Calculate a prediction using the current weight and bias
z = x @ w + b  #  `@` is matrix multiplication
y_pred = torch.sigmoid(z)

# 2. Calculate Loss
# Compare the prediction to the true label
loss = F.binary_cross_entropy(y_pred, y_true)

# 3. Backward Pass
# Calculate the gradients of the loss with respect to w and b
loss.backward()

# 4. Update Parameters
# Manually adjust w and b in the opposite direction of their gradients
with torch.no_grad(): # Temporarily disable gradient tracking for the update
    w -= learning_rate * w.grad
    b -= learning_rate * b.grad

    # Manually zero out the gradients for the next iteration
    w.grad.zero_()
    b.grad.zero_()

print(f"Updated weight:\n{w.data}\n")
print(f"Loss: {loss.item():.4f}")

new_parameter = old_parameter - learning_rate * gradient

import torch
import torch.nn as nn

class NeuralNetwork(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super().__init__()
        self.layers = nn.Sequential(
            # 1st hidden layer
            nn.Linear(num_inputs, 30),
            nn.ReLU(),
            # 2nd hidden layer
            nn.Linear(30, 20),
            nn.ReLU(),
            # Output layer
            nn.Linear(20, num_outputs),
        )

    def forward(self, x):
        logits = self.layers(x)
        return logits

import torch
from torch.utils.data import Dataset, DataLoader

class ToyDataset(Dataset):
    def __init__(self, X, y):
        self.features = X
        self.labels = y

    def __getitem__(self, index):
        one_x = self.features[index]
        one_y = self.labels[index]
        return one_x, one_y

    def __len__(self):
        return self.labels.shape[0]


torch.manual_seed(123)

train_loader = DataLoader(
    dataset=train_ds,
    batch_size=2,
    shuffle=True,
    num_workers=0
)

test_loader = DataLoader(
    dataset=test_ds,
    batch_size=2,
    shuffle=False,
    num_workers=0
)

import torch.nn.functional as F

torch.manual_seed(123)
model = NeuralNetwork(num_inputs=2, num_outputs=2)
optimizer = torch.optim.SGD(model.parameters(), lr=0.5)

num_epochs = 3

for epoch in range(num_epochs):
    
    model.train()
    for batch_idx, (features, labels) in enumerate(train_loader):

        logits = model(features)
        
        loss = F.cross_entropy(logits, labels) # Loss function
        
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    
        ### LOGGING
        print(f"Epoch: {epoch+1:03d}/{num_epochs:03d}"
              f" | Batch {batch_idx:03d}/{len(train_loader):03d}"
              f" | Train/Val Loss: {loss:.2f}")

    model.eval()
    # Optional model evaluation

torch.save(model.state_dict(), "model.pth")

model = NeuralNetwork(2, 2) model.load_state_dict(torch.load("model.pth"))