Skip to content

Networking Labs

Five progressive labs covering Linux network namespaces, raw socket programming, eBPF XDP packet processing, TCP performance tuning, and Kubernetes CNI internals. Each lab includes all commands, expected output, and a troubleshooting guide.

Prerequisites: - Linux system with root access (VM or bare metal) - Packages: iproute2, tcpdump, iperf3, clang, llvm, bpftool, libelf-dev, libbpf-dev - Kernel 5.10+ recommended for eBPF XDP support - For Lab 5: a working Kubernetes cluster (minikube or kind is sufficient)

Lab 1: Network Namespace Experiment

Objective: Create isolated network namespaces, connect them with virtual Ethernet pairs, verify connectivity, and observe how routing tables are namespace-local.

Part A: Create Two Network Namespaces

# Create namespaces
sudo ip netns add ns1
sudo ip netns add ns2

# List them
ip netns list
# Output:
# ns2 (id: 1)
# ns1 (id: 0)

Part B: Create a veth Pair

A veth pair is a virtual Ethernet cable: what goes in one end comes out the other.

# Create a veth pair: veth1 <--> veth2
sudo ip link add veth1 type veth peer name veth2

# Assign each end to a namespace
sudo ip link set veth1 netns ns1
sudo ip link set veth2 netns ns2

# Verify they disappeared from the root namespace
ip link show | grep veth   # should show nothing

Part C: Configure Addresses and Bring Up Interfaces

# In ns1: assign 192.168.100.1/24 to veth1
sudo ip netns exec ns1 ip addr add 192.168.100.1/24 dev veth1
sudo ip netns exec ns1 ip link set veth1 up
sudo ip netns exec ns1 ip link set lo up

# In ns2: assign 192.168.100.2/24 to veth2
sudo ip netns exec ns2 ip addr add 192.168.100.2/24 dev veth2
sudo ip netns exec ns2 ip link set veth2 up
sudo ip netns exec ns2 ip link set lo up

# Show addresses
sudo ip netns exec ns1 ip addr show
sudo ip netns exec ns2 ip addr show

Part D: Verify Connectivity and Observe Routing

# Ping from ns1 to ns2
sudo ip netns exec ns1 ping -c 3 192.168.100.2

# Expected output:
# PING 192.168.100.2 (192.168.100.2) 56(84) bytes of data.
# 64 bytes from 192.168.100.2: icmp_seq=1 ttl=64 time=0.083 ms
# 64 bytes from 192.168.100.2: icmp_seq=2 ttl=64 time=0.072 ms
# 64 bytes from 192.168.100.2: icmp_seq=3 ttl=64 time=0.068 ms

# Show routing table in ns1 — namespace-local
sudo ip netns exec ns1 ip route show
# Output: 192.168.100.0/24 dev veth1 proto kernel scope link src 192.168.100.1

# Root namespace has no route to 192.168.100.0/24
ip route show | grep 192.168.100
# (no output — routes are completely isolated)

Part E: Add Internet Access via NAT (Bridge to Default Network)

# Create a bridge in the root namespace
sudo ip link add br0 type bridge
sudo ip addr add 192.168.100.254/24 dev br0
sudo ip link set br0 up

# Create a third veth pair connecting ns1 to the bridge
sudo ip link add veth1-br type veth peer name veth1-ns
sudo ip link set veth1-br master br0
sudo ip link set veth1-br up
sudo ip link set veth1-ns netns ns1
sudo ip netns exec ns1 ip link set veth1-ns up
sudo ip netns exec ns1 ip addr add 192.168.100.10/24 dev veth1-ns
sudo ip netns exec ns1 ip route add default via 192.168.100.254

# Enable IP forwarding and NAT
sudo sysctl -w net.ipv4.ip_forward=1
sudo iptables -t nat -A POSTROUTING -s 192.168.100.0/24 -j MASQUERADE

# Test: ns1 should reach the internet
sudo ip netns exec ns1 ping -c 2 8.8.8.8

Cleanup

sudo ip netns del ns1
sudo ip netns del ns2
sudo ip link del br0

Troubleshooting Guide

Problem Check Fix
ping fails between namespaces ip netns exec ns1 ip link show — is veth1 UP? ip link set veth1 up inside namespace
ip netns exec fails Namespace doesn't exist ip netns list to verify name
No route to host Check ip route show inside namespace Add route: ip route add ...
Namespace connectivity works but no internet IP forwarding disabled sysctl -w net.ipv4.ip_forward=1

Lab 2: Raw Socket Packet Sniffer

Objective: Write a C program that captures all packets on an interface using AF_PACKET, then parse Ethernet, IP, and TCP headers manually. Compare output with tcpdump.

Complete Code

// sniffer.c
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <net/ethernet.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
#include <linux/if_packet.h>

#define SNAP_LEN 65535

void print_mac(unsigned char *mac) {
    printf("%02x:%02x:%02x:%02x:%02x:%02x",
           mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
}

void handle_packet(unsigned char *buf, int len)
{
    struct ethhdr *eth = (struct ethhdr *)buf;
    struct iphdr  *ip;
    struct tcphdr *tcp;
    struct udphdr *udp;
    char src_ip[INET_ADDRSTRLEN], dst_ip[INET_ADDRSTRLEN];

    printf("ETH: ");
    print_mac(eth->h_source);
    printf(" -> ");
    print_mac(eth->h_dest);
    printf(" proto=0x%04x\n", ntohs(eth->h_proto));

    if (ntohs(eth->h_proto) != ETH_P_IP)
        return;

    ip = (struct iphdr *)(buf + sizeof(struct ethhdr));
    inet_ntop(AF_INET, &ip->saddr, src_ip, INET_ADDRSTRLEN);
    inet_ntop(AF_INET, &ip->daddr, dst_ip, INET_ADDRSTRLEN);

    printf(" IP: %s -> %s proto=%d ttl=%d len=%d\n",
           src_ip, dst_ip, ip->protocol, ip->ttl, ntohs(ip->tot_len));

    int ip_hdr_len = ip->ihl * 4;

    if (ip->protocol == IPPROTO_TCP) {
        tcp = (struct tcphdr *)(buf + sizeof(struct ethhdr) + ip_hdr_len);
        printf(" TCP: sport=%u dport=%u seq=%u ack=%u flags=%s%s%s%s\n",
               ntohs(tcp->source), ntohs(tcp->dest),
               ntohl(tcp->seq), ntohl(tcp->ack_seq),
               tcp->syn ? "SYN " : "",
               tcp->ack ? "ACK " : "",
               tcp->fin ? "FIN " : "",
               tcp->rst ? "RST " : "");
    } else if (ip->protocol == IPPROTO_UDP) {
        udp = (struct udphdr *)(buf + sizeof(struct ethhdr) + ip_hdr_len);
        printf(" UDP: sport=%u dport=%u len=%u\n",
               ntohs(udp->source), ntohs(udp->dest), ntohs(udp->len));
    }
}

int main(int argc, char *argv[])
{
    if (argc < 2) {
        fprintf(stderr, "Usage: %s <interface>\n", argv[0]);
        return 1;
    }

    /* AF_PACKET with ETH_P_ALL captures all Ethernet frames */
    int sock = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL));
    if (sock < 0) {
        perror("socket");
        return 1;
    }

    /* Bind to specific interface (optional — without bind, gets all interfaces) */
    struct sockaddr_ll addr = {0};
    addr.sll_family   = AF_PACKET;
    addr.sll_protocol = htons(ETH_P_ALL);
    addr.sll_ifindex  = if_nametoindex(argv[1]);
    if (!addr.sll_ifindex) {
        fprintf(stderr, "Interface %s not found\n", argv[1]);
        close(sock);
        return 1;
    }
    if (bind(sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
        perror("bind");
        close(sock);
        return 1;
    }

    unsigned char buf[SNAP_LEN];
    printf("Sniffing on %s (Ctrl+C to stop)...\n\n", argv[1]);

    int count = 0;
    while (count < 20) {   /* capture 20 packets */
        int n = recvfrom(sock, buf, SNAP_LEN, 0, NULL, NULL);
        if (n < 0) {
            perror("recvfrom");
            break;
        }
        printf("--- Packet %d (%d bytes) ---\n", ++count, n);
        handle_packet(buf, n);
    }

    close(sock);
    return 0;
}

Build and Test

gcc -O2 -Wall -o sniffer sniffer.c
sudo ./sniffer eth0

# In another terminal, generate traffic
curl -s http://example.com > /dev/null
ping -c 3 8.8.8.8

Expected Output

Sniffing on eth0 (Ctrl+C to stop)...

--- Packet 1 (74 bytes) ---
ETH: 52:54:00:12:34:56 -> ff:ff:ff:ff:ff:ff proto=0x0800
 IP: 10.0.2.15 -> 8.8.8.8 proto=1 ttl=64 len=60

--- Packet 2 (60 bytes) ---
ETH: 52:54:00:12:34:56 -> 52:54:00:ab:cd:ef proto=0x0800
 IP: 10.0.2.15 -> 93.184.216.34 proto=6 ttl=64 len=40
 TCP: sport=54321 dport=80 seq=1234567 ack=0 flags=SYN

Compare with tcpdump

# Run both in parallel — output should describe the same packets
sudo tcpdump -i eth0 -n -c 20 2>/dev/null &
sudo ./sniffer eth0

Troubleshooting Guide

Problem Cause Fix
socket: Operation not permitted Not running as root sudo ./sniffer
No packets captured Wrong interface name ip link show to find correct name
Malformed output Header parsing offset wrong Check ihl * 4 for variable IP header length
Only seeing ARP, no TCP IP filtering too aggressive Check h_proto condition

Lab 3: eBPF XDP Packet Counter

Objective: Write a minimal BPF program in C that counts packets per protocol, attach it to a network interface using XDP, and read counters from userspace.

Part A: BPF Kernel Program

// xdp_counter.c — loaded into the kernel
#include <linux/bpf.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <bpf/bpf_helpers.h>

struct {
    __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
    __type(key, __u32);
    __type(value, __u64);
    __uint(max_entries, 256);   /* indexed by IP protocol number */
} proto_count SEC(".maps");

SEC("xdp")
int xdp_packet_counter(struct xdp_md *ctx)
{
    void *data     = (void *)(long)ctx->data;
    void *data_end = (void *)(long)ctx->data_end;

    struct ethhdr *eth = data;
    if ((void *)(eth + 1) > data_end)
        return XDP_PASS;

    if (eth->h_proto != __constant_htons(ETH_P_IP))
        return XDP_PASS;

    struct iphdr *ip = (void *)(eth + 1);
    if ((void *)(ip + 1) > data_end)
        return XDP_PASS;

    __u32 key = ip->protocol;
    __u64 *count = bpf_map_lookup_elem(&proto_count, &key);
    if (count)
        __sync_fetch_and_add(count, 1);

    return XDP_PASS;
}

char _license[] SEC("license") = "GPL";

Part B: Compile the BPF Program

# Compile to BPF bytecode
clang -O2 -target bpf \
    -I /usr/include/x86_64-linux-gnu \
    -I /usr/include/bpf \
    -c xdp_counter.c \
    -o xdp_counter.o

# Inspect the generated bytecode
llvm-objdump -d xdp_counter.o

Part C: Load and Attach with ip

# Attach to interface (XDP_FLAGS_SKB_MODE = software mode, works everywhere)
sudo ip link set dev eth0 xdpgeneric obj xdp_counter.o sec xdp

# Verify attachment
ip link show eth0
# Output includes: prog/xdp id 42 tag abc123def456

# Generate traffic
ping -c 10 8.8.8.8     # ICMP = protocol 1
curl http://example.com # TCP = protocol 6

Part D: Read Per-CPU Counters with bpftool

# List BPF maps
sudo bpftool map list

# Read the proto_count map (assuming map id 5)
sudo bpftool map dump id 5

# Expected output (showing TCP=6, ICMP=1, UDP=17 counts):
# key: 01 00 00 00  value (per-cpu): [10, 0, 0, 0]    # 10 ICMP on CPU 0
# key: 06 00 00 00  value (per-cpu): [456, 12, 0, 0]   # TCP across CPUs
# key: 11 00 00 00  value (per-cpu): [23, 8, 0, 0]     # UDP across CPUs

Part E: Detach the Program

sudo ip link set dev eth0 xdp off
ip link show eth0   # prog/xdp line disappears

Troubleshooting Guide

Problem Cause Fix
clang: error: unknown target Wrong clang version clang --version — need 10+
Verifier rejects program Out-of-bounds access Add bounds check before every pointer dereference
RTNETLINK: Operation not supported Old kernel without XDP Use xdpgeneric (software mode)
Map lookup returns wrong counts PERCPU_ARRAY — need to sum CPUs Sum values across all CPUs in userspace

Lab 4: TCP Performance Tuning

Objective: Establish iperf3 baseline, apply kernel TCP tuning parameters, enable BBR, and measure improvement at each step.

Part A: Establish Baseline

# Server (run in background or on a separate host)
iperf3 -s -p 5201 &

# Client: baseline measurement
iperf3 -c localhost -t 30 -P 4   # 4 parallel streams, 30 seconds

# Example baseline output:
# [SUM] 0.00-30.00 sec  8.45 GBytes  2.35 Gbits/sec    sender
# [SUM] 0.00-30.00 sec  8.43 GBytes  2.35 Gbits/sec    receiver

Record the baseline. Also record:

ss -n | grep ESTAB | wc -l           # established connections
cat /proc/net/sockstat               # socket memory usage
cat /proc/sys/net/core/rmem_default  # default receive buffer (bytes)

Part B: Tune Buffer Sizes

# Check current limits
sysctl net.core.rmem_max
sysctl net.core.wmem_max
sysctl net.ipv4.tcp_rmem
sysctl net.ipv4.tcp_wmem

# Apply tuning (for 10 Gbps+ networks)
sudo sysctl -w net.core.rmem_max=134217728           # 128 MB
sudo sysctl -w net.core.wmem_max=134217728
sudo sysctl -w net.ipv4.tcp_rmem="4096 87380 134217728"
sudo sysctl -w net.ipv4.tcp_wmem="4096 87380 134217728"
sudo sysctl -w net.core.netdev_max_backlog=5000
sudo sysctl -w net.ipv4.tcp_no_metrics_save=1

# Re-test
iperf3 -c localhost -t 30 -P 4

Typical result: 10–30% throughput improvement for large transfers.

Part C: Enable TCP BBR

BBR (Bottleneck Bandwidth and Round-trip time) is a congestion control algorithm from Google. It significantly outperforms CUBIC on lossy networks and long-haul connections.

# Check current congestion control
sysctl net.ipv4.tcp_congestion_control
# Output: cubic

# Check available algorithms
sysctl net.ipv4.tcp_available_congestion_control

# Enable BBR (requires kernel 4.9+)
sudo modprobe tcp_bbr
echo "tcp_bbr" | sudo tee /proc/sys/net/ipv4/tcp_congestion_control

# Verify
sysctl net.ipv4.tcp_congestion_control
# Output: bbr

# Test again
iperf3 -c localhost -t 30 -P 4

When BBR wins vs. CUBIC:

Scenario Winner Reason
LAN, zero packet loss Similar Both probe bandwidth aggressively
WAN, 1% packet loss BBR CUBIC halves window on any loss
Long RTT (100ms+), good link BBR BBR models bandwidth directly
High concurrency (1000s of flows) BBR Lower buffer occupancy (less bufferbloat)

Part D: Additional Tuning Knobs

# Enable TCP Fast Open (TFO) — reduces latency for repeat connections
sudo sysctl -w net.ipv4.tcp_fastopen=3

# Increase TCP connection backlog
sudo sysctl -w net.core.somaxconn=65535

# Enable receive packet steering
sudo sysctl -w net.core.rps_sock_flow_entries=32768

# Test final configuration
iperf3 -c localhost -t 60 -P 8

# Compare before/after
echo "Baseline: 2.35 Gbits/sec"
echo "After tuning: (your result)"

Tuning Summary Table

Parameter Default Tuned Value Effect
net.core.rmem_max 212992 134217728 Maximum socket receive buffer
net.core.wmem_max 212992 134217728 Maximum socket send buffer
net.ipv4.tcp_rmem 4096 87380 6291456 4096 87380 134217728 TCP receive buffer range
net.ipv4.tcp_congestion_control cubic bbr Congestion algorithm
net.core.somaxconn 4096 65535 Listen backlog
net.ipv4.tcp_tw_reuse 0 1 Reuse TIME_WAIT sockets

Troubleshooting Guide

Problem Check Fix
iperf3 connection refused Server not running iperf3 -s -p 5201 in background
No throughput improvement after buffer tuning Application using small buffers Tune app buffer size too
BBR not available Old kernel or module missing uname -r check; lsmod | grep bbr

Lab 5: Kubernetes CNI Walkthrough

Objective: Manually invoke a CNI plugin to understand what happens when a pod is scheduled. Trace the network path from pod to pod using tcpdump.

Part A: Prerequisites

# Install kind (Kubernetes in Docker)
go install sigs.k8s.io/kind@latest
kind create cluster --name cni-lab

# Verify
kubectl get nodes
# NAME                     STATUS   ROLES           AGE
# cni-lab-control-plane    Ready    control-plane   2m

Part B: Observe Existing Pod Network Path

# Create two test pods
kubectl run pod1 --image=busybox --command -- sleep 3600
kubectl run pod2 --image=busybox --command -- sleep 3600
kubectl wait --for=condition=ready pod/pod1 pod/pod2

# Get pod IPs
kubectl get pod -o wide
# NAME   READY  STATUS   IP             NODE
# pod1   1/1    Running  10.244.0.5     cni-lab-control-plane
# pod2   1/1    Running  10.244.0.6     cni-lab-control-plane

Part C: Trace Pod Network Path

# Get inside the kind node
docker exec -it cni-lab-control-plane bash

# Inside the node: find veth pairs for pods
ip link show type veth
# Each pod gets one veth end inside its netns and one end in the host

# Find the network namespace for pod1
# CNI stores netns under /var/run/netns/ or linked from /proc/<pid>/ns/net
crictl ps | grep pod1
# Get the container ID
CONTAINER_ID=$(crictl ps | grep pod1 | awk '{print $1}')
crictl inspect $CONTAINER_ID | grep pid
# PID: 12345

# Enter pod1's network namespace
nsenter --net=/proc/12345/ns/net -- ip addr show
# Shows pod1's eth0 with IP 10.244.0.5

# From inside pod1's netns, trace packet route
nsenter --net=/proc/12345/ns/net -- ip route show
# default via 10.244.0.1 dev eth0    ← gateway is the CNI bridge

Part D: Manually Invoke CNI

# CNI plugins live here on kind nodes
ls /opt/cni/bin/
# bridge  flannel  host-local  loopback  portmap  ...

# CNI config (kind uses kindnet or flannel)
cat /etc/cni/net.d/*.conf

# Manually invoke the bridge CNI plugin
# First, create a network namespace
ip netns add test-ns

# CNI_COMMAND=ADD invokes the plugin
cat << 'EOF' > /tmp/cni-conf.json
{
  "cniVersion": "0.4.0",
  "name": "mynet",
  "type": "bridge",
  "bridge": "cni-test0",
  "isGateway": true,
  "ipMasq": true,
  "ipam": {
    "type": "host-local",
    "subnet": "10.99.0.0/24",
    "routes": [{"dst": "0.0.0.0/0"}]
  }
}
EOF

CNI_COMMAND=ADD \
CNI_CONTAINERID=test-container-id \
CNI_NETNS=/var/run/netns/test-ns \
CNI_IFNAME=eth0 \
CNI_PATH=/opt/cni/bin \
/opt/cni/bin/bridge < /tmp/cni-conf.json

# Expected JSON output from CNI plugin:
# {
#   "cniVersion": "0.4.0",
#   "interfaces": [{"name": "cni-test0",...},{"name":"eth0",...}],
#   "ips": [{"address": "10.99.0.2/24", "gateway": "10.99.0.1"}]
# }

Part E: Observe veth Pair Creation

# After CNI ADD, observe what was created
ip link show cni-test0    # bridge created
ip link show | grep veth  # veth pair in host namespace
ip netns exec test-ns ip addr show eth0   # 10.99.0.2/24 inside netns

# Trace a packet from test-ns to the internet
ip netns exec test-ns ping -c 2 8.8.8.8
# While it runs, capture on the host bridge
tcpdump -i cni-test0 -n icmp
# Output shows ICMP packets crossing the bridge

Part F: Pod-to-Pod tcpdump

# Back to the kubectl environment
# Capture on the node while pods communicate

docker exec -it cni-lab-control-plane tcpdump -i any -n \
    src 10.244.0.5 -c 20 2>/dev/null &

kubectl exec pod1 -- ping -c 5 10.244.0.6

# Expected tcpdump output:
# 10:00:01 IP 10.244.0.5 > 10.244.0.6: ICMP echo request
# 10:00:01 IP 10.244.0.6 > 10.244.0.5: ICMP echo reply
# (packets appear twice: on the veth in pod1's netns and on the bridge)

Pod Network Path Summary

pod1 (eth0 10.244.0.5)
    └── veth-abc123 (host end)
        └── cni0 bridge (10.244.0.1/24)
            └── veth-def456 (pod2's host end)
                └── pod2 (eth0 10.244.0.6)

All intra-node pod traffic goes: pod1 eth0 → veth → bridge → veth → pod2 eth0. No kernel routing — pure L2 bridging.

Cleanup

# Remove test namespace
ip netns del test-ns
ip link del cni-test0 2>/dev/null

# Delete kind cluster
kind delete cluster --name cni-lab

Troubleshooting Guide

Problem Check Fix
Pod stuck in ContainerCreating kubectl describe pod → CNI error Check CNI plugin logs: journalctl -u kubelet
No connectivity between pods Route missing on node ip route show on node; check CNI config
CNI ADD returns error Plugin config invalid Check JSON syntax; verify ipam block
tcpdump shows packets on bridge but pod can't ping iptables DROP rule iptables -L -n -v | grep DROP

Lab Appendix: Key Commands Reference

Goal Command
List all network namespaces ip netns list
Execute in a namespace ip netns exec <ns> <cmd>
Show veth pairs ip link show type veth
Show bridge members bridge link show
Show all routes ip route show table all
Show socket statistics ss -ntp
Capture packets tcpdump -i eth0 -n -w cap.pcap
Read capture tcpdump -r cap.pcap -n
Show BPF programs bpftool prog list
Show BPF maps bpftool map list
XDP attach ip link set dev eth0 xdpgeneric obj prog.o sec xdp
XDP detach ip link set dev eth0 xdp off