$ sudo dnf install clang llvm
$ sudo dnf install elfutils-libelf-devel libpcap-devel

• Kernel headers 설치 (타겟 커널이 다른 경우 해당 커널의 헤더를 사용, 선택사항)

  $ sudo dnf install kernel-headers
  

• bpftool 설치 (선택사항)

  $ sudo dnf install bpftool
  

• perf 유틸리티 설치 (선택사항)

  $ sudo dnf install perf
  

$ sudo apt install clang llvm
$ sudo apt install libelf-dev libpcap-dev gcc-multilib build-essential

• Kernel headers 설치 (타겟 커널이 다른 경우 해당 커널의 헤더를 사용, 선택사항)

  $ sudo apt install linux-headers-$(uname -r)
  

• bpftool 설치 (선택사항)
  • Ubuntu 환경인 경우

    $ sudo apt install linux-tools-common linux-tools-generic
    

• perf 유틸리티 설치 (선택사항)
  • Debian 환경인 경우

    $ sudo apt install linux-perf
    

  • Ubuntu 환경인 경우

    $ sudo apt install linux-tools-$(uname -r)
    

$ sudo zypper install clang llvm
$ sudo zypper install libelf-devel libpcap-devel

• Kernel headers 설치 (타겟 커널이 다른 경우 해당 커널의 헤더를 사용, 선택사항)

  $ sudo zypper install kernel-devel
  

• bpftool 설치 (선택사항)

  $ sudo zypper install bpftool
  

• perf 유틸리티 설치 (선택사항)

  $ sudo zypper install perf
  

  
  
  

• XDP : eXpress Data Path
• BPF : Berkeley Packet Filter
  • eBPF : Extended Berkeley Packet Filter
    • eBPF로 작성된 프로그램을 "bpf()" 시스템 콜을 통해서 Kernel에 전달하면 Kernel내에서 Sandbox [4] 형태의 인터프리터(Interpreter : JIT virtual machine(https://en.wikipedia.org/wiki/Just-in-time_compilation))로 동작하게 됩니다.
  • cBPF : cBPF(Classic BPF)는 1992년 부터 존재하였으나 이는 구식으로 취급되며 eBPF는 Linux Kernel v3.18(https://kernelnewbies.org/Linux_3.18)부터 등장하였습니다. 현재는 cBPF 도 내부적으로 eBPF로 투명하게 변환되어 처리됩니다.
    • Socket filter 예제
      • 이 예제소스는 cBPF(Classic Berkeley Packet Filter) code 를 이용한 내 Linux box 의 모든 TCP syn packet 인입을 잡아내는 Application side에서의 감시코드입니다.

        More

        /*
          Copyright (C) HWPORT.COM
          All rights reserved.
          Code by JaeHyuk Cho <mailto:minzkn@minzkn.com>
        */
        
        #if !defined(_ISOC99_SOURCE)
        # define _ISOC99_SOURCE (1L)
        #endif
        
        #if !defined(_GNU_SOURCE)
        # define _GNU_SOURCE (1L)
        #endif
        
        #include <sys/types.h>
        #include <sys/socket.h>
        #include <stdio.h>
        #include <stdlib.h>
        #include <unistd.h>
        
        #include <arpa/inet.h>
        
        #include <linux/filter.h>
        #include <linux/if_ether.h>
        #include <linux/if.h>
        
        int main(int s_argc, char **s_argv);
        
        static void * (__mz_dump_ex__)(size_t s_s, size_t s_d, void * s_da, size_t s_si)
        { size_t s_o, s_w, s_lo; unsigned char s_b[ 16 + 1 ];
         if(s_da == NULL)return(NULL);
         s_b[sizeof(s_b) - 1] = (unsigned char)'\0';
         for(s_o = (size_t)0;s_o < s_si;s_o += (size_t)16){
          s_w = ((s_si - s_o) <= ((size_t)16)) ? (s_si - s_o) : ((size_t)16);
          (void)fprintf(stdout, "%08lX", (unsigned long)(s_d + s_o));
          for(s_lo = (size_t)0;s_lo < s_w;s_lo++){
           if(s_lo == ((size_t)(16 >> 1)))(void)fputs(" | ", stdout);
           else (void)fputs(" ", stdout);
           s_b[s_lo] = *(((unsigned char *)s_da) + (s_s + s_o + s_lo));
           (void)fprintf(stdout, "%02X", (int)s_b[s_lo]);
           if((s_b[s_lo] & ((unsigned char)(1 << 7))) || (s_b[s_lo] < ((unsigned char)' ')) ||
              (s_b[s_lo] == ((unsigned char)0x7f)))s_b[s_lo] = (unsigned char)'.';}
          while(s_lo < ((size_t)16)){
           if(s_lo == ((size_t)(16 >> 1)))(void)fputs("     ", stdout);
           else (void)fputs("   ", stdout);
           s_b[s_lo] = (unsigned char)' '; s_lo++;}
          (void)fprintf(stdout, " [%s]\n", (char *)(&s_b[0]));}
         return(s_da);
        }
        static __inline void * (mz_dump_ex)(size_t s_seek_offset, size_t s_display_offset, void * s_data, size_t s_size)
        { return(__mz_dump_ex__(s_seek_offset, s_display_offset, s_data, s_size)); }
        #define mz_dump(m_data,m_size) mz_dump_ex((size_t)0,(size_t)0,(void *)(m_data),(size_t)(m_size))
        
        int main(int s_argc, char **s_argv)
        {
            int s_socket;
            unsigned char s_buffer[ 64 << 10 ];
            ssize_t s_recv_bytes;
        # if 1L
            struct sock_filter s_bpf_code[] = { /* tcpdump -dd 'tcp[tcpflags] & (tcp-syn) != 0' */
                { 0x28, 0, 0, 0x0000000c },
                { 0x15, 0, 8, 0x00000800 },
                { 0x30, 0, 0, 0x00000017 },
                { 0x15, 0, 6, 0x00000006 },
                { 0x28, 0, 0, 0x00000014 },
                { 0x45, 4, 0, 0x00001fff },
                { 0xb1, 0, 0, 0x0000000e },
                { 0x50, 0, 0, 0x0000001b },
                { 0x45, 0, 1, 0x00000002 },
                { 0x6, 0, 0, 0x0000ffff },
                { 0x6, 0, 0, 0x00000000 },
            };
        # else
            struct sock_filter s_bpf_code[] = { /* tcpdump -dd tcp */
                { 0x28, 0, 0, 0x0000000c },
                { 0x15, 0, 2, 0x000086dd },
                { 0x30, 0, 0, 0x00000014 },
                { 0x15, 3, 4, 0x00000006 },
                { 0x15, 0, 3, 0x00000800 },
                { 0x30, 0, 0, 0x00000017 },
                { 0x15, 0, 1, 0x00000006 },
                { 0x6, 0, 0, 0x00000800 },
                { 0x6, 0, 0, 0x00000000 }
            };
        # endif
            struct sock_fprog s_filter;
            int s_check;
        
            (void)s_argc;
            (void)s_argv;
        
            s_socket = socket(PF_PACKET, SOCK_RAW, htons(ETH_P_IP));
            if(s_socket == (-1)) {
                perror("socket");
                return(EXIT_FAILURE);
            }
        
            s_filter.len = sizeof(s_bpf_code) / sizeof(struct sock_filter);
            s_filter.filter = s_bpf_code;
            s_check = setsockopt(s_socket, SOL_SOCKET, SO_ATTACH_FILTER, (const void *)(&s_filter), (socklen_t)sizeof(s_filter));
            (void)s_check;
        
            for(;;) {
                s_recv_bytes = recv(s_socket, (void *)(&s_buffer[0]), sizeof(s_buffer), 0);
                if(s_recv_bytes == ((ssize_t)(-1))) {
                    perror("recv");
                    break;
                }
        
                (void)fprintf(stdout, "recv %ld bytes\n", (long)s_recv_bytes);
                mz_dump(s_buffer, s_recv_bytes);
            }
        
            (void)close(s_socket);
        
            return(EXIT_SUCCESS);
        }
        
        /* vim: set expandtab: */
        /* End of source */
        

      • 하나의 cBPF 는 크게 4가지 요소를 담고 있으며 첫번째 요소 16bits는 OP-CODE이고 두번째 요소 8bits 는 True 조건일 때 분기 Offset, 세번째 요소의 8bits False 조건일 때 분기 Offset, 그리고 마지막 요소 32bits 는 OP-CODE에 따른 특정 값을 넘겨줘야 할 때 사용합니다. OP-CODE 나 sock_filter 자료구조는 "linux/filter.h" 헤더를 참고하면 이해하기 쉽습니다.
        • cBPF 예1) ICMPv4 reply packet 중 특정 ident 값만 통과시키는 filter 예시 (이 경우 IPv4 header 를 건너뛰는 부분 포함)

          More

                      struct sock_filter s_bpf_code[] = { /* ident 가 다르거나 ICMP_ECHOREPLY 가 아닌 경우 버림 */
                          BPF_STMT(BPF_LDX | BPF_B   | BPF_MSH, 0),          /* Skip IP header due BSD, see ping6. */
                          BPF_STMT(BPF_LD  | BPF_H   | BPF_IND, 4),          /* Load icmp echo ident */
                          BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0xAAAA, 1, 0), /* Ours? */
                          BPF_STMT(BPF_RET | BPF_K, 0),                      /* Echo with wrong ident. Reject. */
                          BPF_STMT(BPF_LD  | BPF_B   | BPF_IND, 0),          /* Load icmp type */
                          BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, ICMP_ECHOREPLY, 0, 1), /* Echo? */
                          BPF_STMT(BPF_RET | BPF_K, ~0U),                    /* Yes, it passes. */
                          BPF_STMT(BPF_RET | BPF_K, 0)                       /* Echo with wrong reply. Reject. */
                      };
          

        • cBPF 예2) ICMPv6 reply packet 중 특정 ident 값만 통과시키는 filter 예시

          More

                      struct sock_filter s_bpf_code[] = { /* ident 가 다르거나 ICMP6_ECHO_REPLY 가 아닌 경우 버림 */
                          BPF_STMT(BPF_LD  | BPF_H   | BPF_ABS, 4),          /* Load icmp echo ident */
                          BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0xAAAA, 1, 0), /* Ours? */
                          BPF_STMT(BPF_RET | BPF_K, 0),                      /* Echo with wrong ident. Reject. */
                          BPF_STMT(BPF_LD  | BPF_B   | BPF_ABS, 0),          /* Load icmp type */
                          BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, ICMP6_ECHO_REPLY, 0, 1), /* Echo? */
                          BPF_STMT(BPF_RET | BPF_K, ~0U),                    /* Yes, it passes. */
                          BPF_STMT(BPF_RET | BPF_K, 0)                       /* Echo with wrong reply. Reject. */
                      };
          

• LLVM(https://llvm.org/) : (이전 이름: Low Level Virtual Machine => 지금은 LLVM그 자체가 이름임) 은 컴파일러의 기반구조. 언어에 가상 기계를 생성, 가상 기계가 언어에 독립적인 최적화를 실행
• Clang(https://clang.llvm.org/) : 클랭 - C, C++, 오브젝티브-C, 오브젝티브-C++ 프로그래밍 언어를 위한 컴파일러 프론트엔드. LLVM을 백엔드로 사용
  • 대부분의 C 로 작성된 프로그램들은 CC 환경변수 빌드인 경우 다음과 같이 clang으로 빌드할 수 있음.

    make CC=clang
    

• BCC(https://github.com/iovisor/bcc) : BPF Compiler Collection

  BCC is a toolkit for creating efficient kernel tracing and manipulation programs, and includes several useful tools and examples. It makes use of extended BPF (Berkeley Packet Filters), formally known as eBPF, a new feature that was first added to Linux 3.15. Much of what BCC uses requires Linux 4.1 and above.
  

• kprobes : Kernel dynamic instrumentation
• uprobes : User-level dynamic instrumentation

XDP Project(https://xdp-project.net/)

• XDP Hands-On Tutorial(https://github.com/xdp-project/xdp-tutorial)
• xdp-tools - Library and utilities for use with XDP(https://github.com/xdp-project/xdp-tools)
  • RaspberryPi 환경에서 빌드하는 경우 이슈 - "/usr/include/linux/types.h:5:10: fatal error: 'asm/types.h' file not found"(https://giters.com/xdp-project/xdp-tools/issues/4)
• XDP Paper(https://github.com/xdp-project/xdp-paper)
  • The eXpress Data Path: Fast Programmable Packet Processing in the Operating System Kernel(https://github.com/xdp-project/xdp-paper/blob/master/xdp-the-express-data-path.pdf)

BPF and XDP Reference Guide(https://docs.cilium.io/en/latest/bpf/)

• https://github.com/ebpf-io/ebpf.io

• Programmer’s Guide(https://doc.dpdk.org/guides/prog_guide/index.html)
  • 58. IPsec Packet Processing Library(https://doc.dpdk.org/guides/prog_guide/ipsec_lib.html)
• DPDK PMD for AF_XDP(https://static.sched.com/hosted_files/dpdkuserspace2018/6a/pm%2006%20DPDK%20PMD%20for%20AF_XDP.pdf)

• Day1(https://youtu.be/1GSgyvn4N7E)
• Day2(https://youtu.be/jw8tEPP6jwQ)

• The LLVM Target-Independent Code Generator(https://llvm.org/docs/CodeGenerator.html)

struct xdp_buff *xdp {
...
void *data_meta;
...
}

xdp_set_data_meta_invalid(&xdp);
int bpf_xdp_adjust_meta(struct xdp_buff *xdp_md, int delta);
int bpf_xdp_adjust_head(struct xdp_buff *xdp_md, int delta);

driver on XDP RX packet :
    xdp_buff.data_meta = xdp_buff->data - sizeof(meta_data);
    *xdp_buff.data_meta = meta_data;
XDP user program:
    meta_data = (struct meta_data*)xdp_buff->data_meta;
...

• Kernel and BPF program feature compatibility(https://www.mankier.com/3/libxdp#Kernel_and_BPF_program_feature_compatibility)

  To get the full benefit of all features, libxdp needs to be used with kernel 5.10 or newer, unless the commits mentioned below have been backported.
  ...