Add support for parsing PPP headers with variable lengths

src/packet/packet.cpp

@@ -771,22 +771,95 @@ static inline const char *parse_ether(struct packet *pkt, const char *data, uint
     return parse_l3(pkt, next_proto, layer->pld_ptr, layer->pld_len);
 }
 
+static inline int next_proto_is_ppp(uint16_t next_proto)
+{
+    // /usr/include/linux/ppp_defs.h.html
+    switch (next_proto)
+    {
+    case PPP_IP:         /* Internet Protocol */
+    case PPP_AT:         /* AppleTalk Protocol */
+    case PPP_IPX:        /* IPX protocol */
+    case PPP_VJC_COMP:   /* VJ compressed TCP */
+    case PPP_VJC_UNCOMP: /* VJ uncompressed TCP */
+    case PPP_MP:         /* Multilink protocol */
+    case PPP_IPV6:       /* Internet Protocol Version 6 */
+    case PPP_COMPFRAG:   /* fragment compressed below bundle */
+    case PPP_COMP:       /* compressed packet */
+    case PPP_MPLS_UC:    /* Multi Protocol Label Switching - Unicast */
+    case PPP_MPLS_MC:    /* Multi Protocol Label Switching - Multicast */
+    case PPP_IPCP:       /* IP Control Protocol */
+    case PPP_ATCP:       /* AppleTalk Control Protocol */
+    case PPP_IPXCP:      /* IPX Control Protocol */
+    case PPP_IPV6CP:     /* IPv6 Control Protocol */
+    case PPP_CCPFRAG:    /* CCP at link level (below MP bundle) */
+    // case PPP_CCP:        /* Compression Control Protocol */ (same as PPP_CCPFRAG)
+    case PPP_MPLSCP: /* MPLS Control Protocol */
+    case PPP_LCP:    /* Link Control Protocol */
+    case PPP_PAP:    /* Password Authentication Protocol */
+    case PPP_LQR:    /* Link Quality Report protocol */
+    case PPP_CHAP:   /* Cryptographic Handshake Auth. Protocol */
+    case PPP_CBCP:   /* Callback Control Protocol */
+        return 1;
+    default:
+        return 0;
+    }
+}
+
 static inline const char *parse_ppp(struct packet *pkt, const char *data, uint16_t len)
 {
+    /*
+     * https://datatracker.ietf.org/doc/html/rfc1661#section-2
+     * +----------+-------------+---------+
+     * | Protocol | Information | Padding |
+     * | 8/16 bits|      *      |    *    |
+     * +----------+-------------+---------+
+     *
+     * https://datatracker.ietf.org/doc/html/rfc1331#section-3.1
+     * +----------+----------+----------+----------+------------
+     * |   Flag   | Address  | Control  | Protocol | Information
+     * | 01111110 | 11111111 | 00000011 | 16 bits  |      *
+     * +----------+----------+----------+----------+------------
+     *         ---+----------+----------+-----------------
+     *            |   FCS    |   Flag   | Inter-frame Fill
+     *            | 16 bits  | 01111110 | or next Address
+     *         ---+----------+----------+-----------------
+     */
     if (unlikely(len < 4))
     {
         PACKET_LOG_DATA_INSUFFICIENCY(LAYER_TYPE_PPP);
         return data;
     }
 
+    uint16_t hdr_len = 0;
+    uint16_t next_proto = 0;
+
+    // ppp header 1 byte
+    next_proto = *((uint8_t *)data);
+    if (next_proto_is_ppp(next_proto))
+    {
+        hdr_len = 1;
+        goto success;
+    }
+
+    // ppp header 2 bytes
+    next_proto = ntohs(*((uint16_t *)data));
+    if (next_proto_is_ppp(next_proto))
+    {
+        hdr_len = 2;
+        goto success;
+    }
+
+    // ppp header 4 bytes
+    next_proto = ntohs(*((uint16_t *)data + 1));
+    hdr_len = 4;
+
+success:
     struct packet_layer *layer = get_free_layer(pkt);
     if (unlikely(layer == NULL))
     {
         return data;
     }
-    uint16_t next_proto = ntohs(*((uint16_t *)data + 1));
+    SET_LAYER(pkt, layer, LAYER_TYPE_PPP, hdr_len, data, len, 0);
-    SET_LAYER(pkt, layer, LAYER_TYPE_PPP, 4, data, len, 0);
-
     switch (next_proto)
     {
         // TESTED
@@ -869,10 +942,8 @@ static inline const char *parse_vlan(struct packet *pkt, const char *data, uint1
 
 static inline const char *parse_pppoe_ses(struct packet *pkt, const char *data, uint16_t len)
 {
-#define PPPOE_TYPE_IPV4 0x2100
-#define PPPOE_TYPE_IPV6 0x5700
 
-    if (unlikely(len < 8))
+    if (unlikely(len < 6))
     {
         PACKET_LOG_DATA_INSUFFICIENCY(LAYER_TYPE_PPPOE);
         return data;
@@ -883,20 +954,10 @@ static inline const char *parse_pppoe_ses(struct packet *pkt, const char *data,
     {
         return data;
     }
-    uint16_t next_proto = *((uint16_t *)data + 3);
+    SET_LAYER(pkt, layer, LAYER_TYPE_PPPOE, 6, data, len, 0);
-    SET_LAYER(pkt, layer, LAYER_TYPE_PPPOE, 8, data, len, 0);
 
-    switch (next_proto)
-    {
     // TESTED
-    case PPPOE_TYPE_IPV4:
+    return parse_ppp(pkt, layer->pld_ptr, layer->pld_len);
-        return parse_ipv4(pkt, layer->pld_ptr, layer->pld_len);
-    case PPPOE_TYPE_IPV6:
-        return parse_ipv6(pkt, layer->pld_ptr, layer->pld_len);
-    default:
-        PACKET_LOG_UNSUPPORT_PROTO("pppoe", next_proto);
-        return layer->pld_ptr;
-    }
 }
 
 static inline const char *parse_mpls(struct packet *pkt, const char *data, uint16_t len)

src/packet/test/gtest_packet.cpp

@@ -1602,11 +1602,8 @@ TEST(PACKET, ETH_VLAN_PPPOE_IP4_TCP)
 
 	// LAYER_TYPE_L2
 	const struct packet_layer *outer_l2_record = packet_get_outermost_layer(&handler, LAYER_TYPE_L2);
-	const struct packet_layer *inner_l2_record = packet_get_innermost_layer(&handler, LAYER_TYPE_L2);
 
 	EXPECT_TRUE(outer_l2_record != nullptr);
-	EXPECT_TRUE(inner_l2_record != nullptr);
-	EXPECT_TRUE(outer_l2_record == inner_l2_record);
 	EXPECT_TRUE(outer_l2_record == outer_eth_record);
 
 	// LAYER_TYPE_VLAN
@@ -1634,8 +1631,8 @@ TEST(PACKET, ETH_VLAN_PPPOE_IP4_TCP)
 	EXPECT_TRUE(inner_pppoe_record != nullptr);
 	EXPECT_TRUE(outer_pppoe_record == inner_pppoe_record);
 	EXPECT_TRUE(outer_pppoe_record->hdr_offset == 18);
-	EXPECT_TRUE(outer_pppoe_record->hdr_len == 8);
+	EXPECT_TRUE(outer_pppoe_record->hdr_len == 6);
-	EXPECT_TRUE(outer_pppoe_record->pld_len == 52);
+	EXPECT_TRUE(outer_pppoe_record->pld_len == 54);
 
 	// LAYER_TYPE_L2_TUN
 	const struct packet_layer *inner_l2_tun_record = packet_get_innermost_layer(&handler, LAYER_TYPE_L2_TUN);
@@ -1643,6 +1640,23 @@ TEST(PACKET, ETH_VLAN_PPPOE_IP4_TCP)
 	EXPECT_TRUE(inner_l2_tun_record != nullptr);
 	EXPECT_TRUE(inner_l2_tun_record == outer_pppoe_record);
 
+	// LAYER_TYPE_PPP
+	const struct packet_layer *outer_ppp_record = packet_get_outermost_layer(&handler, LAYER_TYPE_PPP);
+	const struct packet_layer *inner_ppp_record = packet_get_innermost_layer(&handler, LAYER_TYPE_PPP);
+
+	EXPECT_TRUE(outer_ppp_record != nullptr);
+	EXPECT_TRUE(inner_ppp_record != nullptr);
+	EXPECT_TRUE(outer_ppp_record == inner_ppp_record);
+	EXPECT_TRUE(outer_ppp_record->hdr_offset == 24);
+	EXPECT_TRUE(outer_ppp_record->hdr_len == 2);
+	EXPECT_TRUE(outer_ppp_record->pld_len == 52);
+
+	// LAYER_TYPE_L2
+	const struct packet_layer *inner_l2_record = packet_get_innermost_layer(&handler, LAYER_TYPE_L2);
+
+	EXPECT_TRUE(inner_l2_record != nullptr);
+	EXPECT_TRUE(inner_l2_record == outer_ppp_record);
+
 	// LAYER_TYPE_IPV4
 	const struct packet_layer *outer_ipv4_record = packet_get_outermost_layer(&handler, LAYER_TYPE_IPV4);
 	const struct packet_layer *inner_ipv4_record = packet_get_innermost_layer(&handler, LAYER_TYPE_IPV4);

test/packet_parser/cmp_layers.sh

@@ -13,38 +13,143 @@ else
 fi
 
 # remove l7 protocol fields
-function replace() {
+function preprocess_tshark_ouput() {
-    file=$1
+    input_file=$1
-    array=(":data" ":ntp" ":rip" ":isakmp" ":esp" ":udpencap" ":sip" ":sdp" ":rtcp" ":rtp" ":ssh" ":dns" ":ssl" ":gquic" ":http-text-lines" ":http" ":msmms" ":bfd" ":ftp-data-text-lines" ":ftp" ":ssdp" ":mdns" ":radius" ":pop" ":smtp" ":rtmpt" ":bittorrent" ":oicq" ":json" ":media" ":x11" ":telnet" ":nbss:smb" ":memcache" ":rtspi" ":rdt" ":rtsp" ":nbns" ":nbdgm:smb:browser" ":lcp" ":chap" ":ipcp" ":comp_data" ":ccp" ":snmp" ":socks" ":bgp" ":eigrp" ":bootp" ":xml" ":echo" ":vssmonitoring" ":mndp" ":websocket-text-lines" ":websocket" ":image-jfif" ":png" ":pkix1implicit" ":x509sat" ":x509ce" ":pkix1explicit" ":llmnr" ":pkcs-1")
+    output_file=$2
-    for key in "${array[@]}"; do
+    cp ${input_file} ${output_file}
-        sed "s/$key//g" ${file} >.tmp.txt
+    kv_array=(
-        mv .tmp.txt ${file}
+        ":data" ""
+        ":ntp" ""
+        ":rip" ""
+        ":isakmp" ""
+        ":esp" ""
+        ":udpencap" ""
+        ":sip" ""
+        ":sdp" ""
+        ":rtcp" ""
+        ":rtp" ""
+        ":ssh" ""
+        ":dns" ""
+        ":ssl" ""
+        ":gquic" ""
+        ":http-text-lines" ""
+        ":http" ""
+        ":msmms" ""
+        ":bfd" ""
+        ":ftp-data-text-lines" ""
+        ":ftp" ""
+        ":ssdp" ""
+        ":mdns" ""
+        ":radius" ""
+        ":pop" ""
+        ":smtp" ""
+        ":rtmpt" ""
+        ":bittorrent" ""
+        ":oicq" ""
+        ":json" ""
+        ":media" ""
+        ":x11" ""
+        ":telnet" ""
+        ":nbss:smb" ""
+        ":memcache" ""
+        ":rtspi" ""
+        ":rdt" ""
+        ":rtsp" ""
+        ":nbns" ""
+        ":nbdgm:smb:browser" ""
+        ":lcp" ""
+        ":chap" ""
+        ":ipcp" ""
+        ":comp_data" ""
+        ":ccp" ""
+        ":snmp" ""
+        ":socks" ""
+        ":bgp" ""
+        ":eigrp" ""
+        ":bootp" ""
+        ":xml" ""
+        ":echo" ""
+        ":vssmonitoring" ""
+        ":mndp" ""
+        ":websocket-text-lines" ""
+        ":websocket" ""
+        ":image-jfif" ""
+        ":png" ""
+        ":pkix1implicit" ""
+        ":x509sat" ""
+        ":x509ce" ""
+        ":pkix1explicit" ""
+        ":llmnr" ""
+        ":pkcs-1" ""
+        ":bitcoin" ""
+        ":image-gif" ""
+        ":dhcpv6" ""
+        ":tcp:pptp" ":tcp"
+        ":ieee8021ad" ":vlan"
+        ":tcp-text-lines" ":tcp"
+    )
+    for ((i = 0; i < ${#kv_array[@]}; i += 2)); do
+        key=${kv_array[i]}
+        val=${kv_array[i + 1]}
+        sed "s/$key/$val/g" ${output_file} >${output_file}.tmp
+        mv ${output_file}.tmp ${output_file}
     done
 }
 
-output_dir="cmp_output/"
+# When MPLS is nested, packet_parser will output multiple mpls fields, and tshark will only output one mpls field, so we need to preprocess the output
+function preprocess_parser_ouput() {
+    input_file=$1
+    output_file=$2
+    cp ${input_file} ${output_file}
+    kv_array=(
+        ":mpls:mpls" ":mpls"
+    )
+    for ((i = 0; i < ${#kv_array[@]}; i += 2)); do
+        key=${kv_array[i]}
+        val=${kv_array[i + 1]}
+        sed "s/$key/$val/g" ${output_file} >${output_file}.tmp
+        mv ${output_file}.tmp ${output_file}
+    done
+}
+
+err_count=0
+pass_count=0
+curr_count=0
+total_count=${#pcap_files[@]}
+tmp_file_dir="cmp_tmp_files/"
+err_pcap_dir="cmp_err_pcaps/"
+
+rm -rf ${err_pcap_dir} && mkdir ${err_pcap_dir}
 for pcap in "${pcap_files[@]}"; do
-    rm -rf ${output_dir} && mkdir ${output_dir}
+    rm -rf ${tmp_file_dir} && mkdir ${tmp_file_dir}
+
+    curr_count=$((curr_count + 1))
 
     # tshark output frame.protocols
-    tshark -r ${pcap} -T fields -e frame.number -e frame.protocols >>${output_dir}/tshark.txt
+    tshark -r ${pcap} -T fields -e frame.number -e frame.protocols >>${tmp_file_dir}/tshark_output.txt
 
     # packet_parser output frame.protocols
-    ./packet_parser -f ${pcap} -p >>${output_dir}/parser.txt
+    ./packet_parser -f ${pcap} -p >>${tmp_file_dir}/parser_output.txt
 
     # compare tshark and packet_parser output
-    cp ${output_dir}/tshark.txt ${output_dir}/expect.txt
+    preprocess_tshark_ouput ${tmp_file_dir}/tshark_output.txt ${tmp_file_dir}/tshark_format.txt
-    replace ${output_dir}/expect.txt
+    preprocess_parser_ouput ${tmp_file_dir}/parser_output.txt ${tmp_file_dir}/parser_format.txt
-    diff ${output_dir}/expect.txt ${output_dir}/parser.txt >>${output_dir}/diff.txt
+    diff ${tmp_file_dir}/tshark_format.txt ${tmp_file_dir}/parser_format.txt >>${tmp_file_dir}/diff.txt
 
     # print result
-    line_count=$(cat ${output_dir}/diff.txt | wc -l)
+    line_count=$(cat ${tmp_file_dir}/diff.txt | wc -l)
     if [ "$line_count" -ne 0 ]; then
-        printf "\033[31m ${pcap} TEST FAILED \033[0m\n"
+        printf "\033[31m [${curr_count}/${total_count}] ${pcap} TEST FAILED \033[0m\n"
-        cat ${output_dir}/diff.txt | head -n 100
+        cat ${tmp_file_dir}/diff.txt | head -n 100
-        #exit 0
+        cp ${pcap} ${err_pcap_dir}
+        err_count=$((err_count + 1))
     else
-        printf "\033[32m ${pcap} TEST PASSED \033[0m\n"
+        printf "\033[32m [${curr_count}/${total_count}] ${pcap} TEST PASSED \033[0m\n"
+        pass_count=$((pass_count + 1))
     fi
-
 done
+
+printf "\033[32m\nTotal: ${total_count}, Passed: ${pass_count}, Failed: ${err_count}\033[0m\n"
+if [ "$err_count" -ne 0 ]; then
+    printf "\033[31mFailed pcap files are saved in ${err_pcap_dir}\033[0m\n"
+fi

test/packet_parser/packet_parser.cpp

@@ -30,6 +30,9 @@ static int ipv6_proto_to_str(const struct packet_layer *ipv6_layer, char *buff,
     case IPPROTO_DSTOPTS:
         used += snprintf(buff + used, size - used, ":ipv6.dstopts");
         break;
+    case IPPROTO_FRAGMENT:
+        used += snprintf(buff + used, size - used, ":ipv6.fraghdr");
+        break;
     default:
         break;
     }
@@ -62,7 +65,7 @@ static int packet_proto_to_str(const struct packet *pkt, char *buff, int size)
             used += snprintf(buff + used, size - used, "vlan:ethertype");
             break;
         case LAYER_TYPE_PPPOE:
-            used += snprintf(buff + used, size - used, "pppoes:ppp");
+            used += snprintf(buff + used, size - used, "pppoes");
             break;
         case LAYER_TYPE_MPLS:
             used += snprintf(buff + used, size - used, "mpls");