Debug OpenvSwitch (by quqi99)

作者：张华 发表于：2020-12-28
版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明

GDB调试

# will stop service
sysctl kernel.core_pattern
sudo service apport start force_start=1 enabled=1
grep enabled /etc/default/apport
sudo killall -SIGSEGV ovs-vswitchd
sudo cat /var/log/apport.log

# won't stop service
gdb -ex "set pagination 0" -ex "thread apply all bt" -batch -p $(pidof ovs-vswitchd) #for call trace
gcore $(pidof ovs-vswitchd)
sudo apt install openvswitch-dbg cgdb
#cgdb $(which ovs-vswitchd) $(pidof ovs-vswitchd)
cgdb ovs-vswitchd /var/crash/_usr_lib_openvswitch-switch_ovs-vswitchd.0.crash

Debug Log

# log level: emer，err，warn，info，dbg,OFF
# log type: console, syslog, file
sudo ovs-appctl vlog/list |grep vswitchd
sudo ovs-appctl vlog/set vswitchd:file:dbg
sudo ovs-appctl vlog/set vswitchd:file:OFF
sudo ovs-appctl vlog/set ANY:ANY:dbg

# use this one, we can see the realtime log
sudo ovs-appctl vlog/set ANY:file:dbg
tail -f /var/log/openvswitch/ovs-vswitchd.log

CLI Usage

#网桥状态
ovs-vsctl show
#网桥查询
ovs-vsctl list-br
#端口查询
ovs-vsctl list-ports br-int
#接口查询
ovs-vsctl list-ifaces br-int
#端口、接口归属查询
ovs-vsctl port-to-br tap30580aa5-b0
ovs-vsctl iface-to-br tap30580aa5-b0
#查询网桥流表
ovs-ofctl dump-flows br-int
#查询网桥信息
ovs-ofctl show br-int
#Datapath统计信息查询：hit表示datapath命中数，missed未命中，lost表示没有传递到用户空间就丢弃了
ovs-dpctl show
#查询端口详细统计信息
ovs-dpctl show -s
#指定端口统计信息
ovs-ofctl dump-ports br-int 1
#网桥转发规则
ovs-appctl fdb/show br-int

#日志查询
ovsdb-tool show-log -m /var/lib/openvswitch/conf.db

#流表匹配
ovs-appctl ofproto/trace br-tun dl_vlan=1

tcpdump

* On compute node
tcpdump -vvv -neli tapd76aea08-db port 67 or port 68 or arp -w `hostname`-tapd76aea08-db-vmport.pcap

* On dhcp-agent
sudo ip netns exec qdhcp-2be2d2e6-a691-49fb-b260-c4e8abe86fd7 tcpdump -neli any port 67 or port 68 or arp -w `hostname`-dhcpport.pcap

* Run dhclient <interface> on the VM
Check if the instance got IP or not

Flow Debug - ovs-stat

# https://snapcraft.io/ovs-stat
juju ssh nova-compute/0 -- sudo -s
#sudo snap install snapd --edge
sudo snap install ovs-stat

# for analysing ovs itself 
sudo snap connect ovs-stat:openvswitch
sudo snap connect ovs-stat:network-control

# for analysing sosreport ovs data where sosreport is on a seperate filesystem
#sudo snap connect ovs-stat:removable-media

#ovs-stat -p /tmp/results --tree ./sosreport-015 --openstack  #don't use sudo
ovs-stat -p /tmp/results --tree --openstack

root@juju-7c33c2-bionic-7:~# readlink -f /tmp/snap.ovs-stat/tmp/results/juju-7c33c2-bionic-7/ovs/bridges/br-int/ports/*
/tmp/snap.ovs-stat/tmp/results/juju-7c33c2-bionic-7/ovs/ports/int-br-data
/tmp/snap.ovs-stat/tmp/results/juju-7c33c2-bionic-7/ovs/ports/patch-tun
/tmp/snap.ovs-stat/tmp/results/juju-7c33c2-bionic-7/ovs/ports/tapcdda0c13-5e
root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query "openstack.ports"
int-br-data  patch-int  patch-tun  phy-br-data  tapcdda0c13-5e  vxlan-0a05008d
root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query "openstack.bridge br-int"
int-br-data     patch-tun       tapcdda0c13-5e

#demo help
root@juju-7c33c2-bionic-7:~# sudo ls /tmp/snap.ovs-stat/tmp/results
juju-7c33c2-bionic-7
root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query ""
Choose one of the following:
ofproto-trace
openstack
root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query "openstack"
Choose one of the following:
openstack.bridge
openstack.bridge.ports
openstack.bridges
openstack.bridges.list
openstack.dvr
openstack.l2pop
openstack.local-vlans
openstack.local-vlans.info
openstack.port
openstack.port.flows
openstack.port.info
openstack.port.list
openstack.ports
openstack.ports.list

root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query "openstack.port tapcdda0c13-5e"
Port tapcdda0c13-5e:
  - bridge: br-int
  - id: 6
  - local-vlan: 3
  - mac address: fe:16:3e:c4:58:9c
root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query "openstack.l2pop 3"
10.5.0.141

root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query "ofproto-trace"
Choose one of the following:
ofproto-trace.port
ofproto-trace.port.all
root@juju-7c33c2-bionic-7:~# ovs-stat -p /tmp/results --host juju-7c33c2-bionic-7 --query "ofproto-trace.port tapcdda0c13-5e"
IMPORTANT: it looks like this port is attached to a vm so mac address has been converted from fe:16:3e:c4:58:9c to fa:16:3e:c4:58:9c
[arp]
sudo ovs-appctl ofproto/trace br-int in_port=6,arp,arp_spa=192.168.21.7,dl_src=fa:16:3e:c4:58:9c
[icmp]
sudo ovs-appctl ofproto/trace br-int in_port=6,ip,nw_proto=1,nw_src=192.168.21.7,nw_dst=1.1.1.1,dl_src=fa:16:3e:c4:58:9c
[dhcp]
sudo ovs-appctl ofproto/trace br-int udp,in_port=6,dl_src=fa:16:3e:c4:58:9c,dl_dst=ff:ff:ff:ff:ff:ff,nw_src=0.0.0.0,nw_dst=255.255.255.255,udp_src=68,udp_dst=67
[vm-to-vm]
sudo ovs-appctl ofproto/trace br-int in_port=6,tcp,dl_src=fa:16:3e:c4:58:9c,dl_dst=MAC_OF_REMOTE_INSTANCE
sudo ovs-appctl ofproto/trace br-int in_port=6,dl_vlan=3,dl_src=fa:16:3e:c4:58:9c,dl_dst=MAC_OF_REMOTE_INSTANCE


root@juju-7c33c2-bionic-7:~# sudo ovs-appctl ofproto/trace br-int in_port=6,arp,arp_spa=192.168.21.7,dl_src=fa:16:3e:c4:58:9c
Flow: arp,in_port=6,vlan_tci=0x0000,dl_src=fa:16:3e:c4:58:9c,dl_dst=00:00:00:00:00:00,arp_spa=192.168.21.7,arp_tpa=0.0.0.0,arp_op=0,arp_sha=00:00:00:00:00:00,arp_tha=00:00:00:00:00:00
bridge("br-int")
----------------
 0. priority 0, cookie 0xe1185b8d7fdb39b4
    goto_table:60
60. in_port=6, priority 100, cookie 0xe1185b8d7fdb39b4
    set_field:0x6->reg5
    set_field:0x3->reg6
    resubmit(,71)
71. arp,reg5=0x6,in_port=6,dl_src=fa:16:3e:c4:58:9c,arp_spa=192.168.21.7, priority 95, cookie 0xe1185b8d7fdb39b4
    resubmit(,94)
94. priority 1, cookie 0xe1185b8d7fdb39b4
    NORMAL
     -> no learned MAC for destination, flooding

    bridge("br-tun")
    ----------------
         0. in_port=1, priority 1, cookie 0x95591ec120b6fb4b
            goto_table:2
         2. dl_dst=00:00:00:00:00:00/01:00:00:00:00:00, priority 0, cookie 0x95591ec120b6fb4b
            goto_table:20
        20. priority 0, cookie 0x95591ec120b6fb4b
            goto_table:22
        22. dl_vlan=3, priority 1, cookie 0x95591ec120b6fb4b
            pop_vlan
            set_field:0x4d1->tun_id
            output:3
             -> output to kernel tunnel

bridge("br-data")
-----------------
 0. in_port=1, priority 2, cookie 0xf521d3d0416ecc5b
    drop

Final flow: arp,reg5=0x6,reg6=0x3,in_port=6,vlan_tci=0x0000,dl_src=fa:16:3e:c4:58:9c,dl_dst=00:00:00:00:00:00,arp_spa=192.168.21.7,arp_tpa=0.0.0.0,arp_op=0,arp_sha=00:00:00:00:00:00,arp_tha=00:00:00:00:00:00
Megaflow: pkt_mark=0,recirc_id=0,ct_state=-trk,eth,arp,in_port=6,vlan_tci=0x0000/0x1fff,dl_src=fa:16:3e:c4:58:9c,dl_dst=00:00:00:00:00:00,arp_spa=192.168.21.7
Datapath actions: set(tunnel(tun_id=0x4d1,src=10.5.2.207,dst=10.5.0.141,ttl=64,tp_dst=4789,flags(df|key))),6,push_vlan(vid=3,pcp=0),1

收集问题前后流表

ovs-stat --openstack --compress --archive-tag before
<restart neutron-openvswitch-agent>
ovs-stat --openstack --compress --archive-tag after