Merge 1e48c1007e into 8095c51678

Deprecate Rule.Flags field (#304 )
The functionality added in a46119e5 never worked: If you set NFTA_RULE_POSITION to 0, the kernel will just complain that a rule with this handle does not exist. This removes the broken functionality, leaving the field deprecated. The right way to insert a rule at the beginning of a chain is to use InsertRule and leave Position unset. https://github.com/google/nftables/issues/126 mentions that the nft command allows referring to rules by index. But here is a quote from the nft manpage: > The add and insert commands support an optional location specifier, > which is either a handle or the index (starting at zero) of an > existing rule. Internally, rule locations are always identified by > handle and the translation from index happens in userspace. In other words, identifiying rules by index is a feature of nft and is not part of the kernel interface.
2025-03-26 10:02:38 +01:00 · 2025-03-26 09:57:20 +01:00 · 2025-03-26 09:54:08 +01:00 · 2025-03-26 09:24:33 +01:00 · 2025-02-03 15:16:57 +00:00
15 changed files with 1011 additions and 38 deletions
--- a/chain.go
+++ b/chain.go
@ -140,7 +140,7 @@ func (cc *Conn) AddChain(c *Chain) *Chain {
 			{Type: unix.NFTA_CHAIN_TYPE, Data: []byte(c.Type + "\x00")},
 		})...)
 	}
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWCHAIN),
 			Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
@ -161,7 +161,7 @@ func (cc *Conn) DelChain(c *Chain) {
 		{Type: unix.NFTA_CHAIN_NAME, Data: []byte(c.Name + "\x00")},
 	})

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELCHAIN),
 			Flags: netlink.Request | netlink.Acknowledge,
@ -179,7 +179,7 @@ func (cc *Conn) FlushChain(c *Chain) {
 		{Type: unix.NFTA_RULE_TABLE, Data: []byte(c.Table.Name + "\x00")},
 		{Type: unix.NFTA_RULE_CHAIN, Data: []byte(c.Name + "\x00")},
 	})
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELRULE),
 			Flags: netlink.Request | netlink.Acknowledge,
--- a/conn.go
+++ b/conn.go
@ -41,7 +41,7 @@ type Conn struct {

 	lasting      bool       // establish a lasting connection to be used across multiple netlink operations.
 	mu           sync.Mutex // protects the following state
-	messages     []netlink.Message
+	messages     []netlinkMessage
 	err          error
 	nlconn       *netlink.Conn // netlink socket using NETLINK_NETFILTER protocol.
 	sockOptions  []SockOption
@ -49,6 +49,12 @@ type Conn struct {
 	allocatedIDs uint32
 }

+type netlinkMessage struct {
+	Header netlink.Header
+	Data   []byte
+	rule   *Rule
+}
+
 // ConnOption is an option to change the behavior of the nftables Conn returned by Open.
 type ConnOption func(*Conn)

@ -268,6 +274,11 @@ func (cc *Conn) Flush() error {
 		} else if replyIndex < len(cc.messages) {
 			msg := messages[replyIndex+1]
 			if msg.Header.Sequence == reply.Header.Sequence && msg.Header.Type == reply.Header.Type {
+				// The only messages which set the echo flag are rule create messages.
+				err := cc.messages[replyIndex].rule.handleCreateReply(reply)
+				if err != nil {
+					errs = errors.Join(errs, err)
+				}
 				replyIndex++
 				for replyIndex < len(cc.messages) && cc.messages[replyIndex].Header.Flags&netlink.Echo == 0 {
 					replyIndex++
@ -309,7 +320,7 @@ func (cc *Conn) Flush() error {
 func (cc *Conn) FlushRuleset() {
 	cc.mu.Lock()
 	defer cc.mu.Unlock()
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELTABLE),
 			Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
@ -368,26 +379,30 @@ func (cc *Conn) marshalExpr(fam byte, e expr.Any) []byte {
 	return b
 }

-func batch(messages []netlink.Message) []netlink.Message {
-	batch := []netlink.Message{
-		{
+func batch(messages []netlinkMessage) []netlink.Message {
+	batch := make([]netlink.Message, len(messages)+2)
+	batch[0] = netlink.Message{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType(unix.NFNL_MSG_BATCH_BEGIN),
 			Flags: netlink.Request,
 		},
 		Data: extraHeader(0, unix.NFNL_SUBSYS_NFTABLES),
-		},
 	}

-	batch = append(batch, messages...)
+	for i, msg := range messages {
+		batch[i+1] = netlink.Message{
+			Header: msg.Header,
+			Data:   msg.Data,
+		}
+	}

-	batch = append(batch, netlink.Message{
+	batch[len(messages)+1] = netlink.Message{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType(unix.NFNL_MSG_BATCH_END),
 			Flags: netlink.Request,
 		},
 		Data: extraHeader(0, unix.NFNL_SUBSYS_NFTABLES),
-	})
+	}

 	return batch
 }
--- a/expr/ct.go
+++ b/expr/ct.go
@ -27,6 +27,7 @@ import (
 type CtKey uint32

 // Possible CtKey values.
+// Retrieved from https://git.netfilter.org/libnftnl/tree/include/linux/netfilter/nf_tables.h#n1121
 const (
 	CtKeySTATE      CtKey = unix.NFT_CT_STATE
 	CtKeyDIRECTION  CtKey = unix.NFT_CT_DIRECTION
@ -48,6 +49,13 @@ const (
 	CtKeyZONE       CtKey = unix.NFT_CT_ZONE
 	CtKeyEVENTMASK  CtKey = unix.NFT_CT_EVENTMASK

+	// These values seem to be missing from the unix package
+	CtKeySRCIP  CtKey = 19
+	CtKeyDSTIP  CtKey = 20
+	CtKeySRCIP6 CtKey = 21
+	CtKeyDSTIP6 CtKey = 22
+	CtKeyID     CtKey = 23
+
 	// https://sources.debian.org/src//nftables/0.9.8-3/src/ct.c/?hl=39#L39
 	CtStateBitINVALID     uint32 = 1
 	CtStateBitESTABLISHED uint32 = 2
@ -157,7 +165,7 @@ func (e *Ct) marshalData(fam byte) ([]byte, error) {
 	exprData = append(exprData, regData...)

 	switch e.Key {
-	case CtKeySRC, CtKeyDST, CtKeyPROTOSRC, CtKeyPROTODST:
+	case CtKeySRC, CtKeyDST, CtKeyPROTOSRC, CtKeyPROTODST, CtKeySRCIP, CtKeyDSTIP, CtKeySRCIP6, CtKeyDSTIP6:
 		regData, err = netlink.MarshalAttributes(
 			[]netlink.Attribute{
 				{Type: unix.NFTA_CT_DIRECTION, Data: binaryutil.BigEndian.PutUint32(e.Direction)},
--- a/expr/ct_test.go
+++ b/expr/ct_test.go
@ -46,6 +46,38 @@ func TestCt(t *testing.T) {
 				SourceRegister: true,
 			},
 		},
+		{
+			name: "Unmarshal Ct ip direction original case",
+			ct: Ct{
+				Register:  1,
+				Key:       CtKeySRCIP,
+				Direction: 0,
+			},
+		},
+		{
+			name: "Unmarshal Ct ip direction reply case",
+			ct: Ct{
+				Register:  1,
+				Key:       CtKeySRCIP,
+				Direction: 1,
+			},
+		},
+		{
+			name: "Unmarshal Ct ip6 direction original case",
+			ct: Ct{
+				Register:  1,
+				Key:       CtKeySRCIP6,
+				Direction: 0,
+			},
+		},
+		{
+			name: "Unmarshal Ct ip6 direction reply case",
+			ct: Ct{
+				Register:  1,
+				Key:       CtKeyDSTIP6,
+				Direction: 1,
+			},
+		},
 	}

 	for _, tt := range tests {
--- a/flowtable.go
+++ b/flowtable.go
@ -142,7 +142,7 @@ func (cc *Conn) AddFlowtable(f *Flowtable) *Flowtable {
 		{Type: unix.NLA_F_NESTED | NFTA_FLOWTABLE_HOOK, Data: cc.marshalAttr(hookAttr)},
 	})...)

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | NFT_MSG_NEWFLOWTABLE),
 			Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
@ -162,7 +162,7 @@ func (cc *Conn) DelFlowtable(f *Flowtable) {
 		{Type: NFTA_FLOWTABLE_NAME, Data: []byte(f.Name)},
 	})

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | NFT_MSG_DELFLOWTABLE),
 			Flags: netlink.Request | netlink.Acknowledge,
--- a/internal/nf2go/main.go
+++ b/internal/nf2go/main.go
@ -0,0 +1,299 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/nftables"
+	"github.com/vishvananda/netns"
+)
+
+func main() {
+	args := os.Args[1:]
+	if len(args) != 1 {
+		log.Fatalf("need to specify the file to read the \"nft list ruleset\" dump")
+	}
+
+	filename := args[0]
+
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	// Create a new network namespace
+	ns, err := netns.New()
+	if err != nil {
+		log.Fatalf("netns.New() failed: %v", err)
+	}
+	n, err := nftables.New(nftables.WithNetNSFd(int(ns)))
+	if err != nil {
+		log.Fatalf("nftables.New() failed: %v", err)
+	}
+
+	scriptOutput, err := applyNFTRuleset(filename)
+	if err != nil {
+		log.Fatalf("Failed to apply nftables script: %v\noutput:%s", err, scriptOutput)
+	}
+
+	var buf bytes.Buffer
+	// Helper function to print to the file
+	pf := func(format string, a ...interface{}) {
+		_, err := fmt.Fprintf(&buf, format, a...)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	pf("// Code generated by nft2go. DO NOT EDIT.\n")
+	pf("package main\n\n")
+	pf("import (\n")
+	pf("\t\"fmt\"\n")
+	pf("\t\"log\"\n")
+	pf("\t\"github.com/google/nftables\"\n")
+	pf("\t\"github.com/google/nftables/expr\"\n")
+	pf(")\n\n")
+	pf("func main() {\n")
+	pf("\tn, err:= nftables.New()\n")
+	pf("\tif err!= nil {\n")
+	pf("\t\tlog.Fatal(err)\n")
+	pf("\t}\n\n")
+	pf("\n")
+	pf("\tvar expressions []expr.Any\n")
+	pf("\tvar chain *nftables.Chain\n")
+	pf("\tvar table *nftables.Table\n")
+
+	tables, err := n.ListTables()
+	if err != nil {
+		log.Fatalf("ListTables failed: %v", err)
+	}
+
+	chains, err := n.ListChains()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, table := range tables {
+		log.Printf("processing table: %s", table.Name)
+
+		pf("\ttable = n.AddTable(&nftables.Table{Family: %s,Name: \"%s\"})\n", TableFamilyString(table.Family), table.Name)
+		for _, chain := range chains {
+			if chain.Table.Name != table.Name {
+				continue
+			}
+
+			sets, err := n.GetSets(table)
+			if err != nil {
+				log.Fatal(err)
+			}
+			for _, set := range sets {
+				// TODO datatype and the other options
+				pf("\tn.AddSet(&nftables.Set{\n")
+				pf("\t\tTable: table,\n")
+				pf("\t\tName: \"%s\",\n", set.Name)
+				pf("\t}, nil)\n")
+			}
+
+			pf("\tchain = n.AddChain(&nftables.Chain{Name: \"%s\", Table: table, Type: %s, Hooknum: %s, Priority: %s})\n",
+				chain.Name, ChainTypeString(chain.Type), ChainHookRef(chain.Hooknum), ChainPrioRef(chain.Priority))
+
+			rules, err := n.GetRules(table, chain)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			for _, rule := range rules {
+				pf("\texpressions = []expr.Any{\n")
+				for _, exp := range rule.Exprs {
+					pf("\t\t%#v,\n", exp)
+				}
+				pf("\t\t}\n")
+				pf("\tn.AddRule(&nftables.Rule{\n")
+				pf("\t\tTable: table,\n")
+				pf("\t\tChain: chain,\n")
+				pf("\t\tExprs: expressions,\n")
+				pf("\t})\n")
+			}
+		}
+	}
+
+	pf("\n\tif err:= n.Flush(); err!= nil {\n")
+	pf("\t\tlog.Fatal(err)\n")
+	pf("\t}\n\n")
+	pf("\tfmt.Println(\"nft ruleset applied.\")\n")
+	pf("}\n")
+
+	// Program nftables using your Go code
+	if err := flushNFTRuleset(); err != nil {
+		log.Fatalf("Failed to flush nftables ruleset: %v", err)
+	}
+
+	// Create the output file
+	// Create a temporary directory
+	tempDir, err := ioutil.TempDir("", "nftables_gen")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(tempDir) // Clean up the temporary directory
+
+	// Create the temporary Go file
+	tempGoFile := filepath.Join(tempDir, "nftables_recreate.go")
+	f, err := os.Create(tempGoFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	mw := io.MultiWriter(f, os.Stdout)
+	buf.WriteTo(mw)
+
+	// Format the generated code
+	log.Printf("formating file: %s", tempGoFile)
+	cmd := exec.Command("gofmt", "-w", "-s", tempGoFile)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Fatalf("gofmt error: %v\nOutput: %s", err, output)
+	}
+
+	// Run the generated code
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	log.Printf("executing file: %s", tempGoFile)
+	cmd = exec.CommandContext(ctx, "go", "run", tempGoFile)
+	output, err = cmd.CombinedOutput()
+	if err != nil {
+		log.Fatalf("Execution error: %v\nOutput: %s", err, output)
+	}
+
+	// Retrieve nftables state using nft
+	log.Printf("obtain current ruleset: %s", tempGoFile)
+	actualOutput, err := listNFTRuleset()
+	if err != nil {
+		log.Fatalf("Failed to list nftables ruleset: %v\noutput:%s", err, actualOutput)
+	}
+
+	expectedOutput, err := os.ReadFile(filename)
+	if err != nil {
+		log.Fatalf("Failed to list nftables ruleset: %v\noutput:%s", err, actualOutput)
+	}
+
+	if !compareMultilineStringsIgnoreIndentation(string(expectedOutput), actualOutput) {
+		log.Printf("Expected output:\n%s", string(expectedOutput))
+		log.Printf("Actual output:\n%s", actualOutput)
+
+		log.Fatalf("nftables ruleset mismatch:\n%s", cmp.Diff(string(expectedOutput), actualOutput))
+	}
+
+	if err := flushNFTRuleset(); err != nil {
+		log.Fatalf("Failed to flush nftables ruleset: %v", err)
+	}
+}
+
+func applyNFTRuleset(scriptPath string) (string, error) {
+	cmd := exec.Command("nft", "--debug=netlink", "-f", scriptPath)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(out), err
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+func listNFTRuleset() (string, error) {
+	cmd := exec.Command("nft", "list", "ruleset")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return string(out), err
+	}
+	return strings.TrimSpace(string(out)), nil
+}
+
+func flushNFTRuleset() error {
+	cmd := exec.Command("nft", "flush", "ruleset")
+	return cmd.Run()
+}
+
+func ChainHookRef(hookNum *nftables.ChainHook) string {
+	i := uint32(0)
+	if hookNum != nil {
+		i = uint32(*hookNum)
+	}
+	switch i {
+	case 0:
+		return "nftables.ChainHookPrerouting"
+	case 1:
+		return "nftables.ChainHookInput"
+	case 2:
+		return "nftables.ChainHookForward"
+	case 3:
+		return "nftables.ChainHookOutput"
+	case 4:
+		return "nftables.ChainHookPostrouting"
+	case 5:
+		return "nftables.ChainHookIngress"
+	case 6:
+		return "nftables.ChainHookEgress"
+	}
+	return ""
+}
+
+func ChainPrioRef(priority *nftables.ChainPriority) string {
+	i := int32(0)
+	if priority != nil {
+		i = int32(*priority)
+	}
+	return fmt.Sprintf("nftables.ChainPriorityRef(%d)", i)
+}
+
+func ChainTypeString(chaintype nftables.ChainType) string {
+	switch chaintype {
+	case nftables.ChainTypeFilter:
+		return "nftables.ChainTypeFilter"
+	case nftables.ChainTypeRoute:
+		return "nftables.ChainTypeRoute"
+	case nftables.ChainTypeNAT:
+		return "nftables.ChainTypeNAT"
+	default:
+		return "nftables.ChainTypeFilter"
+	}
+}
+
+func TableFamilyString(family nftables.TableFamily) string {
+	switch family {
+	case nftables.TableFamilyUnspecified:
+		return "nftables.TableFamilyUnspecified"
+	case nftables.TableFamilyINet:
+		return "nftables.TableFamilyINet"
+	case nftables.TableFamilyIPv4:
+		return "nftables.TableFamilyIPv4"
+	case nftables.TableFamilyIPv6:
+		return "nftables.TableFamilyIPv6"
+	case nftables.TableFamilyARP:
+		return "nftables.TableFamilyARP"
+	case nftables.TableFamilyNetdev:
+		return "nftables.TableFamilyNetdev"
+	case nftables.TableFamilyBridge:
+		return "nftables.TableFamilyBridge"
+	default:
+		return "nftables.TableFamilyIPv4"
+	}
+}
+
+func compareMultilineStringsIgnoreIndentation(str1, str2 string) bool {
+	// Remove all indentation from both strings
+	re := regexp.MustCompile(`(?m)^\s+`)
+	str1 = re.ReplaceAllString(str1, "")
+	str2 = re.ReplaceAllString(str2, "")
+
+	return str1 == str2
+}
--- a/internal/nf2go/rules.txt
+++ b/internal/nf2go/rules.txt
@ -0,0 +1,550 @@
+table ip nat {
+	chain PREROUTING {
+		type nat hook prerouting priority dstnat; policy accept;
+		 counter packets 769 bytes 46239 jump KUBE-SERVICES
+		ip daddr 192.168.8.1 counter packets 0 bytes 0 jump DOCKER_OUTPUT
+	}
+
+	chain INPUT {
+		type nat hook input priority 100; policy accept;
+	}
+
+	chain OUTPUT {
+		type nat hook output priority -100; policy accept;
+		 counter packets 245348 bytes 14721139 jump KUBE-SERVICES
+		ip daddr 192.168.8.1 counter packets 122 bytes 9151 jump DOCKER_OUTPUT
+	}
+
+	chain POSTROUTING {
+		type nat hook postrouting priority srcnat; policy accept;
+		 counter packets 246110 bytes 14766910 jump KUBE-POSTROUTING
+		ip daddr 192.168.8.1 counter packets 0 bytes 0 jump DOCKER_POSTROUTING
+	}
+
+	chain DOCKER_OUTPUT {
+		ip daddr 192.168.8.1 tcp dport 53 counter packets 0 bytes 0 dnat to 127.0.0.11:36405
+		ip daddr 192.168.8.1 udp dport 53 counter packets 122 bytes 9151 dnat to 127.0.0.11:39066
+	}
+
+	chain DOCKER_POSTROUTING {
+		ip saddr 127.0.0.11 tcp sport 36405 counter packets 0 bytes 0 snat to 192.168.8.1:53
+		ip saddr 127.0.0.11 udp sport 39066 counter packets 0 bytes 0 snat to 192.168.8.1:53
+	}
+
+	chain KUBE-KUBELET-CANARY {
+	}
+
+	chain KUBE-PROXY-CANARY {
+	}
+
+	chain KUBE-SERVICES {
+		meta l4proto tcp ip daddr 10.96.0.10  tcp dport 9153 counter packets 0 bytes 0 jump KUBE-SVC-JD5MR3NA4I4DYORP
+		meta l4proto udp ip daddr 10.96.0.10  udp dport 53 counter packets 0 bytes 0 jump KUBE-SVC-TCOU7JCQXEZGVUNU
+		meta l4proto tcp ip daddr 10.96.106.185  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-D5JKTLXOFYHV5HQZ
+		meta l4proto tcp ip daddr 10.96.70.203  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-IB3WK5BQ64FMB5FP
+		meta l4proto tcp ip daddr 10.96.0.10  tcp dport 53 counter packets 13 bytes 780 jump KUBE-SVC-ERIFXISQEP7F7OF4
+		meta l4proto tcp ip daddr 10.96.86.60  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-QUBDBT4PCRU7S2PI
+		meta l4proto tcp ip daddr 10.96.184.88  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-ZD23KKVZJDKFKTCE
+		meta l4proto tcp ip daddr 10.96.149.162  tcp dport 443 counter packets 0 bytes 0 jump KUBE-SVC-WHNIZNLB5XFXIX2C
+		meta l4proto tcp ip daddr 10.96.225.221  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-ROH4UCJ7RVN2OSM4
+		meta l4proto tcp ip daddr 10.96.50.119  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-SB7WEE53EMIXFNKY
+		meta l4proto tcp ip daddr 10.96.230.205  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-OJLEMCF5KYSTXAAJ
+		meta l4proto tcp ip daddr 10.96.149.162  tcp dport 15014 counter packets 0 bytes 0 jump KUBE-SVC-XHUBMW47Y5G3ICIS
+		meta l4proto tcp ip daddr 10.96.83.127  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-4MYBDLPZ2DFGC5Z6
+		meta l4proto tcp ip daddr 10.96.245.249  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-53SQRANQXVHTJ6HK
+		meta l4proto tcp ip daddr 10.96.85.31  tcp dport 9090 counter packets 0 bytes 0 jump KUBE-SVC-VVO7BBXOSCJQDQML
+		meta l4proto tcp ip daddr 10.96.0.1  tcp dport 443 counter packets 0 bytes 0 jump KUBE-SVC-NPX46M4PTMTKRN6Y
+		meta l4proto tcp ip daddr 10.96.113.49  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-SVC-COV23IKAKYWND6VU
+		meta l4proto tcp ip daddr 10.96.149.162  tcp dport 15010 counter packets 0 bytes 0 jump KUBE-SVC-NVNLZVDQSGQUD3NM
+		meta l4proto tcp ip daddr 10.96.231.15  tcp dport 3000 counter packets 0 bytes 0 jump KUBE-SVC-XUJLWDDTZEWKLHU6
+		meta l4proto tcp ip daddr 10.96.149.162  tcp dport 15012 counter packets 2 bytes 120 jump KUBE-SVC-CG3LQLBYYHBKATGN
+		 fib daddr type local counter packets 329 bytes 19740 jump KUBE-NODEPORTS
+	}
+
+	chain KUBE-POSTROUTING {
+		meta mark & 0x00004000 != 0x00004000 counter packets 1593 bytes 95580 return
+		counter packets 13 bytes 780 meta mark set mark xor 0x4000
+		 counter packets 13 bytes 780 masquerade fully-random
+	}
+
+	chain KUBE-NODEPORTS {
+		meta l4proto tcp  tcp dport 30207 counter packets 0 bytes 0 jump KUBE-EXT-VVO7BBXOSCJQDQML
+		meta l4proto tcp  tcp dport 31182 counter packets 0 bytes 0 jump KUBE-EXT-XUJLWDDTZEWKLHU6
+	}
+
+	chain KUBE-MARK-MASQ {
+		counter packets 13 bytes 780 meta mark set mark or 0x4000
+	}
+
+	chain KUBE-SVC-COV23IKAKYWND6VU {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.113.49  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-UYFG3BP6SBY2ENL5
+	}
+
+	chain KUBE-SVC-OJLEMCF5KYSTXAAJ {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.230.205  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-4Y5RNE5AMASHQ2OZ
+	}
+
+	chain KUBE-SVC-4MYBDLPZ2DFGC5Z6 {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.83.127  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-VLSRDM63ATDZAA3A
+	}
+
+	chain KUBE-SVC-53SQRANQXVHTJ6HK {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.245.249  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 meta random & 2147483647 < 715827883 counter packets 0 bytes 0 jump KUBE-SEP-AYXBHI7HU6SF34DI
+		 meta random & 2147483647 < 1073741824 counter packets 0 bytes 0 jump KUBE-SEP-PBXQV3T6XQVEVBGL
+		 counter packets 0 bytes 0 jump KUBE-SEP-WLEZNEPJCJLP5SOM
+	}
+
+	chain KUBE-EXT-VVO7BBXOSCJQDQML {
+		 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		counter packets 0 bytes 0 jump KUBE-SVC-VVO7BBXOSCJQDQML
+	}
+
+	chain KUBE-SVC-VVO7BBXOSCJQDQML {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.85.31  tcp dport 9090 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-6ZOGXI2ZCDJV5G4O
+	}
+
+	chain KUBE-SVC-NPX46M4PTMTKRN6Y {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.0.1  tcp dport 443 counter packets 29 bytes 1740 jump KUBE-MARK-MASQ
+		 counter packets 30 bytes 1800 jump KUBE-SEP-B5DUEXKFRYN46BFH
+	}
+
+	chain KUBE-SEP-B5DUEXKFRYN46BFH {
+		ip saddr 192.168.8.4  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 30 bytes 1800 dnat to 192.168.8.4:6443
+	}
+
+	chain KUBE-EXT-XUJLWDDTZEWKLHU6 {
+		 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		counter packets 0 bytes 0 jump KUBE-SVC-XUJLWDDTZEWKLHU6
+	}
+
+	chain KUBE-SVC-XUJLWDDTZEWKLHU6 {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.231.15  tcp dport 3000 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-AEOIAT7KXYABTON5
+	}
+
+	chain KUBE-SVC-D5JKTLXOFYHV5HQZ {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.106.185  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-IRX4MQBX4KRVQAZU
+	}
+
+	chain KUBE-SVC-IB3WK5BQ64FMB5FP {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.70.203  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-ME7B5OZS3EMRJJUS
+	}
+
+	chain KUBE-SVC-SB7WEE53EMIXFNKY {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.50.119  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-UBXSLXFPFJOARYRN
+	}
+
+	chain KUBE-SVC-QUBDBT4PCRU7S2PI {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.86.60  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-ENYDTZIVOME6TBBV
+	}
+
+	chain KUBE-SVC-ZD23KKVZJDKFKTCE {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.184.88  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-5KXO57AZTMNCDCVX
+	}
+
+	chain KUBE-SVC-ROH4UCJ7RVN2OSM4 {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.225.221  tcp dport 9080 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-YMBKJHKN7Y2F6666
+	}
+
+	chain KUBE-SEP-YMBKJHKN7Y2F6666 {
+		ip saddr 10.244.1.189  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.189:9080
+	}
+
+	chain KUBE-SEP-5KXO57AZTMNCDCVX {
+		ip saddr 10.244.1.189  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.189:9080
+	}
+
+	chain KUBE-SEP-UBXSLXFPFJOARYRN {
+		ip saddr 10.244.1.108  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.108:9080
+	}
+
+	chain KUBE-SEP-ENYDTZIVOME6TBBV {
+		ip saddr 10.244.1.108  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.108:9080
+	}
+
+	chain KUBE-SEP-6ZOGXI2ZCDJV5G4O {
+		ip saddr 10.244.1.9  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.9:9090
+	}
+
+	chain KUBE-SEP-UYFG3BP6SBY2ENL5 {
+		ip saddr 10.244.1.215  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.215:9080
+	}
+
+	chain KUBE-SEP-4Y5RNE5AMASHQ2OZ {
+		ip saddr 10.244.2.98  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.2.98:9080
+	}
+
+	chain KUBE-SEP-VLSRDM63ATDZAA3A {
+		ip saddr 10.244.1.237  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.237:9080
+	}
+
+	chain KUBE-SEP-AYXBHI7HU6SF34DI {
+		ip saddr 10.244.1.131  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.131:9080
+	}
+
+	chain KUBE-SEP-PBXQV3T6XQVEVBGL {
+		ip saddr 10.244.1.215  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.215:9080
+	}
+
+	chain KUBE-SEP-WLEZNEPJCJLP5SOM {
+		ip saddr 10.244.2.98  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.2.98:9080
+	}
+
+	chain KUBE-SEP-AEOIAT7KXYABTON5 {
+		ip saddr 10.244.1.13  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.13:3000
+	}
+
+	chain KUBE-SEP-IRX4MQBX4KRVQAZU {
+		ip saddr 10.244.1.237  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.237:9080
+	}
+
+	chain KUBE-SEP-ME7B5OZS3EMRJJUS {
+		ip saddr 10.244.1.131  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.1.131:9080
+	}
+
+	chain KUBE-SVC-WHNIZNLB5XFXIX2C {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.149.162  tcp dport 443 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-22OJIEOUHUS2VH36
+	}
+
+	chain KUBE-SEP-22OJIEOUHUS2VH36 {
+		ip saddr 10.244.2.88  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.2.88:15017
+	}
+
+	chain KUBE-SVC-XHUBMW47Y5G3ICIS {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.149.162  tcp dport 15014 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-N6BCKY3MRNRV2ZJ2
+	}
+
+	chain KUBE-SEP-N6BCKY3MRNRV2ZJ2 {
+		ip saddr 10.244.2.88  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.2.88:15014
+	}
+
+	chain KUBE-SVC-CG3LQLBYYHBKATGN {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.149.162  tcp dport 15012 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 259 bytes 15540 jump KUBE-SEP-E675BOVPZS3XINT6
+	}
+
+	chain KUBE-SEP-E675BOVPZS3XINT6 {
+		ip saddr 10.244.2.88  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 259 bytes 15540 dnat to 10.244.2.88:15012
+	}
+
+	chain KUBE-SVC-NVNLZVDQSGQUD3NM {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.149.162  tcp dport 15010 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 counter packets 0 bytes 0 jump KUBE-SEP-E2OCVXQ5RANXZKNO
+	}
+
+	chain KUBE-SEP-E2OCVXQ5RANXZKNO {
+		ip saddr 10.244.2.88  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.2.88:15010
+	}
+
+	chain KUBE-SVC-ERIFXISQEP7F7OF4 {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.0.10  tcp dport 53 counter packets 1718 bytes 103080 jump KUBE-MARK-MASQ
+		 meta random & 2147483647 < 1073741824 counter packets 875 bytes 52500 jump KUBE-SEP-IH2TMTJLEHQTEXG4
+		 counter packets 843 bytes 50580 jump KUBE-SEP-QHHO2BBHA2W6ABVA
+	}
+
+	chain KUBE-SEP-IH2TMTJLEHQTEXG4 {
+		ip saddr 10.244.0.44  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 875 bytes 52500 dnat to 10.244.0.44:53
+	}
+
+	chain KUBE-SVC-JD5MR3NA4I4DYORP {
+		meta l4proto tcp ip saddr != 10.244.0.0/16 ip daddr 10.96.0.10  tcp dport 9153 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 meta random & 2147483647 < 1073741824 counter packets 0 bytes 0 jump KUBE-SEP-KTQ44RRS25IYXDAU
+		 counter packets 0 bytes 0 jump KUBE-SEP-SWQQGKEGIJFNRJRL
+	}
+
+	chain KUBE-SEP-KTQ44RRS25IYXDAU {
+		ip saddr 10.244.0.44  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.0.44:9153
+	}
+
+	chain KUBE-SVC-TCOU7JCQXEZGVUNU {
+		meta l4proto udp ip saddr != 10.244.0.0/16 ip daddr 10.96.0.10  udp dport 53 counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		 meta random & 2147483647 < 1073741824 counter packets 2 bytes 171 jump KUBE-SEP-YGFHBW2DM6N3IEK3
+		 counter packets 0 bytes 0 jump KUBE-SEP-W3GSBK4IMEBEFHPJ
+	}
+
+	chain KUBE-SEP-YGFHBW2DM6N3IEK3 {
+		ip saddr 10.244.0.44  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto udp   counter packets 2 bytes 171 dnat to 10.244.0.44:53
+	}
+
+	chain KUBE-SEP-QHHO2BBHA2W6ABVA {
+		ip saddr 10.244.0.63  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 843 bytes 50580 dnat to 10.244.0.63:53
+	}
+
+	chain KUBE-SEP-SWQQGKEGIJFNRJRL {
+		ip saddr 10.244.0.63  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto tcp   counter packets 0 bytes 0 dnat to 10.244.0.63:9153
+	}
+
+	chain KUBE-SEP-W3GSBK4IMEBEFHPJ {
+		ip saddr 10.244.0.63  counter packets 0 bytes 0 jump KUBE-MARK-MASQ
+		meta l4proto udp   counter packets 0 bytes 0 dnat to 10.244.0.63:53
+	}
+}
+table ip mangle {
+	chain KUBE-IPTABLES-HINT {
+	}
+
+	chain KUBE-KUBELET-CANARY {
+	}
+
+	chain KUBE-PROXY-CANARY {
+	}
+}
+table ip filter {
+	chain KUBE-FIREWALL {
+		ip saddr != 127.0.0.0/8 ip daddr 127.0.0.0/8  ct status dnat counter packets 0 bytes 0 drop
+	}
+
+	chain OUTPUT {
+		type filter hook output priority filter; policy accept;
+		ct state new  counter packets 245383 bytes 14723644 jump KUBE-PROXY-FIREWALL
+		ct state new  counter packets 245383 bytes 14723644 jump KUBE-SERVICES
+		counter packets 3043981 bytes 2633914697 jump KUBE-FIREWALL
+	}
+
+	chain INPUT {
+		type filter hook input priority filter; policy accept;
+		ct state new  counter packets 47332 bytes 2840200 jump KUBE-PROXY-FIREWALL
+		 counter packets 3210147 bytes 435991279 jump KUBE-NODEPORTS
+		ct state new  counter packets 47332 bytes 2840200 jump KUBE-EXTERNAL-SERVICES
+		counter packets 3210791 bytes 436579809 jump KUBE-FIREWALL
+	}
+
+	chain KUBE-KUBELET-CANARY {
+	}
+
+	chain KUBE-PROXY-CANARY {
+	}
+
+	chain KUBE-EXTERNAL-SERVICES {
+	}
+
+	chain FORWARD {
+		type filter hook forward priority filter; policy accept;
+		ct state new  counter packets 774 bytes 46491 jump KUBE-PROXY-FIREWALL
+		 counter packets 287624 bytes 35538980 jump KUBE-FORWARD
+		ct state new  counter packets 774 bytes 46491 jump KUBE-SERVICES
+		ct state new  counter packets 774 bytes 46491 jump KUBE-EXTERNAL-SERVICES
+	}
+
+	chain KUBE-NODEPORTS {
+	}
+
+	chain KUBE-SERVICES {
+		meta l4proto tcp ip daddr 10.96.65.65  tcp dport 80 counter packets 0 bytes 0 reject
+	}
+
+	chain KUBE-FORWARD {
+		 meta mark & 0x00004000 == 0x00004000 counter packets 0 bytes 0 accept
+		 ct state related,established counter packets 1866 bytes 233783 accept
+	}
+
+	chain KUBE-PROXY-FIREWALL {
+	}
+}
+table ip6 mangle {
+	chain KUBE-IPTABLES-HINT {
+	}
+
+	chain KUBE-KUBELET-CANARY {
+	}
+
+	chain KUBE-PROXY-CANARY {
+	}
+}
+table ip6 nat {
+	chain KUBE-KUBELET-CANARY {
+	}
+
+	chain POSTROUTING {
+		type nat hook postrouting priority srcnat; policy accept;
+		 counter packets 0 bytes 0 jump KUBE-POSTROUTING
+	}
+
+	chain KUBE-PROXY-CANARY {
+	}
+
+	chain KUBE-SERVICES {
+		ip6 daddr != ::1  fib daddr type local counter packets 0 bytes 0 jump KUBE-NODEPORTS
+	}
+
+	chain OUTPUT {
+		type nat hook output priority -100; policy accept;
+		 counter packets 0 bytes 0 jump KUBE-SERVICES
+	}
+
+	chain PREROUTING {
+		type nat hook prerouting priority dstnat; policy accept;
+		 counter packets 0 bytes 0 jump KUBE-SERVICES
+	}
+
+	chain KUBE-POSTROUTING {
+		meta mark & 0x00004000 != 0x00004000 counter packets 0 bytes 0 return
+		counter packets 0 bytes 0 meta mark set mark xor 0x4000
+		 counter packets 0 bytes 0 
+	}
+
+	chain KUBE-NODEPORTS {
+	}
+
+	chain KUBE-MARK-MASQ {
+		counter packets 0 bytes 0 meta mark set mark or 0x4000
+	}
+}
+table ip6 filter {
+	chain KUBE-KUBELET-CANARY {
+	}
+
+	chain KUBE-PROXY-CANARY {
+	}
+
+	chain KUBE-EXTERNAL-SERVICES {
+	}
+
+	chain INPUT {
+		type filter hook input priority filter; policy accept;
+		counter packets 8 bytes 552 jump KUBE-FIREWALL
+		ct state new  counter packets 0 bytes 0 jump KUBE-PROXY-FIREWALL
+		 counter packets 8 bytes 552 jump KUBE-NODEPORTS
+		ct state new  counter packets 0 bytes 0 jump KUBE-EXTERNAL-SERVICES
+	}
+
+	chain FORWARD {
+		type filter hook forward priority filter; policy accept;
+		ct state new  counter packets 0 bytes 0 jump KUBE-PROXY-FIREWALL
+		 counter packets 0 bytes 0 jump KUBE-FORWARD
+		ct state new  counter packets 0 bytes 0 jump KUBE-SERVICES
+		ct state new  counter packets 0 bytes 0 jump KUBE-EXTERNAL-SERVICES
+	}
+
+	chain KUBE-NODEPORTS {
+	}
+
+	chain KUBE-SERVICES {
+	}
+
+	chain OUTPUT {
+		type filter hook output priority filter; policy accept;
+		counter packets 139 bytes 7888 jump KUBE-FIREWALL
+		ct state new  counter packets 0 bytes 0 jump KUBE-PROXY-FIREWALL
+		ct state new  counter packets 0 bytes 0 jump KUBE-SERVICES
+	}
+
+	chain KUBE-FORWARD {
+		 meta mark & 0x00004000 == 0x00004000 counter packets 0 bytes 0 accept
+		 ct state related,established counter packets 0 bytes 0 accept
+	}
+
+	chain KUBE-PROXY-FIREWALL {
+	}
+
+	chain KUBE-FIREWALL {
+	}
+}
+table inet kindnet-network-policies {
+	set podips-v4 {
+		type ipv4_addr
+	}
+
+	set podips-v6 {
+		type ipv6_addr
+	}
+
+	chain postrouting {
+		type filter hook postrouting priority srcnat - 5; policy accept;
+		udp dport 53 accept
+		meta l4proto ipv6-icmp accept
+		meta skuid 0 accept
+		ct state established,related accept
+		ip saddr @podips-v4 queue flags bypass to 102
+		ip daddr @podips-v4 queue flags bypass to 102
+		ip6 saddr @podips-v6 queue flags bypass to 102
+		ip6 daddr @podips-v6 queue flags bypass to 102
+	}
+
+	chain prerouting {
+		type filter hook prerouting priority dstnat + 5; policy accept;
+		meta l4proto != udp accept
+		udp dport != 53 accept
+		ip saddr @podips-v4 queue flags bypass to 102
+		ip daddr @podips-v4 queue flags bypass to 102
+		ip6 saddr @podips-v6 queue flags bypass to 102
+		ip6 daddr @podips-v6 queue flags bypass to 102
+	}
+}
+table inet kindnet-dnscache {
+	set set-v4-nameservers {
+		type ipv4_addr
+		elements = { 10.96.0.10 }
+	}
+
+	chain prerouting {
+		type filter hook prerouting priority raw; policy accept;
+		ip saddr 10.244.2.0/24 ip daddr @set-v4-nameservers udp dport 53 queue flags bypass to 103
+	}
+
+	chain output {
+		type filter hook output priority raw; policy accept;
+		meta mark 0x0000006e udp sport 53 notrack
+	}
+}
+table inet kindnet-ipmasq {
+	set noMasqV4 {
+		type ipv4_addr
+		flags interval
+		auto-merge
+		elements = { 10.244.0.0/24, 10.244.1.0/24,
+			     10.244.2.0/24 }
+	}
+
+	set noMasqV6 {
+		type ipv6_addr
+		flags interval
+		auto-merge
+	}
+
+	chain postrouting {
+		type nat hook postrouting priority srcnat - 10; policy accept;
+		ct state established,related accept
+		fib saddr type local accept
+		ip daddr @noMasqV4 accept
+		ip6 daddr @noMasqV6 accept
+		masquerade
+	}
+}
--- a/internal/nf2go/rules_simple.txt
+++ b/internal/nf2go/rules_simple.txt
@ -0,0 +1,17 @@
+table ip filter {
+	chain output {
+		type filter hook output priority 100; policy accept;
+	}
+
+	chain input {
+		type filter hook input priority 0; policy accept;
+		iifname "lan0" accept
+		iifname "wan0" drop
+	}
+
+	chain forward {
+		type filter hook forward priority 0; policy drop;
+		iifname "lan0" oifname "wan0" accept
+		iifname "wan0" oifname "lan0" ct state related,established accept
+	}
+}
--- a/internal/nftest/nftest.go
+++ b/internal/nftest/nftest.go
@ -8,7 +8,9 @@ import (
 	"testing"

 	"github.com/google/nftables"
+	"github.com/google/nftables/binaryutil"
 	"github.com/mdlayher/netlink"
+	"golang.org/x/sys/unix"
 )

 // Recorder provides an nftables connection that does not send to the Linux
@ -21,6 +23,7 @@ type Recorder struct {
 // Conn opens an nftables connection that records netlink messages into the
 // Recorder.
 func (r *Recorder) Conn() (*nftables.Conn, error) {
+	nextHandle := uint64(1)
 	return nftables.New(nftables.WithTestDial(
 		func(req []netlink.Message) ([]netlink.Message, error) {
 			r.requests = append(r.requests, req...)
@ -30,6 +33,14 @@ func (r *Recorder) Conn() (*nftables.Conn, error) {
 			for _, msg := range req {
 				if msg.Header.Flags&netlink.Echo != 0 {
 					data := append([]byte{}, msg.Data...)
+					switch msg.Header.Type {
+					case netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWRULE):
+						attrs, _ := netlink.MarshalAttributes([]netlink.Attribute{
+							{Type: unix.NFTA_RULE_HANDLE, Data: binaryutil.BigEndian.PutUint64(nextHandle)},
+						})
+						nextHandle++
+						data = append(data, attrs...)
+					}
 					replies = append(replies, netlink.Message{
 						Header: msg.Header,
 						Data:   data,
--- a/nftables_test.go
+++ b/nftables_test.go
@ -80,6 +80,7 @@ func linediff(a, b string) string {
 }

 func expectMessages(t *testing.T, want [][]byte) nftables.ConnOption {
+	nextHandle := uint64(1)
 	return nftables.WithTestDial(func(req []netlink.Message) ([]netlink.Message, error) {
 		var replies []netlink.Message
 		for idx, msg := range req {
@ -103,6 +104,14 @@ func expectMessages(t *testing.T, want [][]byte) nftables.ConnOption {
 			// Generate replies.
 			if msg.Header.Flags&netlink.Echo != 0 {
 				data := append([]byte{}, msg.Data...)
+				switch msg.Header.Type {
+				case netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWRULE):
+					attrs, _ := netlink.MarshalAttributes([]netlink.Attribute{
+						{Type: unix.NFTA_RULE_HANDLE, Data: binaryutil.BigEndian.PutUint64(nextHandle)},
+					})
+					nextHandle++
+					data = append(data, attrs...)
+				}
 				replies = append(replies, netlink.Message{
 					Header: msg.Header,
 					Data:   data,
@ -316,7 +325,7 @@ func TestRuleHandle(t *testing.T) {
 	}

 	for _, tt := range tests {
-		for _, afterFlush := range []bool{false} {
+		for _, afterFlush := range []bool{false, true} {
 			flushName := map[bool]string{
 				false: "-before-flush",
 				true:  "-after-flush",
--- a/obj.go
+++ b/obj.go
@ -124,7 +124,7 @@ func (cc *Conn) AddObj(o Obj) Obj {
 		attrs = append(attrs, netlink.Attribute{Type: unix.NLA_F_NESTED | unix.NFTA_OBJ_DATA, Data: data})
 	}

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWOBJ),
 			Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
@ -146,7 +146,7 @@ func (cc *Conn) DeleteObject(o Obj) {
 	data := cc.marshalAttr(attrs)
 	data = append(data, cc.marshalAttr([]netlink.Attribute{{Type: unix.NLA_F_NESTED | unix.NFTA_OBJ_DATA}})...)

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELOBJ),
 			Flags: netlink.Request | netlink.Acknowledge,
--- a/rule.go
+++ b/rule.go
@ -48,10 +48,13 @@ const (
 type Rule struct {
 	Table *Table
 	Chain *Chain
-	// Handle identifies an existing Rule.
+	// Handle identifies an existing Rule. For a new Rule, this field is set
+	// during the Flush() in which the rule is committed. Make sure to not access
+	// this field concurrently with this Flush() to avoid data races.
 	Handle uint64
 	// ID is an identifier for a new Rule, which is assigned by
 	// AddRule/InsertRule, and only valid before the rule is committed by Flush().
+	// The field is set to 0 during Flush().
 	ID uint32
 	// Position can be set to the Handle of another Rule to insert the new Rule
 	// before (InsertRule) or after (AddRule) the existing rule.
@ -59,10 +62,8 @@ type Rule struct {
 	// PositionID can be set to the ID of another Rule, same as Position, for when
 	// the existing rule is not yet committed.
 	PositionID uint32
-	// The list of possible flags are specified by nftnl_rule_attr, see
-	// https://git.netfilter.org/libnftnl/tree/include/libnftnl/rule.h#n21
-	// Current nftables go implementation supports only
-	// NFTNL_RULE_POSITION flag for setting rule at position 0
+	// Deprecated: The feature for which this field was added never worked.
+	// The field may be removed in a later version.
 	Flags    uint32
 	Exprs    []expr.Any
 	UserData []byte
@ -171,16 +172,19 @@ func (cc *Conn) newRule(r *Rule, op ruleOperation) *Rule {
 	}

 	var flags netlink.HeaderFlags
+	var ruleRef *Rule
 	switch op {
 	case operationAdd:
 		flags = netlink.Request | netlink.Acknowledge | netlink.Create | netlink.Echo | netlink.Append
+		ruleRef = r
 	case operationInsert:
 		flags = netlink.Request | netlink.Acknowledge | netlink.Create | netlink.Echo
+		ruleRef = r
 	case operationReplace:
 		flags = netlink.Request | netlink.Acknowledge | netlink.Replace
 	}

-	if r.Position != 0 || (r.Flags&(1<<unix.NFTA_RULE_POSITION)) != 0 {
+	if r.Position != 0 {
 		msgData = append(msgData, cc.marshalAttr([]netlink.Attribute{
 			{Type: unix.NFTA_RULE_POSITION, Data: binaryutil.BigEndian.PutUint64(r.Position)},
 		})...)
@ -190,17 +194,42 @@ func (cc *Conn) newRule(r *Rule, op ruleOperation) *Rule {
 		})...)
 	}

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  newRuleHeaderType,
 			Flags: flags,
 		},
 		Data: append(extraHeader(uint8(r.Table.Family), 0), msgData...),
+		rule: ruleRef,
 	})

 	return r
 }

+func (r *Rule) handleCreateReply(reply netlink.Message) error {
+	ad, err := netlink.NewAttributeDecoder(reply.Data[4:])
+	if err != nil {
+		return err
+	}
+	ad.ByteOrder = binary.BigEndian
+	var handle uint64
+	for ad.Next() {
+		switch ad.Type() {
+		case unix.NFTA_RULE_HANDLE:
+			handle = ad.Uint64()
+		}
+	}
+	if ad.Err() != nil {
+		return ad.Err()
+	}
+	if handle == 0 {
+		return fmt.Errorf("missing rule handle in create reply")
+	}
+	r.Handle = handle
+	r.ID = 0
+	return nil
+}
+
 func (cc *Conn) ReplaceRule(r *Rule) *Rule {
 	return cc.newRule(r, operationReplace)
 }
@ -247,7 +276,7 @@ func (cc *Conn) DelRule(r *Rule) error {
 	}
 	flags := netlink.Request | netlink.Acknowledge

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  delRuleHeaderType,
 			Flags: flags,
--- a/set.go
+++ b/set.go
@ -506,7 +506,7 @@ func (cc *Conn) appendElemList(s *Set, vals []SetElement, hdrType uint16) error
 			{Type: unix.NFTA_SET_ELEM_LIST_ELEMENTS | unix.NLA_F_NESTED, Data: encodedElem},
 		}

-		cc.messages = append(cc.messages, netlink.Message{
+		cc.messages = append(cc.messages, netlinkMessage{
 			Header: netlink.Header{
 				Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | hdrType),
 				Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
@ -680,7 +680,7 @@ func (cc *Conn) AddSet(s *Set, vals []SetElement) error {
 		tableInfo = append(tableInfo, netlink.Attribute{Type: unix.NLA_F_NESTED | NFTA_SET_ELEM_EXPRESSIONS, Data: data})
 	}

-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWSET),
 			Flags: netlink.Request | netlink.Acknowledge | netlink.Create,
@ -700,7 +700,7 @@ func (cc *Conn) DelSet(s *Set) {
 		{Type: unix.NFTA_SET_TABLE, Data: []byte(s.Table.Name + "\x00")},
 		{Type: unix.NFTA_SET_NAME, Data: []byte(s.Name + "\x00")},
 	})
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELSET),
 			Flags: netlink.Request | netlink.Acknowledge,
@ -717,7 +717,7 @@ func (cc *Conn) FlushSet(s *Set) {
 		{Type: unix.NFTA_SET_TABLE, Data: []byte(s.Table.Name + "\x00")},
 		{Type: unix.NFTA_SET_NAME, Data: []byte(s.Name + "\x00")},
 	})
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELSETELEM),
 			Flags: netlink.Request | netlink.Acknowledge,
--- a/set_test.go
+++ b/set_test.go
@ -254,7 +254,10 @@ func TestMarshalSet(t *testing.T) {
 			}
 			msg := c.messages[connMsgSetIdx]

-			nset, err := setsFromMsg(msg)
+			nset, err := setsFromMsg(netlink.Message{
+				Header: msg.Header,
+				Data:   msg.Data,
+			})
 			if err != nil {
 				t.Fatalf("setsFromMsg() error: %+v", err)
 			}
--- a/table.go
+++ b/table.go
@ -57,7 +57,7 @@ func (cc *Conn) DelTable(t *Table) {
 		{Type: unix.NFTA_TABLE_NAME, Data: []byte(t.Name + "\x00")},
 		{Type: unix.NFTA_TABLE_FLAGS, Data: []byte{0, 0, 0, 0}},
 	})
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELTABLE),
 			Flags: netlink.Request | netlink.Acknowledge,
@ -73,7 +73,7 @@ func (cc *Conn) addTable(t *Table, flag netlink.HeaderFlags) *Table {
 		{Type: unix.NFTA_TABLE_NAME, Data: []byte(t.Name + "\x00")},
 		{Type: unix.NFTA_TABLE_FLAGS, Data: []byte{0, 0, 0, 0}},
 	})
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWTABLE),
 			Flags: netlink.Request | netlink.Acknowledge | flag,
@ -103,7 +103,7 @@ func (cc *Conn) FlushTable(t *Table) {
 	data := cc.marshalAttr([]netlink.Attribute{
 		{Type: unix.NFTA_RULE_TABLE, Data: []byte(t.Name + "\x00")},
 	})
-	cc.messages = append(cc.messages, netlink.Message{
+	cc.messages = append(cc.messages, netlinkMessage{
 		Header: netlink.Header{
 			Type:  netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELRULE),
 			Flags: netlink.Request | netlink.Acknowledge,
Author	SHA1	Message	Date
Antonio Ojea	89f7b14695	Merge `1e48c1007e` into `8095c51678`	2025-03-26 10:02:38 +01:00
Jan Schär	8095c51678	Deprecate Rule.Flags field (#304 ) The functionality added in `a46119e5` never worked: If you set NFTA_RULE_POSITION to 0, the kernel will just complain that a rule with this handle does not exist. This removes the broken functionality, leaving the field deprecated. The right way to insert a rule at the beginning of a chain is to use InsertRule and leave Position unset. https://github.com/google/nftables/issues/126 mentions that the nft command allows referring to rules by index. But here is a quote from the nft manpage: > The add and insert commands support an optional location specifier, > which is either a handle or the index (starting at zero) of an > existing rule. Internally, rule locations are always identified by > handle and the translation from index happens in userspace. In other words, identifiying rules by index is a feature of nft and is not part of the kernel interface.	2025-03-26 09:57:20 +01:00
Nick Garlis	e0bb410d54	Add missing ct keys (#310 ) The following keys were missing from the CtKey type: - `NFT_CT_SRC_IP` - `NFT_CT_DST_IP` - `NFT_CT_SRC_IP6` - `NFT_CT_DST_IP6` - `NFT_CT_ID` Since they also seem to be missing from the unix package, their actual values were added.	2025-03-26 09:54:08 +01:00
Jan Schär	207a46354c	Set rule handle during flush (#299 ) This change makes it possible to delete rules after inserting them, without needing to query the rules first. Additionally, this allows positioning a new rule next to an existing rule. There are two ways to refer to a rule: Either by ID or by handle. The ID is assigned by userspace, and is only valid within a transaction, so it can only be used before the flush. The handle is assigned by the kernel when the transaction is committed, and can thus only be used after the flush. We thus need to set an ID on each newly created rule, and retrieve the handle of the rule during the flush. I extended the message struct with a pointer to the Rule which the message creates. This allows calling the reply handler callback which sets the handle. I updated tests to add a handle to generated replies for the NFT_MSG_NEWRULE messages.	2025-03-26 09:24:33 +01:00
Antonio Ojea	1e48c1007e	nf2go: convert nftables rules to golang code One of the biggest barriers to adopt the netlink format for nftables is the complexity of writing bytecode. This commits adds a tool that allows to take an nftables dump and generate the corresponding golang code and validating that the generated code produces the exact same output. Change-Id: I491b35e0d8062de33c67091dd4126d843b231838 Signed-off-by: Antonio Ojea <aojea@google.com>	2025-02-03 15:16:57 +00:00