whatcanGOwrong

go/pkg/mod/github.com/cilium/ebpf@v0.11.0/examples/tcprtt_sockops/bpf_bpfeb.go

@@ -0,0 +1,143 @@
+// Code generated by bpf2go; DO NOT EDIT.
+//go:build (arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64) && linux
+
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+
+	"github.com/cilium/ebpf"
+)
+
+type bpfRttEvent struct {
+	Sport uint16
+	Dport uint16
+	Saddr uint32
+	Daddr uint32
+	Srtt  uint32
+}
+
+type bpfSkInfo struct {
+	SkKey  bpfSkKey
+	SkType uint8
+	_      [3]byte
+}
+
+type bpfSkKey struct {
+	LocalIp4   uint32
+	RemoteIp4  uint32
+	LocalPort  uint32
+	RemotePort uint32
+}
+
+// loadBpf returns the embedded CollectionSpec for bpf.
+func loadBpf() (*ebpf.CollectionSpec, error) {
+	reader := bytes.NewReader(_BpfBytes)
+	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("can't load bpf: %w", err)
+	}
+
+	return spec, err
+}
+
+// loadBpfObjects loads bpf and converts it into a struct.
+//
+// The following types are suitable as obj argument:
+//
+//	*bpfObjects
+//	*bpfPrograms
+//	*bpfMaps
+//
+// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
+func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
+	spec, err := loadBpf()
+	if err != nil {
+		return err
+	}
+
+	return spec.LoadAndAssign(obj, opts)
+}
+
+// bpfSpecs contains maps and programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfSpecs struct {
+	bpfProgramSpecs
+	bpfMapSpecs
+}
+
+// bpfSpecs contains programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfProgramSpecs struct {
+	BpfSockopsCb *ebpf.ProgramSpec `ebpf:"bpf_sockops_cb"`
+}
+
+// bpfMapSpecs contains maps before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfMapSpecs struct {
+	MapEstabSk *ebpf.MapSpec `ebpf:"map_estab_sk"`
+	RttEvents  *ebpf.MapSpec `ebpf:"rtt_events"`
+}
+
+// bpfObjects contains all objects after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfObjects struct {
+	bpfPrograms
+	bpfMaps
+}
+
+func (o *bpfObjects) Close() error {
+	return _BpfClose(
+		&o.bpfPrograms,
+		&o.bpfMaps,
+	)
+}
+
+// bpfMaps contains all maps after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfMaps struct {
+	MapEstabSk *ebpf.Map `ebpf:"map_estab_sk"`
+	RttEvents  *ebpf.Map `ebpf:"rtt_events"`
+}
+
+func (m *bpfMaps) Close() error {
+	return _BpfClose(
+		m.MapEstabSk,
+		m.RttEvents,
+	)
+}
+
+// bpfPrograms contains all programs after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfPrograms struct {
+	BpfSockopsCb *ebpf.Program `ebpf:"bpf_sockops_cb"`
+}
+
+func (p *bpfPrograms) Close() error {
+	return _BpfClose(
+		p.BpfSockopsCb,
+	)
+}
+
+func _BpfClose(closers ...io.Closer) error {
+	for _, closer := range closers {
+		if err := closer.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Do not access this directly.
+//
+//go:embed bpf_bpfeb.o
+var _BpfBytes []byte

go/pkg/mod/github.com/cilium/ebpf@v0.11.0/examples/tcprtt_sockops/bpf_bpfel.go

@@ -0,0 +1,143 @@
+// Code generated by bpf2go; DO NOT EDIT.
+//go:build (386 || amd64 || amd64p32 || arm || arm64 || loong64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64) && linux
+
+package main
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"io"
+
+	"github.com/cilium/ebpf"
+)
+
+type bpfRttEvent struct {
+	Sport uint16
+	Dport uint16
+	Saddr uint32
+	Daddr uint32
+	Srtt  uint32
+}
+
+type bpfSkInfo struct {
+	SkKey  bpfSkKey
+	SkType uint8
+	_      [3]byte
+}
+
+type bpfSkKey struct {
+	LocalIp4   uint32
+	RemoteIp4  uint32
+	LocalPort  uint32
+	RemotePort uint32
+}
+
+// loadBpf returns the embedded CollectionSpec for bpf.
+func loadBpf() (*ebpf.CollectionSpec, error) {
+	reader := bytes.NewReader(_BpfBytes)
+	spec, err := ebpf.LoadCollectionSpecFromReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("can't load bpf: %w", err)
+	}
+
+	return spec, err
+}
+
+// loadBpfObjects loads bpf and converts it into a struct.
+//
+// The following types are suitable as obj argument:
+//
+//	*bpfObjects
+//	*bpfPrograms
+//	*bpfMaps
+//
+// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
+func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
+	spec, err := loadBpf()
+	if err != nil {
+		return err
+	}
+
+	return spec.LoadAndAssign(obj, opts)
+}
+
+// bpfSpecs contains maps and programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfSpecs struct {
+	bpfProgramSpecs
+	bpfMapSpecs
+}
+
+// bpfSpecs contains programs before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfProgramSpecs struct {
+	BpfSockopsCb *ebpf.ProgramSpec `ebpf:"bpf_sockops_cb"`
+}
+
+// bpfMapSpecs contains maps before they are loaded into the kernel.
+//
+// It can be passed ebpf.CollectionSpec.Assign.
+type bpfMapSpecs struct {
+	MapEstabSk *ebpf.MapSpec `ebpf:"map_estab_sk"`
+	RttEvents  *ebpf.MapSpec `ebpf:"rtt_events"`
+}
+
+// bpfObjects contains all objects after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfObjects struct {
+	bpfPrograms
+	bpfMaps
+}
+
+func (o *bpfObjects) Close() error {
+	return _BpfClose(
+		&o.bpfPrograms,
+		&o.bpfMaps,
+	)
+}
+
+// bpfMaps contains all maps after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfMaps struct {
+	MapEstabSk *ebpf.Map `ebpf:"map_estab_sk"`
+	RttEvents  *ebpf.Map `ebpf:"rtt_events"`
+}
+
+func (m *bpfMaps) Close() error {
+	return _BpfClose(
+		m.MapEstabSk,
+		m.RttEvents,
+	)
+}
+
+// bpfPrograms contains all programs after they have been loaded into the kernel.
+//
+// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
+type bpfPrograms struct {
+	BpfSockopsCb *ebpf.Program `ebpf:"bpf_sockops_cb"`
+}
+
+func (p *bpfPrograms) Close() error {
+	return _BpfClose(
+		p.BpfSockopsCb,
+	)
+}
+
+func _BpfClose(closers ...io.Closer) error {
+	for _, closer := range closers {
+		if err := closer.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Do not access this directly.
+//
+//go:embed bpf_bpfel.o
+var _BpfBytes []byte

go/pkg/mod/github.com/cilium/ebpf@v0.11.0/examples/tcprtt_sockops/bpf_sockops.h

@@ -0,0 +1,87 @@
+/*
+ * Note that this header file contains a subset of kernel 
+ * definitions needed for the tcprtt_sockops example.
+ */
+#ifndef BPF_SOCKOPS_H
+#define BPF_SOCKOPS_H
+
+/*
+ * Copy of TCP states.
+ * See: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/bpf.h#L6347.
+ */
+enum {
+	TCP_ESTABLISHED = 1,
+	TCP_SYN_SENT = 2,
+	TCP_SYN_RECV = 3,
+	TCP_FIN_WAIT1 = 4,
+	TCP_FIN_WAIT2 = 5,
+	TCP_TIME_WAIT = 6,
+	TCP_CLOSE = 7,
+	TCP_CLOSE_WAIT = 8,
+	TCP_LAST_ACK = 9,
+	TCP_LISTEN = 10,
+	TCP_CLOSING = 11,
+	TCP_NEW_SYN_RECV = 12,
+	TCP_MAX_STATES = 13,
+};
+
+/*
+ * Copy of sock_ops operations.
+ * See: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/bpf.h#L6233.
+ */
+enum {
+	BPF_SOCK_OPS_VOID                   = 0,
+	BPF_SOCK_OPS_TIMEOUT_INIT           = 1,
+	BPF_SOCK_OPS_RWND_INIT              = 2,
+	BPF_SOCK_OPS_TCP_CONNECT_CB         = 3,
+	BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB  = 4,
+	BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB = 5,
+	BPF_SOCK_OPS_NEEDS_ECN              = 6,
+	BPF_SOCK_OPS_BASE_RTT               = 7,
+	BPF_SOCK_OPS_RTO_CB                 = 8,
+	BPF_SOCK_OPS_RETRANS_CB             = 9,
+	BPF_SOCK_OPS_STATE_CB               = 10,
+	BPF_SOCK_OPS_TCP_LISTEN_CB          = 11,
+	BPF_SOCK_OPS_RTT_CB                 = 12,
+	BPF_SOCK_OPS_PARSE_HDR_OPT_CB       = 13,
+	BPF_SOCK_OPS_HDR_OPT_LEN_CB         = 14,
+	BPF_SOCK_OPS_WRITE_HDR_OPT_CB       = 15,
+};
+
+/*
+ * Copy of definitions for bpf_sock_ops_cb_flags.
+ * See: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/bpf.h#L6178.
+ */
+enum {
+	BPF_SOCK_OPS_RTO_CB_FLAG                   = 1,
+	BPF_SOCK_OPS_RETRANS_CB_FLAG               = 2,
+	BPF_SOCK_OPS_STATE_CB_FLAG                 = 4,
+	BPF_SOCK_OPS_RTT_CB_FLAG                   = 8,
+	BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG     = 16,
+	BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG = 32,
+	BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG         = 64,
+	BPF_SOCK_OPS_ALL_CB_FLAGS                  = 127,
+};
+
+/*
+ * Copy of bpf.h's bpf_sock_ops with minimal subset 
+ * of fields used by the tcprtt_sockops example.
+ * See: https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/bpf.h#L6101.
+ */
+struct bpf_sock_ops {
+	__u32 op;
+	union {
+		__u32 args[4];
+		__u32 reply;
+		__u32 replylong[4];
+	};
+	__u32 family;
+	__u32 remote_ip4;
+	__u32 local_ip4;
+	__u32 remote_port;
+	__u32 local_port;
+	__u32 srtt_us;
+    __u32 bpf_sock_ops_cb_flags;
+} __attribute__((preserve_access_index));
+
+#endif

go/pkg/mod/github.com/cilium/ebpf@v0.11.0/examples/tcprtt_sockops/main.go

@@ -0,0 +1,150 @@
+//go:build linux
+
+// This program demonstrates attaching an eBPF program to
+// a cgroupv2 path and using sockops to process TCP socket events.
+// It prints the IPs/ports/RTT information every time TCP sockets
+// update their internal RTT value.
+// It supports only IPv4 for this example.
+//
+// Sample output:
+//
+// examples# go run -exec sudo ./tcprtt_sockops
+// 2022/08/14 20:58:03 eBPF program loaded and attached on cgroup /sys/fs/cgroup/unified
+// 2022/08/14 20:58:03 Src addr        Port   -> Dest addr       Port   RTT (ms)
+// 2022/08/14 20:58:09 10.0.1.205      54844  -> 20.42.73.25     443    67
+// 2022/08/14 20:58:09 10.0.1.205      54844  -> 20.42.73.25     443    67
+// 2022/08/14 20:58:33 10.0.1.205      38620  -> 140.82.121.4    443    26
+// 2022/08/14 20:58:33 10.0.1.205      38620  -> 140.82.121.4    443    26
+// 2022/08/14 20:58:43 34.67.40.146    45380  -> 10.0.1.205      5201   106
+// 2022/08/14 20:58:43 34.67.40.146    45380  -> 10.0.1.205      5201   106
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"log"
+	"net"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/internal"
+	"github.com/cilium/ebpf/link"
+	"github.com/cilium/ebpf/ringbuf"
+	"github.com/cilium/ebpf/rlimit"
+
+	"golang.org/x/sys/unix"
+)
+
+// $BPF_CLANG and $BPF_CFLAGS are set by the Makefile.
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -tags "linux" -cc $BPF_CLANG -cflags $BPF_CFLAGS -type rtt_event bpf tcprtt_sockops.c -- -I../headers
+
+func main() {
+	stopper := make(chan os.Signal, 1)
+	signal.Notify(stopper, os.Interrupt, syscall.SIGTERM)
+
+	// Allow the current process to lock memory for eBPF resources.
+	if err := rlimit.RemoveMemlock(); err != nil {
+		log.Fatal(err)
+	}
+
+	// Find the path to a cgroup enabled to version 2
+	cgroupPath, err := findCgroupPath()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Load pre-compiled programs and maps into the kernel.
+	objs := bpfObjects{}
+	if err := loadBpfObjects(&objs, nil); err != nil {
+		log.Fatalf("loading objects: %v", err)
+	}
+	defer objs.Close()
+
+	// Attach ebpf program to a cgroupv2
+	link, err := link.AttachCgroup(link.CgroupOptions{
+		Path:    cgroupPath,
+		Program: objs.bpfPrograms.BpfSockopsCb,
+		Attach:  ebpf.AttachCGroupSockOps,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer link.Close()
+
+	log.Printf("eBPF program loaded and attached on cgroup %s\n", cgroupPath)
+
+	rd, err := ringbuf.NewReader(objs.bpfMaps.RttEvents)
+	if err != nil {
+		log.Fatalf("opening ringbuf reader: %s", err)
+	}
+	defer rd.Close()
+
+	log.Printf("%-15s %-6s -> %-15s %-6s %-6s",
+		"Src addr",
+		"Port",
+		"Dest addr",
+		"Port",
+		"RTT (ms)",
+	)
+	go readLoop(rd)
+
+	// Wait
+	<-stopper
+}
+
+func findCgroupPath() (string, error) {
+	cgroupPath := "/sys/fs/cgroup"
+
+	var st syscall.Statfs_t
+	err := syscall.Statfs(cgroupPath, &st)
+	if err != nil {
+		return "", err
+	}
+	isCgroupV2Enabled := st.Type == unix.CGROUP2_SUPER_MAGIC
+	if !isCgroupV2Enabled {
+		cgroupPath = filepath.Join(cgroupPath, "unified")
+	}
+	return cgroupPath, nil
+}
+
+func readLoop(rd *ringbuf.Reader) {
+	// bpfRttEvent is generated by bpf2go.
+	var event bpfRttEvent
+	for {
+		record, err := rd.Read()
+		if err != nil {
+			if errors.Is(err, ringbuf.ErrClosed) {
+				log.Println("received signal, exiting..")
+				return
+			}
+			log.Printf("reading from reader: %s", err)
+			continue
+		}
+
+		// Parse the ringbuf event entry into a bpfRttEvent structure.
+		if err := binary.Read(bytes.NewBuffer(record.RawSample), internal.NativeEndian, &event); err != nil {
+			log.Printf("parsing ringbuf event: %s", err)
+			continue
+		}
+
+		log.Printf("%-15s %-6d -> %-15s %-6d %-6d",
+			intToIP(event.Saddr),
+			event.Sport,
+			intToIP(event.Daddr),
+			event.Dport,
+			event.Srtt,
+		)
+	}
+}
+
+// intToIP converts IPv4 number to net.IP
+func intToIP(ipNum uint32) net.IP {
+	ip := make(net.IP, 4)
+	binary.BigEndian.PutUint32(ip, ipNum)
+	return ip
+}

go/pkg/mod/github.com/cilium/ebpf@v0.11.0/examples/tcprtt_sockops/tcprtt_sockops.c

@@ -0,0 +1,161 @@
+//go:build ignore
+
+#include "common.h"
+
+#include "bpf_endian.h"
+#include "bpf_sockops.h"
+#include "bpf_tracing.h"
+
+#define AF_INET 2
+#define SOCKOPS_MAP_SIZE 65535
+
+char __license[] SEC("license") = "Dual MIT/GPL";
+
+enum {
+	SOCK_TYPE_ACTIVE  = 0,
+	SOCK_TYPE_PASSIVE = 1,
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, SOCKOPS_MAP_SIZE);
+	__type(key, struct sk_key);
+	__type(value, struct sk_info);
+} map_estab_sk SEC(".maps");
+
+struct sk_key {
+	u32 local_ip4;
+	u32 remote_ip4;
+	u32 local_port;
+	u32 remote_port;
+};
+
+struct sk_info {
+	struct sk_key sk_key;
+	u8 sk_type;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_RINGBUF);
+	__uint(max_entries, 1 << 24);
+} rtt_events SEC(".maps");
+
+struct rtt_event {
+	u16 sport;
+	u16 dport;
+	u32 saddr;
+	u32 daddr;
+	u32 srtt;
+};
+struct rtt_event *unused_event __attribute__((unused));
+
+static inline void init_sk_key(struct bpf_sock_ops *skops, struct sk_key *sk_key) {
+	sk_key->local_ip4   = bpf_ntohl(skops->local_ip4);
+	sk_key->remote_ip4  = bpf_ntohl(skops->remote_ip4);
+	sk_key->local_port  = skops->local_port;
+	sk_key->remote_port = bpf_ntohl(skops->remote_port);
+}
+
+static inline void bpf_sock_ops_establish_cb(struct bpf_sock_ops *skops, u8 sock_type) {
+	int err;
+	struct sk_info sk_info = {};
+	// Only process IPv4 sockets
+	if (skops == NULL || skops->family != AF_INET)
+		return;
+
+	// Initialize the 4-tuple key
+	init_sk_key(skops, &sk_info.sk_key);
+	sk_info.sk_type = sock_type;
+
+	// Store the socket info in map using the 4-tuple as key
+	// We keep track of TCP connections in 'established' state
+	err = bpf_map_update_elem(&map_estab_sk, &sk_info.sk_key, &sk_info, BPF_NOEXIST);
+	if (err != 0) {
+		// Storing the 4-tuple in map has failed, return early.
+		// This can happen in case the 4-tuple already exists in the map (i.e. BPF_NOEXIST flag)
+		return;
+	}
+
+	// Enable sockops callbacks for RTT and TCP state change
+	bpf_sock_ops_cb_flags_set(skops, BPF_SOCK_OPS_RTT_CB_FLAG | BPF_SOCK_OPS_STATE_CB_FLAG);
+}
+
+static inline void bpf_sock_ops_rtt_cb(struct bpf_sock_ops *skops) {
+	struct sk_key sk_key = {};
+	struct sk_info *sk_info;
+	struct rtt_event *rtt_event;
+
+	// Initialize the 4-tuple key
+	init_sk_key(skops, &sk_key);
+
+	// Retrieve the socket info from map of established connections
+	sk_info = bpf_map_lookup_elem(&map_estab_sk, &sk_key);
+	if (!sk_info)
+		return;
+
+	rtt_event = bpf_ringbuf_reserve(&rtt_events, sizeof(struct rtt_event), 0);
+	if (!rtt_event) {
+		return;
+	}
+
+	switch (sk_info->sk_type) {
+	case SOCK_TYPE_ACTIVE:
+		// If socket is 'active', 'local' means 'source'
+		// and 'remote' means 'destination'
+		rtt_event->saddr = sk_info->sk_key.local_ip4;
+		rtt_event->daddr = sk_info->sk_key.remote_ip4;
+		rtt_event->sport = sk_info->sk_key.local_port;
+		rtt_event->dport = sk_info->sk_key.remote_port;
+		break;
+	case SOCK_TYPE_PASSIVE:
+		// If socket is 'passive', 'local' means 'destination'
+		// and 'remote' means 'source'
+		rtt_event->saddr = sk_info->sk_key.remote_ip4;
+		rtt_event->daddr = sk_info->sk_key.local_ip4;
+		rtt_event->sport = sk_info->sk_key.remote_port;
+		rtt_event->dport = sk_info->sk_key.local_port;
+		break;
+	}
+
+	// Extract smoothed RTT
+	rtt_event->srtt = skops->srtt_us >> 3;
+	rtt_event->srtt /= 1000;
+
+	// Send RTT event data to userspace app via ring buffer
+	bpf_ringbuf_submit(rtt_event, 0);
+}
+
+static inline void bpf_sock_ops_state_cb(struct bpf_sock_ops *skops) {
+	struct sk_key sk_key = {};
+
+	// Socket changed state. args[0] stores the previous state.
+	// Perform cleanup of map entry if socket is exiting
+	// the 'established' state,
+	if (skops->args[0] == TCP_ESTABLISHED) {
+		init_sk_key(skops, &sk_key);
+		bpf_map_delete_elem(&map_estab_sk, &sk_key);
+	}
+}
+
+SEC("sockops")
+int bpf_sockops_cb(struct bpf_sock_ops *skops) {
+	u32 op;
+	op = skops->op;
+
+	switch (op) {
+	case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:
+		bpf_sock_ops_establish_cb(skops, SOCK_TYPE_ACTIVE);
+		break;
+	case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
+		bpf_sock_ops_establish_cb(skops, SOCK_TYPE_PASSIVE);
+		break;
+	case BPF_SOCK_OPS_RTT_CB:
+		bpf_sock_ops_rtt_cb(skops);
+		break;
+	case BPF_SOCK_OPS_STATE_CB:
+		bpf_sock_ops_state_cb(skops);
+		break;
+	}
+
+	return 0;
+}