container/podman
container/podman
1
0
Fork 0
mirror of https://github.com/containers/podman synced 2024-10-21 01:34:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

netavark IPAM assignment

Browse source 
Add a new boltdb to handle IPAM assignment.

The db structure is the following:
Each network has their own bucket with the network name as bucket key.
Inside the network bucket there is an ID bucket which maps the container ID (key)
to a json array of ip addresses (value).
The network bucket also has a bucket for each subnet, the subnet is used as key.
Inside the subnet bucket an ip is used as key and the container ID as value.

The db should be stored on a tmpfs to ensure we always have a clean
state after a reboot.

Signed-off-by: Paul Holzinger <pholzing@redhat.com>
This commit is contained in:
Paul Holzinger 2021-10-20 15:55:22 +02:00
parent eaae294628
commit 4febe55769
No known key found for this signature in database
GPG key ID: EB145DD938A3CAF2
12 changed files with 964 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
libpod/network/cni/cni_conversion.go
View file

@ -157,7 +157,7 @@ func convertIPAMConfToNetwork(network *types.Network, ipam ipamConfig, confPath
return errors.Errorf("failed to parse gateway ip %s", ipam.Gateway)
}
// convert to 4 byte if ipv4
internalutil.NormalizeIP(&gateway)
util.NormalizeIP(&gateway)
} else if !network.Internal {
// only add a gateway address if the network is not internal
gateway, err = util.FirstIPInSubnet(sub)

56
libpod/network/internal/util/bridge.go
View file

@ -27,41 +27,43 @@ func CreateBridge(n NetUtil, network *types.Network, usedNetworks []*net.IPNet)
}
}
if len(network.Subnets) == 0 {
freeSubnet, err := GetFreeIPv4NetworkSubnet(usedNetworks)
if err != nil {
return err
}
network.Subnets = append(network.Subnets, *freeSubnet)
}
// ipv6 enabled means dual stack, check if we already have
// a ipv4 or ipv6 subnet and add one if not.
if network.IPv6Enabled {
ipv4 := false
ipv6 := false
for _, subnet := range network.Subnets {
if util.IsIPv6(subnet.Subnet.IP) {
ipv6 = true
}
if util.IsIPv4(subnet.Subnet.IP) {
ipv4 = true
}
}
if !ipv4 {
if network.IPAMOptions["driver"] != types.DHCPIPAMDriver {
if len(network.Subnets) == 0 {
freeSubnet, err := GetFreeIPv4NetworkSubnet(usedNetworks)
if err != nil {
return err
}
network.Subnets = append(network.Subnets, *freeSubnet)
}
if !ipv6 {
freeSubnet, err := GetFreeIPv6NetworkSubnet(usedNetworks)
if err != nil {
return err
// ipv6 enabled means dual stack, check if we already have
// a ipv4 or ipv6 subnet and add one if not.
if network.IPv6Enabled {
ipv4 := false
ipv6 := false
for _, subnet := range network.Subnets {
if util.IsIPv6(subnet.Subnet.IP) {
ipv6 = true
}
if util.IsIPv4(subnet.Subnet.IP) {
ipv4 = true
}
}
if !ipv4 {
freeSubnet, err := GetFreeIPv4NetworkSubnet(usedNetworks)
if err != nil {
return err
}
network.Subnets = append(network.Subnets, *freeSubnet)
}
if !ipv6 {
freeSubnet, err := GetFreeIPv6NetworkSubnet(usedNetworks)
if err != nil {
return err
}
network.Subnets = append(network.Subnets, *freeSubnet)
}
network.Subnets = append(network.Subnets, *freeSubnet)
}
network.IPAMOptions["driver"] = types.HostLocalIPAMDriver
}
network.IPAMOptions["driver"] = types.HostLocalIPAMDriver
return nil
}

8
libpod/network/internal/util/ip.go
View file

@ -68,11 +68,3 @@ func getRandomIPv6Subnet() (net.IPNet, error) {
ip = append(ip, make([]byte, 8)...)
return net.IPNet{IP: ip, Mask: net.CIDRMask(64, 128)}, nil
}
// NormalizeIP will transform the given ip to the 4 byte len ipv4 if possible
func NormalizeIP(ip *net.IP) {
ipv4 := ip.To4()
if ipv4 != nil {
*ip = ipv4
}
}

6
libpod/network/internal/util/validate.go
View file

@ -38,7 +38,7 @@ func ValidateSubnet(s *types.Subnet, addGateway bool, usedNetworks []*net.IPNet)
if !s.Subnet.Contains(s.Gateway) {
return errors.Errorf("gateway %s not in subnet %s", s.Gateway, &s.Subnet)
}
NormalizeIP(&s.Gateway)
util.NormalizeIP(&s.Gateway)
} else if addGateway {
ip, err := util.FirstIPInSubnet(net)
if err != nil {
@ -52,13 +52,13 @@ func ValidateSubnet(s *types.Subnet, addGateway bool, usedNetworks []*net.IPNet)
if !s.Subnet.Contains(s.LeaseRange.StartIP) {
return errors.Errorf("lease range start ip %s not in subnet %s", s.LeaseRange.StartIP, &s.Subnet)
}
NormalizeIP(&s.LeaseRange.StartIP)
util.NormalizeIP(&s.LeaseRange.StartIP)
}
if s.LeaseRange.EndIP != nil {
if !s.Subnet.Contains(s.LeaseRange.EndIP) {
return errors.Errorf("lease range end ip %s not in subnet %s", s.LeaseRange.EndIP, &s.Subnet)
}
NormalizeIP(&s.LeaseRange.EndIP)
util.NormalizeIP(&s.LeaseRange.EndIP)
}
}
return nil

368
libpod/network/netavark/ipam.go Normal file
View file

@ -0,0 +1,368 @@
package netavark
import (
"encoding/json"
"fmt"
"net"
"github.com/containers/podman/v3/libpod/network/types"
"github.com/containers/podman/v3/libpod/network/util"
"github.com/pkg/errors"
"go.etcd.io/bbolt"
)
// IPAM boltdb structure
// Each network has their own bucket with the network name as bucket key.
// Inside the network bucket there is an ID bucket which maps the container ID (key)
// to a json array of ip addresses (value).
// The network bucket also has a bucket for each subnet, the subnet is used as key.
// Inside the subnet bucket an ip is used as key and the container ID as value.
const (
idBucket = "ids"
// lastIP this is used as key to store the last allocated ip
// note that this string should not be 4 or 16 byte long
lastIP = "lastIP"
)
var (
idBucketKey = []byte(idBucket)
lastIPKey = []byte(lastIP)
)
type ipamError struct {
msg string
cause error
}
func (e *ipamError) Error() string {
msg := "IPAM error"
if e.msg != "" {
msg += ": " + e.msg
}
if e.cause != nil {
msg += ": " + e.cause.Error()
}
return msg
}
func newIPAMError(cause error, msg string, args ...interface{}) *ipamError {
return &ipamError{
msg: fmt.Sprintf(msg, args...),
cause: cause,
}
}
// openDB will open the ipam database
// Note that the caller has to Close it.
func (n *netavarkNetwork) openDB() (*bbolt.DB, error) {
db, err := bbolt.Open(n.ipamDBPath, 0600, nil)
if err != nil {
return nil, newIPAMError(err, "failed to open database %s", n.ipamDBPath)
}
return db, nil
}
// allocIPs will allocate ips for the the container. It will change the
// NetworkOptions in place. When static ips are given it will validate
// that these are free to use and will allocate them to the container.
func (n *netavarkNetwork) allocIPs(opts *types.NetworkOptions) error {
db, err := n.openDB()
if err != nil {
return err
}
defer db.Close()
err = db.Update(func(tx *bbolt.Tx) error {
for netName, netOpts := range opts.Networks {
network := n.networks[netName]
if network == nil {
return newIPAMError(nil, "could not find network %q", netName)
}
// check if we have to alloc ips
if !requiresIPAMAlloc(network) {
continue
}
// create/get network bucket
netBkt, err := tx.CreateBucketIfNotExists([]byte(netName))
if err != nil {
return newIPAMError(err, "failed to create/get network bucket for network %s", netName)
}
// requestIPs is the list of ips which should be used for this container
requestIPs := make([]net.IP, 0, len(network.Subnets))
for _, subnet := range network.Subnets {
subnetBkt, err := netBkt.CreateBucketIfNotExists([]byte(subnet.Subnet.String()))
if err != nil {
return newIPAMError(err, "failed to create/get subnet bucket for network %s", netName)
}
// search for a static ip which matches the current subnet
// in this case the user wants this one and we should not assign a free one
var ip net.IP
for _, staticIP := range netOpts.StaticIPs {
if subnet.Subnet.Contains(staticIP) {
ip = staticIP
break
}
}
// when static ip is requested for this network
if ip != nil {
// convert to 4 byte ipv4 if needed
util.NormalizeIP(&ip)
id := subnetBkt.Get(ip)
if id != nil {
return newIPAMError(nil, "requested ip address %s is already allocated to container ID %s", ip.String(), string(id))
}
} else {
ip, err = getFreeIPFromBucket(subnetBkt, subnet)
if err != nil {
return err
}
err = subnetBkt.Put(lastIPKey, ip)
if err != nil {
return newIPAMError(err, "failed to store last ip in database")
}
}
err = subnetBkt.Put(ip, []byte(opts.ContainerID))
if err != nil {
return newIPAMError(err, "failed to store ip in database")
}
requestIPs = append(requestIPs, ip)
}
idsBucket, err := netBkt.CreateBucketIfNotExists(idBucketKey)
if err != nil {
return newIPAMError(err, "failed to create/get id bucket for network %s", netName)
}
ipsBytes, err := json.Marshal(requestIPs)
if err != nil {
return newIPAMError(err, "failed to marshal ips")
}
err = idsBucket.Put([]byte(opts.ContainerID), ipsBytes)
if err != nil {
return newIPAMError(err, "failed to store ips in database")
}
netOpts.StaticIPs = requestIPs
opts.Networks[netName] = netOpts
}
return nil
})
return err
}
func getFreeIPFromBucket(bucket *bbolt.Bucket, subnet types.Subnet) (net.IP, error) {
var rangeStart net.IP
var rangeEnd net.IP
if subnet.LeaseRange != nil {
rangeStart = subnet.LeaseRange.StartIP
rangeEnd = subnet.LeaseRange.EndIP
}
if rangeStart == nil {
// let start with the first ip in subnet
rangeStart = util.NextIP(subnet.Subnet.IP)
}
if rangeEnd == nil {
lastIP, err := util.LastIPInSubnet(&subnet.Subnet.IPNet)
// this error should never happen but lets check anyways to prevent panics
if err != nil {
return nil, errors.Wrap(err, "failed to get lastIP")
}
// ipv4 uses the last ip in a subnet for broadcast so we cannot use it
if util.IsIPv4(lastIP) {
lastIP = util.PrevIP(lastIP)
}
rangeEnd = lastIP
}
lastIPByte := bucket.Get(lastIPKey)
curIP := net.IP(lastIPByte)
if curIP == nil {
curIP = rangeStart
} else {
curIP = util.NextIP(curIP)
}
// store the start ip to make sure we know when we looped over all available ips
startIP := curIP
for {
// skip the gateway
if subnet.Gateway != nil {
if util.Cmp(curIP, subnet.Gateway) == 0 {
curIP = util.NextIP(curIP)
continue
}
}
// if we are at the end we need to jump back to the start ip
if util.Cmp(curIP, rangeEnd) > 0 {
if util.Cmp(rangeStart, startIP) == 0 {
return nil, newIPAMError(nil, "failed to find free IP in range: %s - %s", rangeStart.String(), rangeEnd.String())
}
curIP = rangeStart
continue
}
// check if ip is already used by another container
// if not return it
if bucket.Get(curIP) == nil {
return curIP, nil
}
curIP = util.NextIP(curIP)
if util.Cmp(curIP, startIP) == 0 {
return nil, newIPAMError(nil, "failed to find free IP in range: %s - %s", rangeStart.String(), rangeEnd.String())
}
}
}
// getAssignedIPs will read the ipam database and will fill in the used ips for this container.
// It will change the NetworkOptions in place.
func (n *netavarkNetwork) getAssignedIPs(opts *types.NetworkOptions) error {
db, err := n.openDB()
if err != nil {
return err
}
defer db.Close()
err = db.View(func(tx *bbolt.Tx) error {
for netName, netOpts := range opts.Networks {
network := n.networks[netName]
if network == nil {
return newIPAMError(nil, "could not find network %q", netName)
}
// check if we have to alloc ips
if !requiresIPAMAlloc(network) {
continue
}
// get network bucket
netBkt := tx.Bucket([]byte(netName))
if netBkt == nil {
return newIPAMError(nil, "failed to get network bucket for network %s", netName)
}
idBkt := netBkt.Bucket(idBucketKey)
if idBkt == nil {
return newIPAMError(nil, "failed to get id bucket for network %s", netName)
}
ipJSON := idBkt.Get([]byte(opts.ContainerID))
if ipJSON == nil {
return newIPAMError(nil, "failed to get ips for container ID %s on network %s", opts.ContainerID, netName)
}
// assignedIPs is the list of ips which should be used for this container
assignedIPs := make([]net.IP, 0, len(network.Subnets))
err = json.Unmarshal(ipJSON, &assignedIPs)
if err != nil {
return newIPAMError(err, "failed to unmarshal ips from database")
}
for i := range assignedIPs {
util.NormalizeIP(&assignedIPs[i])
}
netOpts.StaticIPs = assignedIPs
opts.Networks[netName] = netOpts
}
return nil
})
return err
}
// deallocIPs will release the ips in the network options from the DB so that
// they can be reused by other containers. It expects that the network options
// are already filled with the correct ips. Use getAssignedIPs() for this.
func (n *netavarkNetwork) deallocIPs(opts *types.NetworkOptions) error {
db, err := n.openDB()
if err != nil {
return err
}
defer db.Close()
err = db.Update(func(tx *bbolt.Tx) error {
for netName, netOpts := range opts.Networks {
network := n.networks[netName]
if network == nil {
return newIPAMError(nil, "could not find network %q", netName)
}
// check if we have to alloc ips
if !requiresIPAMAlloc(network) {
continue
}
// get network bucket
netBkt := tx.Bucket([]byte(netName))
if netBkt == nil {
return newIPAMError(nil, "failed to get network bucket for network %s", netName)
}
for _, subnet := range network.Subnets {
subnetBkt := netBkt.Bucket([]byte(subnet.Subnet.String()))
if subnetBkt == nil {
return newIPAMError(nil, "failed to get subnet bucket for network %s", netName)
}
// search for a static ip which matches the current subnet
// in this case the user wants this one and we should not assign a free one
var ip net.IP
for _, staticIP := range netOpts.StaticIPs {
if subnet.Subnet.Contains(staticIP) {
ip = staticIP
break
}
}
if ip == nil {
return newIPAMError(nil, "failed to find ip for subnet %s on network %s", subnet.Subnet.String(), netName)
}
util.NormalizeIP(&ip)
err = subnetBkt.Delete(ip)
if err != nil {
return newIPAMError(err, "failed to remove ip %s from subnet bucket for network %s", ip.String(), netName)
}
}
idBkt := netBkt.Bucket(idBucketKey)
if idBkt == nil {
return newIPAMError(nil, "failed to get id bucket for network %s", netName)
}
err = idBkt.Delete([]byte(opts.ContainerID))
if err != nil {
return newIPAMError(err, "failed to remove allocated ips for container ID %s on network %s", opts.ContainerID, netName)
}
}
return nil
})
return err
}
// requiresIPAMAlloc return true when we have to allocate ips for this network
// it checks the ipam driver and if subnets are set
func requiresIPAMAlloc(network *types.Network) bool {
// only do host allocation when driver is set to HostLocalIPAMDriver or unset
switch network.IPAMOptions["driver"] {
case "", types.HostLocalIPAMDriver:
default:
return false
}
// no subnets == no ips to assign
return len(network.Subnets) > 0
}

433
libpod/network/netavark/ipam_test.go Normal file
View file

@ -0,0 +1,433 @@
package netavark
import (
"bytes"
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"github.com/containers/podman/v3/libpod/network/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/sirupsen/logrus"
)
var _ = Describe("IPAM", func() {
var (
networkInterface *netavarkNetwork
networkConfDir string
logBuffer bytes.Buffer
)
BeforeEach(func() {
var err error
networkConfDir, err = ioutil.TempDir("", "podman_netavark_test")
if err != nil {
Fail("Failed to create tmpdir")
}
logBuffer = bytes.Buffer{}
logrus.SetOutput(&logBuffer)
})
JustBeforeEach(func() {
libpodNet, err := NewNetworkInterface(InitConfig{
NetworkConfigDir: networkConfDir,
IPAMDBPath: filepath.Join(networkConfDir, "ipam.db"),
LockFile: filepath.Join(networkConfDir, "netavark.lock"),
})
if err != nil {
Fail("Failed to create NewCNINetworkInterface")
}
networkInterface = libpodNet.(*netavarkNetwork)
// run network list to force a network load
networkInterface.NetworkList()
})
AfterEach(func() {
os.RemoveAll(networkConfDir)
})
It("simple ipam alloc", func() {
netName := types.DefaultNetworkName
for i := 2; i < 100; i++ {
opts := &types.NetworkOptions{
ContainerID: "someContainerID",
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err := networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP(fmt.Sprintf("10.88.0.%d", i)).To4()))
}
})
It("ipam try to alloc same ip", func() {
netName := types.DefaultNetworkName
opts := &types.NetworkOptions{
ContainerID: "someContainerID",
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err := networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP("10.88.0.2").To4()))
opts = &types.NetworkOptions{
ContainerID: "otherID",
Networks: map[string]types.PerNetworkOptions{
netName: {StaticIPs: []net.IP{net.ParseIP("10.88.0.2")}},
},
}
err = networkInterface.allocIPs(opts)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("IPAM error: requested ip address 10.88.0.2 is already allocated to container ID someContainerID"))
})
It("ipam try to alloc more ips as in range", func() {
s, _ := types.ParseCIDR("10.0.0.1/24")
network, err := networkInterface.NetworkCreate(types.Network{
Subnets: []types.Subnet{
{
Subnet: s,
LeaseRange: &types.LeaseRange{
StartIP: net.ParseIP("10.0.0.10"),
EndIP: net.ParseIP("10.0.0.20"),
},
},
},
})
Expect(err).ToNot(HaveOccurred())
netName := network.Name
for i := 10; i < 21; i++ {
opts := &types.NetworkOptions{
ContainerID: fmt.Sprintf("someContainerID-%d", i),
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err = networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP(fmt.Sprintf("10.0.0.%d", i)).To4()))
}
opts := &types.NetworkOptions{
ContainerID: "someContainerID-22",
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
// now this should fail because all free ips are already assigned
err = networkInterface.allocIPs(opts)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("IPAM error: failed to find free IP in range: 10.0.0.10 - 10.0.0.20"))
})
It("ipam basic setup", func() {
netName := types.DefaultNetworkName
opts := &types.NetworkOptions{
ContainerID: "someContainerID",
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
expectedIP := net.ParseIP("10.88.0.2").To4()
err := networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(expectedIP))
// remove static ips from opts
netOpts := opts.Networks[netName]
netOpts.StaticIPs = nil
opts.Networks[netName] = netOpts
err = networkInterface.getAssignedIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(expectedIP))
err = networkInterface.allocIPs(opts)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("IPAM error: requested ip address 10.88.0.2 is already allocated to container ID someContainerID"))
// dealloc the ip
err = networkInterface.deallocIPs(opts)
Expect(err).ToNot(HaveOccurred())
err = networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(expectedIP))
})
It("ipam dual stack", func() {
s1, _ := types.ParseCIDR("10.0.0.0/26")
s2, _ := types.ParseCIDR("fd80::/24")
network, err := networkInterface.NetworkCreate(types.Network{
Subnets: []types.Subnet{
{
Subnet: s1,
},
{
Subnet: s2,
},
},
})
Expect(err).ToNot(HaveOccurred())
netName := network.Name
opts := &types.NetworkOptions{
ContainerID: "someContainerID",
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err = networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(2))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP("10.0.0.2").To4()))
Expect(opts.Networks[netName].StaticIPs[1]).To(Equal(net.ParseIP("fd80::2")))
// remove static ips from opts
netOpts := opts.Networks[netName]
netOpts.StaticIPs = nil
opts.Networks[netName] = netOpts
err = networkInterface.getAssignedIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(2))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP("10.0.0.2").To4()))
Expect(opts.Networks[netName].StaticIPs[1]).To(Equal(net.ParseIP("fd80::2")))
err = networkInterface.deallocIPs(opts)
Expect(err).ToNot(HaveOccurred())
// try to alloc the same again
err = networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(2))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP("10.0.0.2").To4()))
Expect(opts.Networks[netName].StaticIPs[1]).To(Equal(net.ParseIP("fd80::2")))
})
It("ipam with two networks", func() {
s, _ := types.ParseCIDR("10.0.0.0/24")
network, err := networkInterface.NetworkCreate(types.Network{
Subnets: []types.Subnet{
{
Subnet: s,
},
},
})
Expect(err).ToNot(HaveOccurred())
netName1 := network.Name
s, _ = types.ParseCIDR("10.0.1.0/24")
network, err = networkInterface.NetworkCreate(types.Network{
Subnets: []types.Subnet{
{
Subnet: s,
},
},
})
Expect(err).ToNot(HaveOccurred())
netName2 := network.Name
opts := &types.NetworkOptions{
ContainerID: "someContainerID",
Networks: map[string]types.PerNetworkOptions{
netName1: {},
netName2: {},
},
}
err = networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName1))
Expect(opts.Networks[netName1].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName1].StaticIPs[0]).To(Equal(net.ParseIP("10.0.0.2").To4()))
Expect(opts.Networks).To(HaveKey(netName2))
Expect(opts.Networks[netName2].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName2].StaticIPs[0]).To(Equal(net.ParseIP("10.0.1.2").To4()))
// remove static ips from opts
netOpts := opts.Networks[netName1]
netOpts.StaticIPs = nil
opts.Networks[netName1] = netOpts
netOpts = opts.Networks[netName2]
netOpts.StaticIPs = nil
opts.Networks[netName2] = netOpts
err = networkInterface.getAssignedIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName1))
Expect(opts.Networks[netName1].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName1].StaticIPs[0]).To(Equal(net.ParseIP("10.0.0.2").To4()))
Expect(opts.Networks).To(HaveKey(netName2))
Expect(opts.Networks[netName2].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName2].StaticIPs[0]).To(Equal(net.ParseIP("10.0.1.2").To4()))
err = networkInterface.deallocIPs(opts)
Expect(err).ToNot(HaveOccurred())
// try to alloc the same again
err = networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName1))
Expect(opts.Networks[netName1].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName1].StaticIPs[0]).To(Equal(net.ParseIP("10.0.0.2").To4()))
Expect(opts.Networks).To(HaveKey(netName2))
Expect(opts.Networks[netName2].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName2].StaticIPs[0]).To(Equal(net.ParseIP("10.0.1.2").To4()))
})
It("ipam alloc more ips as in subnet", func() {
s, _ := types.ParseCIDR("10.0.0.0/26")
network, err := networkInterface.NetworkCreate(types.Network{
Subnets: []types.Subnet{
{
Subnet: s,
},
},
})
Expect(err).ToNot(HaveOccurred())
netName := network.Name
for i := 2; i < 64; i++ {
opts := &types.NetworkOptions{
ContainerID: fmt.Sprintf("id-%d", i),
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err = networkInterface.allocIPs(opts)
if i < 63 {
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP(fmt.Sprintf("10.0.0.%d", i)).To4()))
} else {
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("IPAM error: failed to find free IP in range: 10.0.0.1 - 10.0.0.62"))
}
}
})
It("ipam alloc -> dealloc -> alloc", func() {
s, _ := types.ParseCIDR("10.0.0.0/27")
network, err := networkInterface.NetworkCreate(types.Network{
Subnets: []types.Subnet{
{
Subnet: s,
},
},
})
Expect(err).ToNot(HaveOccurred())
netName := network.Name
for i := 2; i < 10; i++ {
opts := types.NetworkOptions{
ContainerID: fmt.Sprintf("id-%d", i),
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err = networkInterface.allocIPs(&opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP(fmt.Sprintf("10.0.0.%d", i)).To4()))
err = networkInterface.deallocIPs(&opts)
Expect(err).ToNot(HaveOccurred())
}
for i := 0; i < 30; i++ {
opts := types.NetworkOptions{
ContainerID: fmt.Sprintf("id-%d", i),
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err = networkInterface.allocIPs(&opts)
if i < 29 {
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(1))
// The (i+8)%29+2 part looks cryptic but it is actually simple, we already have 8 ips allocated above
// so we expect the 8 available ip. We have 29 assignable ip addresses in this subnet because "i"+8 can
// be greater than 30 we have to modulo by 29 to go back to the beginning. Also the first free ip is
// network address + 2, so we have to add 2 to the result
Expect(opts.Networks[netName].StaticIPs[0]).To(Equal(net.ParseIP(fmt.Sprintf("10.0.0.%d", (i+8)%29+2)).To4()))
} else {
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(Equal("IPAM error: failed to find free IP in range: 10.0.0.1 - 10.0.0.30"))
}
}
})
It("ipam with dhcp driver should not set ips", func() {
network, err := networkInterface.NetworkCreate(types.Network{
IPAMOptions: map[string]string{
"driver": types.DHCPIPAMDriver,
},
})
Expect(err).ToNot(HaveOccurred())
netName := network.Name
opts := &types.NetworkOptions{
ContainerID: "someContainerID",
Networks: map[string]types.PerNetworkOptions{
netName: {},
},
}
err = networkInterface.allocIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(0))
err = networkInterface.getAssignedIPs(opts)
Expect(err).ToNot(HaveOccurred())
Expect(opts.Networks).To(HaveKey(netName))
Expect(opts.Networks[netName].StaticIPs).To(HaveLen(0))
// dealloc the ip
err = networkInterface.deallocIPs(opts)
Expect(err).ToNot(HaveOccurred())
})
})

1
libpod/network/netavark/netavark_suite_test.go
View file

@ -32,6 +32,7 @@ func getNetworkInterface(confDir string, machine bool) (types.ContainerNetwork,
NetworkConfigDir: confDir,
IsMachine: machine,
NetavarkBinary: netavarkBinary,
IPAMDBPath: filepath.Join(confDir, "ipam.db"),
LockFile: filepath.Join(confDir, "netavark.lock"),
})
}

26
libpod/network/netavark/network.go
View file

@ -13,6 +13,7 @@ import (
"github.com/containers/podman/v3/libpod/define"
"github.com/containers/podman/v3/libpod/network/internal/util"
"github.com/containers/podman/v3/libpod/network/types"
pkgutil "github.com/containers/podman/v3/pkg/util"
"github.com/containers/storage/pkg/lockfile"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@ -30,6 +31,9 @@ type netavarkNetwork struct {
// defaultSubnet is the default subnet for the default network.
defaultSubnet types.IPNet
// ipamDBPath is the path to the ip allocation bolt db
ipamDBPath string
// isMachine describes whenever podman runs in a podman machine environment.
isMachine bool
@ -50,6 +54,10 @@ type InitConfig struct {
// NetavarkBinary is the path to the netavark binary.
NetavarkBinary string
// IPAMDBPath is the path to the ipam database. This should be on a tmpfs.
// If empty defaults to XDG_RUNTIME_DIR/netavark/ipam.db or /run/netavark/ipam.db as root.
IPAMDBPath string
// DefaultNetwork is the name for the default network.
DefaultNetwork string
// DefaultSubnet is the default subnet for the default network.
@ -85,9 +93,27 @@ func NewNetworkInterface(conf InitConfig) (types.ContainerNetwork, error) {
return nil, errors.Wrap(err, "failed to parse default subnet")
}
ipamdbPath := conf.IPAMDBPath
if ipamdbPath == "" {
runDir, err := pkgutil.GetRuntimeDir()
if err != nil {
return nil, err
}
// as root runtimeDir is empty so use /run
if runDir == "" {
runDir = "/run"
}
ipamdbPath = filepath.Join(runDir, "netavark")
if err := os.MkdirAll(ipamdbPath, 0700); err != nil {
return nil, errors.Wrap(err, "failed to create ipam db path")
}
ipamdbPath = filepath.Join(ipamdbPath, "ipam.db")
}
n := &netavarkNetwork{
networkConfigDir: conf.NetworkConfigDir,
netavarkBinary: conf.NetavarkBinary,
ipamDBPath: ipamdbPath,
defaultNetwork: defaultNetworkName,
defaultSubnet: defaultNet,
isMachine: conf.IsMachine,

43
libpod/network/netavark/run.go
View file

@ -32,21 +32,29 @@ func (n *netavarkNetwork) Setup(namespacePath string, options types.SetupOptions
return nil, err
}
// TODO IP address assignment
// allocate IPs in the IPAM db
err = n.allocIPs(&options.NetworkOptions)
if err != nil {
return nil, err
}
netavarkOpts, err := n.convertNetOpts(options.NetworkOptions)
if err != nil {
return nil, errors.Wrap(err, "failed to convert net opts")
}
b, err := json.Marshal(&netavarkOpts)
if err != nil {
return nil, err
// trace output to get the json
if logrus.IsLevelEnabled(logrus.TraceLevel) {
b, err := json.Marshal(&netavarkOpts)
if err != nil {
return nil, err
}
// show the full netavark command so we can easily reproduce errors from the cli
logrus.Tracef("netavark command: printf '%s' | %s setup %s", string(b), n.netavarkBinary, namespacePath)
}
fmt.Println(string(b))
result := map[string]types.StatusBlock{}
err = execNetavark(n.netavarkBinary, []string{"setup", namespacePath}, netavarkOpts, result)
err = execNetavark(n.netavarkBinary, []string{"setup", namespacePath}, netavarkOpts, &result)
if len(result) != len(options.Networks) {
logrus.Errorf("unexpected netavark result: %v", result)
@ -65,12 +73,33 @@ func (n *netavarkNetwork) Teardown(namespacePath string, options types.TeardownO
return err
}
// get IPs from the IPAM db
err = n.getAssignedIPs(&options.NetworkOptions)
if err != nil {
// when there is an error getting the ips we should still continue
// to call teardown for netavark to prevent leaking network interfaces
logrus.Error(err)
}
netavarkOpts, err := n.convertNetOpts(options.NetworkOptions)
if err != nil {
return errors.Wrap(err, "failed to convert net opts")
}
return execNetavark(n.netavarkBinary, []string{"teardown", namespacePath}, netavarkOpts, nil)
retErr := execNetavark(n.netavarkBinary, []string{"teardown", namespacePath}, netavarkOpts, nil)
// when netavark returned an error we still free the used ips
// otherwise we could end up in a state where block the ips forever
err = n.deallocIPs(&netavarkOpts.NetworkOptions)
if err != nil {
if retErr != nil {
logrus.Error(err)
} else {
retErr = err
}
}
return retErr
}
func (n *netavarkNetwork) convertNetOpts(opts types.NetworkOptions) (*netavarkOptions, error) {

10
libpod/network/netavark/run_test.go
View file

@ -14,8 +14,6 @@ package netavark_test
// })
import (
"bytes"
"fmt"
"io/ioutil"
"net"
"os"
@ -39,7 +37,6 @@ var _ = Describe("run netavark", func() {
var (
libpodNet types.ContainerNetwork
confDir string
logBuffer bytes.Buffer
netNSTest ns.NetNS
netNSContainer ns.NetNS
)
@ -60,8 +57,11 @@ var _ = Describe("run netavark", func() {
}
BeforeEach(func() {
// set the logrus settings
logrus.SetLevel(logrus.TraceLevel)
// disable extra quotes so we can easily copy the netavark command
logrus.SetFormatter(&logrus.TextFormatter{DisableQuote: true})
logrus.SetOutput(os.Stderr)
// The tests need root privileges.
// Technically we could work around that by using user namespaces and
// the rootless cni code but this is to much work to get it right for a unit test.
@ -74,8 +74,6 @@ var _ = Describe("run netavark", func() {
if err != nil {
Fail("Failed to create tmpdir")
}
logBuffer = bytes.Buffer{}
logrus.SetOutput(&logBuffer)
netNSTest, err = netns.NewNS()
if err != nil {
@ -106,8 +104,6 @@ var _ = Describe("run netavark", func() {
netns.UnmountNS(netNSContainer)
netNSContainer.Close()
fmt.Println(logBuffer.String())
})
It("test basic setup", func() {

12
libpod/network/util/ip.go
View file

@ -1,6 +1,8 @@
package util
import "net"
import (
"net"
)
// IsIPv6 returns true if netIP is IPv6.
func IsIPv6(netIP net.IP) bool {
@ -44,3 +46,11 @@ func FirstIPInSubnet(addr *net.IPNet) (net.IP, error) { //nolint:interfacer
cidr.IP[len(cidr.IP)-1]++
return cidr.IP, nil
}
// NormalizeIP will transform the given ip to the 4 byte len ipv4 if possible
func NormalizeIP(ip *net.IP) {
ipv4 := ip.To4()
if ipv4 != nil {
*ip = ipv4
}
}

53
libpod/network/util/ip_calc.go Normal file
View file

@ -0,0 +1,53 @@
// Copyright 2015 CNI authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package util
import (
"math/big"
"net"
)
// NextIP returns IP incremented by 1
func NextIP(ip net.IP) net.IP {
i := ipToInt(ip)
return intToIP(i.Add(i, big.NewInt(1)))
}
// PrevIP returns IP decremented by 1
func PrevIP(ip net.IP) net.IP {
i := ipToInt(ip)
return intToIP(i.Sub(i, big.NewInt(1)))
}
// Cmp compares two IPs, returning the usual ordering:
// a < b : -1
// a == b : 0
// a > b : 1
func Cmp(a, b net.IP) int {
aa := ipToInt(a)
bb := ipToInt(b)
return aa.Cmp(bb)
}
func ipToInt(ip net.IP) *big.Int {
if v := ip.To4(); v != nil {
return big.NewInt(0).SetBytes(v)
}
return big.NewInt(0).SetBytes(ip.To16())
}
func intToIP(i *big.Int) net.IP {
return net.IP(i.Bytes())
}