FreeBSD 12.1, vnet jail, and internet access

classic Classic list List threaded Threaded
9 messages Options
Reply | Threaded
Open this post in threaded view
|

FreeBSD 12.1, vnet jail, and internet access

David Mehler
Hello,

I'm trying to get vnet jails going on FreeBSD 12.1-p6. I can start and
stop it and interfaces come up and go down, on the jail it can ping
the gateway but pings fail. Am I missing a step? I've got a single
IPv4 address and am using private IPv4 addresses. As of now I have not
set an IPv6 address to this jail. The routing tables all look good.
Here's my configuration:

On the host:
#ifconfig bridge0
ifconfig: interface bridge0 does not exist

#ifconfig epair0a
ifconfig: interface epair0a does not exist

#ifconfig epair0b
ifconfig: interface epair0b does not exist

#cat rc.conf
hostname="xxxxxxxxxxxxxx"
ifconfig_vtnet0="DHCP"
ifconfig_vtnet0_ipv6="inet6 accept_rtadv"
jail_enable="YES"

#ifconfig vtnet0
vtnet0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=6c07bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWTSO,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
        ether f2:3c:92:bc:54:37
        inet6 fe80::f03c:92ff:febc:5437%vtnet0 prefixlen 64 scopeid 0x1
        inet6 xxx prefixlen 64 autoconf
        inet xxx.xxx.xxx.xxx netmask 0xffffff00 broadcast xxx.xxx.xxx.xxx
        media: Ethernet 10Gbase-T <full-duplex>
        status: active
        nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>

#cat jail.conf
loghost {
host.hostname     = "loghost";
path              = "/jail/loghost";
mount.devfs;
devfs_ruleset     = "4";
exec.consolelog   = "/var/log/console.loghost";
vnet              = "new";
exec.clean;
vnet.interface    = "epair0b";
exec.prestart     = "ifconfig epair0  create up";
exec.prestart    += "ifconfig bridge0 create up";
exec.prestart    += "ifconfig bridge0 inet 192.168.122.1/24 addm vtnet0";
exec.prestart    += "ifconfig bridge0 addm epair0a";
exec.start        = "/bin/sh /etc/rc";
exec.start       += "ifconfig epair0b inet 192.168.122.50 netmask
255.255.255.0";
exec.start       += "route add default 192.168.122.1";
exec.stop         = "/bin/sh /etc/rc.shutdown";
exec.poststop     = "ifconfig epair0a destroy";
exec.poststop    += "ifconfig bridge0 deletem epair0a";
exec.poststop    += "ifconfig bridge0 destroy";
}

#service jail start
Starting jails: loghost.

#jls
   JID  IP Address      Hostname                      Path
     3                  loghost                       /jail/loghost

#ifconfig bridge0
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        ether 02:bf:cf:92:2c:00
        inet 192.168.122.1 netmask 0xffffff00 broadcast 192.168.122.255
        id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
        maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
        root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
        member: epair0a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 3 priority 128 path cost 2000
        member: vtnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
                ifmaxaddr 0 port 1 priority 128 path cost 2000
        groups: bridge
        nd6 options=1<PERFORMNUD>

#ifconfig epair0a
epair0a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
metric 0 mtu 1500
        options=8<VLAN_MTU>
        ether 02:c0:11:e6:99:0a
        inet6 fe80::c0:11ff:fee6:990a%epair0a prefixlen 64 tentative scopeid 0x3
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>

#ifconfig epair0b
ifconfig: interface epair0b does not exist

#netstat -rn
Routing tables

Internet:
Destination        Gateway            Flags     Netif Expire
default            xxx.xxx.xxx.xxx       UGS      vtnet0
xxx.xxx.xxx.xxx/24     link#1             U        vtnet0
xxx.xxx.xxx.xxx      link#1             UHS         lo0
127.0.0.1          link#2             UH          lo0
192.168.122.0/24   link#5             U       bridge0
192.168.122.1      link#5             UHS         lo0

In the jail:
#jexec loghost /bin/tcsh
#ifconfig -a
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
        options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
        inet6 ::1 prefixlen 128
        inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
        inet 127.0.0.1 netmask 0xff000000
        groups: lo
        nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
epair0b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
        options=8<VLAN_MTU>
        ether 02:c0:11:e6:99:0b
        inet 192.168.122.50 netmask 0xffffff00 broadcast 192.168.122.255
        groups: epair
        media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
        status: active
        nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>

#cat /etc/rc.conf
hostname="loghost"

#ping -c 1 192.168.122.1
PING 192.168.122.1 (192.168.122.1): 56 data bytes
64 bytes from 192.168.122.1: icmp_seq=0 ttl=64 time=0.111 ms

--- 192.168.122.1 ping statistics ---
1 packets transmitted, 1 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 0.111/0.111/0.111/0.000 ms

#ping -c 1 1.1.1.1
PING 1.1.1.1 (1.1.1.1): 56 data bytes

--- 1.1.1.1 ping statistics ---
1 packets transmitted, 0 packets received, 100.0% packet loss

#netstat -rn
Routing tables

Internet:
Destination        Gateway            Flags     Netif Expire
default            192.168.122.1      UGS     epair0b
127.0.0.1          link#1             UH          lo0
192.168.122.0/24   link#2             U       epair0b
192.168.122.50     link#2             UHS         lo0

Am I missing a step with vnet? I was under the impression that vnet
jails have there own TCPIP stack separate from the host's stack.

Thanks.
Dave.
_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

JÁKÓ András
 Hello David,

> I'm trying to get vnet jails going on FreeBSD 12.1-p6. I can start and
> stop it and interfaces come up and go down, on the jail it can ping
> the gateway but pings fail. Am I missing a step? I've got a single
> IPv4 address and am using private IPv4 addresses.

Do you have NAT set up on the host?

András
_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

David Mehler
Hello,

Thanks for your reply. Can you please clarify? I was under the
impression that the two stacks were separate? Should I nat on the
bridge or epair?

Thanks.
Dave.


On 6/27/20, JÁKÓ András <[hidden email]> wrote:

>  Hello David,
>
>> I'm trying to get vnet jails going on FreeBSD 12.1-p6. I can start and
>> stop it and interfaces come up and go down, on the jail it can ping
>> the gateway but pings fail. Am I missing a step? I've got a single
>> IPv4 address and am using private IPv4 addresses.
>
> Do you have NAT set up on the host?
>
> András
>
_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

JÁKÓ András
> I was under the impression that the two stacks were separate?

They are. But I don't think your ISP knows anything about your private
subnet, so they won't send IP packets with your private destination
address to you. And most probably they won't accept IP packets with your
private source address from you. So you have to translate these private
addresses if you want your ISP (and others) to forward them.

> Should I nat on the bridge or epair?

On the bridge, I guess.

András
_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

Ernie Luzar
JÁKÓ András wrote:

>> I was under the impression that the two stacks were separate?
>
> They are. But I don't think your ISP knows anything about your private
> subnet, so they won't send IP packets with your private destination
> address to you. And most probably they won't accept IP packets with your
> private source address from you. So you have to translate these private
> addresses if you want your ISP (and others) to forward them.
>
>> Should I nat on the bridge or epair?
>
> On the bridge, I guess.
>

Have 2 questions.

If there were no ip addresses on the bridge and the epair0b in the vnet
jail would packets pass out the bridge member external interface?

How would I setup a public domain name to target the vnet jail?
_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

JÁKÓ András
> > > I was under the impression that the two stacks were separate?
> >
> > They are. But I don't think your ISP knows anything about your private
> > subnet, so they won't send IP packets with your private destination
> > address to you. And most probably they won't accept IP packets with your
> > private source address from you. So you have to translate these private
> > addresses if you want your ISP (and others) to forward them.
> >
> > > Should I nat on the bridge or epair?
> >
> > On the bridge, I guess.
> >
>
> Have 2 questions.
>
> If there were no ip addresses on the bridge and the epair0b in the vnet jail
> would packets pass out the bridge member external interface?

It's a 802.1 bridge, it can pass frames to the external interface
(according to its MAC address table).

> How would I setup a public domain name to target the vnet jail?

A public domain name should point to a public IP address. If your jail's
IP address is a private one, and you do NAT, then use your public IP
address (the one that is translated to the jail's private address). If
you have a public address in the jail and you don't use address
translation, then use the jail's public IP address in the DNS.

András
_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

Ernie Luzar
JÁKÓ András wrote:

>>>> I was under the impression that the two stacks were separate?
>>> They are. But I don't think your ISP knows anything about your private
>>> subnet, so they won't send IP packets with your private destination
>>> address to you. And most probably they won't accept IP packets with your
>>> private source address from you. So you have to translate these private
>>> addresses if you want your ISP (and others) to forward them.
>>>
>>>> Should I nat on the bridge or epair?
>>> On the bridge, I guess.
>>>
>> Have 2 questions.
>>
>> If there were no ip addresses on the bridge and the epair0b in the vnet jail
>> would packets pass out the bridge member external interface?
>
> It's a 802.1 bridge, it can pass frames to the external interface
> (according to its MAC address table).
>
>> How would I setup a public domain name to target the vnet jail?
>
> A public domain name should point to a public IP address. If your jail's
> IP address is a private one, and you do NAT, then use your public IP
> address (the one that is translated to the jail's private address). If
> you have a public address in the jail and you don't use address
> translation, then use the jail's public IP address in the DNS.
>
> András
>

I think I have determined what your talking about. All the vnet
literature talks about a vnet jail having it's own separate ip stack. I
interpreted this to mean that the vnet jail's stack was connected
directly to the  epair0b / bridge0 / host external interface WITHOUT the
host's firewall knowing anything about that vnet traffic.

Now for the first time I hear you saying that this is not correct. That
all external interface traffic passes through the hosts firewall
including vnet traffic before its handed off to the vnet stack.

I am running FBSD 12.1-p6 on real hardware. em0 is the host interface
connected to the public network with a dynamic ip address by DHCP. To
populate my working vnet jail directory tree I did this.

# download the base.txz file to the host
cd /usr
fetch -avrA
http://ftp.freebsd.org/pub/FreeBSD/releases/amd64/amd64/12.1-RELEASE/base.txz

# unpack base.txz to directory tree
mkdir -p /usr/jails/jailname
cd /usr/jails
xzdec base.txz | tar --unlink -xpJf - -C /usr/jails/jailname

# prep jail directory
cp /etc/localtime jailname/etc
cp /etc/resolv.conf jailname/etc
echo "sendmail_enable="none"" > jailname/etc/rc.conf
echo "sendmail_submit_enable="none"" >> jailname/etc/rc.conf
echo "sendmail_outbound_enable="none"" >> jailname/etc/rc.conf
echo "sendmail_msp_queue_enable="none"" >> jailname/etc/rc.conf


/etc/jail.conf
#
# Using manual command method FBSD 12.1
# with assigned ip address for epairb and bridge.
# start and stop vnet jail works without crashing the host because
# of the embedded sleep commands that work around the teardown bug that
# is now fixed in soon to be released FBSD 13.
# From within the vnet jail can ping the bridge private ip,
# host public ip and the public internet. ping -c 2 1.1.1.1 0% packet loss
#
# Very important detail; host firewall must NAT the private
# ip addresses used.
#
# Issue the following console commands to prep the bridge instead of
# cloned_interfaces="bridge0"
# ifconfig_bridge0="inet 10.0.100.1/24 addm em0 up"
# in rc.conf
#
# ifconfig bridge0 create up
# ifconfig bridge0 inet 10.0.100.1/24 addm em0
#
# using native jail command for start and stop of vnet jail
# -v = verbose outputs log of what start process is really doing
# jail -vc jailname  to start      jail -vr jailname  to stop
# service jail [start stop] jailname  works also.
#
# jexec jailname login -f root   to login to the vnet jail from host
#

testjail {
host.hostname     =  "vnet_testjail";
path              =  "/usr/jails/testjail";
exec.consolelog   =  "/var/log/vnet_testjail.console.log";
mount.devfs       =  "true";
devfs_ruleset     =  "4";
vnet              =  "new";
vnet.interface    =  "epair1b";
exec.prestart     =  "ifconfig epair1  create up";
exec.prestart    +=  "ifconfig bridge0 addm epair1a";
exec.start        =  "/bin/sh /etc/rc";
exec.start       +=  "ifconfig epair1b inet 10.0.100.55 netmask
255.255.255.0";
exec.start       +=  "route add default 10.0.100.1";
exec.stop         =  "/bin/sh /etc/rc.shutdown";
exec.poststop     =  "sleep 2";
exec.poststop    +=  "ifconfig bridge0 deletem epair1a";
exec.poststop    +=  "sleep 2";
exec.poststop    +=  "ifconfig epair1a destroy";
}


Now to get back to your post statement that a 802.1 bridge can pass
frames to the external interface according to MAC address table. I
interpreted this to mean that ip addresses are not needed in the
jail.conf jail definitions to accomplish this. I think that what you are
talking about is the jib method shown in /usr/share/examples/jails. I
have tried getting this jib method to work many times without any success.

There is no bridge to begin with because the jib will create it on the
first vnet jail being started. This is the jail.conf I tried.

testjail2 {
host.hostname     =  "vnet_testjail2";
path              =  "/usr/jails/testjail2";
exec.consolelog   =  "/var/log/vnet_testjail2.console.log";
mount.devfs       =  "true";
devfs_ruleset     =  "4";
vnet              =  "new";
vnet.interface    =  "e0b_testjail2";
exec.prestart     =  "jib addm testjail2 em0";
exec.start        =  "/bin/sh /etc/rc";
exec.stop         =  "/bin/sh /etc/rc.shutdown";
exec.poststop     =  "jib destroy testjail2";
}

I can start and stop this jib jail but when I login to the this vnet
jail and issue ping -c2 1.1.1.1   I get this message
ping: sendto: Network is unreachable.

What changes to the above jib vnet jail config are needed to make it an
MAC address driven vnet jail?

Thanks for the info you have already provided and for your continued help.


















_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

Dan Langille
On Tue, Jun 30, 2020, at 8:30 PM, Ernie Luzar wrote:

> I think I have determined what your talking about. All the vnet
> literature talks about a vnet jail having it's own separate ip stack. I
> interpreted this to mean that the vnet jail's stack was connected
> directly to the  epair0b / bridge0 / host external interface WITHOUT the
> host's firewall knowing anything about that vnet traffic.

FYI, you are not alone.  I have tried to get this working.

A colleague too. We are not novices.

When we get this figured out, it will get documented with a simple
working example.  I promise that.

--
  Dan Langille
  [hidden email]
_______________________________________________
[hidden email] mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-jail
To unsubscribe, send any mail to "[hidden email]"
Reply | Threaded
Open this post in threaded view
|

Re: FreeBSD 12.1, vnet jail, and internet access

freebsd-jail mailing list
Quoting Dan Langille <[hidden email]> (from Tue, 30 Jun 2020  
21:02:24 -0400):

> On Tue, Jun 30, 2020, at 8:30 PM, Ernie Luzar wrote:
>
>> I think I have determined what your talking about. All the vnet
>> literature talks about a vnet jail having it's own separate ip stack. I
>> interpreted this to mean that the vnet jail's stack was connected
>> directly to the  epair0b / bridge0 / host external interface WITHOUT the
>> host's firewall knowing anything about that vnet traffic.
>
> FYI, you are not alone.  I have tried to get this working.
>
> A colleague too. We are not novices.
>
> When we get this figured out, it will get documented with a simple
> working example.  I promise that.
Think about the host as your hypervisor on steroids.

And with this in mind:
  - Your host has a network stack "N0".
  - Your vnet jail has a seperate network stack "N1".
  - The kernel of the "hypervisor" has a firewall and automatically  
makes it see all physical hardware (remember, it depends upon the  
rules if it does something there or not).
  - Without doing anything, they are not connected (= separate), and  
N1 not even to hardware.
  - On the host you create a virtual network device "bridge0". By  
creating it, it is created in the "namespace of the hypervisor" =  
inside N0. This means the firewall of the host is able to do something  
there, if the rules are setup accordingly.
  - When you create the epair, it is also created in N0, like the  
bridge. On the host all commands you do are operating in the namespace  
of the "hypervisor". The firewall sees both ends of the epair and can  
react to it.
  - When you then give epairXb to N1, you remove it from the N0, which means:
    * you have a P2P connection between N0 and N1
    * the host firewall can not inspect packets on epairXb but still on epairXa
    * you could give an IP to epairXa and have only the host  
communicate with the jail, or do some other things like giving epairXa  
to another jail and have a P2P connection between jails (host firewall  
doesn't see both epair ends anymore) or e.g. the next point
  - Then you connect epairXa to the bridge. If there are other jails  
connected you can have them communicate between each other in this  
virtual network, with the host being able to intercept packets which  
show up on the bridge (it is still in the N0 namespace).
  - If you want to communicate with the outside, you can:
    * connect a network interface (which is inside the namespace of  
the host) to the bridge and the packets leaving the physical device  
have the IP from the jail.
    * give the bridge an IP address and have the host route between  
the bridge and the outside (or have it route between bridge A and  
bridge B but not to the outside).
   - In all the above cases, the bridge(s) and the physical interface  
live in the namespace of N0. As such the firewall of N0 can inspect  
packets there, and you can do NAT (the jail doesn't know what is  
outside, so it makes sense to do the NAT on the host).

Bye,
Alexander.

--
http://www.Leidinger.net [hidden email]: PGP 0x8F31830F9F2772BF
http://www.FreeBSD.org    [hidden email]  : PGP 0x8F31830F9F2772BF

attachment0 (836 bytes) Download Attachment