Xen4 CentOS6 QuickStart
yum install -y CentOS-Extras
yum install -y centos-release-xen
yum install -y xen
/usr/bin/grub-bootxen.sh
init 6

More info here.


lomount is great for mounting disk images


Hack job of XEN-ifying CentOS 4.x

co4732xenUG.tar.bz2

Unpack in root (/) of image to be XEN-ified.
Do as noted in README in root.


Restoring a server from a tar file

Mon Mar 3 14:21:24 NZDT 2008
MYSERVER=someserver
mount -o acl /dev/vg/$MYSERVER /t
ssh root@backups
    cat /cache/someserver.ctru.auckland.ac.nz.f.20080301075727.tar.gz | \
        ssh root@xentest tar --numeric-owner -C /t -xvzf -
mkdir /t/{sys,proc,tmp,mnt}
chmod a-w /t/proc
chmod 1777 /t/tmp
mkdir /tt
mount -o acl /dev/vg/co51i386 /tt
/bin/cp -R /tt/lib/modules/2.6.18-53.1.13.el5xen/ /t/lib/modules/
cp -ax /tt/boot/vmlinuz-2.6.18-53.1.13.el5xen /t/boot/
cp -ax /tt/boot/initrd-2.6.18-53.1.13.el5xen.img /t/boot/
cp -ax /tt/boot/System.map-2.6.18-53.1.13.el5xen /t/boot/
cd /t/boot/
rm -f System.map
ln -s System.map-2.6.18-53.1.13.el5xen System.map
cd
vi /t/boot/grub/menu.lst
        default=0
        timeout=5
        title CentOS (2.6.18-8.el5xen)
            root (hd0,0)
            kernel /boot/vmlinuz-2.6.18-53.1.13.el5xen ro root=/dev/sda1
            initrd /boot/initrd-2.6.18-53.1.13.el5xen.img
vi /t/etc/fstab
        /dev/sda1       /               ext3    defaults        1 1
        none            /dev/pts        devpts  gid=5,mode=620  0 0
        none            /dev/shm        tmpfs   defaults        0 0
        none            /proc           proc    defaults        0 0
        none            /sys            sysfs   defaults        0 0
        /dev/sda2       swap            swap    defaults        0 0
vi /t/etc/modprobe.conf
        alias eth0 xennet
        alias scsi_hostadapter xenblk
vi /etc/xen/${MYSERVER}
        name = "someserver"
        memory = 3000
        uuid = "33333329-7527-492c-2b36-fae49ae7d076"
        bootloader="/usr/bin/pygrub"
        disk = ['phy:vg/someserver,sda1,w',
                'phy:vg/someserverswap,sda2,w']
        vcpus=1
        on_reboot   = 'restart'
        on_crash    = 'restart'
vi /t/etc/selinux/config
        # SELINUX=enforcing
        SELINUX=disabled
vi /t/etc/inittab
        co:2345:respawn:/sbin/agetty xvc0 9600 vt100-nav
vi /t/etc/securetty
        xvc0
umount /t /tt
xm create -c ${MYSERVER}

Attach a drive to a guest:
# xm block-attach co51i386 file://somedir/cdimg.iso /dev/hdb r
Detach a drive from a guest:
# xm block-detach co51i386 /dev/hdb

Recovering a guest from a backup image


# On the XEN server
SITE=somesite
DISK=4G
VG=vg2

lvcreate -L 1G -n ${SITE}swap $VG
mkswap /dev/$VG/${SITE}swap
lvcreate -L $DISK -n $SITE $VG
mke2fs -j -L $SITE /dev/$VG/$SITE
mount -o acl /dev/$VG/$SITE /t

# On the BACKUP image server
SITE=somesite
cd /data/$SITE.ctru.auckland.ac.nz/LIVE/
rsync -e ssh -A -a --numeric-ids --delete . root@xentest:/t/
~/mirror.sh $SITE.ctru.auckland.ac.nz
rsync -e ssh -A -a --numeric-ids --delete . root@xentest:/t/
scp /data/xentest2.ctru.auckland.ac.nz/LIVE/etc/xen/$SITE xentest:/etc/xen/

# On the XEN server
cd /t
mkdir sys proc mnt
cd
umount /t
vi /etc/xen/$SITE
xm create -c $SITE
	# Single user mode
	fixfiles relabel
	init 0


# On restored virtual server we need to re-associate with the domain
net join -U administrator --server=PDC --workgroup=MYDOMAIN --ipaddress=10.20.30.40

# A default basic xen guest install
mount -o ro /dev/vg/i386 /tt

lvcreate -L 1G -n cmsswap vg
mkswap /dev/vg/cmsswap

lvcreate -L 12G -n cms vg
mke2fs -j /dev/vg/cms

mount /dev/vg/cms /t
cd /t
tar --numeric-owner -xzf ~/cms.ctru.auckland.ac.nz.f.20071125034509.tar.gz
mkdir proc sys mnt

vi /t/etc/securetty
        xvc0
vi /t/etc/inittab
        co:2345:respawn:/sbin/agetty xvc0 9600 vt100-nav
vi /t/etc/rc.local
	killall nash-hotplug
vi /t/etc/fstab

cd /tt/
find boot lib/modules -depth -print0 | cpio -pvumd0 /t

umount /t
vi /etc/xen/cms
        name = "cms"
        memory = 334
        vif = ['mac=00:14:3E:00:00:44, bridge=xenbr0']
        uuid = "44444426-7527-492c-2b36-fae49ae7d076"
        bootloader="/usr/bin/pygrub"
        disk = ['phy:vg/cms,sda1,w',
                'phy:vg/cmsswap,sda2,w']
xm create -c cms

Starting udev:  [FAILED]    error
vi /t/etc/rc.local
	killall nash-hotplug
  

Stability issues fixup (suggested, not confirmed)

title Xen
        root (hd0,0)
        kernel /xen.gz dom0_mem=262144
        module /vmlinuz-2.6-xen ro root=/dev/vg/root max_loop=64 nousb
        module /initrd-2.6-xen.img

Kernel errors

kernel: 4gb seg fixup, process ...

Fix (on guest):
echo "hwcap 0 nosegneg" > /etc/ld.so.conf.d/nosegneg.conf
ldconfig

Starting the graphical host installer

python /usr/share/virt-manager/virt-manager.py

Restoring a Server (CentOS 5.0)

MYSERVER=myserver
lvcreate -n ${MYSERVER} -L10G vg
mke2fs -j -L ${MYSERVER} /dev/vg/${MYSERVER}
mkdir /t /tt
mount /dev/vg/${MYSERVER} /t
cd /t
tar --numeric-owner -xzf ~/myserver-full.tar.gz
mkdir sys proc tmp mnt
chmod a-w proc
chmod 1777 tmp
mount /dev/vg/basic-para-virt-install /tt
/bin/cp -R /tt/lib/modules/2.6.18-8.el5xen/ /t/lib/modules/
mkdir -p /t/boot/grub/
vi /t/boot/grub/menu.lst
default=0 timeout=5 title CentOS (2.6.18-8.el5xen) root (hd0,0) kernel /boot/vmlinuz-2.6.18-8.el5xen ro root=/dev/sda1 initrd /boot/initrd-2.6.18-8.el5xen.img
cp -ax /tt/boot/vmlinuz-2.6.18-8.el5xen /t/boot/ cp -ax /tt/boot/initrd-2.6.18-8.el5xen.img /t/boot/ cp -ax /tt/boot/System.map-2.6.18-8.el5xen /t/boot/ cd /t/boot/ rm -f System.map ln -s System.map-2.6.18-8.el5xen System.map cd vi /t/etc/fstab
# LABEL=/boot /boot ext3 defaults 1 2 # /dev/vg/swap swap swap defaults 0 0 # /dev/vg/root / ext3 defaults 1 1 /dev/sda1 / ext3 defaults 1 1
vi /t/etc/modprobe.conf
alias eth0 xennet alias scsi_hostadapter xenblk
vi /t/etc/sysconfig/network-scripts/ifcfg-eth0 # MAC match & IP set anew as needed vi /etc/xen/${MYSERVER} /t/etc/sysconfig/network-scripts/ifcfg-eth0 vi /etc/xen/${MYSERVER}
name = "myserver" memory = "500" disk = [ 'phy:/dev/vg/myserver,sda1,w', ] vif = [ 'mac=00:16:3e:00:e1:9e, bridge=xenbr0', ] vfb = ["type=vnc,vncunused=1"] uuid = "5ddd0eed-0b6c-8334-89ae-08f4ee319b49" bootloader="/usr/bin/pygrub" vcpus=1 on_reboot = 'restart' on_crash = 'restart'
chroot /t
fixfiles relabel exit
umount /t /tt rmdir /t /tt xm create -c ${MYSERVER}

Binary install of Xen on Centos 4.4

Wed May 16 13:13:17 NZST 2007
yum install bridge-utils zlib-devel xorg-x11-devel openssl-devel \
        ncurses-devel python-devel tetex-latex ghostscript transfig \
        gcc cpp glibc-devel glibc-headers glibc-kernheaders
mkdir ~/Build
cd ~/Build/
gtar -xvzf ~/xen-3.0.4_1-install-x86_32p.tgz
cd dist/
sh ./install.sh
new-kernel-pkg --mkinitrd --depmod --install 2.6.16.33-xen
cd /boot/
ln -s initrd-2.6.16.33-xen.img initrd-2.6-xen.img
vi /boot/grub/menu.lst
        title Xen
                root (hd0,0)
                kernel /xen.gz dom0_mem=262144
                module /vmlinuz-2.6-xen ro root=/dev/VG/root max_loop=64
                module /initrd-2.6-xen.img
init 6

Xen 3 on Centos 4.3

Notes for the steps I use for creating virtual servers.
The notes here are for upgrading and virtualising of the guests, not just virtualising.
I assume a bare metal server (ie. a completely fresh start).
# The real server gets a fresh O/S load
Installation Type: (o) Custom
Disk Partitioning Setup: (o) Manually Partition with Disk Druid

	/dev/hda1	/boot	Ext3	256M
	/dev/hda2		LVM	Remainder

	/dev/vg/root	/	Ext3	8G
	/dev/vg/swap	swap 	Swap	4G

Firewall Configuration: [X] Remote login
Package Group Selection: [X] Minimal

# Server installs and then reboots

# Set up proxy (if needed) echo ftp_proxy=ftp://proxy:8080 >> /root/.bash_profile echo http_proxy=http://proxy:8080 >> /root/.bash_profile echo export ftp_proxy http_proxy >> /root/.bash_profile . /root/.bash_profile # Install Xen rpm -e lksctp-tools yum install bridge-utils wget http://project-xen.web.cern.ch/project-xen/xen/rpm/xen-3.0.1-12.1_slc4.1.i386.rpm wget http://project-xen.web.cern.ch/project-xen/xen/rpm/xen-kernel-2.6.12-xen3_12.1_slc4.1.i686.rpm wget http://apt.sw.be/redhat/el4/en/i386/RPMS.dag/python-twisted-1.3.0-1.2.el4.rf.i386.rpm rpm -ivh /root/python-twisted-1.3.0-1.2.el4.rf.i386.rpm rpm -ivh /root/xen-3.0.1-12.1_slc4.1.i386.rpm rpm -ivh /root/xen-kernel-2.6.12-xen3_12.1_slc4.1.i686.rpm vi /boot/grub/menu.lst
title Xen
	root (hd0,0)
	kernel /xen.gz dom0_mem=262144
	module /vmlinuz-2.6-xen ro root=/dev/vg/root
	module /initrd-2.6-xen.img
chkconfig nfslock off chkconfig netfs off chkconfig gpm off chkconfig cups off chkconfig isdn off chkconfig pcmcia off chkconfig apmd off chkconfig smartd off chkconfig kudzu off chkconfig portmap off chkconfig mdmonitor off chkconfig autofs off chkconfig rpcidmapd off chkconfig rpcgssd off chkconfig readahead off chkconfig readahead_early off mv /lib/tls /lib/tls.disabled init 6

Create a virtual server

NEWSERVER=qwerty lvcreate --size 128M --name ${NEWSERVER}swap /dev/vg mkswap /dev/vg/${NEWSERVER}swap lvcreate --size 4G --name ${NEWSERVER} /dev/vg mkfs.ext3 /dev/vg/${NEWSERVER} mkdir /t mount /dev/vg/${NEWSERVER} /t cp -ax /{dev,var,etc,usr,bin,sbin,lib} /t mkdir /t/{proc,sys,home,tmp,root,opt,mnt} chown root:root /t/{proc,sys,home,tmp,root,opt,mnt} chmod 755 /t/{proc,sys,home,root,opt} chmod 1777 /t/tmp vi /t/etc/selinux/config
# SELINUX=enforcing
SELINUX=disabled
vi /t/etc/fstab
/dev/sda1               /                       ext3    defaults        1 1
/dev/sda2               swap                    swap    defaults        0 0
# /dev/vg/root          /                       ext3    defaults        1 1
# LABEL=/boot           /boot                   ext3    defaults        1 2
# /dev/vg/swap          swap                    swap    defaults        0 0
vi /t/etc/sysconfig/network vi /t/etc/sysconfig/network-scripts/ifcfg-eth0 vi /t/etc/hosts vi /t/var/spool/cron/* vi /etc/xen/${NEWSERVER}
name = "newserver"
hostname = "newserver"
memory = 128
vif = ['mac=00:16:3E:FF:00:05']
ip = "10.0.0.1"
netmask = "255.255.255.0"
gateway = "10.0.0.254"
root = "/dev/sda1 ro"
extra = "3 fastboot"
kernel = "/boot/vmlinuz-2.6-xen"
ramdisk = "/boot/initrd-2.6-xen.img"
disk = ['phy:vg/newserver,sda1,w',
        'phy:vg/newserverswap,sda2,w']
# I am upgrading the servers in this case so I create a /OLD folder on the # virtual server and copy the entire content of the old server across. # A truly virtualised server would be a lot simpler. # Stuff you may wish to do if you are upgrading mkdir /t/OLD/ cd /t/OLD/ gtar --numeric-owner -xvzf ~/${NEWSERVER}.tar.gz # Unmount the guest filesystem cd umount /t rmdir /t # Start the guest with console "-c" attached xm create -c ${NEWSERVER}

Configure a Virtual Server from within

You are logged onto newserver as root for this.

chkconfig xend off service xend stop chkconfig xendomains off service xendomains stop # Allow clock updates echo '/bin/echo 1 > /proc/sys/xen/independent_wallclock' >> /etc/rc.local echo 1 > /proc/sys/xen/independent_wallclock # Stuff that you may want to do if you're upgrading/moving mv /OLD/home/* /home/ cat /OLD/etc/passwd >> /etc/passwd vi /etc/passwd cat /OLD/etc/shadow >> /etc/shadow vi /etc/shadow cat /OLD/etc/group >> /etc/group vi /etc/group cat /OLD/etc/sudoers >> /etc/sudoers vi /etc/sudoers cat /OLD/etc/hosts.deny >> /etc/hosts.deny vi /etc/hosts.deny cat /OLD/etc/hosts.allow >> /etc/hosts.allow echo "sendmail: 127." >> /etc/hosts.allow vi /etc/hosts.allow cat /OLD/etc/hosts >> /etc/hosts vi /etc/hosts cat /OLD/etc/aliases >> /etc/aliases vi /etc/aliases newaliases cat /OLD/var/spool/cron/root >> /var/spool/cron/root crontab -e # May want to look at other users crontab entries also in /OLD/var/spool/cron/ cp -ax /OLD/root/.ssh /root/ cp -ax /OLD/opt/openssh/P/etc/*_key* /etc/ssh/ cp -ax /OLD/opt/openssh/P/etc/moduli /etc/ssh/ service sshd restart


Install on Centos 4.3 minimal from source

# wget ... linux-2.6.16.33.tar.bz2
# wget ... xen-3.0.4_1-src.tgz
echo ftp_proxy=ftp://proxy:8080 >> /root/.bash_profile
echo http_proxy=http://proxy:8080 >> /root/.bash_profile
echo export ftp_proxy http_proxy >> /root/.bash_profile
. /root/.bash_profile

rpm -e lksctp-tools
yum install bridge-utils zlib-devel xorg-x11-devel openssl-devel ncurses-devel python-devel \
        tetex-latex ghostscript transfig gcc cpp glibc-devel glibc-headers \
        glibc-kernheaders

mkdir ~/Build/
cd ~/Build/
gtar -xvzf ~/xen-3.0.4_1-src.tgz
ln ~/linux-2.6.16.33.tar.bz2 ~/Build/xen-3.0.4_1-src/
cd ~/Build/xen-3.0.4_1-src/

make KERNELS="linux-2.6-xen0 linux-2.6-xenU" XEN_TARGET_X86_PAE=y world
make KERNELS="linux-2.6-xen0 linux-2.6-xenU" XEN_TARGET_X86_PAE=y install
mkinitrd /boot/initrd-2.6.16.33-xen0.img 2.6.16.33-xen0
mkinitrd /boot/initrd-2.6.16.33-xenU.img 2.6.16.33-xenU
vi /etc/grub.conf
        title Xen 3.0 / XenLinux 2.6.16
                root (hd0,0)
                kernel /xen.gz dom0_mem=128000 noreboot console=tty0
                module /vmlinuz-2.6.16.33-xen0 root=/dev/vg/root nousb
                module /initrd-2.6.16.33-xen0.img
mv /lib/tls /lib/tls.disabled
init 6

Document History
Tue Oct 10 14:21:25 NZDT 2006 Clark Mills Created
Fri Oct 27 08:26:11 NZDT 2006 Clark Mills Moved & edited for public site
© Clinical Trials Research Unit, The University of Auckland, 2006