2014-02-26 05:43:34 +01:00
|
|
|
{ stdenv, writeScript, vmTools, makeInitrd
|
2016-10-30 02:52:47 +02:00
|
|
|
, samba, vde2, openssh, socat, netcat-gnu, coreutils, gnugrep, gzip
|
2014-02-26 05:43:34 +01:00
|
|
|
}:
|
|
|
|
|
2014-02-15 23:23:47 +01:00
|
|
|
{ sshKey
|
|
|
|
, qemuArgs ? []
|
|
|
|
, command ? "sync"
|
|
|
|
, suspendTo ? null
|
|
|
|
, resumeFrom ? null
|
|
|
|
, installMode ? false
|
|
|
|
}:
|
|
|
|
|
2014-02-26 05:43:34 +01:00
|
|
|
with stdenv.lib;
|
2014-02-15 23:23:47 +01:00
|
|
|
|
2014-02-26 05:43:34 +01:00
|
|
|
let
|
2014-02-15 23:23:47 +01:00
|
|
|
preInitScript = writeScript "preinit.sh" ''
|
|
|
|
#!${vmTools.initrdUtils}/bin/ash -e
|
|
|
|
export PATH=${vmTools.initrdUtils}/bin
|
|
|
|
mount -t proc none /proc
|
|
|
|
mount -t sysfs none /sys
|
|
|
|
for arg in $(cat /proc/cmdline); do
|
|
|
|
if [ "x''${arg#command=}" != "x$arg" ]; then
|
|
|
|
command="''${arg#command=}"
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
|
|
|
|
for i in $(cat ${modulesClosure}/insmod-list); do
|
|
|
|
insmod $i
|
|
|
|
done
|
|
|
|
|
2014-02-16 16:53:02 +01:00
|
|
|
mkdir -p /dev /fs
|
|
|
|
|
|
|
|
mount -t tmpfs none /dev
|
2014-02-15 23:23:47 +01:00
|
|
|
mknod /dev/null c 1 3
|
|
|
|
mknod /dev/zero c 1 5
|
|
|
|
mknod /dev/random c 1 8
|
|
|
|
mknod /dev/urandom c 1 9
|
|
|
|
mknod /dev/tty c 5 0
|
|
|
|
|
|
|
|
ifconfig lo up
|
|
|
|
ifconfig eth0 up 192.168.0.2
|
|
|
|
|
2014-02-16 16:53:02 +01:00
|
|
|
mount -t tmpfs none /fs
|
|
|
|
mkdir -p /fs/nix/store /fs/xchg /fs/dev /fs/sys /fs/proc /fs/etc /fs/tmp
|
2014-02-15 23:23:47 +01:00
|
|
|
|
2014-02-16 16:53:02 +01:00
|
|
|
mount -o bind /dev /fs/dev
|
|
|
|
mount -t sysfs none /fs/sys
|
|
|
|
mount -t proc none /fs/proc
|
2014-02-15 23:23:47 +01:00
|
|
|
|
2014-02-16 03:19:40 +01:00
|
|
|
mount -t 9p \
|
nixos/vm-tests: Remove msize mount option
This seems to be the root cause of the random page allocation failures
and @wizeman did a very good job on not only finding the root problem
but also giving a detailed explanation of it in #10828.
Here is an excerpt:
The problem here is that the kernel is trying to allocate a contiguous
section of 2^7=128 pages, which is 512 KB. This is way too much:
kernel pages tend to get fragmented over time and kernel developers
often go to great lengths to try allocating at most only 1 contiguous
page at a time whenever they can.
From the error message, it looks like the culprit is unionfs, but this
is misleading: unionfs is the name of the userspace process that was
running when the system ran out of memory, but it wasn't unionfs who
was allocating the memory: it was the kernel; specifically it was the
v9fs_dir_readdir_dotl() function, which is the code for handling the
readdir() function in the 9p filesystem (the filesystem that is used
to share a directory structure between a qemu host and its VM).
If you look at the code, here's what it's doing at the moment it tries
to allocate memory:
buflen = fid->clnt->msize - P9_IOHDRSZ;
rdir = v9fs_alloc_rdir_buf(file, buflen);
If you look into v9fs_alloc_rdir_buf(), you will see that it will try
to allocate a contiguous buffer of memory (using kzalloc(), which is a
wrapper around kmalloc()) of size buflen + 8 bytes or so.
So in reality, this code actually allocates a buffer of size
proportional to fid->clnt->msize. What is this msize? If you follow
the definition of the structures, you will see that it's the
negotiated buffer transfer size between 9p client and 9p server. On
the client side, it can be controlled with the msize mount option.
What this all means is that, the reason for running out of memory is
that the code (which we can't easily change) tries to allocate a
contiguous buffer of size more or less equal to "negotiated 9p
protocol buffer size", which seems to be way too big (in our NixOS
tests, at least).
After that initial finding, @lethalman tested the gnome3 gdm test
without setting the msize parameter at all and it seems to have resolved
the problem.
The reason why I'm committing this without testing against all of the
NixOS VM test is basically that I think we can only go better but not
worse than the current state.
Signed-off-by: aszlig <aszlig@redmoonstudios.org>
2015-12-14 17:26:24 +01:00
|
|
|
-o trans=virtio,version=9p2000.L,cache=loose \
|
2014-02-16 16:53:02 +01:00
|
|
|
store /fs/nix/store
|
2014-02-16 03:19:40 +01:00
|
|
|
|
2014-02-15 23:23:47 +01:00
|
|
|
mount -t 9p \
|
nixos/vm-tests: Remove msize mount option
This seems to be the root cause of the random page allocation failures
and @wizeman did a very good job on not only finding the root problem
but also giving a detailed explanation of it in #10828.
Here is an excerpt:
The problem here is that the kernel is trying to allocate a contiguous
section of 2^7=128 pages, which is 512 KB. This is way too much:
kernel pages tend to get fragmented over time and kernel developers
often go to great lengths to try allocating at most only 1 contiguous
page at a time whenever they can.
From the error message, it looks like the culprit is unionfs, but this
is misleading: unionfs is the name of the userspace process that was
running when the system ran out of memory, but it wasn't unionfs who
was allocating the memory: it was the kernel; specifically it was the
v9fs_dir_readdir_dotl() function, which is the code for handling the
readdir() function in the 9p filesystem (the filesystem that is used
to share a directory structure between a qemu host and its VM).
If you look at the code, here's what it's doing at the moment it tries
to allocate memory:
buflen = fid->clnt->msize - P9_IOHDRSZ;
rdir = v9fs_alloc_rdir_buf(file, buflen);
If you look into v9fs_alloc_rdir_buf(), you will see that it will try
to allocate a contiguous buffer of memory (using kzalloc(), which is a
wrapper around kmalloc()) of size buflen + 8 bytes or so.
So in reality, this code actually allocates a buffer of size
proportional to fid->clnt->msize. What is this msize? If you follow
the definition of the structures, you will see that it's the
negotiated buffer transfer size between 9p client and 9p server. On
the client side, it can be controlled with the msize mount option.
What this all means is that, the reason for running out of memory is
that the code (which we can't easily change) tries to allocate a
contiguous buffer of size more or less equal to "negotiated 9p
protocol buffer size", which seems to be way too big (in our NixOS
tests, at least).
After that initial finding, @lethalman tested the gnome3 gdm test
without setting the msize parameter at all and it seems to have resolved
the problem.
The reason why I'm committing this without testing against all of the
NixOS VM test is basically that I think we can only go better but not
worse than the current state.
Signed-off-by: aszlig <aszlig@redmoonstudios.org>
2015-12-14 17:26:24 +01:00
|
|
|
-o trans=virtio,version=9p2000.L,cache=loose \
|
2014-02-16 16:53:02 +01:00
|
|
|
xchg /fs/xchg
|
|
|
|
|
2014-02-16 18:58:08 +01:00
|
|
|
echo root:x:0:0::/root:/bin/false > /fs/etc/passwd
|
2014-02-15 23:23:47 +01:00
|
|
|
|
2014-02-16 16:53:02 +01:00
|
|
|
set +e
|
|
|
|
chroot /fs $command $out
|
|
|
|
echo $? > /fs/xchg/in-vm-exit
|
|
|
|
|
|
|
|
poweroff -f
|
2014-02-15 23:23:47 +01:00
|
|
|
'';
|
|
|
|
|
|
|
|
initrd = makeInitrd {
|
2014-02-26 05:43:34 +01:00
|
|
|
contents = singleton {
|
2014-02-15 23:23:47 +01:00
|
|
|
object = preInitScript;
|
|
|
|
symlink = "/init";
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
2014-02-16 03:19:40 +01:00
|
|
|
loopForever = "while :; do ${coreutils}/bin/sleep 1; done";
|
|
|
|
|
2014-02-15 23:23:47 +01:00
|
|
|
initScript = writeScript "init.sh" (''
|
|
|
|
#!${stdenv.shell}
|
2014-02-16 16:47:23 +01:00
|
|
|
${coreutils}/bin/cp -L "${sshKey}" /ssh.key
|
|
|
|
${coreutils}/bin/chmod 600 /ssh.key
|
|
|
|
'' + (if installMode then ''
|
|
|
|
echo -n "Waiting for Windows installation to finish..."
|
2016-10-30 02:52:47 +02:00
|
|
|
while ! ${netcat-gnu}/bin/netcat -z 192.168.0.1 22; do
|
2014-02-16 16:47:23 +01:00
|
|
|
echo -n .
|
|
|
|
# Print a dot every 10 seconds only to shorten line length.
|
|
|
|
${coreutils}/bin/sleep 10
|
|
|
|
done
|
2014-05-07 03:21:55 +02:00
|
|
|
${coreutils}/bin/touch /xchg/waiting_done
|
2014-02-16 16:47:23 +01:00
|
|
|
echo " success."
|
|
|
|
# Loop forever, because this VM is going to be killed.
|
|
|
|
${loopForever}
|
|
|
|
'' else ''
|
2014-02-16 16:53:02 +01:00
|
|
|
${coreutils}/bin/mkdir -p /etc/samba /etc/samba/private \
|
|
|
|
/var/lib/samba /var/log /var/run
|
2014-02-15 23:23:47 +01:00
|
|
|
${coreutils}/bin/cat > /etc/samba/smb.conf <<CONFIG
|
|
|
|
[global]
|
|
|
|
security = user
|
|
|
|
map to guest = Bad User
|
2014-02-16 18:58:08 +01:00
|
|
|
guest account = root
|
2014-02-15 23:23:47 +01:00
|
|
|
workgroup = cygwin
|
|
|
|
netbios name = controller
|
|
|
|
server string = %h
|
|
|
|
log level = 1
|
|
|
|
max log size = 1000
|
|
|
|
log file = /var/log/samba.log
|
|
|
|
|
|
|
|
[nixstore]
|
|
|
|
path = /nix/store
|
2014-02-16 18:58:08 +01:00
|
|
|
writable = yes
|
2014-02-15 23:23:47 +01:00
|
|
|
guest ok = yes
|
2014-02-16 03:19:40 +01:00
|
|
|
|
|
|
|
[xchg]
|
|
|
|
path = /xchg
|
2014-02-16 18:58:08 +01:00
|
|
|
writable = yes
|
2014-02-16 03:19:40 +01:00
|
|
|
guest ok = yes
|
2014-02-15 23:23:47 +01:00
|
|
|
CONFIG
|
|
|
|
|
|
|
|
${samba}/sbin/nmbd -D
|
|
|
|
${samba}/sbin/smbd -D
|
2014-02-16 16:53:02 +01:00
|
|
|
|
2014-02-15 23:23:47 +01:00
|
|
|
echo -n "Waiting for Windows VM to become available..."
|
2016-10-30 02:52:47 +02:00
|
|
|
while ! ${netcat-gnu}/bin/netcat -z 192.168.0.1 22; do
|
2014-02-15 23:23:47 +01:00
|
|
|
echo -n .
|
|
|
|
${coreutils}/bin/sleep 1
|
|
|
|
done
|
2014-05-07 03:21:55 +02:00
|
|
|
${coreutils}/bin/touch /xchg/waiting_done
|
2014-02-15 23:23:47 +01:00
|
|
|
echo " success."
|
|
|
|
|
|
|
|
${openssh}/bin/ssh \
|
|
|
|
-o UserKnownHostsFile=/dev/null \
|
|
|
|
-o StrictHostKeyChecking=no \
|
|
|
|
-i /ssh.key \
|
|
|
|
-l Administrator \
|
2016-06-12 19:11:37 +02:00
|
|
|
192.168.0.1 -- ${lib.escapeShellArg command}
|
2014-02-26 05:43:34 +01:00
|
|
|
'') + optionalString (suspendTo != null) ''
|
2014-02-16 03:19:40 +01:00
|
|
|
${coreutils}/bin/touch /xchg/suspend_now
|
|
|
|
${loopForever}
|
2014-02-17 00:34:19 +01:00
|
|
|
'');
|
2014-02-15 23:23:47 +01:00
|
|
|
|
2014-02-26 05:43:34 +01:00
|
|
|
kernelAppend = concatStringsSep " " [
|
2014-02-15 23:23:47 +01:00
|
|
|
"panic=1"
|
|
|
|
"loglevel=4"
|
|
|
|
"console=tty1"
|
|
|
|
"console=ttyS0"
|
|
|
|
"command=${initScript}"
|
|
|
|
];
|
|
|
|
|
2014-02-26 05:43:34 +01:00
|
|
|
controllerQemuArgs = concatStringsSep " " (maybeKvm64 ++ [
|
2014-05-07 03:21:55 +02:00
|
|
|
"-pidfile $CTRLVM_PIDFILE"
|
2014-02-15 23:23:47 +01:00
|
|
|
"-nographic"
|
|
|
|
"-no-reboot"
|
|
|
|
"-virtfs local,path=/nix/store,security_model=none,mount_tag=store"
|
2014-02-16 03:19:40 +01:00
|
|
|
"-virtfs local,path=$XCHG_DIR,security_model=none,mount_tag=xchg"
|
2014-02-15 23:23:47 +01:00
|
|
|
"-kernel ${modulesClosure.kernel}/bzImage"
|
|
|
|
"-initrd ${initrd}/initrd"
|
|
|
|
"-append \"${kernelAppend}\""
|
|
|
|
"-net nic,vlan=0,macaddr=52:54:00:12:01:02,model=virtio"
|
|
|
|
"-net vde,vlan=0,sock=$QEMU_VDE_SOCKET"
|
|
|
|
]);
|
|
|
|
|
2014-02-26 05:43:34 +01:00
|
|
|
maybeKvm64 = optional (stdenv.system == "x86_64-linux") "-cpu kvm64";
|
2014-02-15 23:23:47 +01:00
|
|
|
|
2014-02-26 05:43:34 +01:00
|
|
|
cygwinQemuArgs = concatStringsSep " " (maybeKvm64 ++ [
|
2014-02-15 23:23:47 +01:00
|
|
|
"-monitor unix:$MONITOR_SOCKET,server,nowait"
|
2014-05-07 03:21:55 +02:00
|
|
|
"-pidfile $WINVM_PIDFILE"
|
2014-02-15 23:23:47 +01:00
|
|
|
"-nographic"
|
|
|
|
"-net nic,vlan=0,macaddr=52:54:00:12:01:01"
|
|
|
|
"-net vde,vlan=0,sock=$QEMU_VDE_SOCKET"
|
|
|
|
"-rtc base=2010-01-01,clock=vm"
|
2014-02-26 05:43:34 +01:00
|
|
|
] ++ qemuArgs ++ optionals (resumeFrom != null) [
|
2014-02-15 23:23:47 +01:00
|
|
|
"-incoming 'exec: ${gzip}/bin/gzip -c -d \"${resumeFrom}\"'"
|
|
|
|
]);
|
|
|
|
|
2014-02-26 05:43:34 +01:00
|
|
|
modulesClosure = overrideDerivation vmTools.modulesClosure (o: {
|
|
|
|
rootModules = o.rootModules ++ singleton "virtio_net";
|
2014-02-15 23:23:47 +01:00
|
|
|
});
|
|
|
|
|
|
|
|
preVM = ''
|
2014-02-16 22:34:45 +01:00
|
|
|
(set; declare -p) > saved-env
|
2014-02-16 03:19:40 +01:00
|
|
|
XCHG_DIR="$(${coreutils}/bin/mktemp -d nix-vm.XXXXXXXXXX --tmpdir)"
|
2014-02-16 22:34:45 +01:00
|
|
|
${coreutils}/bin/mv saved-env "$XCHG_DIR/"
|
2014-02-17 00:34:19 +01:00
|
|
|
|
|
|
|
eval "$preVM"
|
|
|
|
|
2014-02-15 23:23:47 +01:00
|
|
|
QEMU_VDE_SOCKET="$(pwd)/vde.ctl"
|
|
|
|
MONITOR_SOCKET="$(pwd)/monitor"
|
2014-05-07 03:21:55 +02:00
|
|
|
WINVM_PIDFILE="$(pwd)/winvm.pid"
|
|
|
|
CTRLVM_PIDFILE="$(pwd)/ctrlvm.pid"
|
2014-02-15 23:23:47 +01:00
|
|
|
${vde2}/bin/vde_switch -s "$QEMU_VDE_SOCKET" &
|
2014-02-16 00:27:43 +01:00
|
|
|
echo 'alive?' | ${socat}/bin/socat - \
|
|
|
|
UNIX-CONNECT:$QEMU_VDE_SOCKET/ctl,retry=20
|
2014-02-15 23:23:47 +01:00
|
|
|
'';
|
|
|
|
|
2014-05-07 03:21:55 +02:00
|
|
|
vmExec = ''
|
2014-02-15 23:23:47 +01:00
|
|
|
${vmTools.qemuProg} ${controllerQemuArgs} &
|
|
|
|
${vmTools.qemuProg} ${cygwinQemuArgs} &
|
2014-05-07 03:21:55 +02:00
|
|
|
echo -n "Waiting for VMs to start up..."
|
|
|
|
timeout=60
|
|
|
|
while ! test -e "$WINVM_PIDFILE" -a -e "$CTRLVM_PIDFILE"; do
|
|
|
|
timeout=$(($timeout - 1))
|
|
|
|
echo -n .
|
|
|
|
if test $timeout -le 0; then
|
|
|
|
echo " timed out."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
${coreutils}/bin/sleep 1
|
|
|
|
done
|
|
|
|
echo " done."
|
|
|
|
'';
|
|
|
|
|
|
|
|
checkDropOut = ''
|
|
|
|
if ! test -e "$XCHG_DIR/waiting_done" &&
|
|
|
|
! kill -0 $(< "$WINVM_PIDFILE"); then
|
|
|
|
echo "Windows VM has dropped out early, bailing out!" >&2
|
|
|
|
exit 1
|
|
|
|
fi
|
2014-02-17 00:34:19 +01:00
|
|
|
'';
|
|
|
|
|
2014-05-07 06:48:53 +02:00
|
|
|
toMonitor = "${socat}/bin/socat - UNIX-CONNECT:$MONITOR_SOCKET";
|
|
|
|
|
2014-02-17 00:34:19 +01:00
|
|
|
postVM = if suspendTo != null then ''
|
2014-05-07 03:21:55 +02:00
|
|
|
while ! test -e "$XCHG_DIR/suspend_now"; do
|
|
|
|
${checkDropOut}
|
|
|
|
${coreutils}/bin/sleep 1
|
|
|
|
done
|
2014-05-07 06:48:53 +02:00
|
|
|
${toMonitor} <<CMD
|
2014-02-15 23:23:47 +01:00
|
|
|
stop
|
|
|
|
migrate_set_speed 4095m
|
|
|
|
migrate "exec:${gzip}/bin/gzip -c > '${suspendTo}'"
|
|
|
|
CMD
|
2014-05-07 06:48:53 +02:00
|
|
|
echo -n "Waiting for memory dump to finish..."
|
|
|
|
while ! echo info migrate | ${toMonitor} | \
|
|
|
|
${gnugrep}/bin/grep -qi '^migration *status: *complete'; do
|
|
|
|
${coreutils}/bin/sleep 1
|
|
|
|
echo -n .
|
|
|
|
done
|
|
|
|
echo " done."
|
|
|
|
echo quit | ${toMonitor}
|
2014-05-07 03:21:55 +02:00
|
|
|
wait $(< "$WINVM_PIDFILE")
|
2014-02-17 00:34:19 +01:00
|
|
|
eval "$postVM"
|
|
|
|
exit 0
|
|
|
|
'' else if installMode then ''
|
2014-05-07 03:21:55 +02:00
|
|
|
wait $(< "$WINVM_PIDFILE")
|
2014-02-17 00:34:19 +01:00
|
|
|
eval "$postVM"
|
|
|
|
exit 0
|
|
|
|
'' else ''
|
2014-05-07 03:21:55 +02:00
|
|
|
while kill -0 $(< "$CTRLVM_PIDFILE"); do
|
|
|
|
${checkDropOut}
|
|
|
|
done
|
2014-02-17 00:34:19 +01:00
|
|
|
if ! test -e "$XCHG_DIR/in-vm-exit"; then
|
|
|
|
echo "Virtual machine didn't produce an exit code."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
eval "$postVM"
|
|
|
|
exit $(< "$XCHG_DIR/in-vm-exit")
|
2014-02-15 23:23:47 +01:00
|
|
|
'';
|
|
|
|
|
|
|
|
in writeScript "run-cygwin-vm.sh" ''
|
|
|
|
#!${stdenv.shell} -e
|
|
|
|
${preVM}
|
|
|
|
${vmExec}
|
2014-02-17 00:34:19 +01:00
|
|
|
${postVM}
|
2014-02-15 23:23:47 +01:00
|
|
|
''
|