cafkafk/nixpkgs
cafkafk/nixpkgs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

python2 and python3: build unoptimized bytecode again

Browse source 
In 9d03ff5222 I made the CPython builds
reproducible. This required not generating default unoptimized bytecode.
I was under the impression the optimized bytecode would be used then,
but you need to opt-in on that. Not having the default bytecode resulted
in a significant performance hit. Therefore, bytecode is generated again
in this commit, and thereby the builds are no longer reproducible.

https://bugs.python.org/issue29708
This commit is contained in:
Frederik Rietdijk 2021-04-27 11:42:10 +02:00 committed by Frederik Rietdijk
parent 57ef4455c1
commit 23e348bfe2
2 changed files with 19 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
pkgs/development/interpreters/python/cpython/2.7/default.nix
View file

@ -27,9 +27,9 @@
, sha256
, passthruFun
, static ? false
, stripBytecode ? reproducibleBuild
, stripBytecode ? true
, rebuildBytecode ? true
, reproducibleBuild ? true
, reproducibleBuild ? false
, enableOptimizations ? false
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}:
@ -48,6 +48,8 @@ assert lib.assertMsg (reproducibleBuild -> stripBytecode)
assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
"Deterministic builds are not achieved when optimizations are enabled.";
assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
"Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
with lib;
@ -296,8 +298,10 @@ in with passthru; stdenv.mkDerivation ({
# First we delete all old bytecode.
find $out -name "*.pyc" -delete
'' + optionalString rebuildBytecode ''
# Then, we build for the two optimization levels.
# We do not build unoptimized bytecode, because its not entirely deterministic yet.
# We build 3 levels of optimized bytecode. Note the default level, without optimizations,
# is not reproducible yet. https://bugs.python.org/issue29708
# Not creating bytecode will result in a large performance loss however, so we do build it.
find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'' + optionalString stdenv.hostPlatform.isCygwin ''

16
pkgs/development/interpreters/python/cpython/default.nix
View file

@ -35,7 +35,7 @@
, stripTests ? false
, stripTkinter ? false
, rebuildBytecode ? true
, stripBytecode ? reproducibleBuild
, stripBytecode ? true
, includeSiteCustomize ? true
, static ? stdenv.hostPlatform.isStatic
, enableOptimizations ? false
@ -46,7 +46,7 @@
# enabling LTO on 32bit arch causes downstream packages to fail when linking
# enabling LTO on *-darwin causes python3 to fail when linking.
, enableLTO ? stdenv.is64bit && stdenv.isLinux
, reproducibleBuild ? true
, reproducibleBuild ? false
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}:
@ -73,6 +73,9 @@ assert lib.assertMsg (reproducibleBuild -> stripBytecode)
assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
"Deterministic builds are not achieved when optimizations are enabled.";
assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
"Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
with lib;
let
@ -422,11 +425,14 @@ in with passthru; stdenv.mkDerivation {
# First we delete all old bytecode.
find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
'' + optionalString rebuildBytecode ''
# Then, we build for the two optimization levels.
# We do not build unoptimized bytecode, because its not entirely deterministic yet.
# Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
# compileall uses this checked-hash method by default when `SOURCE_DATE_EPOCH` is set.
# compileall uses the therein introduced checked-hash method by default when
# `SOURCE_DATE_EPOCH` is set.
# We exclude lib2to3 because that's Python 2 code which fails
# We build 3 levels of optimized bytecode. Note the default level, without optimizations,
# is not reproducible yet. https://bugs.python.org/issue29708
# Not creating bytecode will result in a large performance loss however, so we do build it.
find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'';