Merge pull request #65294 from steshaw/linkchecker-9.4.0
linkchecker: 9.3.1 -> 9.4.0
This commit is contained in:
Peter Hoeg
GitHub
commit
b9412f4fdb
2 changed files with 31 additions and 106 deletions
|
|
@ -1,60 +0,0 @@
|
|||
diff --git a/linkcheck/checker/httpurl.py b/linkcheck/checker/httpurl.py
|
||||
index 6f207b6..161619c 100644
|
||||
--- a/linkcheck/checker/httpurl.py
|
||||
+++ b/linkcheck/checker/httpurl.py
|
||||
@@ -75,7 +75,7 @@ def allows_robots (self, url):
|
||||
@return: True if access is granted, otherwise False
|
||||
@rtype: bool
|
||||
"""
|
||||
- return self.aggregate.robots_txt.allows_url(self)
|
||||
+ return not self.aggregate.config['robotstxt'] or self.aggregate.robots_txt.allows_url(self)
|
||||
|
||||
def content_allows_robots (self):
|
||||
"""
|
||||
diff --git a/linkcheck/configuration/__init__.py b/linkcheck/configuration/__init__.py
|
||||
index fc2c148..234fa05 100644
|
||||
--- a/linkcheck/configuration/__init__.py
|
||||
+++ b/linkcheck/configuration/__init__.py
|
||||
@@ -163,6 +163,7 @@ def __init__ (self):
|
||||
## checking options
|
||||
self["allowedschemes"] = []
|
||||
self['cookiefile'] = None
|
||||
+ self['robotstxt'] = True
|
||||
self["debugmemory"] = False
|
||||
self["localwebroot"] = None
|
||||
self["maxfilesizeparse"] = 1*1024*1024
|
||||
diff --git a/linkcheck/configuration/confparse.py b/linkcheck/configuration/confparse.py
|
||||
index 67751ed..845fa95 100644
|
||||
--- a/linkcheck/configuration/confparse.py
|
||||
+++ b/linkcheck/configuration/confparse.py
|
||||
@@ -149,6 +149,7 @@ def read_checking_config (self):
|
||||
self.get(section, 'allowedschemes').split(',')]
|
||||
self.read_boolean_option(section, "debugmemory")
|
||||
self.read_string_option(section, "cookiefile")
|
||||
+ self.read_boolean_option(section, "robotstxt")
|
||||
self.read_string_option(section, "localwebroot")
|
||||
try:
|
||||
self.read_boolean_option(section, "sslverify")
|
||||
diff --git a/linkchecker b/linkchecker
|
||||
index 199532c..9e91fa5 100755
|
||||
--- a/linkchecker
|
||||
+++ b/linkchecker
|
||||
@@ -321,6 +321,9 @@ group.add_argument("--cookiefile", dest="cookiefile", metavar="FILENAME",
|
||||
help=_(
|
||||
"""Read a file with initial cookie data. The cookie data format is
|
||||
explained below."""))
|
||||
+# const because store_false doesn't detect absent flags
|
||||
+group.add_argument("--no-robots", action="store_const", const=False,
|
||||
+ dest="norobotstxt", help=_("Disable robots.txt checks"))
|
||||
group.add_argument("--check-extern", action="store_true",
|
||||
dest="checkextern", help=_("""Check external URLs."""))
|
||||
group.add_argument("--ignore-url", action="append", metavar="REGEX",
|
||||
@@ -431,6 +434,8 @@ if options.externstrict:
|
||||
if options.extern:
|
||||
pats = [linkcheck.get_link_pat(arg) for arg in options.extern]
|
||||
config["externlinks"].extend(pats)
|
||||
+if options.norobotstxt is not None:
|
||||
+ config['robotstxt'] = options.norobotstxt
|
||||
if options.checkextern:
|
||||
config["checkextern"] = True
|
||||
elif not config["checkextern"]:
|
||||
|
|
@ -1,64 +1,49 @@
|
|||
{ stdenv, lib, fetchFromGitHub, fetchpatch, python2, gettext }:
|
||||
let
|
||||
# pin requests version until next release.
|
||||
# see: https://github.com/linkcheck/linkchecker/issues/76
|
||||
python2Packages = (python2.override {
|
||||
packageOverrides = self: super: {
|
||||
requests = super.requests.overridePythonAttrs(oldAttrs: rec {
|
||||
version = "2.14.2";
|
||||
src = oldAttrs.src.override {
|
||||
inherit version;
|
||||
sha256 = "0lyi82a0ijs1m7k9w1mqwbmq1qjsac35fazx7xqyh8ws76xanx52";
|
||||
};
|
||||
});
|
||||
};
|
||||
}).pkgs;
|
||||
in
|
||||
{ stdenv, lib, fetchFromGitHub, python2Packages, gettext }:
|
||||
|
||||
python2Packages.buildPythonApplication rec {
|
||||
pname = "LinkChecker";
|
||||
version = "9.3.1";
|
||||
pname = "linkchecker";
|
||||
version = "9.4.0";
|
||||
|
||||
nativeBuildInputs = [ gettext ];
|
||||
pythonPath = (with python2Packages; [
|
||||
requests
|
||||
]) ++ [ gettext ];
|
||||
|
||||
checkInputs = with python2Packages; [ pytest ];
|
||||
|
||||
# the original repository is abandoned, development is now happening here:
|
||||
src = fetchFromGitHub {
|
||||
owner = "linkcheck";
|
||||
repo = "linkchecker";
|
||||
owner = pname;
|
||||
repo = pname;
|
||||
rev = "v${version}";
|
||||
sha256 = "080mv4iwvlsfnm7l9basd6i8p4q8990mdhkwick9s6javrbf1r1d";
|
||||
sha256 = "1vbwl2vb8dyzki27z3sl5yf9dhdd2cpkg10vbgaz868dhpqlshgs";
|
||||
};
|
||||
|
||||
# 1. upstream refuses to support ignoring robots.txt
|
||||
# 2. fix build: https://github.com/linkcheck/linkchecker/issues/10
|
||||
patches =
|
||||
let
|
||||
fix-setup-py = fetchpatch {
|
||||
name = "fix-setup-py.patch";
|
||||
url = https://github.com/linkcheck/linkchecker/commit/e62e630.patch;
|
||||
sha256 = "046q1whg715w2yv33xx6rkj7fspvvz60cl978ax92lnf8j101czx";
|
||||
};
|
||||
in [
|
||||
./add-no-robots-flag.patch
|
||||
fix-setup-py
|
||||
];
|
||||
nativeBuildInputs = [ gettext ];
|
||||
|
||||
postInstall = ''
|
||||
rm $out/bin/linkchecker-gui
|
||||
propagatedBuildInputs = with python2Packages; [
|
||||
ConfigArgParse
|
||||
argcomplete
|
||||
dnspython
|
||||
pyxdg
|
||||
requests
|
||||
];
|
||||
|
||||
checkInputs = with python2Packages; [
|
||||
parameterized
|
||||
pytest
|
||||
];
|
||||
|
||||
postPatch = ''
|
||||
sed -i 's/^requests.*$/requests>=2.2/' requirements.txt
|
||||
sed -i "s/'request.*'/'requests >= 2.2'/" setup.py
|
||||
sed -i 's~/usr/lib/python2.7/argparse.py~~g' po/Makefile
|
||||
'';
|
||||
|
||||
checkPhase = ''
|
||||
runHook preCheck
|
||||
|
||||
# the mime test fails for me...
|
||||
rm tests/test_mimeutil.py
|
||||
${lib.optionalString stdenv.isDarwin ''
|
||||
# network tests fails on darwin
|
||||
rm tests/test_network.py
|
||||
# network tests fails on darwin
|
||||
rm tests/test_network.py
|
||||
''}
|
||||
make test PYTESTOPTS="--tb=short" TESTS="tests/test_*.py tests/logger/test_*.py"
|
||||
|
||||
runHook postCheck
|
||||
'';
|
||||
|
||||
meta = {
|
||||
|
|
|
|||
Loading…
Reference in a new issue