From 43403682772d0dac1781ad6560a43631e1ac50fb Mon Sep 17 00:00:00 2001 From: Marc Joliet Date: Tue, 24 Sep 2013 15:42:32 +0200 Subject: [PATCH 1/7] Support bzip2 and lzma/xz compressed man pages Add support for bzip2 and lzma/xz compressed man pages. Support for bzip2 is part of the Python standard library (at least for 2.7 and >=3.2), while lzma/xz is only in Python >=3.3; however, there is a backports module for Python 2.7 and 3.2. --- share/tools/create_manpage_completions.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/share/tools/create_manpage_completions.py b/share/tools/create_manpage_completions.py index 14bf13ee7..b568a97d2 100755 --- a/share/tools/create_manpage_completions.py +++ b/share/tools/create_manpage_completions.py @@ -17,9 +17,14 @@ Redistributions in binary form must reproduce the above copyright notice, this l THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ -import string, sys, re, os.path, gzip, traceback, getopt, errno, codecs +import string, sys, re, os.path, bz2, gzip, traceback, getopt, errno, codecs from deroff import Deroffer +try: + import backports.lzma as lzma +except: + import lzma + # Whether we're Python 3 IS_PY3 = sys.version_info[0] >= 3 @@ -717,6 +722,14 @@ def parse_manpage_at_path(manpage_path, output_directory): fd = gzip.open(manpage_path, 'r') manpage = fd.read() if IS_PY3: manpage = manpage.decode('latin-1') + if manpage_path.endswith('.bz2'): + fd = bz2.BZ2File(manpage_path, 'r') + manpage = fd.read() + if IS_PY3: manpage = manpage.decode('latin-1') + elif manpage_path.endswith('.xz') or manpage_path.endswith('.lzma'): + fd = lzma.LZMAFile(str(manpage_path), 'r') + manpage = fd.read() + if IS_PY3: manpage = manpage.decode('latin-1') else: if IS_PY3: fd = open(manpage_path, 'r', encoding='latin-1') From 7d0722bc1850e47cc8d83aa21450aa142fbb683e Mon Sep 17 00:00:00 2001 From: Marc Joliet Date: Wed, 25 Sep 2013 01:35:32 +0200 Subject: [PATCH 2/7] Change an "if" to more appropriate "elif" I overlooked an "if" that should have been an "elif". Oops. --- share/tools/create_manpage_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/tools/create_manpage_completions.py b/share/tools/create_manpage_completions.py index b568a97d2..b5ff43d23 100755 --- a/share/tools/create_manpage_completions.py +++ b/share/tools/create_manpage_completions.py @@ -722,7 +722,7 @@ def parse_manpage_at_path(manpage_path, output_directory): fd = gzip.open(manpage_path, 'r') manpage = fd.read() if IS_PY3: manpage = manpage.decode('latin-1') - if manpage_path.endswith('.bz2'): + elif manpage_path.endswith('.bz2'): fd = bz2.BZ2File(manpage_path, 'r') manpage = fd.read() if IS_PY3: manpage = manpage.decode('latin-1') From 4856567a2aaf55f98a8f2ab621b3767ca45d565d Mon Sep 17 00:00:00 2001 From: Marc Joliet Date: Wed, 25 Sep 2013 14:35:11 +0200 Subject: [PATCH 3/7] Only try "lzma" module on ImportError --- share/tools/create_manpage_completions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/tools/create_manpage_completions.py b/share/tools/create_manpage_completions.py index b5ff43d23..e1b7ccaa4 100755 --- a/share/tools/create_manpage_completions.py +++ b/share/tools/create_manpage_completions.py @@ -22,7 +22,7 @@ from deroff import Deroffer try: import backports.lzma as lzma -except: +except ImportError: import lzma # Whether we're Python 3 From fc7c489ab68d1fbc399b11f3a03db10cd95704f8 Mon Sep 17 00:00:00 2001 From: Marc Joliet Date: Wed, 25 Sep 2013 14:36:42 +0200 Subject: [PATCH 4/7] Skip lzma/xz manpages if lzma module not available Skip man pages compressed with lzma/xz if the lzma module is not available; also print a corresponding diagnostic message. --- share/tools/create_manpage_completions.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/share/tools/create_manpage_completions.py b/share/tools/create_manpage_completions.py index e1b7ccaa4..4b9ad88f9 100755 --- a/share/tools/create_manpage_completions.py +++ b/share/tools/create_manpage_completions.py @@ -20,10 +20,14 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND import string, sys, re, os.path, bz2, gzip, traceback, getopt, errno, codecs from deroff import Deroffer +lzma_available = True try: - import backports.lzma as lzma + try: + import backports.lzma as lzma + except ImportError: + import lzma except ImportError: - import lzma + lzma_available = False # Whether we're Python 3 IS_PY3 = sys.version_info[0] >= 3 @@ -727,6 +731,8 @@ def parse_manpage_at_path(manpage_path, output_directory): manpage = fd.read() if IS_PY3: manpage = manpage.decode('latin-1') elif manpage_path.endswith('.xz') or manpage_path.endswith('.lzma'): + if not lzma_available: + return fd = lzma.LZMAFile(str(manpage_path), 'r') manpage = fd.read() if IS_PY3: manpage = manpage.decode('latin-1') @@ -829,6 +835,8 @@ def parse_and_output_man_pages(paths, output_directory, show_progress): last_progress_string_length = 0 if show_progress and not WRITE_TO_STDOUT: print("Parsing man pages and writing completions to {0}".format(output_directory)) + if not lzma_available and not WRITE_TO_STDOUT: + print('"lzma" module not available, cannot parse man pages compressed with xz or lzma') for manpage_path in paths: index += 1 From 1c8c9a10b5d7d42731befa6ef2eb9ba60b4c7f1d Mon Sep 17 00:00:00 2001 From: Marc Joliet Date: Wed, 25 Sep 2013 18:09:23 +0200 Subject: [PATCH 5/7] Only print an error when an lzma/xz manpage occurs Only print an error when an lzma/xz compressed man page occurs. Also, use add_diagnostic instead of print. --- share/tools/create_manpage_completions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/share/tools/create_manpage_completions.py b/share/tools/create_manpage_completions.py index 4b9ad88f9..f12d5c147 100755 --- a/share/tools/create_manpage_completions.py +++ b/share/tools/create_manpage_completions.py @@ -20,6 +20,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND import string, sys, re, os.path, bz2, gzip, traceback, getopt, errno, codecs from deroff import Deroffer +lzma_printed_msg = False lzma_available = True try: try: @@ -714,7 +715,7 @@ def parse_manpage_at_path(manpage_path, output_directory): filename = os.path.basename(manpage_path) # Clear diagnostics - global diagnostic_indent + global diagnostic_indent, lzma_printed_msg diagnostic_output[:] = [] diagnostic_indent = 0 @@ -732,6 +733,11 @@ def parse_manpage_at_path(manpage_path, output_directory): if IS_PY3: manpage = manpage.decode('latin-1') elif manpage_path.endswith('.xz') or manpage_path.endswith('.lzma'): if not lzma_available: + if not lzma_printed_msg: + add_diagnostic('A man page is compressed with lzma or xz, but the "lzma" module is not available.' + ' Skipping. (This message is only printed on the first occurrence.)', + NOT_VERBOSE) + lzma_printed_msg = True return fd = lzma.LZMAFile(str(manpage_path), 'r') manpage = fd.read() @@ -835,8 +841,6 @@ def parse_and_output_man_pages(paths, output_directory, show_progress): last_progress_string_length = 0 if show_progress and not WRITE_TO_STDOUT: print("Parsing man pages and writing completions to {0}".format(output_directory)) - if not lzma_available and not WRITE_TO_STDOUT: - print('"lzma" module not available, cannot parse man pages compressed with xz or lzma') for manpage_path in paths: index += 1 From 6de9a9258207ca38d320cc0bc3c3653deef12d7d Mon Sep 17 00:00:00 2001 From: Marc Joliet Date: Thu, 26 Sep 2013 17:15:53 +0200 Subject: [PATCH 6/7] Change the way xz/lzma man pages are detected This avoids the use of the global and puts the diagnostic message in a self-contained location. --- share/tools/create_manpage_completions.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/share/tools/create_manpage_completions.py b/share/tools/create_manpage_completions.py index f12d5c147..41fae2edf 100755 --- a/share/tools/create_manpage_completions.py +++ b/share/tools/create_manpage_completions.py @@ -20,7 +20,6 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND import string, sys, re, os.path, bz2, gzip, traceback, getopt, errno, codecs from deroff import Deroffer -lzma_printed_msg = False lzma_available = True try: try: @@ -715,7 +714,7 @@ def parse_manpage_at_path(manpage_path, output_directory): filename = os.path.basename(manpage_path) # Clear diagnostics - global diagnostic_indent, lzma_printed_msg + global diagnostic_indent diagnostic_output[:] = [] diagnostic_indent = 0 @@ -733,11 +732,6 @@ def parse_manpage_at_path(manpage_path, output_directory): if IS_PY3: manpage = manpage.decode('latin-1') elif manpage_path.endswith('.xz') or manpage_path.endswith('.lzma'): if not lzma_available: - if not lzma_printed_msg: - add_diagnostic('A man page is compressed with lzma or xz, but the "lzma" module is not available.' - ' Skipping. (This message is only printed on the first occurrence.)', - NOT_VERBOSE) - lzma_printed_msg = True return fd = lzma.LZMAFile(str(manpage_path), 'r') manpage = fd.read() @@ -841,6 +835,15 @@ def parse_and_output_man_pages(paths, output_directory, show_progress): last_progress_string_length = 0 if show_progress and not WRITE_TO_STDOUT: print("Parsing man pages and writing completions to {0}".format(output_directory)) + + man_page_suffixes = set([os.path.splitext(m)[1][1:] for m in paths]) + lzma_xz_occurs = "xz" in man_page_suffixes or "lzma" in man_page_suffixes + if lzma_xz_occurs and not lzma_available: + add_diagnostic('At least one man page is compressed with lzma or xz, but the "lzma" module is not available.' + ' Any man page compressed with either will be skipped.', + NOT_VERBOSE) + flush_diagnostics(sys.stderr) + for manpage_path in paths: index += 1 From f4f36e356fdfc73d52409efa5b9bd35a5218041b Mon Sep 17 00:00:00 2001 From: Marc Joliet Date: Thu, 26 Sep 2013 20:00:01 +0200 Subject: [PATCH 7/7] Document the optional dependency to backports.lzma. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c74624656..522b0d91b 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,12 @@ To switch your default shell back, you can run: Substitute /bin/bash with /bin/tcsh or /bin/zsh as appropriate. +## Optional Dependencies + +In order to generate completions from man pages compressed with either lzma or xz, you may need to install an extra Python package. + +Python versions prior to 2.6 are not supported. For Python versions 2.6 to 3.2 you need to install the module `backports.lzma`. How to install it depends on your system and how you installed Python. Most Linux distributions should include it as a package named `backports-lzma` (or similar). From version 3.3 onwards, Python already includes the required module. + ## Contact Us Questions, comments, rants and raves can be posted to the official fish mailing list at or join us on our IRC channel [#fish at irc.oftc.net](https://webchat.oftc.net/?channels=fish).