from __future__ import print_function import logging import re import os from pyhhi.build.common.error import InvalidInputParameterError class CMakeRstUtilParams(object): def __init__(self): self.dry_run = False self.rst_module_filenm = None self.update_action = 'add' self.extension_module_names = [] self.extension_section_title = "Extension Modules" self.output_rst_filenm = None class RstModuleSection(object): def __init__(self): self.section_title = None self.section_title_marker_line = None self.section_header = [] self.section_content = [] class CMakeManualRstUtil(object): def __init__(self, dry_run=False): self._logger = logging.getLogger(__name__) self._dry_run = dry_run self._section_title_cmake_modules_orig = "All Modules" self._section_title_cmake_modules = "Standard Modules" self._reserved_section_titles = [self._section_title_cmake_modules, self._section_title_cmake_modules_orig, "Legacy CPack Modules"] self._section_title_marker = '=' self._is_cmake_source_tree = False # Location of .rst self._cmake_help_module_dir = None # Automatic removal of cmake module wrapper files. self._autoremove = True self._re_section_content = re.compile(r'\s+/module/([a-zA-Z0-9_-]+)\s*$') def is_cmake_source_tree(self): return self._is_cmake_source_tree def add_extension_modules(self, rst_module_filenm, extension_module_names, section_title="Extension Modules", output_rst_filenm=None): if not os.path.exists(rst_module_filenm): raise InvalidInputParameterError("CMake RST module file {} does not exist.".format(rst_module_filenm)) if output_rst_filenm is None: # in-place replacement output_rst_filenm = rst_module_filenm # Empty sections are not permitted, use one of the remove_xxx() methods to get rid of a section. assert extension_module_names self._detect_cmake_source_tree(rst_module_filenm) self._check_section_title(section_title) (module_file_header, section_list) = self._parse_rst_module_file(rst_module_filenm) if len(section_list) > 0: # CMake's original module file has one or more sections. All extension sections are inserted in front of # section titled "All Modules" or "Standard Modules". The latter title is used after the first extension section has been inserted. # Get cmake module names cmake_module_names_set = self._get_cmake_module_names(section_list) # Extension module names must not replace existing cmake module names extension_module_names_set = set(extension_module_names) if not cmake_module_names_set.isdisjoint(extension_module_names_set): conflicting_names = cmake_module_names_set.intersection(extension_module_names_set) msg = "CMake builtin module names cannot be part of an extension section.\n" msg += "Please check the extension module names: " + " ".join(conflicting_names) raise Exception(msg) else: # Looks like a corrupted cmake-modules.7.rst assert False # Get a list of RST extension module wrapper files belonging to section "section_title". existing_extension_module_names = self._get_extension_module_names(section_list, section_title) if existing_extension_module_names: self._logger.debug('existing extensions: %s', ' '.join(existing_extension_module_names)) modified = self._update_section(section_list, section_title, extension_module_names) if modified: # print("new content lines:") # self._dump_section_list(section_list[0:-1]) # Change title of the CMake module section from "All Modules" to "Standard Modules". for section in section_list: if section.section_title == self._section_title_cmake_modules_orig: section.section_title = self._section_title_cmake_modules section.section_title_marker_line = '{0:=>{width}}'.format(self._section_title_marker, width=len(section.section_title)) break section_list = self._sort_section_list(section_list) # Save top-level RST module file self._save_rst_module_file(output_rst_filenm, module_file_header, section_list) # Remove RST extension module wrapper files not belonging to this section anymore. outdated_extension_module_names = [] for mod_nm in existing_extension_module_names: if mod_nm not in extension_module_names: outdated_extension_module_names.append(mod_nm) if outdated_extension_module_names: self._logger.debug('outdated extensions: %s', ' '.join(outdated_extension_module_names)) # Remove outdated RST extension module wrapper files. self._remove_rst_extension_module_files(outdated_extension_module_names) def remove_extension_module_section(self, rst_module_filenm, section_title="Extension Modules", output_rst_filenm=None): if not os.path.exists(rst_module_filenm): raise Exception("CMake RST module file " + rst_module_filenm + " does not exist.") if output_rst_filenm is None: output_rst_filenm = rst_module_filenm self._detect_cmake_source_tree(rst_module_filenm) self._check_section_title(section_title) (module_file_header, section_list) = self._parse_rst_module_file(rst_module_filenm) assert len(section_list) > 0 # Get a list of RST extension module wrapper files belonging to section "section_title". existing_extension_module_names = self._get_extension_module_names(section_list, section_title) if existing_extension_module_names: self._logger.debug('existing extensions: %s', ' '.join(existing_extension_module_names)) modified = False new_section_list = [] for sect in section_list: if section_title == sect.section_title: modified = True else: new_section_list.append(sect) assert len(new_section_list) > 0 sect = new_section_list[0] if sect.section_title == self._section_title_cmake_modules: modified = True # Recover the original section title sect.section_title = self._section_title_cmake_modules_orig sect.section_title_marker_line = '{0:=>{width}}'.format(self._section_title_marker, width=len(sect.section_title)) if modified: # Save top-level RST module file self._save_rst_module_file(output_rst_filenm, module_file_header, new_section_list) # Remove RST extension module wrapper files belonging to the section just removed. self._remove_rst_extension_module_files(existing_extension_module_names) def remove_all_extension_modules(self, rst_module_filenm, output_rst_filenm=None): if not os.path.exists(rst_module_filenm): raise Exception("CMake RST module file " + rst_module_filenm + " does not exist.") if output_rst_filenm is None: output_rst_filenm = rst_module_filenm self._detect_cmake_source_tree(rst_module_filenm) (module_file_header, section_list) = self._parse_rst_module_file(rst_module_filenm) existing_extension_module_names = self._get_extension_module_names(section_list) if existing_extension_module_names: self._logger.debug('existing extensions: %s', ' '.join(existing_extension_module_names)) modified = False while section_list: sect = section_list[0] if sect.section_title == self._section_title_cmake_modules: # Found first original section modified = True sect.section_title = self._section_title_cmake_modules_orig sect.section_title_marker_line = '{0:=>{width}}'.format(self._section_title_marker, width=len(sect.section_title)) break elif sect.section_title == self._section_title_cmake_modules_orig: # Found first original section break else: modified = True section_list.pop(0) assert len(section_list) > 0 if modified: # Save top-level RST module file self._save_rst_module_file(output_rst_filenm, module_file_header, section_list) # Remove all RST extension module wrapper files self._remove_rst_extension_module_files(existing_extension_module_names) def _detect_cmake_source_tree(self, rst_module_filenm): rst_manual_dir = os.path.dirname(rst_module_filenm) rst_module_dir = os.path.join(rst_manual_dir, '..', 'module') if (os.path.basename(rst_manual_dir) == 'manual') and (os.path.exists(rst_module_dir) and os.path.isdir(rst_module_dir)): self._is_cmake_source_tree = True self._cmake_help_module_dir = self._get_rst_module_dir(rst_module_filenm) assert os.path.exists(self._cmake_help_module_dir) and os.path.isdir(self._cmake_help_module_dir) else: self._is_cmake_source_tree = False self._cmake_help_module_dir = None def _get_rst_module_dir(self, rst_module_filenm): rst_manual_dir = os.path.dirname(rst_module_filenm) rst_module_dir = os.path.join(rst_manual_dir, '..', 'module') return os.path.normpath(rst_module_dir) def _check_section_title(self, section_title): if section_title in self._reserved_section_titles: raise InvalidInputParameterError("section {} is protected and cannot be modified, please contact technical support.".format(section_title)) def _check_extension_module_names(self, extension_module_names): if not self.is_cmake_source_tree(): return for mod_nm in extension_module_names: rst_module_file = os.path.join(self._cmake_help_module_dir, mod_nm + '.rst') if not os.path.exists(rst_module_file): raise Exception("file {} does not exist.".format(rst_module_file)) def _update_section(self, section_list, section_title, extension_module_names): section = None modified = False assert extension_module_names extension_module_names.sort() for sec in section_list: if sec.section_title == section_title: section = sec break if not section: # add a new section modified = True section = RstModuleSection() section.section_title = section_title section.section_title_marker_line = '{0:=>{width}}'.format(self._section_title_marker, width=len(section.section_title)) section.section_header.append('') section.section_header.append('.. toctree::') section.section_header.append(' :maxdepth: 1') section.section_header.append('') section_list.insert(0, section) else: # Section already present, update only if section content is different. existing_extension_module_names = [] # re_section_content = re.compile(r'\s+/module/([a-zA-Z0-9_-]+)\s*$') for line in section.section_content: re_match = self._re_section_content.match(line) if re_match: existing_extension_module_names.append(re_match.group(1)) existing_extension_module_names.sort() if len(existing_extension_module_names) != len(extension_module_names): modified = True else: for module_nm in existing_extension_module_names: if module_nm not in extension_module_names: modified = True break if modified: # replace the section section.section_content = [] if modified: # replace the content of an existing section for module_nm in extension_module_names: section.section_content.append(' /module/' + module_nm) return modified def _sort_section_list(self, section_list): """Sort section list by title and keep CMake's original sections at the end.""" section_list_ext = [] while section_list: sect = section_list[0] if sect.section_title in [self._section_title_cmake_modules_orig, self._section_title_cmake_modules]: break else: section_list_ext.append(sect) section_list.pop(0) section_title_dict = {} section_list_sorted = [] for sect in section_list_ext: section_title_dict[sect.section_title] = sect title_list = list(section_title_dict.keys()) title_list.sort() for title in title_list: section_list_sorted.append(section_title_dict[title]) # and append CMake's original sections. section_list_sorted.extend(section_list) return section_list_sorted def _parse_rst_module_file(self, rst_module_filenm): module_file_header = [] section_list = [] section = None previous_line = None re_empty_line = re.compile(r'^\s*$') with open(rst_module_filenm) as f: for line in f: line = line.rstrip() if not section_list: # Processing lines in front of the first section header if line.startswith(self._section_title_marker): section = RstModuleSection() section.section_title = previous_line section.section_title_marker_line = line self._logger.debug("found section: {}".format(previous_line)) section_list.append(section) # Remove the title line from the module header. module_file_header.pop() if module_file_header and re_empty_line.match(module_file_header[-1]): # Remove the last empty line from the module header to simplify saving sections later on. module_file_header.pop() else: module_file_header.append(line) else: if section.section_content: if self._re_section_content.match(line): section.section_content.append(line) else: section = RstModuleSection() else: # No section content found yet. if not section.section_title: if line.startswith(self._section_title_marker): section.section_title = previous_line section.section_title_marker_line = line section_list.append(section) self._logger.debug("found section: {}".format(previous_line)) else: pass else: if self._re_section_content.match(line): section.section_content.append(line) else: section.section_header.append(line) previous_line = line # self._dump_module_file_header(module_file_header) # self._dump_section_list(section_list) return module_file_header, section_list def _get_extension_module_names(self, section_list, section_title=None): extension_module_names = [] for sect in section_list: if sect.section_title in [self._section_title_cmake_modules_orig, self._section_title_cmake_modules]: continue if (section_title is None) or (section_title == sect.section_title): for line in sect.section_content: re_match = self._re_section_content.match(line) if re_match: extension_module_names.append(re_match.group(1)) return extension_module_names def _get_cmake_module_names(self, section_list): cmake_module_names = set() section_all_modules_fnd = False for sect in section_list: if not section_all_modules_fnd: if sect.section_title in [self._section_title_cmake_modules_orig, self._section_title_cmake_modules]: section_all_modules_fnd = True else: # One of the extension sections in front of "All Modules" or "Standard Modules" continue for line in sect.section_content: re_match = self._re_section_content.match(line) if re_match: cmake_module_names.add(re_match.group(1)) return cmake_module_names def _save_rst_module_file(self, output_rst_filenm, module_file_header, section_list): if self._dry_run: self._logger.debug("leaving without saving anything to %s", output_rst_filenm) return self._logger.debug("saving updated content to %s", output_rst_filenm) with open(output_rst_filenm, "w") as f: for line in module_file_header: f.write(line + '\n') for sect in section_list: f.write('\n') f.write(sect.section_title + '\n') f.write(sect.section_title_marker_line + '\n') for line in sect.section_header: f.write(line + '\n') for line in sect.section_content: f.write(line + '\n') def _remove_rst_extension_module_files(self, extension_module_names): if not self._autoremove: return if not self.is_cmake_source_tree(): return for mod_nm in extension_module_names: rst_module_file = os.path.join(self._cmake_help_module_dir, mod_nm + '.rst') if os.path.exists(rst_module_file): self._logger.debug("removing %s", rst_module_file) if not self._dry_run: os.remove(rst_module_file) def _dump_module_file_header(self, module_file_header): for line in module_file_header: print("file header: " + line) def _dump_section_list(self, section_list): print("number of sections: %d" %(len(section_list))) for section in section_list: print("section title: " + section.section_title) print("section title marker: " + section.section_title_marker_line) for line in section.section_header: print("section header: " + line) for line in section.section_content: print("section content: " + line)