From 799f742e9d4685534c39a537d8451610e2be8196 Mon Sep 17 00:00:00 2001 From: abk16 Date: Fri, 30 Dec 2022 18:37:01 +0100 Subject: [PATCH 1/6] [SPLIT] util.convert_bootstrap: move etree converter to views_convert Refactored BootstrapConverter code, separating conversion class and operations from bootstrap-specific code. The new EtreeConverter can be used to implement conversions of any kind, reusing the pre-defined ElementOperations by defining an operations list. --- src/util/convert_bootstrap.py | 713 ++------------------------------ src/util/views_convert.py | 758 ++++++++++++++++++++++++++++++++++ tools/compile23.py | 1 + 3 files changed, 785 insertions(+), 687 deletions(-) create mode 100644 src/util/views_convert.py diff --git a/src/util/convert_bootstrap.py b/src/util/convert_bootstrap.py index b86d001b..14f1d639 100644 --- a/src/util/convert_bootstrap.py +++ b/src/util/convert_bootstrap.py @@ -1,10 +1,7 @@ """Convert an XML/HTML document Bootstrap code from an older version to a newer one.""" import logging -import os.path -import re from functools import lru_cache -from typing import Iterable from lxml import etree @@ -13,544 +10,29 @@ except ImportError: from distutils.version import StrictVersion as Version # N.B. deprecated, will be removed in py3.12 +from .views_convert import ( + ALL, + BE, + BS, + CSS, + AddClasses, + ElementOperation, + EtreeConverter, + PullUp, + RegexReplaceClass, + RemoveClasses, + RemoveElement, + RenameAttribute, + ReplaceClasses, + adapt_xpath_for_qweb, + edit_element_classes, + get_classes, + regex_xpath, +) _logger = logging.getLogger(__name__) -# Regex boundary patterns for class names within attributes (`\b` is not good enough -# because it matches `-` etc.). It's probably enough to detect if the class name -# is surrounded by either whitespace or beginning/end of string. -# Includes boundary chars that can appear in t-att(f)-* attributes. -BS = r"(?:^|(?<=[\s}'\"]))" -BE = r"(?:$|(?=[\s{#'\"]))" -B = rf"(?:{BS}|{BE})" - - -def _xpath_has_class(context, *cls): - """Extension function for xpath to check if the context node has all the classes passed as arguments.""" - node_classes = set(context.context_node.attrib.get("class", "").split()) - return node_classes.issuperset(cls) - - -@lru_cache(maxsize=128) # does >99% hits over ~10mln calls -def _xpath_has_t_class_inner(attrs_values, classes): - """ - Inner implementation of :func:`_xpath_has_t_class`, suitable for caching. - - :param tuple[str] attrs_values: the values of the ``t-att-class`` and ``t-attf-class`` attributes - :param tuple[str] classes: the classes to check - """ - return any( - all(re.search(rf"{BS}{escaped_cls}{BE}", attr_value or "") for escaped_cls in map(re.escape, classes)) - for attr_value in attrs_values - ) - - -def _xpath_has_t_class(context, *cls): - """Extension function for xpath to check if the context node has all the classes passed as arguments in one of ``class`` or ``t-att(f)-class`` attributes.""" - return _xpath_has_class(context, *cls) or _xpath_has_t_class_inner( - tuple(map(context.context_node.attrib.get, ("t-att-class", "t-attf-class"))), cls - ) - - -@lru_cache(maxsize=1024) # does >88% hits over ~1.5mln calls -def _xpath_regex_inner(pattern, item): - """Inner implementation of :func:`_xpath_regex`, suitable for caching.""" - return bool(re.search(pattern, item)) - - -def _xpath_regex(context, item, pattern): - """Extension function for xpath to check if the passed item (attribute or text) matches the passed regex pattern.""" - if not item: - return False - if isinstance(item, list): - item = item[0] # only first attribute is valid - return _xpath_regex_inner(pattern, item) - - -xpath_utils = etree.FunctionNamespace(None) -xpath_utils["hasclass"] = _xpath_has_class -xpath_utils["has-class"] = _xpath_has_class -xpath_utils["has-t-class"] = _xpath_has_t_class -xpath_utils["regex"] = _xpath_regex - -html_utf8_parser = etree.HTMLParser(encoding="utf-8") - - -def innerxml(element, is_html=False): - """ - Return the inner XML of an element as a string. - - :param etree.ElementBase element: the element to convert. - :param bool is_html: whether to use HTML for serialization, XML otherwise. Defaults to False. - :rtype: str - """ - return (element.text or "") + "".join( - etree.tostring(child, encoding=str, method="html" if is_html else None) for child in element - ) - - -def split_classes(*joined_classes): - """Return a list of classes given one or more strings of joined classes separated by spaces.""" - return [c for classes in joined_classes for c in (classes or "").split(" ") if c] - - -def get_classes(element): - """Return the list of classes from the ``class`` attribute of an element.""" - return split_classes(element.get("class", "")) - - -def join_classes(classes): - """Return a string of classes joined by space given a list of classes.""" - return " ".join(classes) - - -def set_classes(element, classes): - """ - Set the ``class`` attribute of an element from a list of classes. - - If the list is empty, the attribute is removed. - """ - if classes: - element.attrib["class"] = join_classes(classes) - else: - element.attrib.pop("class", None) - - -def edit_classlist(classes, add, remove): - """ - Edit a class list, adding and removing classes. - - :param str | typing.List[str] classes: the original classes list or str to edit. - :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the list. - :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) - from the list. The `ALL` sentinel value can be specified to remove all classes. - :rtype: typing.List[str] - :return: the new class list. - """ - if remove is ALL: - classes = [] - remove = None - else: - if isinstance(classes, str): - classes = [classes] - classes = split_classes(*classes) - - remove_index = None - if isinstance(remove, str): - remove = [remove] - for classname in remove or []: - while classname in classes: - remove_index = max(remove_index or 0, classes.index(classname)) - classes.remove(classname) - - insert_index = remove_index if remove_index is not None else len(classes) - if isinstance(add, str): - add = [add] - for classname in add or []: - if classname not in classes: - classes.insert(insert_index, classname) - insert_index += 1 - - return classes - - -def edit_element_t_classes(element, add, remove): - """ - Edit inplace qweb ``t-att-class`` and ``t-attf-class`` attributes of an element, adding and removing the specified classes. - - N.B. adding new classes will not work if neither ``t-att-class`` nor ``t-attf-class`` are present. - - :param etree.ElementBase element: the element to edit. - :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. - :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) - from the element. The `ALL` sentinel value can be specified to remove all classes. - """ - if isinstance(add, str): - add = [add] - if add: - add = split_classes(*add) - if isinstance(remove, str): - remove = [remove] - if remove and remove is not ALL: - remove = split_classes(*remove) - - if not add and not remove: - return # nothing to do - - for attr in ("t-att-class", "t-attf-class"): - if attr in element.attrib: - value = element.attrib.pop(attr) - - # if we need to remove all classes, just remove or replace the attribute - # with the literal string of classes to add, removing all logic - if remove is ALL: - if not add: - value = None - else: - value = " ".join(add or []) - if attr.startswith("t-att-"): - value = f"'{value}'" - - else: - joined_adds = join_classes(add or []) - # if there's no classes to remove, try to append the new classes in a sane way - if not remove: - if add: - if attr.startswith("t-att-"): - value = f"{value} + ' {joined_adds}'" if value else f"'{joined_adds}'" - else: - value = f"{value} {joined_adds}" - # otherwise use regexes to replace the classes in the attribute - # if there's just one replace, do a string replacement with the new classes - elif len(remove) == 1: - value = re.sub(rf"{BS}{re.escape(remove[0])}{BE}", joined_adds, value) - # else there's more than one class to remove and split at matches, - # then rejoin with new ones at the position of the last removed one. - else: - value = re.split(rf"{BS}(?:{'|'.join(map(re.escape, remove))}){BE}", value) - value = "".join(value[:-1] + [joined_adds] + value[-1:]) - - if value is not None: - element.attrib[attr] = value - - -def edit_element_classes(element, add, remove, is_qweb=False): - """ - Edit inplace the "class" attribute of an element, adding and removing classes. - - :param etree.ElementBase element: the element to edit. - :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. - :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) - from the element. The `ALL` sentinel value can be specified to remove all classes. - :param bool is_qweb: if True also edit classes in ``t-att-class`` and ``t-attf-class`` attributes. - Defaults to False. - """ - if not is_qweb or not set(element.attrib) & {"t-att-class", "t-attf-class"}: - set_classes(element, edit_classlist(get_classes(element), add, remove)) - if is_qweb: - edit_element_t_classes(element, add, remove) - - -ALL = object() -"""Sentinel object to indicate "all items" in a collection""" - - -def simple_css_selector_to_xpath(selector, prefix="//"): - """ - Convert a basic CSS selector cases to an XPath expression. - - Supports node names, classes, ``>`` and ``,`` combinators. - - :param str selector: the CSS selector to convert. - :param str prefix: the prefix to add to the XPath expression. Defaults to ``//``. - :return: the resulting XPath expression. - :rtype: str - """ - separator = prefix - xpath_parts = [] - combinators = "+>,~ " - for selector_part in map(str.strip, re.split(rf"(\s*[{combinators}]\s*)", selector)): - if not selector_part: - separator = "//" - elif selector_part == ">": - separator = "/" - elif selector_part == ",": - separator = "|" + prefix - elif re.search(r"^(?:[a-z](-?\w+)*|[*.])", selector_part, flags=re.I): - element, *classes = selector_part.split(".") - if not element: - element = "*" - class_predicates = [f"[hasclass('{classname}')]" for classname in classes if classname] - xpath_parts += [separator, element + "".join(class_predicates)] - else: - raise NotImplementedError(f"Unsupported CSS selector syntax: {selector}") - - return "".join(xpath_parts) - - -CSS = simple_css_selector_to_xpath - - -def regex_xpath(pattern, attr=None, xpath=None): - """ - Return an XPath expression that matches elements with an attribute value matching the given regex pattern. - - :param str pattern: a regex pattern to match the attribute value against. - :param str | None attr: the attribute to match the pattern against. - If not given, the pattern is matched against the element's text. - :param str | None xpath: an optional XPath expression to further filter the elements to match. - :rtype: str - """ - # TODO abt: investigate lxml xpath variables interpolation (xpath.setcontext? registerVariables?) - if "'" in pattern and '"' in pattern: - quoted_pattern = "concat('" + "', \"'\", '".join(pattern.split("'")) + "')" - elif "'" in pattern: - quoted_pattern = '"' + pattern + '"' - else: - quoted_pattern = "'" + pattern + "'" - xpath_pre = xpath or "//*" - attr_or_text = f"@{attr}" if attr is not None else "text()" - return xpath_pre + f"[regex({attr_or_text}, {quoted_pattern})]" - - -def adapt_xpath_for_qweb(xpath): - """Adapts a xpath to enable matching on qweb ``t-att(f)-`` attributes.""" - xpath = re.sub(r"\bhas-?class(?=\()", "has-t-class", xpath) - # supposing that there's only one of `class`, `t-att-class`, `t-attf-class`, - # joining all of them with a space and removing trailing whitespace should behave - # similarly to COALESCE, and result in ORing the values for matching - xpath = re.sub( - r"(?<=\()\s*@(? - ... - - - after:: - .. code-block:: html - - ... - - """ - - def __call__(self, element, converter): - parent = element.getparent() - if parent is None: - raise ValueError(f"Cannot pull up contents of xml element with no parent: {element}") - - prev_sibling = element.getprevious() - if prev_sibling is not None: - prev_sibling.tail = ((prev_sibling.tail or "") + (element.text or "")) or None - else: - parent.text = ((parent.text or "") + (element.text or "")) or None - - for child in element: - element.addprevious(child) - - parent.remove(element) - - -class RenameAttribute(ElementOperation): - """Rename an attribute. Silently ignores elements that do not have the attribute.""" - - def __init__(self, old_name, new_name, extra_xpath=""): - self.old_name = old_name - self.new_name = new_name - self.extra_xpath = extra_xpath - - def __call__(self, element, converter): - rename_map = {self.old_name: self.new_name} - if converter.is_qweb: - rename_map = { - f"{prefix}{old}": f"{prefix}{new}" - for old, new in rename_map.items() - for prefix in ("",) + (("t-att-", "t-attf-") if not old.startswith("t-") else ()) - if f"{prefix}{old}" in element.attrib - } - if rename_map: - # to preserve attributes order, iterate+rename on a copy and reassign (clear+update, bc readonly property) - attrib_before = dict(element.attrib) - element.attrib.clear() - element.attrib.update({rename_map.get(k, k): v for k, v in attrib_before.items()}) - return element - - def xpath(self, xpath=None): - """ - Return an XPath expression that matches elements with the old attribute name. - - :param str | None xpath: an optional XPath expression to further filter the elements to match. - :rtype: str - """ - return (xpath or "//*") + f"[@{self.old_name}]{self.extra_xpath}" - - -class RegexReplace(ElementOperation): - """ - Uses `re.sub` to modify an attribute or the text of an element. - - N.B. no checks are made to ensure the attribute to replace is actually present on the elements. - - :param str pattern: the regex pattern to match. - :param str repl: the replacement string. - :param str | None attr: the attribute to replace. If not specified, the text of the element is replaced. - """ - - def __init__(self, pattern, sub, attr=None): - self.pattern = pattern - self.repl = sub - self.attr = attr - - def __call__(self, element, converter): - if self.attr is None: - # TODO abt: what about tail? - element.text = re.sub(self.pattern, self.repl, element.text or "") - else: - for attr in (self.attr,) + ((f"t-att-{self.attr}", f"t-attf-{self.attr}") if converter.is_qweb else ()): - if attr in element.attrib: - element.attrib[attr] = re.sub(self.pattern, self.repl, element.attrib[attr]) - return element - - def xpath(self, xpath=None): - """ - Return an XPath expression that matches elements with the old attribute name. - - :param str | None xpath: an optional XPath expression to further filter the elements to match. - :rtype: str - """ - return regex_xpath(self.pattern, self.attr, xpath) - - -class RegexReplaceClass(RegexReplace): - """ - Uses `re.sub` to modify the class. - - Basically, same as `RegexReplace`, but with `attr="class"`. - """ - - def __init__(self, pattern, sub, attr="class"): - super().__init__(pattern, sub, attr) - - class BS3to4ConvertBlockquote(ElementOperation): """Convert a BS3 ``
`` element to a BS4 ``
`` element with the ``blockquote`` class.""" @@ -739,12 +221,14 @@ def __call__(self, element, converter): return element -class BootstrapConverter: +class BootstrapConverter(EtreeConverter): """ Class for converting XML or HTML Bootstrap code across versions. - :param etree.ElementTree tree: the parsed XML or HTML tree to convert. + :param str src_version: the source Bootstrap version to convert from. + :param str dst_version: the destination Bootstrap version to convert to. :param bool is_html: whether the tree is an HTML document. + :para bool is_qweb: whether the tree contains QWeb directives. See :class:`EtreeConverter` for more info. """ MIN_VERSION = "3.0" @@ -929,10 +413,9 @@ class BootstrapConverter: ], } - def __init__(self, tree, is_html=False, is_qweb=False): - self.tree = tree - self.is_html = is_html - self.is_qweb = is_qweb + def __init__(self, src_version, dst_version, *, is_html=False, is_qweb=False): + conversions = self._get_conversions(src_version, dst_version) + super().__init__(conversions, is_html=is_html, is_qweb=is_qweb) @classmethod def _get_sorted_conversions(cls): @@ -967,150 +450,6 @@ def get_conversions(cls, src_ver, dst_ver, is_qweb=False): result = [(adapt_xpath_for_qweb(xpath), conversions) for xpath, conversions in result] return [(etree.XPath(xpath), conversions) for xpath, conversions in result] - def convert(self, src_version, dst_version): - """ - Convert the loaded document inplace from the source version to the destination, returning the converted document and the number of conversion operations applied. - - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :rtype: etree.ElementTree, int - """ - conversions = self.get_conversions(src_version, dst_version, is_qweb=self.is_qweb) - applied_operations_count = 0 - for xpath, operations in conversions: - for element in xpath(self.tree): - for operation in operations: - if element is None: # previous operations that returned None (i.e. deleted element) - raise ValueError("Matched xml element is not available anymore! Check operations.") - element = operation(element, self) # noqa: PLW2901 - applied_operations_count += 1 - return self.tree, applied_operations_count - - @classmethod - def convert_arch(cls, arch, src_version, dst_version, is_html=False, **converter_kwargs): - """ - Class method for converting a string of XML or HTML code. - - :param str arch: the XML or HTML code to convert. - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :param bool is_html: whether the arch is an HTML document. - :param dict converter_kwargs: additional keyword arguments to pass to the converter. - :return: the converted XML or HTML code. - :rtype: str - """ - stripped_arch = arch.strip() - doc_header_match = re.search(r"^<\?xml .+\?>\s*", stripped_arch) - doc_header = doc_header_match.group(0) if doc_header_match else "" - stripped_arch = stripped_arch[doc_header_match.end() :] if doc_header_match else stripped_arch - - tree = etree.fromstring(f"{stripped_arch}", parser=html_utf8_parser if is_html else None) - - tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) - if not ops_count: - return arch - - wrap_node = tree.xpath("//wrap")[0] - return doc_header + "\n".join( - etree.tostring(child, encoding="unicode", with_tail=True, method="html" if is_html else None) - for child in wrap_node - ) - - @classmethod - def convert_file(cls, path, src_version, dst_version, is_html=None, **converter_kwargs): - """ - Class method for converting an XML or HTML file inplace. - - :param str path: the path to the XML or HTML file to convert. - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :param bool is_html: whether the file is an HTML document. - If not set, will be detected from the file extension. - :param dict converter_kwargs: additional keyword arguments to pass to the converter. - :rtype: None - """ - if is_html is None: - is_html = os.path.splitext(path)[1].startswith("htm") - tree = etree.parse(path, parser=html_utf8_parser if is_html else None) - - tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) - if not ops_count: - logging.info("No conversion operations applied, skipping file %s", path) - return - - tree.write(path, encoding="utf-8", method="html" if is_html else None, xml_declaration=not is_html) - - def element_factory(self, *args, **kwargs): - """ - Create new elements using the correct document type. - - Basically a wrapper for either etree.XML or etree.HTML depending on the type of document loaded. - - :param args: positional arguments to pass to the etree.XML or etree.HTML function. - :param kwargs: keyword arguments to pass to the etree.XML or etree.HTML function. - :return: the created element. - """ - return etree.HTML(*args, **kwargs) if self.is_html else etree.XML(*args, **kwargs) - - def build_element(self, tag, classes=None, contents=None, **attributes): - """ - Create a new element with the given tag, classes, contents and attributes. - - Like :meth:`~.element_factory`, can be used by operations to create elements abstracting away the document type. - - :param str tag: the tag of the element to create. - :param typing.Iterable[str] | None classes: the classes to set on the new element. - :param str | None contents: the contents of the new element (i.e. inner text/HTML/XML). - :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. - :return: the created element. - :rtype: etree.ElementBase - """ - element = self.element_factory(f"<{tag}>{contents or ''}") - for name, value in attributes.items(): - element.attrib[name] = value - if classes: - set_classes(element, classes) - return element - - def copy_element( - self, - element, - tag=None, - add_classes=None, - remove_classes=None, - copy_attrs=True, - copy_contents=True, - **attributes, - ): - """ - Create a copy of an element, optionally changing the tag, classes, contents and attributes. - - Like :meth:`~.element_factory`, can be used by operations to copy elements abstracting away the document type. - - :param etree.ElementBase element: the element to copy. - :param str | None tag: if specified, overrides the tag of the new element. - :param str | typing.Iterable[str] | None add_classes: if specified, adds the given class(es) to the new element. - :param str | typing.Iterable[str] | ALL | None remove_classes: if specified, removes the given class(es) - from the new element. The `ALL` sentinel value can be specified to remove all classes. - :param bool copy_attrs: if True, copies the attributes of the source element to the new one. Defaults to True. - :param bool copy_contents: if True, copies the contents of the source element to the new one. Defaults to True. - :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. - Will be str merged with the attributes of the source element, overriding the latter. - :return: the new copied element. - :rtype: etree.ElementBase - """ - tag = tag or element.tag - contents = innerxml(element, is_html=self.is_html) if copy_contents else None - if copy_attrs: - attributes = {**element.attrib, **attributes} - new_element = self.build_element(tag, contents=contents, **attributes) - edit_element_classes(new_element, add_classes, remove_classes, is_qweb=self.is_qweb) - return new_element - - def adapt_xpath(self, xpath): - """Adapts an xpath to match qweb ``t-att(f)-*`` attributes, if ``is_qweb`` is True.""" - return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath - def convert_tree(tree, src_version, dst_version, **converter_kwargs): """ diff --git a/src/util/views_convert.py b/src/util/views_convert.py new file mode 100644 index 00000000..aa5d4d4b --- /dev/null +++ b/src/util/views_convert.py @@ -0,0 +1,758 @@ +"""Helpers to manipulate views/templates.""" + +import logging +import os.path +import re +from abc import ABC, abstractmethod +from functools import lru_cache +from typing import Iterable + +from lxml import etree + +_logger = logging.getLogger(__name__) + + +# Regex boundary patterns for class names within attributes (`\b` is not good enough +# because it matches `-` etc.). It's probably enough to detect if the class name +# is surrounded by either whitespace or beginning/end of string. +# Includes boundary chars that can appear in t-att(f)-* attributes. +BS = r"(?:^|(?<=[\s}'\"]))" +BE = r"(?:$|(?=[\s{#'\"]))" +B = rf"(?:{BS}|{BE})" + + +def _xpath_has_class(context, *cls): + """Extension function for xpath to check if the context node has all the classes passed as arguments.""" + node_classes = set(context.context_node.attrib.get("class", "").split()) + return node_classes.issuperset(cls) + + +@lru_cache(maxsize=128) # does >99% hits over ~10mln calls +def _xpath_has_t_class_inner(attrs_values, classes): + """ + Inner implementation of :func:`_xpath_has_t_class`, suitable for caching. + + :param tuple[str] attrs_values: the values of the ``t-att-class`` and ``t-attf-class`` attributes + :param tuple[str] classes: the classes to check + """ + return any( + all(re.search(rf"{BS}{escaped_cls}{BE}", attr_value or "") for escaped_cls in map(re.escape, classes)) + for attr_value in attrs_values + ) + + +def _xpath_has_t_class(context, *cls): + """Extension function for xpath to check if the context node has all the classes passed as arguments in one of ``class`` or ``t-att(f)-class`` attributes.""" + return _xpath_has_class(context, *cls) or _xpath_has_t_class_inner( + tuple(map(context.context_node.attrib.get, ("t-att-class", "t-attf-class"))), cls + ) + + +@lru_cache(maxsize=1024) # does >88% hits over ~1.5mln calls +def _xpath_regex_inner(pattern, item): + """Inner implementation of :func:`_xpath_regex`, suitable for caching.""" + return bool(re.search(pattern, item)) + + +def _xpath_regex(context, item, pattern): + """Extension function for xpath to check if the passed item (attribute or text) matches the passed regex pattern.""" + if not item: + return False + if isinstance(item, list): + item = item[0] # only first attribute is valid + return _xpath_regex_inner(pattern, item) + + +xpath_utils = etree.FunctionNamespace(None) +xpath_utils["hasclass"] = _xpath_has_class +xpath_utils["has-class"] = _xpath_has_class +xpath_utils["has-t-class"] = _xpath_has_t_class +xpath_utils["regex"] = _xpath_regex + +html_utf8_parser = etree.HTMLParser(encoding="utf-8") + + +def innerxml(element, is_html=False): + """ + Return the inner XML of an element as a string. + + :param etree.ElementBase element: the element to convert. + :param bool is_html: whether to use HTML for serialization, XML otherwise. Defaults to False. + :rtype: str + """ + return (element.text or "") + "".join( + etree.tostring(child, encoding=str, method="html" if is_html else None) for child in element + ) + + +def split_classes(*joined_classes): + """Return a list of classes given one or more strings of joined classes separated by spaces.""" + return [c for classes in joined_classes for c in (classes or "").split(" ") if c] + + +def get_classes(element): + """Return the list of classes from the ``class`` attribute of an element.""" + return split_classes(element.get("class", "")) + + +def join_classes(classes): + """Return a string of classes joined by space given a list of classes.""" + return " ".join(classes) + + +def set_classes(element, classes): + """ + Set the ``class`` attribute of an element from a list of classes. + + If the list is empty, the attribute is removed. + """ + if classes: + element.attrib["class"] = join_classes(classes) + else: + element.attrib.pop("class", None) + + +def edit_classlist(classes, add, remove): + """ + Edit a class list, adding and removing classes. + + :param str | typing.List[str] classes: the original classes list or str to edit. + :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the list. + :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) + from the list. The `ALL` sentinel value can be specified to remove all classes. + :rtype: typing.List[str] + :return: the new class list. + """ + if remove is ALL: + classes = [] + remove = None + else: + if isinstance(classes, str): + classes = [classes] + classes = split_classes(*classes) + + remove_index = None + if isinstance(remove, str): + remove = [remove] + for classname in remove or []: + while classname in classes: + remove_index = max(remove_index or 0, classes.index(classname)) + classes.remove(classname) + + insert_index = remove_index if remove_index is not None else len(classes) + if isinstance(add, str): + add = [add] + for classname in add or []: + if classname not in classes: + classes.insert(insert_index, classname) + insert_index += 1 + + return classes + + +def edit_element_t_classes(element, add, remove): + """ + Edit inplace qweb ``t-att-class`` and ``t-attf-class`` attributes of an element, adding and removing the specified classes. + + N.B. adding new classes will not work if neither ``t-att-class`` nor ``t-attf-class`` are present. + + :param etree.ElementBase element: the element to edit. + :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. + :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) + from the element. The `ALL` sentinel value can be specified to remove all classes. + """ + if isinstance(add, str): + add = [add] + if add: + add = split_classes(*add) + if isinstance(remove, str): + remove = [remove] + if remove and remove is not ALL: + remove = split_classes(*remove) + + if not add and not remove: + return # nothing to do + + for attr in ("t-att-class", "t-attf-class"): + if attr in element.attrib: + value = element.attrib.pop(attr) + + # if we need to remove all classes, just remove or replace the attribute + # with the literal string of classes to add, removing all logic + if remove is ALL: + if not add: + value = None + else: + value = " ".join(add or []) + if attr.startswith("t-att-"): + value = f"'{value}'" + + else: + joined_adds = join_classes(add or []) + # if there's no classes to remove, try to append the new classes in a sane way + if not remove: + if add: + if attr.startswith("t-att-"): + value = f"{value} + ' {joined_adds}'" if value else f"'{joined_adds}'" + else: + value = f"{value} {joined_adds}" + # otherwise use regexes to replace the classes in the attribute + # if there's just one replace, do a string replacement with the new classes + elif len(remove) == 1: + value = re.sub(rf"{BS}{re.escape(remove[0])}{BE}", joined_adds, value) + # else there's more than one class to remove and split at matches, + # then rejoin with new ones at the position of the last removed one. + else: + value = re.split(rf"{BS}(?:{'|'.join(map(re.escape, remove))}){BE}", value) + value = "".join(value[:-1] + [joined_adds] + value[-1:]) + + if value is not None: + element.attrib[attr] = value + + +def edit_element_classes(element, add, remove, is_qweb=False): + """ + Edit inplace the "class" attribute of an element, adding and removing classes. + + :param etree.ElementBase element: the element to edit. + :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. + :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) + from the element. The `ALL` sentinel value can be specified to remove all classes. + :param bool is_qweb: if True also edit classes in ``t-att-class`` and ``t-attf-class`` attributes. + Defaults to False. + """ + if not is_qweb or not set(element.attrib) & {"t-att-class", "t-attf-class"}: + set_classes(element, edit_classlist(get_classes(element), add, remove)) + if is_qweb: + edit_element_t_classes(element, add, remove) + + +ALL = object() +"""Sentinel object to indicate "all items" in a collection""" + + +def simple_css_selector_to_xpath(selector, prefix="//"): + """ + Convert a basic CSS selector cases to an XPath expression. + + Supports node names, classes, ``>`` and ``,`` combinators. + + :param str selector: the CSS selector to convert. + :param str prefix: the prefix to add to the XPath expression. Defaults to ``//``. + :return: the resulting XPath expression. + :rtype: str + """ + separator = prefix + xpath_parts = [] + combinators = "+>,~ " + for selector_part in map(str.strip, re.split(rf"(\s*[{combinators}]\s*)", selector)): + if not selector_part: + separator = "//" + elif selector_part == ">": + separator = "/" + elif selector_part == ",": + separator = "|" + prefix + elif re.search(r"^(?:[a-z](-?\w+)*|[*.])", selector_part, flags=re.I): + element, *classes = selector_part.split(".") + if not element: + element = "*" + class_predicates = [f"[hasclass('{classname}')]" for classname in classes if classname] + xpath_parts += [separator, element + "".join(class_predicates)] + else: + raise NotImplementedError(f"Unsupported CSS selector syntax: {selector}") + + return "".join(xpath_parts) + + +CSS = simple_css_selector_to_xpath + + +def regex_xpath(pattern, attr=None, xpath=None): + """ + Return an XPath expression that matches elements with an attribute value matching the given regex pattern. + + :param str pattern: a regex pattern to match the attribute value against. + :param str | None attr: the attribute to match the pattern against. + If not given, the pattern is matched against the element's text. + :param str | None xpath: an optional XPath expression to further filter the elements to match. + :rtype: str + """ + # TODO abt: investigate lxml xpath variables interpolation (xpath.setcontext? registerVariables?) + if "'" in pattern and '"' in pattern: + quoted_pattern = "concat('" + "', \"'\", '".join(pattern.split("'")) + "')" + elif "'" in pattern: + quoted_pattern = '"' + pattern + '"' + else: + quoted_pattern = "'" + pattern + "'" + xpath_pre = xpath or "//*" + attr_or_text = f"@{attr}" if attr is not None else "text()" + return xpath_pre + f"[regex({attr_or_text}, {quoted_pattern})]" + + +def adapt_xpath_for_qweb(xpath): + """Adapts a xpath to enable matching on qweb ``t-att(f)-`` attributes.""" + xpath = re.sub(r"\bhas-?class(?=\()", "has-t-class", xpath) + # supposing that there's only one of `class`, `t-att-class`, `t-attf-class`, + # joining all of them with a space and removing trailing whitespace should behave + # similarly to COALESCE, and result in ORing the values for matching + xpath = re.sub( + r"(?<=\()\s*@(? + ... +
+ + after:: + .. code-block:: html + + ... + + """ + + def __call__(self, element, converter): + parent = element.getparent() + if parent is None: + raise ValueError(f"Cannot pull up contents of xml element with no parent: {element}") + + prev_sibling = element.getprevious() + if prev_sibling is not None: + prev_sibling.tail = ((prev_sibling.tail or "") + (element.text or "")) or None + else: + parent.text = ((parent.text or "") + (element.text or "")) or None + + for child in element: + element.addprevious(child) + + parent.remove(element) + + +class RenameAttribute(ElementOperation): + """Rename an attribute. Silently ignores elements that do not have the attribute.""" + + def __init__(self, old_name, new_name, extra_xpath=""): + self.old_name = old_name + self.new_name = new_name + self.extra_xpath = extra_xpath + + def __call__(self, element, converter): + rename_map = {self.old_name: self.new_name} + if converter.is_qweb: + rename_map = { + f"{prefix}{old}": f"{prefix}{new}" + for old, new in rename_map.items() + for prefix in ("",) + (("t-att-", "t-attf-") if not old.startswith("t-") else ()) + if f"{prefix}{old}" in element.attrib + } + if rename_map: + # to preserve attributes order, iterate+rename on a copy and reassign (clear+update, bc readonly property) + attrib_before = dict(element.attrib) + element.attrib.clear() + element.attrib.update({rename_map.get(k, k): v for k, v in attrib_before.items()}) + return element + + def xpath(self, xpath=None): + """ + Return an XPath expression that matches elements with the old attribute name. + + :param str | None xpath: an optional XPath expression to further filter the elements to match. + :rtype: str + """ + return (xpath or "//*") + f"[@{self.old_name}]{self.extra_xpath}" + + +class RegexReplace(ElementOperation): + """ + Uses `re.sub` to modify an attribute or the text of an element. + + N.B. no checks are made to ensure the attribute to replace is actually present on the elements. + + :param str pattern: the regex pattern to match. + :param str repl: the replacement string. + :param str | None attr: the attribute to replace. If not specified, the text of the element is replaced. + """ + + def __init__(self, pattern, sub, attr=None): + self.pattern = pattern + self.repl = sub + self.attr = attr + + def __call__(self, element, converter): + if self.attr is None: + # TODO abt: what about tail? + element.text = re.sub(self.pattern, self.repl, element.text or "") + else: + for attr in (self.attr,) + ((f"t-att-{self.attr}", f"t-attf-{self.attr}") if converter.is_qweb else ()): + if attr in element.attrib: + element.attrib[attr] = re.sub(self.pattern, self.repl, element.attrib[attr]) + return element + + def xpath(self, xpath=None): + """ + Return an XPath expression that matches elements with the old attribute name. + + :param str | None xpath: an optional XPath expression to further filter the elements to match. + :rtype: str + """ + return regex_xpath(self.pattern, self.attr, xpath) + + +class RegexReplaceClass(RegexReplace): + """ + Uses `re.sub` to modify the class. + + Basically, same as `RegexReplace`, but with `attr="class"`. + """ + + def __init__(self, pattern, sub, attr="class"): + super().__init__(pattern, sub, attr) + + +class EtreeConverter(ABC): + """ + Class for converting lxml etree documents, applying a bunch of operations on them. + + :param etree.ElementTree tree: the parsed XML or HTML tree to convert. + :param bool is_html: whether the tree is an HTML document. + :para bool is_qweb: whether the tree contains QWeb directives. + If this is enabled, XPaths will be auto-transformed to try to also match ``t-att*`` attributes. + """ + + def __init__(self, tree, is_html=False, is_qweb=False): + self.tree = tree + self.is_html = is_html + self.is_qweb = is_qweb + + @classmethod + @abstractmethod + def get_conversions(cls, *args, **kwargs): + """Return the conversions to apply to the tree.""" + + @classmethod + @lru_cache(maxsize=32) + def _compile_conversions(cls, conversions, is_qweb): + """ + Compile the given conversions to a list of ``(xpath, operations)`` tuples, with pre-compiled XPaths. + + The conversions must be provided as tuples instead of lists to allow for caching. + + :param tuple[ElementOperation | (str, ElementOperation | tuple[ElementOperation, ...]), ...] conversions: + the conversions to compile. + :param bool is_qweb: whether the conversions are for QWeb. + :rtype: list[(str, list[ElementOperation])] + """ + + def process_spec(spec): + xpath, ops = None, None + if isinstance(spec, ElementOperation): # single operation with its own XPath + xpath, ops = spec.xpath, [spec] + elif isinstance(spec, tuple) and len(spec) == 2: # (xpath, operation | operations) tuple + xpath, ops_spec = spec + if isinstance(ops_spec, ElementOperation): # single operation + ops = [ops_spec] + elif isinstance(ops_spec, tuple): # multiple operations + ops = list(ops_spec) + + if xpath is None or ops is None: + raise ValueError(f"Invalid conversion specification: {spec!r}") + + if is_qweb: + xpath = adapt_xpath_for_qweb(xpath) + + return etree.XPath(xpath), ops + + return [process_spec(spec) for spec in conversions] + + @classmethod + def prepare_conversions(cls, conversions, is_qweb): + """Prepare and compile the conversions into a list of ``(xpath, operations)`` tuples, with pre-compiled XPaths.""" + # make sure conversions list and nested lists of operations are converted to tuples for caching + conversions = tuple( + (spec[0], tuple(spec[1])) + if isinstance(spec, tuple) and len(spec) == 2 and isinstance(spec[1], list) + else spec + for spec in conversions + ) + return cls._compile_conversions(conversions, is_qweb) + + def convert(self, src_version, dst_version): + """ + Convert the loaded document inplace from the source version to the destination, returning the converted document and the number of conversion operations applied. + + :param str src_version: the source Bootstrap version. + :param str dst_version: the destination Bootstrap version. + :rtype: etree.ElementTree, int + """ + conversions = self.get_conversions(src_version, dst_version, is_qweb=self.is_qweb) + applied_operations_count = 0 + for xpath, operations in conversions: + for element in xpath(self.tree): + for operation in operations: + if element is None: # previous operations that returned None (i.e. deleted element) + raise ValueError("Matched xml element is not available anymore! Check operations.") + element = operation(element, self) # noqa: PLW2901 + applied_operations_count += 1 + return self.tree, applied_operations_count + + @classmethod + def convert_arch(cls, arch, src_version, dst_version, is_html=False, **converter_kwargs): + """ + Class method for converting a string of XML or HTML code. + + :param str arch: the XML or HTML code to convert. + :param str src_version: the source Bootstrap version. + :param str dst_version: the destination Bootstrap version. + :param bool is_html: whether the arch is an HTML document. + :param dict converter_kwargs: additional keyword arguments to pass to the converter. + :return: the converted XML or HTML code. + :rtype: str + """ + stripped_arch = arch.strip() + doc_header_match = re.search(r"^<\?xml .+\?>\s*", stripped_arch) + doc_header = doc_header_match.group(0) if doc_header_match else "" + stripped_arch = stripped_arch[doc_header_match.end() :] if doc_header_match else stripped_arch + + tree = etree.fromstring(f"{stripped_arch}", parser=html_utf8_parser if is_html else None) + + tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) + if not ops_count: + return arch + + wrap_node = tree.xpath("//wrap")[0] + return doc_header + "\n".join( + etree.tostring(child, encoding="unicode", with_tail=True, method="html" if is_html else None) + for child in wrap_node + ) + + @classmethod + def convert_file(cls, path, src_version, dst_version, is_html=None, **converter_kwargs): + """ + Class method for converting an XML or HTML file inplace. + + :param str path: the path to the XML or HTML file to convert. + :param str src_version: the source Bootstrap version. + :param str dst_version: the destination Bootstrap version. + :param bool is_html: whether the file is an HTML document. + If not set, will be detected from the file extension. + :param dict converter_kwargs: additional keyword arguments to pass to the converter. + :rtype: None + """ + if is_html is None: + is_html = os.path.splitext(path)[1].startswith("htm") + tree = etree.parse(path, parser=html_utf8_parser if is_html else None) + + tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) + if not ops_count: + logging.info("No conversion operations applied, skipping file %s", path) + return + + tree.write(path, encoding="utf-8", method="html" if is_html else None, xml_declaration=not is_html) + + def element_factory(self, *args, **kwargs): + """ + Create new elements using the correct document type. + + Basically a wrapper for either etree.XML or etree.HTML depending on the type of document loaded. + + :param args: positional arguments to pass to the etree.XML or etree.HTML function. + :param kwargs: keyword arguments to pass to the etree.XML or etree.HTML function. + :return: the created element. + """ + return etree.HTML(*args, **kwargs) if self.is_html else etree.XML(*args, **kwargs) + + def build_element(self, tag, classes=None, contents=None, **attributes): + """ + Create a new element with the given tag, classes, contents and attributes. + + Like :meth:`~.element_factory`, can be used by operations to create elements abstracting away the document type. + + :param str tag: the tag of the element to create. + :param typing.Iterable[str] | None classes: the classes to set on the new element. + :param str | None contents: the contents of the new element (i.e. inner text/HTML/XML). + :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. + :return: the created element. + :rtype: etree.ElementBase + """ + element = self.element_factory(f"<{tag}>{contents or ''}") + for name, value in attributes.items(): + element.attrib[name] = value + if classes: + set_classes(element, classes) + return element + + def copy_element( + self, + element, + tag=None, + add_classes=None, + remove_classes=None, + copy_attrs=True, + copy_contents=True, + **attributes, + ): + """ + Create a copy of an element, optionally changing the tag, classes, contents and attributes. + + Like :meth:`~.element_factory`, can be used by operations to copy elements abstracting away the document type. + + :param etree.ElementBase element: the element to copy. + :param str | None tag: if specified, overrides the tag of the new element. + :param str | typing.Iterable[str] | None add_classes: if specified, adds the given class(es) to the new element. + :param str | typing.Iterable[str] | ALL | None remove_classes: if specified, removes the given class(es) + from the new element. The `ALL` sentinel value can be specified to remove all classes. + :param bool copy_attrs: if True, copies the attributes of the source element to the new one. Defaults to True. + :param bool copy_contents: if True, copies the contents of the source element to the new one. Defaults to True. + :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. + Will be str merged with the attributes of the source element, overriding the latter. + :return: the new copied element. + :rtype: etree.ElementBase + """ + tag = tag or element.tag + contents = innerxml(element, is_html=self.is_html) if copy_contents else None + if copy_attrs: + attributes = {**element.attrib, **attributes} + new_element = self.build_element(tag, contents=contents, **attributes) + edit_element_classes(new_element, add_classes, remove_classes, is_qweb=self.is_qweb) + return new_element + + def adapt_xpath(self, xpath): + """Adapts an xpath to match qweb ``t-att(f)-*`` attributes, if ``is_qweb`` is True.""" + return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath diff --git a/tools/compile23.py b/tools/compile23.py index b5bc369e..f39841c6 100755 --- a/tools/compile23.py +++ b/tools/compile23.py @@ -13,6 +13,7 @@ "src/testing.py", "src/util/jinja_to_qweb.py", "src/util/snippets.py", + "src/util/views_convert.py", "src/util/convert_bootstrap.py", "src/*/tests/*.py", "src/*/17.0.*/*.py", From d47d0f666e0d3039200d3fc100e35be6f3fa604e Mon Sep 17 00:00:00 2001 From: abk16 Date: Fri, 30 Dec 2022 18:37:01 +0100 Subject: [PATCH 2/6] [REF] util.views_convert: simplify ElementOperation and EtreeConverter Simplified ElementOperation class, removing unnecessary features, such as `.op()` method. Made `xpath` attribute part of the base class, subclasses must honor it to restrict their scope of operation. Reworked EtreeConverter class methods, generalized to work with any kind of converter operations (not just Bootstrap conversions). Also made docstrings private (excluded from online docs). --- src/util/convert_bootstrap.py | 341 ++++++++++++++++++---------------- src/util/views_convert.py | 297 ++++++++++++++++++++--------- 2 files changed, 390 insertions(+), 248 deletions(-) diff --git a/src/util/convert_bootstrap.py b/src/util/convert_bootstrap.py index 14f1d639..45c6f11c 100644 --- a/src/util/convert_bootstrap.py +++ b/src/util/convert_bootstrap.py @@ -24,7 +24,6 @@ RemoveElement, RenameAttribute, ReplaceClasses, - adapt_xpath_for_qweb, edit_element_classes, get_classes, regex_xpath, @@ -34,7 +33,11 @@ class BS3to4ConvertBlockquote(ElementOperation): - """Convert a BS3 ``
`` element to a BS4 ``
`` element with the ``blockquote`` class.""" + """ + Convert a BS3 ``
`` element to a BS4 ``
`` element with the ``blockquote`` class. + + :meta private: exclude from online docs + """ def __call__(self, element, converter): blockquote = converter.copy_element(element, tag="div", add_classes="blockquote", copy_attrs=False) @@ -49,6 +52,8 @@ class BS3to4MakeCard(ElementOperation): Pre-processe a BS3 panel, thumbnail, or well element to be converted to a BS4 card. Card components conversion is then handled by the ``ConvertCard`` operation class. + + :meta private: exclude from online docs """ def __call__(self, element, converter): @@ -64,7 +69,11 @@ def __call__(self, element, converter): # TODO abt: refactor code class BS3to4ConvertCard(ElementOperation): - """Fully convert a BS3 panel, thumbnail, or well element and their contents to a BS4 card.""" + """ + Fully convert a BS3 panel, thumbnail, or well element and their contents to a BS4 card. + + :meta private: exclude from online docs + """ POST_CONVERSIONS = { "title": ["card-title"], @@ -149,6 +158,8 @@ class BS4to5ConvertCloseButton(ElementOperation): Convert BS4 ``button.close`` elements to BS5 ``button.btn-close``. Also fixes the ``data-dismiss`` attribute to ``data-bs-dismiss``, and removes any inner contents. + + :meta private: exclude from online docs """ def __call__(self, element, converter): @@ -165,7 +176,11 @@ def __call__(self, element, converter): class BS4to5ConvertCardDeck(ElementOperation): - """Convert BS4 ``.card-deck`` elements to grid components (``.row``, ``.col``, etc.).""" + """ + Convert BS4 ``.card-deck`` elements to grid components (``.row``, ``.col``, etc.). + + :meta private: exclude from online docs + """ def __call__(self, element, converter): cards = element.xpath(converter.adapt_xpath("./*[hasclass('card')]")) @@ -182,7 +197,11 @@ def __call__(self, element, converter): class BS4to5ConvertFormInline(ElementOperation): - """Convert BS4 ``.form-inline`` elements to grid components (``.row``, ``.col``, etc.).""" + """ + Convert BS4 ``.form-inline`` elements to grid components (``.row``, ``.col``, etc.). + + :meta private: exclude from online docs + """ def __call__(self, element, converter): edit_element_classes(element, add="row row-cols-lg-auto", remove="form-inline", is_qweb=converter.is_qweb) @@ -228,122 +247,122 @@ class BootstrapConverter(EtreeConverter): :param str src_version: the source Bootstrap version to convert from. :param str dst_version: the destination Bootstrap version to convert to. :param bool is_html: whether the tree is an HTML document. - :para bool is_qweb: whether the tree contains QWeb directives. See :class:`EtreeConverter` for more info. + :param bool is_qweb: whether the tree contains QWeb directives. See :class:`EtreeConverter` for more info. + + :meta private: exclude from online docs """ MIN_VERSION = "3.0" """Minimum supported Bootstrap version.""" # Conversions definitions by destination version. - # It's a dictionary of version strings to a list of (xpath, operations_list) tuples. - # For operations that implement the `xpath()` method, the `op()` class method can be used - # to directly define the operation and return the tuple with the corresponding XPath expression - # and the operation list. - # The `convert()` method will then process the conversions list in order, and for each tuple - # match the elements in the tree using the XPath expression and apply the operations list to them. + # It's a dictionary of version strings to a conversions list. + # Each item in the conversions list must either be an :class:`ElementOperation`, + # that can provide its own XPath, or a tuple of ``(xpath, operation(s))`` with the XPath + # and a single operation or a list of operations to apply to the nodes matching the XPath. CONVERSIONS = { "4.0": [ # inputs - (CSS(".form-group .control-label"), [ReplaceClasses("control-label", "form-control-label")]), - (CSS(".form-group .text-help"), [ReplaceClasses("text-help", "form-control-feedback")]), - (CSS(".control-group .help-block"), [ReplaceClasses("help-block", "form-text")]), - ReplaceClasses.op("form-group-sm", "form-control-sm"), - ReplaceClasses.op("form-group-lg", "form-control-lg"), - (CSS(".form-control .input-sm"), [ReplaceClasses("input-sm", "form-control-sm")]), - (CSS(".form-control .input-lg"), [ReplaceClasses("input-lg", "form-control-lg")]), + (CSS(".form-group .control-label"), ReplaceClasses("control-label", "form-control-label")), + (CSS(".form-group .text-help"), ReplaceClasses("text-help", "form-control-feedback")), + (CSS(".control-group .help-block"), ReplaceClasses("help-block", "form-text")), + ReplaceClasses("form-group-sm", "form-control-sm"), + ReplaceClasses("form-group-lg", "form-control-lg"), + (CSS(".form-control .input-sm"), ReplaceClasses("input-sm", "form-control-sm")), + (CSS(".form-control .input-lg"), ReplaceClasses("input-lg", "form-control-lg")), # hide - ReplaceClasses.op("hidden-xs", "d-none"), - ReplaceClasses.op("hidden-sm", "d-sm-none"), - ReplaceClasses.op("hidden-md", "d-md-none"), - ReplaceClasses.op("hidden-lg", "d-lg-none"), - ReplaceClasses.op("visible-xs", "d-block d-sm-none"), - ReplaceClasses.op("visible-sm", "d-block d-md-none"), - ReplaceClasses.op("visible-md", "d-block d-lg-none"), - ReplaceClasses.op("visible-lg", "d-block d-xl-none"), + ReplaceClasses("hidden-xs", "d-none"), + ReplaceClasses("hidden-sm", "d-sm-none"), + ReplaceClasses("hidden-md", "d-md-none"), + ReplaceClasses("hidden-lg", "d-lg-none"), + ReplaceClasses("visible-xs", "d-block d-sm-none"), + ReplaceClasses("visible-sm", "d-block d-md-none"), + ReplaceClasses("visible-md", "d-block d-lg-none"), + ReplaceClasses("visible-lg", "d-block d-xl-none"), # image - ReplaceClasses.op("img-rounded", "rounded"), - ReplaceClasses.op("img-circle", "rounded-circle"), - ReplaceClasses.op("img-responsive", ("d-block", "img-fluid")), + ReplaceClasses("img-rounded", "rounded"), + ReplaceClasses("img-circle", "rounded-circle"), + ReplaceClasses("img-responsive", ("d-block", "img-fluid")), # buttons - ReplaceClasses.op("btn-default", "btn-secondary"), - ReplaceClasses.op("btn-xs", "btn-sm"), - (CSS(".btn-group.btn-group-xs"), [ReplaceClasses("btn-group-xs", "btn-group-sm")]), - (CSS(".dropdown .divider"), [ReplaceClasses("divider", "dropdown-divider")]), - ReplaceClasses.op("badge", "badge badge-pill"), - ReplaceClasses.op("label", "badge"), - RegexReplaceClass.op(rf"{BS}label-(default|primary|success|info|warning|danger){BE}", r"badge-\1"), - (CSS(".breadcrumb > li"), [ReplaceClasses("breadcrumb", "breadcrumb-item")]), + ReplaceClasses("btn-default", "btn-secondary"), + ReplaceClasses("btn-xs", "btn-sm"), + (CSS(".btn-group.btn-group-xs"), ReplaceClasses("btn-group-xs", "btn-group-sm")), + (CSS(".dropdown .divider"), ReplaceClasses("divider", "dropdown-divider")), + ReplaceClasses("badge", "badge badge-pill"), + ReplaceClasses("label", "badge"), + RegexReplaceClass(rf"{BS}label-(default|primary|success|info|warning|danger){BE}", r"badge-\1"), + (CSS(".breadcrumb > li"), ReplaceClasses("breadcrumb", "breadcrumb-item")), # li - (CSS(".list-inline > li"), [AddClasses("list-inline-item")]), + (CSS(".list-inline > li"), AddClasses("list-inline-item")), # pagination - (CSS(".pagination > li"), [AddClasses("page-item")]), - (CSS(".pagination > li > a"), [AddClasses("page-link")]), + (CSS(".pagination > li"), AddClasses("page-item")), + (CSS(".pagination > li > a"), AddClasses("page-link")), # carousel - (CSS(".carousel .carousel-inner > .item"), [ReplaceClasses("item", "carousel-item")]), + (CSS(".carousel .carousel-inner > .item"), ReplaceClasses("item", "carousel-item")), # pull - ReplaceClasses.op("pull-right", "float-right"), - ReplaceClasses.op("pull-left", "float-left"), - ReplaceClasses.op("center-block", "mx-auto"), + ReplaceClasses("pull-right", "float-right"), + ReplaceClasses("pull-left", "float-left"), + ReplaceClasses("center-block", "mx-auto"), # well - (CSS(".well"), [BS3to4MakeCard()]), - (CSS(".thumbnail"), [BS3to4MakeCard()]), + (CSS(".well"), BS3to4MakeCard()), + (CSS(".thumbnail"), BS3to4MakeCard()), # blockquote - (CSS("blockquote"), [BS3to4ConvertBlockquote()]), - (CSS(".blockquote.blockquote-reverse"), [ReplaceClasses("blockquote-reverse", "text-right")]), + (CSS("blockquote"), BS3to4ConvertBlockquote()), + (CSS(".blockquote.blockquote-reverse"), ReplaceClasses("blockquote-reverse", "text-right")), # dropdown - (CSS(".dropdown-menu > li > a"), [AddClasses("dropdown-item")]), - (CSS(".dropdown-menu > li"), [PullUp()]), + (CSS(".dropdown-menu > li > a"), AddClasses("dropdown-item")), + (CSS(".dropdown-menu > li"), PullUp()), # in - ReplaceClasses.op("in", "show"), + ReplaceClasses("in", "show"), # table - (CSS("tr.active, td.active"), [ReplaceClasses("active", "table-active")]), - (CSS("tr.success, td.success"), [ReplaceClasses("success", "table-success")]), - (CSS("tr.info, td.info"), [ReplaceClasses("info", "table-info")]), - (CSS("tr.warning, td.warning"), [ReplaceClasses("warning", "table-warning")]), - (CSS("tr.danger, td.danger"), [ReplaceClasses("danger", "table-danger")]), - (CSS("table.table-condesed"), [ReplaceClasses("table-condesed", "table-sm")]), + (CSS("tr.active, td.active"), ReplaceClasses("active", "table-active")), + (CSS("tr.success, td.success"), ReplaceClasses("success", "table-success")), + (CSS("tr.info, td.info"), ReplaceClasses("info", "table-info")), + (CSS("tr.warning, td.warning"), ReplaceClasses("warning", "table-warning")), + (CSS("tr.danger, td.danger"), ReplaceClasses("danger", "table-danger")), + (CSS("table.table-condesed"), ReplaceClasses("table-condesed", "table-sm")), # navbar - (CSS(".nav.navbar > li > a"), [AddClasses("nav-link")]), - (CSS(".nav.navbar > li"), [AddClasses("nav-intem")]), - ReplaceClasses.op("navbar-btn", "nav-item"), - (CSS(".navbar-nav"), [ReplaceClasses("navbar-right nav", "ml-auto")]), - ReplaceClasses.op("navbar-toggler-right", "ml-auto"), - (CSS(".navbar-nav > li > a"), [AddClasses("nav-link")]), - (CSS(".navbar-nav > li"), [AddClasses("nav-item")]), - (CSS(".navbar-nav > a"), [AddClasses("navbar-brand")]), - ReplaceClasses.op("navbar-fixed-top", "fixed-top"), - ReplaceClasses.op("navbar-toggle", "navbar-toggler"), - ReplaceClasses.op("nav-stacked", "flex-column"), - (CSS("nav.navbar"), [AddClasses("navbar-expand-lg")]), - (CSS("button.navbar-toggle"), [ReplaceClasses("navbar-toggle", "navbar-expand-md")]), + (CSS(".nav.navbar > li > a"), AddClasses("nav-link")), + (CSS(".nav.navbar > li"), AddClasses("nav-intem")), + ReplaceClasses("navbar-btn", "nav-item"), + (CSS(".navbar-nav"), ReplaceClasses("navbar-right nav", "ml-auto")), + ReplaceClasses("navbar-toggler-right", "ml-auto"), + (CSS(".navbar-nav > li > a"), AddClasses("nav-link")), + (CSS(".navbar-nav > li"), AddClasses("nav-item")), + (CSS(".navbar-nav > a"), AddClasses("navbar-brand")), + ReplaceClasses("navbar-fixed-top", "fixed-top"), + ReplaceClasses("navbar-toggle", "navbar-toggler"), + ReplaceClasses("nav-stacked", "flex-column"), + (CSS("nav.navbar"), AddClasses("navbar-expand-lg")), + (CSS("button.navbar-toggle"), ReplaceClasses("navbar-toggle", "navbar-expand-md")), # card - (CSS(".panel"), [BS3to4ConvertCard()]), - (CSS(".card"), [BS3to4ConvertCard()]), + (CSS(".panel"), BS3to4ConvertCard()), + (CSS(".card"), BS3to4ConvertCard()), # grid - RegexReplaceClass.op(rf"{BS}col((?:-\w{{2}})?)-offset-(\d{{1,2}}){BE}", r"offset\1-\2"), + RegexReplaceClass(rf"{BS}col((?:-\w{{2}})?)-offset-(\d{{1,2}}){BE}", r"offset\1-\2"), ], "5.0": [ # links - RegexReplaceClass.op(rf"{BS}text-(?!o-)", "link-", xpath=CSS("a")), - (CSS(".nav-item.active > .nav-link"), [AddClasses("active")]), - (CSS(".nav-link.active") + CSS(".nav-item.active", prefix="/parent::"), [RemoveClasses("active")]), + RegexReplaceClass(rf"{BS}text-(?!o-)", "link-", xpath=CSS("a")), + (CSS(".nav-item.active > .nav-link"), AddClasses("active")), + (CSS(".nav-link.active") + CSS(".nav-item.active", prefix="/parent::"), RemoveClasses("active")), # badges - ReplaceClasses.op("badge-pill", "rounded-pill"), - RegexReplaceClass.op(rf"{BS}badge-", r"text-bg-"), + ReplaceClasses("badge-pill", "rounded-pill"), + RegexReplaceClass(rf"{BS}badge-", r"text-bg-"), # buttons - ("//*[hasclass('btn-block')]/parent::div", [AddClasses("d-grid gap-2")]), - ("//*[hasclass('btn-block')]/parent::p[count(./*)=1]", [AddClasses("d-grid gap-2")]), - RemoveClasses.op("btn-block"), - (CSS("button.close"), [BS4to5ConvertCloseButton()]), + ("//*[hasclass('btn-block')]/parent::div", AddClasses("d-grid gap-2")), + ("//*[hasclass('btn-block')]/parent::p[count(./*)=1]", AddClasses("d-grid gap-2")), + RemoveClasses("btn-block"), + (CSS("button.close"), BS4to5ConvertCloseButton()), # card # TODO abt: .card-columns (unused in odoo) - (CSS(".card-deck"), [BS4to5ConvertCardDeck()]), + (CSS(".card-deck"), BS4to5ConvertCardDeck()), # jumbotron - ReplaceClasses.op("jumbotron", "container-fluid py-5"), + ReplaceClasses("jumbotron", "container-fluid py-5"), # new data-bs- attributes - RenameAttribute.op("data-display", "data-bs-display", "[not(@data-snippet='s_countdown')]"), + RenameAttribute("data-display", "data-bs-display", xpath="//*[not(@data-snippet='s_countdown')]"), *[ - RenameAttribute.op(f"data-{attr}", f"data-bs-{attr}") + RenameAttribute(f"data-{attr}", f"data-bs-{attr}") for attr in ( "animation attributes autohide backdrop body container content delay dismiss focus" " interval margin-right no-jquery offset original-title padding-right parent placement" @@ -351,65 +370,65 @@ class BootstrapConverter(EtreeConverter): ).split(" ") ], # popover - (CSS(".popover .arrow"), [ReplaceClasses("arrow", "popover-arrow")]), + (CSS(".popover .arrow"), ReplaceClasses("arrow", "popover-arrow")), # form - ReplaceClasses.op("form-row", "row"), - ("//*[hasclass('form-group')]/parent::form", [AddClasses("row")]), - ReplaceClasses.op("form-group", "col-12 py-2"), - (CSS(".form-inline"), [BS4to5ConvertFormInline()]), - ReplaceClasses.op("custom-checkbox", "form-check"), - RegexReplaceClass.op(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), - RegexReplaceClass.op(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), - RegexReplaceClass.op(rf"{BS}custom-(check|select|range){BE}", r"form-\1"), - (CSS(".custom-switch"), [ReplaceClasses("custom-switch", "form-check form-switch")]), - ReplaceClasses.op("custom-radio", "form-check"), - RemoveClasses.op("custom-control"), - (CSS(".custom-file"), [PullUp()]), - RegexReplaceClass.op(rf"{BS}custom-file-", r"form-file-"), - RegexReplaceClass.op(rf"{BS}form-file(?:-input)?{BE}", r"form-control"), - (CSS("label.form-file-label"), [RemoveElement()]), - (regex_xpath(rf"{BS}input-group-(prepend|append){BE}", "class"), [PullUp()]), - ("//label[not(hasclass('form-check-label'))]", [AddClasses("form-label")]), - ReplaceClasses.op("form-control-file", "form-control"), - ReplaceClasses.op("form-control-range", "form-range"), + ReplaceClasses("form-row", "row"), + ("//*[hasclass('form-group')]/parent::form", AddClasses("row")), + ReplaceClasses("form-group", "col-12 py-2"), + (CSS(".form-inline"), BS4to5ConvertFormInline()), + ReplaceClasses("custom-checkbox", "form-check"), + RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), + RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), + RegexReplaceClass(rf"{BS}custom-(check|select|range){BE}", r"form-\1"), + (CSS(".custom-switch"), ReplaceClasses("custom-switch", "form-check form-switch")), + ReplaceClasses("custom-radio", "form-check"), + RemoveClasses("custom-control"), + (CSS(".custom-file"), PullUp()), + RegexReplaceClass(rf"{BS}custom-file-", r"form-file-"), + RegexReplaceClass(rf"{BS}form-file(?:-input)?{BE}", r"form-control"), + (CSS("label.form-file-label"), RemoveElement()), + (regex_xpath(rf"{BS}input-group-(prepend|append){BE}", "class"), PullUp()), + ("//label[not(hasclass('form-check-label'))]", AddClasses("form-label")), + ReplaceClasses("form-control-file", "form-control"), + ReplaceClasses("form-control-range", "form-range"), # TODO abt: .form-text no loger sets display, add some class? # table - RegexReplaceClass.op(rf"{BS}thead-(light|dark){BE}", r"table-\1"), + RegexReplaceClass(rf"{BS}thead-(light|dark){BE}", r"table-\1"), # grid - RegexReplaceClass.op(rf"{BS}col-((?:\w{{2}}-)?)offset-(\d{{1,2}}){BE}", r"offset-\1\2"), # from BS4 + RegexReplaceClass(rf"{BS}col-((?:\w{{2}}-)?)offset-(\d{{1,2}}){BE}", r"offset-\1\2"), # from BS4 # gutters - ReplaceClasses.op("no-gutters", "g-0"), + ReplaceClasses("no-gutters", "g-0"), # logical properties - RegexReplaceClass.op(rf"{BS}left-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"start-\1"), - RegexReplaceClass.op(rf"{BS}right-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"end-\1"), - RegexReplaceClass.op(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-left{BE}", r"\1-start"), - RegexReplaceClass.op(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-right{BE}", r"\1-end"), - RegexReplaceClass.op(rf"{BS}rounded-sm(-(?:start|end|top|bottom))?", r"rounded\1-1"), - RegexReplaceClass.op(rf"{BS}rounded-lg(-(?:start|end|top|bottom))?", r"rounded\1-3"), - RegexReplaceClass.op(rf"{BS}([mp])l-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1s-\2"), - RegexReplaceClass.op(rf"{BS}([mp])r-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1e-\2"), - ReplaceClasses.op("dropdown-menu-left", "dropdown-menu-start"), - ReplaceClasses.op("dropdown-menu-right", "dropdown-menu-end"), - ReplaceClasses.op("dropleft", "dropstart"), - ReplaceClasses.op("dropright", "dropend"), + RegexReplaceClass(rf"{BS}left-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"start-\1"), + RegexReplaceClass(rf"{BS}right-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"end-\1"), + RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-left{BE}", r"\1-start"), + RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-right{BE}", r"\1-end"), + RegexReplaceClass(rf"{BS}rounded-sm(-(?:start|end|top|bottom))?", r"rounded\1-1"), + RegexReplaceClass(rf"{BS}rounded-lg(-(?:start|end|top|bottom))?", r"rounded\1-3"), + RegexReplaceClass(rf"{BS}([mp])l-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1s-\2"), + RegexReplaceClass(rf"{BS}([mp])r-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1e-\2"), + ReplaceClasses("dropdown-menu-left", "dropdown-menu-start"), + ReplaceClasses("dropdown-menu-right", "dropdown-menu-end"), + ReplaceClasses("dropleft", "dropstart"), + ReplaceClasses("dropright", "dropend"), # tooltips ( "//*[hasclass('tooltip') or @role='tooltip']//*[hasclass('arrow')]", - [ReplaceClasses("arrow", "tooltip-arrow")], + ReplaceClasses("arrow", "tooltip-arrow"), ), # utilities - ReplaceClasses.op("text-monospace", "font-monospace"), - RegexReplaceClass.op(rf"{BS}font-weight-", r"fw-"), - RegexReplaceClass.op(rf"{BS}font-style-", r"fst-"), - ReplaceClasses.op("font-italic", "fst-italic"), + ReplaceClasses("text-monospace", "font-monospace"), + RegexReplaceClass(rf"{BS}font-weight-", r"fw-"), + RegexReplaceClass(rf"{BS}font-style-", r"fst-"), + ReplaceClasses("font-italic", "fst-italic"), # helpers - RegexReplaceClass.op(rf"{BS}embed-responsive-(\d+)by(\d+)", r"ratio-\1x\2"), - RegexReplaceClass.op(rf"{BS}ratio-(\d+)by(\d+)", r"ratio-\1x\2"), - RegexReplaceClass.op(rf"{BS}embed-responsive(?!-)", r"ratio"), - RegexReplaceClass.op(rf"{BS}sr-only(-focusable)?", r"visually-hidden\1"), + RegexReplaceClass(rf"{BS}embed-responsive-(\d+)by(\d+)", r"ratio-\1x\2"), + RegexReplaceClass(rf"{BS}ratio-(\d+)by(\d+)", r"ratio-\1x\2"), + RegexReplaceClass(rf"{BS}embed-responsive(?!-)", r"ratio"), + RegexReplaceClass(rf"{BS}sr-only(-focusable)?", r"visually-hidden\1"), # media - ReplaceClasses.op("media-body", "flex-grow-1"), - ReplaceClasses.op("media", "d-flex"), + ReplaceClasses("media-body", "flex-grow-1"), + ReplaceClasses("media", "d-flex"), ], } @@ -419,57 +438,53 @@ def __init__(self, src_version, dst_version, *, is_html=False, is_qweb=False): @classmethod def _get_sorted_conversions(cls): - """Return the conversions dict sorted by version, from oldest to newest.""" + """ + Return the conversions dict sorted by version, from oldest to newest. + + :meta private: exclude from online docs + """ return sorted(cls.CONVERSIONS.items(), key=lambda kv: Version(kv[0])) @classmethod @lru_cache(maxsize=8) - def get_conversions(cls, src_ver, dst_ver, is_qweb=False): + def _get_conversions(cls, src_version, dst_version): """ - Return the list of conversions to convert Bootstrap from ``src_ver`` to ``dst_ver``, with compiled XPaths. + Return the list of conversions to convert Bootstrap from ``src_version`` to ``dst_version``. + + :param str src_version: the source Bootstrap version. + :param str dst_version: the destination Bootstrap version. + :rtype: list[ElementOperation | (str, ElementOperation | list[ElementOperation])] - :param str src_ver: the source Bootstrap version. - :param str dst_ver: the destination Bootstrap version. - :param bool is_qweb: whether to adapt conversions for QWeb (to support ``t-att(f)-`` conversions). - :rtype: list[(etree.XPath, list[ElementOperation])] + :meta private: exclude from online docs """ - if Version(dst_ver) < Version(src_ver): + if Version(dst_version) < Version(src_version): raise NotImplementedError("Downgrading Bootstrap versions is not supported.") - if Version(src_ver) < Version(cls.MIN_VERSION): - raise NotImplementedError(f"Conversion from Bootstrap version {src_ver} is not supported") + if Version(src_version) < Version(cls.MIN_VERSION): + raise NotImplementedError(f"Conversion from Bootstrap version {src_version} is not supported") + result = [] for version, conversions in BootstrapConverter._get_sorted_conversions(): - if Version(src_ver) < Version(version) <= Version(dst_ver): + if Version(src_version) < Version(version) <= Version(dst_version): result.extend(conversions) + if not result: - if Version(src_ver) == Version(dst_ver): + if Version(src_version) == Version(dst_version): _logger.info("Source and destination versions are the same, no conversion needed.") else: - raise NotImplementedError(f"Conversion from {src_ver} to {dst_ver} is not supported") - if is_qweb: - result = [(adapt_xpath_for_qweb(xpath), conversions) for xpath, conversions in result] - return [(etree.XPath(xpath), conversions) for xpath, conversions in result] - + raise NotImplementedError(f"Conversion from {src_version} to {dst_version} is not supported") -def convert_tree(tree, src_version, dst_version, **converter_kwargs): - """ - Convert an already parsed lxml tree from Bootstrap v3 to v4 inplace. - - :param etree.ElementTree tree: the lxml tree to convert. - :param str src_version: the version of Bootstrap the document is currently using. - :param str dst_version: the version of Bootstrap to convert the document to. - :param dict converter_kwargs: additional keyword arguments to initialize :class:`~.BootstrapConverter`. - :return: the converted lxml tree. - :rtype: etree.ElementTree - """ - tree, ops_count = BootstrapConverter(tree, **converter_kwargs).convert(src_version, dst_version) - return tree + return result +convert_tree = BootstrapConverter.convert_tree convert_arch = BootstrapConverter.convert_arch convert_file = BootstrapConverter.convert_file +bs3to4_converter = BootstrapConverter("3.0", "4.0") +bs4to5_converter = BootstrapConverter("4.0", "5.0") + +# TODO abt: remove this / usages -> replace with refactored converter classes class BootstrapHTMLConverter: def __init__(self, src, dst): self.src = src diff --git a/src/util/views_convert.py b/src/util/views_convert.py index aa5d4d4b..c839c2aa 100644 --- a/src/util/views_convert.py +++ b/src/util/views_convert.py @@ -79,6 +79,8 @@ def innerxml(element, is_html=False): :param etree.ElementBase element: the element to convert. :param bool is_html: whether to use HTML for serialization, XML otherwise. Defaults to False. :rtype: str + + :meta private: exclude from online docs """ return (element.text or "") + "".join( etree.tostring(child, encoding=str, method="html" if is_html else None) for child in element @@ -86,17 +88,29 @@ def innerxml(element, is_html=False): def split_classes(*joined_classes): - """Return a list of classes given one or more strings of joined classes separated by spaces.""" + """ + Return a list of classes given one or more strings of joined classes separated by spaces. + + :meta private: exclude from online docs + """ return [c for classes in joined_classes for c in (classes or "").split(" ") if c] def get_classes(element): - """Return the list of classes from the ``class`` attribute of an element.""" + """ + Return the list of classes from the ``class`` attribute of an element. + + :meta private: exclude from online docs + """ return split_classes(element.get("class", "")) def join_classes(classes): - """Return a string of classes joined by space given a list of classes.""" + """ + Return a string of classes joined by space given a list of classes. + + :meta private: exclude from online docs + """ return " ".join(classes) @@ -105,6 +119,8 @@ def set_classes(element, classes): Set the ``class`` attribute of an element from a list of classes. If the list is empty, the attribute is removed. + + :meta private: exclude from online docs """ if classes: element.attrib["class"] = join_classes(classes) @@ -122,6 +138,8 @@ def edit_classlist(classes, add, remove): from the list. The `ALL` sentinel value can be specified to remove all classes. :rtype: typing.List[str] :return: the new class list. + + :meta private: exclude from online docs """ if remove is ALL: classes = [] @@ -160,6 +178,8 @@ def edit_element_t_classes(element, add, remove): :param str | typing.Iterable[str] | None add: if specified, adds the given class(es) to the element. :param str | typing.Iterable[str] | ALL | None remove: if specified, removes the given class(es) from the element. The `ALL` sentinel value can be specified to remove all classes. + + :meta private: exclude from online docs """ if isinstance(add, str): add = [add] @@ -220,6 +240,8 @@ def edit_element_classes(element, add, remove, is_qweb=False): from the element. The `ALL` sentinel value can be specified to remove all classes. :param bool is_qweb: if True also edit classes in ``t-att-class`` and ``t-attf-class`` attributes. Defaults to False. + + :meta private: exclude from online docs """ if not is_qweb or not set(element.attrib) & {"t-att-class", "t-attf-class"}: set_classes(element, edit_classlist(get_classes(element), add, remove)) @@ -241,6 +263,8 @@ def simple_css_selector_to_xpath(selector, prefix="//"): :param str prefix: the prefix to add to the XPath expression. Defaults to ``//``. :return: the resulting XPath expression. :rtype: str + + :meta private: exclude from online docs """ separator = prefix xpath_parts = [] @@ -276,6 +300,8 @@ def regex_xpath(pattern, attr=None, xpath=None): If not given, the pattern is matched against the element's text. :param str | None xpath: an optional XPath expression to further filter the elements to match. :rtype: str + + :meta private: exclude from online docs """ # TODO abt: investigate lxml xpath variables interpolation (xpath.setcontext? registerVariables?) if "'" in pattern and '"' in pattern: @@ -290,7 +316,11 @@ def regex_xpath(pattern, attr=None, xpath=None): def adapt_xpath_for_qweb(xpath): - """Adapts a xpath to enable matching on qweb ``t-att(f)-`` attributes.""" + """ + Adapts a xpath to enable matching on qweb ``t-att(f)-`` attributes. + + :meta private: exclude from online docs + """ xpath = re.sub(r"\bhas-?class(?=\()", "has-t-class", xpath) # supposing that there's only one of `class`, `t-att-class`, `t-attf-class`, # joining all of them with a space and removing trailing whitespace should behave @@ -303,9 +333,22 @@ def adapt_xpath_for_qweb(xpath): return re.sub(r"\[@(?... + :meta private: exclude from online docs """ def __call__(self, element, converter): @@ -466,12 +530,20 @@ def __call__(self, element, converter): class RenameAttribute(ElementOperation): - """Rename an attribute. Silently ignores elements that do not have the attribute.""" + """ + Rename an attribute. Silently ignores elements that do not have the attribute. + + :param str old_name: the name of the attribute to rename. + :param str new_name: the new name of the attribute. + :param str | None xpath: see :class:`ElementOperation` ``xpath`` parameter. + + :meta private: exclude from online docs + """ - def __init__(self, old_name, new_name, extra_xpath=""): + def __init__(self, old_name, new_name, *, xpath=None): + super().__init__(xpath=xpath) self.old_name = old_name self.new_name = new_name - self.extra_xpath = extra_xpath def __call__(self, element, converter): rename_map = {self.old_name: self.new_name} @@ -489,14 +561,16 @@ def __call__(self, element, converter): element.attrib.update({rename_map.get(k, k): v for k, v in attrib_before.items()}) return element - def xpath(self, xpath=None): + @property + def xpath(self): """ Return an XPath expression that matches elements with the old attribute name. - :param str | None xpath: an optional XPath expression to further filter the elements to match. :rtype: str + + :meta private: exclude from online docs """ - return (xpath or "//*") + f"[@{self.old_name}]{self.extra_xpath}" + return (self._xpath or "//*") + f"[@{self.old_name}]" class RegexReplace(ElementOperation): @@ -506,11 +580,15 @@ class RegexReplace(ElementOperation): N.B. no checks are made to ensure the attribute to replace is actually present on the elements. :param str pattern: the regex pattern to match. - :param str repl: the replacement string. + :param str sub: the replacement string. :param str | None attr: the attribute to replace. If not specified, the text of the element is replaced. + :param str | None xpath: see :class:`ElementOperation` ``xpath`` parameter. + + :meta private: exclude from online docs """ - def __init__(self, pattern, sub, attr=None): + def __init__(self, pattern, sub, attr=None, *, xpath=None): + super().__init__(xpath=xpath) self.pattern = pattern self.repl = sub self.attr = attr @@ -525,14 +603,16 @@ def __call__(self, element, converter): element.attrib[attr] = re.sub(self.pattern, self.repl, element.attrib[attr]) return element - def xpath(self, xpath=None): + @property + def xpath(self): """ Return an XPath expression that matches elements with the old attribute name. - :param str | None xpath: an optional XPath expression to further filter the elements to match. :rtype: str + + :meta private: exclude from online docs """ - return regex_xpath(self.pattern, self.attr, xpath) + return regex_xpath(self.pattern, self.attr, self._xpath) class RegexReplaceClass(RegexReplace): @@ -540,32 +620,35 @@ class RegexReplaceClass(RegexReplace): Uses `re.sub` to modify the class. Basically, same as `RegexReplace`, but with `attr="class"`. + + :meta private: exclude from online docs """ - def __init__(self, pattern, sub, attr="class"): - super().__init__(pattern, sub, attr) + def __init__(self, pattern, sub, attr="class", *, xpath=None): + super().__init__(pattern, sub, attr, xpath=xpath) -class EtreeConverter(ABC): +class EtreeConverter: """ Class for converting lxml etree documents, applying a bunch of operations on them. - :param etree.ElementTree tree: the parsed XML or HTML tree to convert. + :param list[ElementOperation | (str, ElementOperation | list[ElementOperation])] conversions: + the operations to apply to the tree. + Each item in the conversions list must either be an :class:`ElementOperation` that can provide its own XPath, + or a tuple of ``(xpath, operation)`` or ``(xpath, operations)`` with the XPath and an operation + or a list of operations to apply to the nodes matching the XPath. :param bool is_html: whether the tree is an HTML document. :para bool is_qweb: whether the tree contains QWeb directives. If this is enabled, XPaths will be auto-transformed to try to also match ``t-att*`` attributes. + + :meta private: exclude from online docs """ - def __init__(self, tree, is_html=False, is_qweb=False): - self.tree = tree + def __init__(self, conversions, *, is_html=False, is_qweb=False): + self.conversions = self.prepare_conversions(conversions, is_qweb) self.is_html = is_html self.is_qweb = is_qweb - @classmethod - @abstractmethod - def get_conversions(cls, *args, **kwargs): - """Return the conversions to apply to the tree.""" - @classmethod @lru_cache(maxsize=32) def _compile_conversions(cls, conversions, is_qweb): @@ -578,6 +661,8 @@ def _compile_conversions(cls, conversions, is_qweb): the conversions to compile. :param bool is_qweb: whether the conversions are for QWeb. :rtype: list[(str, list[ElementOperation])] + + :meta private: exclude from online docs """ def process_spec(spec): @@ -603,7 +688,11 @@ def process_spec(spec): @classmethod def prepare_conversions(cls, conversions, is_qweb): - """Prepare and compile the conversions into a list of ``(xpath, operations)`` tuples, with pre-compiled XPaths.""" + """ + Prepare and compile the conversions into a list of ``(xpath, operations)`` tuples, with pre-compiled XPaths. + + :meta private: exclude from online docs + """ # make sure conversions list and nested lists of operations are converted to tuples for caching conversions = tuple( (spec[0], tuple(spec[1])) @@ -613,38 +702,62 @@ def prepare_conversions(cls, conversions, is_qweb): ) return cls._compile_conversions(conversions, is_qweb) - def convert(self, src_version, dst_version): + def convert(self, tree): """ - Convert the loaded document inplace from the source version to the destination, returning the converted document and the number of conversion operations applied. + Convert an etree document inplace with the prepared conversions. + + Returns the converted document and the number of conversion operations applied. - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. + :param etree.ElementTree tree: the parsed XML or HTML tree to convert. :rtype: etree.ElementTree, int + + :meta private: exclude from online docs """ - conversions = self.get_conversions(src_version, dst_version, is_qweb=self.is_qweb) applied_operations_count = 0 - for xpath, operations in conversions: - for element in xpath(self.tree): + for xpath, operations in self.conversions: + for element in xpath(tree): for operation in operations: if element is None: # previous operations that returned None (i.e. deleted element) raise ValueError("Matched xml element is not available anymore! Check operations.") element = operation(element, self) # noqa: PLW2901 applied_operations_count += 1 - return self.tree, applied_operations_count + return tree, applied_operations_count + + @classmethod + def convert_tree(cls, tree, *converter_args, **converter_kwargs): + """ + Class method for converting an already parsed lxml tree inplace. + + :param etree.ElementTree tree: the lxml tree to convert. + :param dict converter_args: additional positional arguments to pass to the converter. + See :class:`EtreeConverter` for more details. + :param dict converter_kwargs: additional keyword arguments to pass to the converter. + See :class:`EtreeConverter` for more details. + :return: the converted lxml tree. + :rtype: etree.ElementTree + + :meta private: exclude from online docs + """ + tree, ops_count = cls(*converter_args, **converter_kwargs).convert(tree) + return tree @classmethod - def convert_arch(cls, arch, src_version, dst_version, is_html=False, **converter_kwargs): + def convert_arch(cls, arch, *converter_args, **converter_kwargs): """ Class method for converting a string of XML or HTML code. :param str arch: the XML or HTML code to convert. - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :param bool is_html: whether the arch is an HTML document. + :param dict converter_args: additional positional arguments to pass to the converter. + See :class:`EtreeConverter` for more details. :param dict converter_kwargs: additional keyword arguments to pass to the converter. + See :class:`EtreeConverter` for more details. :return: the converted XML or HTML code. :rtype: str + + :meta private: exclude from online docs """ + is_html = converter_kwargs.pop("is_html", False) # is_html is keyword-only, so it wouldn't be in converter_args + stripped_arch = arch.strip() doc_header_match = re.search(r"^<\?xml .+\?>\s*", stripped_arch) doc_header = doc_header_match.group(0) if doc_header_match else "" @@ -652,7 +765,7 @@ def convert_arch(cls, arch, src_version, dst_version, is_html=False, **converter tree = etree.fromstring(f"{stripped_arch}", parser=html_utf8_parser if is_html else None) - tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) + tree, ops_count = cls(*converter_args, is_html=is_html, **converter_kwargs).convert(tree) if not ops_count: return arch @@ -663,29 +776,33 @@ def convert_arch(cls, arch, src_version, dst_version, is_html=False, **converter ) @classmethod - def convert_file(cls, path, src_version, dst_version, is_html=None, **converter_kwargs): + def convert_file(cls, path, *converter_args, **converter_kwargs): """ Class method for converting an XML or HTML file inplace. :param str path: the path to the XML or HTML file to convert. - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :param bool is_html: whether the file is an HTML document. - If not set, will be detected from the file extension. + :param dict converter_args: additional positional arguments to pass to the converter. + See :class:`EtreeConverter` for more details. :param dict converter_kwargs: additional keyword arguments to pass to the converter. + See :class:`EtreeConverter` for more details. :rtype: None + + :meta private: exclude from online docs """ + is_html = converter_kwargs.pop("is_html", None) if is_html is None: is_html = os.path.splitext(path)[1].startswith("htm") tree = etree.parse(path, parser=html_utf8_parser if is_html else None) - tree, ops_count = cls(tree, is_html, **converter_kwargs).convert(src_version, dst_version) + tree, ops_count = cls(*converter_args, is_html=is_html, **converter_kwargs).convert(tree) if not ops_count: - logging.info("No conversion operations applied, skipping file %s", path) + logging.info("No conversion operations applied, skipping file: %s", path) return tree.write(path, encoding="utf-8", method="html" if is_html else None, xml_declaration=not is_html) + # -- Operations helper methods, useful where operations need some converter-specific info or logic (e.g. is_html) -- + def element_factory(self, *args, **kwargs): """ Create new elements using the correct document type. @@ -695,6 +812,8 @@ def element_factory(self, *args, **kwargs): :param args: positional arguments to pass to the etree.XML or etree.HTML function. :param kwargs: keyword arguments to pass to the etree.XML or etree.HTML function. :return: the created element. + + :meta private: exclude from online docs """ return etree.HTML(*args, **kwargs) if self.is_html else etree.XML(*args, **kwargs) @@ -710,6 +829,8 @@ def build_element(self, tag, classes=None, contents=None, **attributes): :param dict[str, str] attributes: attributes to set on the new element, provided as keyword arguments. :return: the created element. :rtype: etree.ElementBase + + :meta private: exclude from online docs """ element = self.element_factory(f"<{tag}>{contents or ''}") for name, value in attributes.items(): @@ -744,6 +865,8 @@ def copy_element( Will be str merged with the attributes of the source element, overriding the latter. :return: the new copied element. :rtype: etree.ElementBase + + :meta private: exclude from online docs """ tag = tag or element.tag contents = innerxml(element, is_html=self.is_html) if copy_contents else None @@ -754,5 +877,9 @@ def copy_element( return new_element def adapt_xpath(self, xpath): - """Adapts an xpath to match qweb ``t-att(f)-*`` attributes, if ``is_qweb`` is True.""" + """ + Adapts an XPath to match qweb ``t-att(f)-*`` attributes, if ``is_qweb`` is True. + + :meta private: exclude from online docs + """ return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath From 3e66cea552b4b1791a967a643b215c2ed495194a Mon Sep 17 00:00:00 2001 From: abk16 Date: Thu, 5 Jan 2023 17:09:11 +0100 Subject: [PATCH 3/6] [REF] util.views_convert: move views/html convert helpers from BS5 mig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move xpath keywords / where clause functions from base/16.0 pre-90-convert-bootstrap5.py script into util.views and add shortcut methods for them in converter class. Also refactored to make them usable with arbitrary converters. Revise converter API, especially simplified classmethods / remove oneshot aliases (it's easier to just instantiate the converter first). Refactor parse/unparse string arch code into separate helper functions, and add a `ArchEditor` context manager to parse-edit-unparse an arch within a context (similar to edit_views). Make sure EtreeConverter is pickle-able for multiprocessing: concurrent.futures ProcessPoolExecutor uses multiprocessing to spawn its subprocess workers. That requires memory data from the main process to be transferred by serializing and deserializing, using the pickle library. To make this work: - the converter function must not be an anonymous "factory" function,   e.g. one generated as a nested function, so unaccessible from the   outer scope. - instance values in the serialized objects must also be pickleable. Therefore the following changes have been done: - the converter function is now a method of the converter class - to keep @lru_cache decorator on the method, the entire class is   made hashable, this is done by computing a hash of the provided   arguments to `__init__`, making some attributes "private", and   adding read-only properties to access them. - to make the instances pickable, the compiled coversions are first   removed from the pickle-able `__dict__`, because `lxml.XPath`   objects are not python-native, and they're re-compiled when the   instances are de-serialized. Made docstrings for new functions private (excluded from online docs). upgrade PR: odoo/upgrade#5431 --- src/util/convert_bootstrap.py | 12 +- src/util/views_convert.py | 468 +++++++++++++++++++++++++++++----- 2 files changed, 405 insertions(+), 75 deletions(-) diff --git a/src/util/convert_bootstrap.py b/src/util/convert_bootstrap.py index 45c6f11c..f84dc875 100644 --- a/src/util/convert_bootstrap.py +++ b/src/util/convert_bootstrap.py @@ -433,6 +433,8 @@ class BootstrapConverter(EtreeConverter): } def __init__(self, src_version, dst_version, *, is_html=False, is_qweb=False): + self.src_version = src_version + self.dst_version = dst_version conversions = self._get_conversions(src_version, dst_version) super().__init__(conversions, is_html=is_html, is_qweb=is_qweb) @@ -476,14 +478,6 @@ def _get_conversions(cls, src_version, dst_version): return result -convert_tree = BootstrapConverter.convert_tree -convert_arch = BootstrapConverter.convert_arch -convert_file = BootstrapConverter.convert_file - -bs3to4_converter = BootstrapConverter("3.0", "4.0") -bs4to5_converter = BootstrapConverter("4.0", "5.0") - - # TODO abt: remove this / usages -> replace with refactored converter classes class BootstrapHTMLConverter: def __init__(self, src, dst): @@ -493,5 +487,5 @@ def __init__(self, src, dst): def __call__(self, content): if not content: return False, content - converted_content = convert_arch(content, self.src, self.dst, is_html=True, is_qweb=True) + converted_content = BootstrapConverter.convert_arch(content, self.src, self.dst, is_html=True, is_qweb=True) return content != converted_content, converted_content diff --git a/src/util/views_convert.py b/src/util/views_convert.py index c839c2aa..d705c385 100644 --- a/src/util/views_convert.py +++ b/src/util/views_convert.py @@ -9,6 +9,13 @@ from lxml import etree +from odoo import modules + +from . import snippets +from .misc import log_progress +from .pg import get_value_or_en_translation +from .records import edit_view + _logger = logging.getLogger(__name__) @@ -72,6 +79,80 @@ def _xpath_regex(context, item, pattern): html_utf8_parser = etree.HTMLParser(encoding="utf-8") +def parse_arch(arch, is_html=False): + """ + Parse a string of XML or HTML into a lxml :class:`etree.ElementTree`. + + :param str arch: the XML or HTML code to convert. + :param bool is_html: whether the code is HTML or XML. + :return: the parsed etree and the original document header (if any, removed from the tree). + :rtype: (etree.ElementTree, str) + + :meta private: exclude from online docs + """ + stripped_arch = arch.strip() + doc_header_match = re.search(r"^<\?xml .+\?>\s*", stripped_arch) + doc_header = doc_header_match.group(0) if doc_header_match else "" + stripped_arch = stripped_arch[doc_header_match.end() :] if doc_header_match else stripped_arch + + return etree.fromstring(f"{stripped_arch}", parser=html_utf8_parser if is_html else None), doc_header + + +def unparse_arch(tree, doc_header="", is_html=False): + """ + Convert an etree into a string of XML or HTML. + + :param etree.ElementTree tree: the etree to convert. + :param str doc_header: the document header (if any). + :param bool is_html: whether the code is HTML or XML. + :return: the XML or HTML code. + :rtype: str + + :meta private: exclude from online docs + """ + wrap_node = tree.xpath("//wrap")[0] + return doc_header + "\n".join( + etree.tostring(child, encoding="unicode", with_tail=True, method="html" if is_html else None) + for child in wrap_node + ) + + +class ArchEditor: + """ + Context manager to edit an XML or HTML string. + + It will parse an XML or HTML string into an etree, and return it to its original + string representation when exiting the context. + + The etree is available as the ``tree`` attribute of the context manager. + The arch is available as the ``arch`` attribute of the context manager, + and is updated when exiting the context. + + :param str arch: the XML or HTML code to convert. + :param bool is_html: whether the code is HTML or XML. + + :meta private: exclude from online docs + """ + + def __init__(self, arch, is_html=False): + self.arch = arch + self.is_html = is_html + self.doc_header = "" + self.tree = None + + def __enter__(self): + self.tree, self.doc_header = parse_arch(self.arch, is_html=self.is_html) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type: + return + self.arch = unparse_arch(self.tree, self.doc_header, is_html=self.is_html) + + def __str__(self): + return self.arch + + def innerxml(element, is_html=False): """ Return the inner XML of an element as a string. @@ -333,6 +414,118 @@ def adapt_xpath_for_qweb(xpath): return re.sub(r"\[@(?\s*", stripped_arch) - doc_header = doc_header_match.group(0) if doc_header_match else "" - stripped_arch = stripped_arch[doc_header_match.end() :] if doc_header_match else stripped_arch - - tree = etree.fromstring(f"{stripped_arch}", parser=html_utf8_parser if is_html else None) - - tree, ops_count = cls(*converter_args, is_html=is_html, **converter_kwargs).convert(tree) - if not ops_count: - return arch - - wrap_node = tree.xpath("//wrap")[0] - return doc_header + "\n".join( - etree.tostring(child, encoding="unicode", with_tail=True, method="html" if is_html else None) - for child in wrap_node - ) + return self.convert_callback(arch)[1] - @classmethod - def convert_file(cls, path, *converter_args, **converter_kwargs): + def convert_file(self, path): """ - Class method for converting an XML or HTML file inplace. + Convert an XML or HTML file inplace. :param str path: the path to the XML or HTML file to convert. - :param dict converter_args: additional positional arguments to pass to the converter. - See :class:`EtreeConverter` for more details. - :param dict converter_kwargs: additional keyword arguments to pass to the converter. - See :class:`EtreeConverter` for more details. :rtype: None :meta private: exclude from online docs """ - is_html = converter_kwargs.pop("is_html", None) - if is_html is None: - is_html = os.path.splitext(path)[1].startswith("htm") - tree = etree.parse(path, parser=html_utf8_parser if is_html else None) + file_is_html = os.path.splitext(path)[1].startswith("htm") + if self.is_html != file_is_html: + raise ValueError(f"File {path!r} is not a {'HTML' if self.is_html else 'XML'} file!") + + tree = etree.parse(path, parser=html_utf8_parser if self.is_html else None) - tree, ops_count = cls(*converter_args, is_html=is_html, **converter_kwargs).convert(tree) + tree, ops_count = self.convert_tree(tree) if not ops_count: logging.info("No conversion operations applied, skipping file: %s", path) return - tree.write(path, encoding="utf-8", method="html" if is_html else None, xml_declaration=not is_html) + tree.write(path, encoding="utf-8", method="html" if self.is_html else None, xml_declaration=not self.is_html) # -- Operations helper methods, useful where operations need some converter-specific info or logic (e.g. is_html) -- @@ -883,3 +1110,112 @@ def adapt_xpath(self, xpath): :meta private: exclude from online docs """ return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath + + +def convert_views(cr, views_ids, converter): + """ + Convert the specified views xml arch using the provided converter. + + :param psycopg2.cursor cr: the database cursor. + :param typing.Collection[int] views_ids: the ids of the views to convert. + :param EtreeConverter converter: the converter to use. + :rtype: None + + :meta private: exclude from online docs + """ + if converter.is_html: + raise TypeError(f"Cannot convert xml views with provided ``is_html`` converter {converter!r}") + + _logger.info("Converting %s views/templates using %s", len(views_ids), repr(converter)) + for view_id in views_ids: + with edit_view(cr, view_id=view_id, active=None) as tree: + converter.convert_tree(tree) + # TODO abt: maybe notify in the log or report that custom views with noupdate=False were converted? + + +def convert_qweb_views(cr, converter): + """ + Convert QWeb views / templates using the provided converter. + + :param psycopg2.cursor cr: the database cursor. + :param EtreeConverter converter: the converter to use. + :rtype: None + + :meta private: exclude from online docs + """ + if not converter.is_qweb: + raise TypeError("Converter for xml views must be ``is_qweb``, got %s", repr(converter)) + + # views to convert must have `website_id` set and not come from standard modules + standard_modules = set(modules.get_modules()) - {"studio_customization", "__export__", "__cloc_exclude__"} + converter_where = converter.build_where_clause(cr, "v.arch_db") + + # Search for custom/cow'ed views (they have no external ID)... but also + # search for views with external ID that have a related COW'ed view. Indeed, + # when updating a generic view after this script, the archs are compared to + # know if the related COW'ed views must be updated too or not: if we only + # convert COW'ed views they won't get the generic view update as they will be + # judged different from them (user customization) because of the changes + # that were made. + # E.g. + # - In 15.0, install website_sale + # - Enable eCommerce categories: a COW'ed view is created to enable the + # feature (it leaves the generic disabled and creates an exact copy but + # enabled) + # - Migrate to 16.0: you expect your enabled COW'ed view to get the new 16.0 + # version of eCommerce categories... but if the COW'ed view was converted + # while the generic was not, they won't be considered the same + # anymore and only the generic view will get the 16.0 update. + cr.execute( + f""" + WITH keys AS ( + SELECT key + FROM ir_ui_view + GROUP BY key + HAVING COUNT(*) > 1 + ) + SELECT v.id + FROM ir_ui_view v + LEFT JOIN ir_model_data imd + ON imd.model = 'ir.ui.view' + AND imd.module IN %s + AND imd.res_id = v.id + LEFT JOIN keys + ON v.key = keys.key + WHERE v.type = 'qweb' + AND ({converter_where}) + AND ( + imd.id IS NULL + OR ( + keys.key IS NOT NULL + AND imd.noupdate = FALSE + ) + ) + """, + [tuple(standard_modules)], + ) + views_ids = [view_id for (view_id,) in cr.fetchall()] + if views_ids: + convert_views(cr, views_ids, converter) + + +def convert_html_fields(cr, converter): + """ + Convert all html fields data in the database using the provided converter. + + :param psycopg2.cursor cr: the database cursor. + :param EtreeConverter converter: the converter to use. + :rtype: None + + :meta private: exclude from online docs + """ + _logger.info("Converting html fields data using %s", repr(converter)) + + html_fields = list(snippets.html_fields(cr)) + for table, columns in log_progress(html_fields, _logger, "tables", log_hundred_percent=True): + if table not in ("mail_message", "mail_activity"): + extra_where = " OR ".join( + f"({converter.build_where_clause(cr, get_value_or_en_translation(cr, table, column))})" + for column in columns + ) + snippets.convert_html_columns(cr, table, columns, converter.convert_callback, extra_where=extra_where) From 596861b1eb4913f982ac7e9777627a567c66e200 Mon Sep 17 00:00:00 2001 From: abk16 Date: Thu, 12 Jan 2023 12:29:44 +0100 Subject: [PATCH 4/6] [REF] util.views: create new package, move views-related code Changes: - move `util.views_convert` -> `util.views.convert` - move records-related views helpers into `util.views.records` - move `util.convert_bootstrap` -> `util.views.bootstrap` - move some helpers from `util.records` -> `util.views.records` - add version checks for views/html conversion helpers --- src/util/convert_bootstrap.py | 495 +----------------- src/util/domains.py | 2 +- src/util/models.py | 3 +- src/util/modules.py | 3 +- src/util/records.py | 247 +-------- src/util/views/__init__.py | 0 src/util/views/bootstrap.py | 491 +++++++++++++++++ .../{views_convert.py => views/convert.py} | 216 ++++---- src/util/views/records.py | 276 ++++++++++ tools/compile23.py | 4 +- 10 files changed, 898 insertions(+), 839 deletions(-) create mode 100644 src/util/views/__init__.py create mode 100644 src/util/views/bootstrap.py rename src/util/{views_convert.py => views/convert.py} (88%) create mode 100644 src/util/views/records.py diff --git a/src/util/convert_bootstrap.py b/src/util/convert_bootstrap.py index f84dc875..424ef7fb 100644 --- a/src/util/convert_bootstrap.py +++ b/src/util/convert_bootstrap.py @@ -1,491 +1,10 @@ -"""Convert an XML/HTML document Bootstrap code from an older version to a newer one.""" +import warnings -import logging -from functools import lru_cache +from .views.bootstrap import * # noqa: F403 -from lxml import etree - -try: - from packaging.version import Version -except ImportError: - from distutils.version import StrictVersion as Version # N.B. deprecated, will be removed in py3.12 - -from .views_convert import ( - ALL, - BE, - BS, - CSS, - AddClasses, - ElementOperation, - EtreeConverter, - PullUp, - RegexReplaceClass, - RemoveClasses, - RemoveElement, - RenameAttribute, - ReplaceClasses, - edit_element_classes, - get_classes, - regex_xpath, +warnings.warn( + "`util.convert_bootstrap` module has been deprecated in favor of `util.views.bootstrap`. " + "Consider adjusting your imports.", + category=DeprecationWarning, + stacklevel=1, ) - -_logger = logging.getLogger(__name__) - - -class BS3to4ConvertBlockquote(ElementOperation): - """ - Convert a BS3 ``
`` element to a BS4 ``
`` element with the ``blockquote`` class. - - :meta private: exclude from online docs - """ - - def __call__(self, element, converter): - blockquote = converter.copy_element(element, tag="div", add_classes="blockquote", copy_attrs=False) - element.addnext(blockquote) - element.getparent().remove(element) - return blockquote - - -# TODO abt: merge MakeCard and ConvertCard into one operation class -class BS3to4MakeCard(ElementOperation): - """ - Pre-processe a BS3 panel, thumbnail, or well element to be converted to a BS4 card. - - Card components conversion is then handled by the ``ConvertCard`` operation class. - - :meta private: exclude from online docs - """ - - def __call__(self, element, converter): - card = converter.element_factory("
") - card_body = converter.copy_element( - element, tag="div", add_classes="card-body", remove_classes=ALL, copy_attrs=False - ) - card.append(card_body) - element.addnext(card) - element.getparent().remove(element) - return card - - -# TODO abt: refactor code -class BS3to4ConvertCard(ElementOperation): - """ - Fully convert a BS3 panel, thumbnail, or well element and their contents to a BS4 card. - - :meta private: exclude from online docs - """ - - POST_CONVERSIONS = { - "title": ["card-title"], - "description": ["card-description"], - "category": ["card-category"], - "panel-danger": ["card", "bg-danger", "text-white"], - "panel-warning": ["card", "bg-warning"], - "panel-info": ["card", "bg-info", "text-white"], - "panel-success": ["card", "bg-success", "text-white"], - "panel-primary": ["card", "bg-primary", "text-white"], - "panel-footer": ["card-footer"], - "panel-body": ["card-body"], - "panel-title": ["card-title"], - "panel-heading": ["card-header"], - "panel-default": [], - "panel": ["card"], - } - - def _convert_child(self, child, old_card, new_card, converter): - old_card_classes = get_classes(old_card) - - classes = get_classes(child) - - if "header" in classes or ("image" in classes and len(child)): - add_classes = "card-header" - remove_classes = ["header", "image"] - elif "content" in classes: - add_classes = "card-img-overlay" if "card-background" in old_card_classes else "card-body" - remove_classes = "content" - elif {"card-footer", "footer", "text-center"} & set(classes): - add_classes = "card-footer" - remove_classes = "footer" - else: - new_card.append(child) - return - - new_child = converter.copy_element( - child, "div", add_classes=add_classes, remove_classes=remove_classes, copy_attrs=True - ) - - if "image" in classes: - [img_el] = new_child.xpath("./img")[:1] or [None] - if img_el is not None and "src" in img_el: - new_child.attrib["style"] = ( - f'background-image: url("{img_el.attrib["src"]}"); ' - "background-position: center center; " - "background-size: cover;" - ) - new_child.remove(img_el) - - new_card.append(new_child) - - if "content" in classes: # TODO abt: consider skipping for .card-background - [footer] = new_child.xpath(converter.adapt_xpath("./*[hasclass('footer')]"))[:1] or [None] - if footer is not None: - self._convert_child(footer, old_card, new_card, converter) - new_child.remove(footer) - - def _postprocess(self, new_card, converter): - for old_class, new_classes in self.POST_CONVERSIONS.items(): - for element in new_card.xpath(converter.adapt_xpath(f"(.|.//*)[hasclass('{old_class}')]")): - edit_element_classes(element, add=new_classes, remove=old_class) - - def __call__(self, element, converter): - classes = get_classes(element) - new_card = converter.copy_element(element, tag="div", copy_attrs=True, copy_contents=False) - wrapper = new_card - if "card-horizontal" in classes: - wrapper = etree.SubElement(new_card, "div", {"class": "row"}) - - for child in element: - self._convert_child(child, element, wrapper, converter) - - self._postprocess(new_card, converter) - element.addnext(new_card) - element.getparent().remove(element) - return new_card - - -class BS4to5ConvertCloseButton(ElementOperation): - """ - Convert BS4 ``button.close`` elements to BS5 ``button.btn-close``. - - Also fixes the ``data-dismiss`` attribute to ``data-bs-dismiss``, and removes any inner contents. - - :meta private: exclude from online docs - """ - - def __call__(self, element, converter): - new_btn = converter.copy_element(element, remove_classes="close", add_classes="btn-close", copy_contents=False) - - if "data-dismiss" in element.attrib: - new_btn.attrib["data-bs-dismiss"] = element.attrib["data-dismiss"] - del new_btn.attrib["data-dismiss"] - - element.addnext(new_btn) - element.getparent().remove(element) - - return new_btn - - -class BS4to5ConvertCardDeck(ElementOperation): - """ - Convert BS4 ``.card-deck`` elements to grid components (``.row``, ``.col``, etc.). - - :meta private: exclude from online docs - """ - - def __call__(self, element, converter): - cards = element.xpath(converter.adapt_xpath("./*[hasclass('card')]")) - - cols_class = f"row-cols-{len(cards)}" if len(cards) in range(1, 7) else "row-cols-auto" - edit_element_classes(element, add=["row", cols_class], remove="card-deck", is_qweb=converter.is_qweb) - - for card in cards: - new_col = converter.build_element("div", classes=["col"]) - card.addprevious(new_col) - new_col.append(card) - - return element - - -class BS4to5ConvertFormInline(ElementOperation): - """ - Convert BS4 ``.form-inline`` elements to grid components (``.row``, ``.col``, etc.). - - :meta private: exclude from online docs - """ - - def __call__(self, element, converter): - edit_element_classes(element, add="row row-cols-lg-auto", remove="form-inline", is_qweb=converter.is_qweb) - - children_selector = converter.adapt_xpath( - CSS(".form-control,.form-group,.form-check,.input-group,.custom-select,button", prefix="./") - ) - indexed_children = sorted([(element.index(c), c) for c in element.xpath(children_selector)], key=lambda x: x[0]) - - nest_groups = [] - last_idx = -1 - for idx, child in indexed_children: - nest_start, nest_end = idx, idx - labels = [label for label in child.xpath("preceding-sibling::label") if element.index(label) > last_idx] - labels = [ - label - for label in labels - if "for" in label.attrib and child.xpath(f"descendant-or-self::*[@id='{label.attrib['for']}']") - ] or labels[-1:] - if labels: - first_label = labels[0] - assert last_idx < element.index(first_label) < idx, "label must be between last group and current" - nest_start = element.index(first_label) - - assert nest_start <= nest_end, f"expected start {nest_start} to be <= end {nest_end}" - nest_groups.append(element[nest_start : nest_end + 1]) - last_idx = nest_end - - for els in nest_groups: - wrapper = converter.build_element("div", classes=["col-12"]) - els[0].addprevious(wrapper) - for el in els: - wrapper.append(el) - assert el not in element, f"expected {el!r} to be removed from {element!r}" - - return element - - -class BootstrapConverter(EtreeConverter): - """ - Class for converting XML or HTML Bootstrap code across versions. - - :param str src_version: the source Bootstrap version to convert from. - :param str dst_version: the destination Bootstrap version to convert to. - :param bool is_html: whether the tree is an HTML document. - :param bool is_qweb: whether the tree contains QWeb directives. See :class:`EtreeConverter` for more info. - - :meta private: exclude from online docs - """ - - MIN_VERSION = "3.0" - """Minimum supported Bootstrap version.""" - - # Conversions definitions by destination version. - # It's a dictionary of version strings to a conversions list. - # Each item in the conversions list must either be an :class:`ElementOperation`, - # that can provide its own XPath, or a tuple of ``(xpath, operation(s))`` with the XPath - # and a single operation or a list of operations to apply to the nodes matching the XPath. - CONVERSIONS = { - "4.0": [ - # inputs - (CSS(".form-group .control-label"), ReplaceClasses("control-label", "form-control-label")), - (CSS(".form-group .text-help"), ReplaceClasses("text-help", "form-control-feedback")), - (CSS(".control-group .help-block"), ReplaceClasses("help-block", "form-text")), - ReplaceClasses("form-group-sm", "form-control-sm"), - ReplaceClasses("form-group-lg", "form-control-lg"), - (CSS(".form-control .input-sm"), ReplaceClasses("input-sm", "form-control-sm")), - (CSS(".form-control .input-lg"), ReplaceClasses("input-lg", "form-control-lg")), - # hide - ReplaceClasses("hidden-xs", "d-none"), - ReplaceClasses("hidden-sm", "d-sm-none"), - ReplaceClasses("hidden-md", "d-md-none"), - ReplaceClasses("hidden-lg", "d-lg-none"), - ReplaceClasses("visible-xs", "d-block d-sm-none"), - ReplaceClasses("visible-sm", "d-block d-md-none"), - ReplaceClasses("visible-md", "d-block d-lg-none"), - ReplaceClasses("visible-lg", "d-block d-xl-none"), - # image - ReplaceClasses("img-rounded", "rounded"), - ReplaceClasses("img-circle", "rounded-circle"), - ReplaceClasses("img-responsive", ("d-block", "img-fluid")), - # buttons - ReplaceClasses("btn-default", "btn-secondary"), - ReplaceClasses("btn-xs", "btn-sm"), - (CSS(".btn-group.btn-group-xs"), ReplaceClasses("btn-group-xs", "btn-group-sm")), - (CSS(".dropdown .divider"), ReplaceClasses("divider", "dropdown-divider")), - ReplaceClasses("badge", "badge badge-pill"), - ReplaceClasses("label", "badge"), - RegexReplaceClass(rf"{BS}label-(default|primary|success|info|warning|danger){BE}", r"badge-\1"), - (CSS(".breadcrumb > li"), ReplaceClasses("breadcrumb", "breadcrumb-item")), - # li - (CSS(".list-inline > li"), AddClasses("list-inline-item")), - # pagination - (CSS(".pagination > li"), AddClasses("page-item")), - (CSS(".pagination > li > a"), AddClasses("page-link")), - # carousel - (CSS(".carousel .carousel-inner > .item"), ReplaceClasses("item", "carousel-item")), - # pull - ReplaceClasses("pull-right", "float-right"), - ReplaceClasses("pull-left", "float-left"), - ReplaceClasses("center-block", "mx-auto"), - # well - (CSS(".well"), BS3to4MakeCard()), - (CSS(".thumbnail"), BS3to4MakeCard()), - # blockquote - (CSS("blockquote"), BS3to4ConvertBlockquote()), - (CSS(".blockquote.blockquote-reverse"), ReplaceClasses("blockquote-reverse", "text-right")), - # dropdown - (CSS(".dropdown-menu > li > a"), AddClasses("dropdown-item")), - (CSS(".dropdown-menu > li"), PullUp()), - # in - ReplaceClasses("in", "show"), - # table - (CSS("tr.active, td.active"), ReplaceClasses("active", "table-active")), - (CSS("tr.success, td.success"), ReplaceClasses("success", "table-success")), - (CSS("tr.info, td.info"), ReplaceClasses("info", "table-info")), - (CSS("tr.warning, td.warning"), ReplaceClasses("warning", "table-warning")), - (CSS("tr.danger, td.danger"), ReplaceClasses("danger", "table-danger")), - (CSS("table.table-condesed"), ReplaceClasses("table-condesed", "table-sm")), - # navbar - (CSS(".nav.navbar > li > a"), AddClasses("nav-link")), - (CSS(".nav.navbar > li"), AddClasses("nav-intem")), - ReplaceClasses("navbar-btn", "nav-item"), - (CSS(".navbar-nav"), ReplaceClasses("navbar-right nav", "ml-auto")), - ReplaceClasses("navbar-toggler-right", "ml-auto"), - (CSS(".navbar-nav > li > a"), AddClasses("nav-link")), - (CSS(".navbar-nav > li"), AddClasses("nav-item")), - (CSS(".navbar-nav > a"), AddClasses("navbar-brand")), - ReplaceClasses("navbar-fixed-top", "fixed-top"), - ReplaceClasses("navbar-toggle", "navbar-toggler"), - ReplaceClasses("nav-stacked", "flex-column"), - (CSS("nav.navbar"), AddClasses("navbar-expand-lg")), - (CSS("button.navbar-toggle"), ReplaceClasses("navbar-toggle", "navbar-expand-md")), - # card - (CSS(".panel"), BS3to4ConvertCard()), - (CSS(".card"), BS3to4ConvertCard()), - # grid - RegexReplaceClass(rf"{BS}col((?:-\w{{2}})?)-offset-(\d{{1,2}}){BE}", r"offset\1-\2"), - ], - "5.0": [ - # links - RegexReplaceClass(rf"{BS}text-(?!o-)", "link-", xpath=CSS("a")), - (CSS(".nav-item.active > .nav-link"), AddClasses("active")), - (CSS(".nav-link.active") + CSS(".nav-item.active", prefix="/parent::"), RemoveClasses("active")), - # badges - ReplaceClasses("badge-pill", "rounded-pill"), - RegexReplaceClass(rf"{BS}badge-", r"text-bg-"), - # buttons - ("//*[hasclass('btn-block')]/parent::div", AddClasses("d-grid gap-2")), - ("//*[hasclass('btn-block')]/parent::p[count(./*)=1]", AddClasses("d-grid gap-2")), - RemoveClasses("btn-block"), - (CSS("button.close"), BS4to5ConvertCloseButton()), - # card - # TODO abt: .card-columns (unused in odoo) - (CSS(".card-deck"), BS4to5ConvertCardDeck()), - # jumbotron - ReplaceClasses("jumbotron", "container-fluid py-5"), - # new data-bs- attributes - RenameAttribute("data-display", "data-bs-display", xpath="//*[not(@data-snippet='s_countdown')]"), - *[ - RenameAttribute(f"data-{attr}", f"data-bs-{attr}") - for attr in ( - "animation attributes autohide backdrop body container content delay dismiss focus" - " interval margin-right no-jquery offset original-title padding-right parent placement" - " ride sanitize show slide slide-to spy target toggle touch trigger whatever" - ).split(" ") - ], - # popover - (CSS(".popover .arrow"), ReplaceClasses("arrow", "popover-arrow")), - # form - ReplaceClasses("form-row", "row"), - ("//*[hasclass('form-group')]/parent::form", AddClasses("row")), - ReplaceClasses("form-group", "col-12 py-2"), - (CSS(".form-inline"), BS4to5ConvertFormInline()), - ReplaceClasses("custom-checkbox", "form-check"), - RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), - RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), - RegexReplaceClass(rf"{BS}custom-(check|select|range){BE}", r"form-\1"), - (CSS(".custom-switch"), ReplaceClasses("custom-switch", "form-check form-switch")), - ReplaceClasses("custom-radio", "form-check"), - RemoveClasses("custom-control"), - (CSS(".custom-file"), PullUp()), - RegexReplaceClass(rf"{BS}custom-file-", r"form-file-"), - RegexReplaceClass(rf"{BS}form-file(?:-input)?{BE}", r"form-control"), - (CSS("label.form-file-label"), RemoveElement()), - (regex_xpath(rf"{BS}input-group-(prepend|append){BE}", "class"), PullUp()), - ("//label[not(hasclass('form-check-label'))]", AddClasses("form-label")), - ReplaceClasses("form-control-file", "form-control"), - ReplaceClasses("form-control-range", "form-range"), - # TODO abt: .form-text no loger sets display, add some class? - # table - RegexReplaceClass(rf"{BS}thead-(light|dark){BE}", r"table-\1"), - # grid - RegexReplaceClass(rf"{BS}col-((?:\w{{2}}-)?)offset-(\d{{1,2}}){BE}", r"offset-\1\2"), # from BS4 - # gutters - ReplaceClasses("no-gutters", "g-0"), - # logical properties - RegexReplaceClass(rf"{BS}left-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"start-\1"), - RegexReplaceClass(rf"{BS}right-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"end-\1"), - RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-left{BE}", r"\1-start"), - RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-right{BE}", r"\1-end"), - RegexReplaceClass(rf"{BS}rounded-sm(-(?:start|end|top|bottom))?", r"rounded\1-1"), - RegexReplaceClass(rf"{BS}rounded-lg(-(?:start|end|top|bottom))?", r"rounded\1-3"), - RegexReplaceClass(rf"{BS}([mp])l-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1s-\2"), - RegexReplaceClass(rf"{BS}([mp])r-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1e-\2"), - ReplaceClasses("dropdown-menu-left", "dropdown-menu-start"), - ReplaceClasses("dropdown-menu-right", "dropdown-menu-end"), - ReplaceClasses("dropleft", "dropstart"), - ReplaceClasses("dropright", "dropend"), - # tooltips - ( - "//*[hasclass('tooltip') or @role='tooltip']//*[hasclass('arrow')]", - ReplaceClasses("arrow", "tooltip-arrow"), - ), - # utilities - ReplaceClasses("text-monospace", "font-monospace"), - RegexReplaceClass(rf"{BS}font-weight-", r"fw-"), - RegexReplaceClass(rf"{BS}font-style-", r"fst-"), - ReplaceClasses("font-italic", "fst-italic"), - # helpers - RegexReplaceClass(rf"{BS}embed-responsive-(\d+)by(\d+)", r"ratio-\1x\2"), - RegexReplaceClass(rf"{BS}ratio-(\d+)by(\d+)", r"ratio-\1x\2"), - RegexReplaceClass(rf"{BS}embed-responsive(?!-)", r"ratio"), - RegexReplaceClass(rf"{BS}sr-only(-focusable)?", r"visually-hidden\1"), - # media - ReplaceClasses("media-body", "flex-grow-1"), - ReplaceClasses("media", "d-flex"), - ], - } - - def __init__(self, src_version, dst_version, *, is_html=False, is_qweb=False): - self.src_version = src_version - self.dst_version = dst_version - conversions = self._get_conversions(src_version, dst_version) - super().__init__(conversions, is_html=is_html, is_qweb=is_qweb) - - @classmethod - def _get_sorted_conversions(cls): - """ - Return the conversions dict sorted by version, from oldest to newest. - - :meta private: exclude from online docs - """ - return sorted(cls.CONVERSIONS.items(), key=lambda kv: Version(kv[0])) - - @classmethod - @lru_cache(maxsize=8) - def _get_conversions(cls, src_version, dst_version): - """ - Return the list of conversions to convert Bootstrap from ``src_version`` to ``dst_version``. - - :param str src_version: the source Bootstrap version. - :param str dst_version: the destination Bootstrap version. - :rtype: list[ElementOperation | (str, ElementOperation | list[ElementOperation])] - - :meta private: exclude from online docs - """ - if Version(dst_version) < Version(src_version): - raise NotImplementedError("Downgrading Bootstrap versions is not supported.") - if Version(src_version) < Version(cls.MIN_VERSION): - raise NotImplementedError(f"Conversion from Bootstrap version {src_version} is not supported") - - result = [] - for version, conversions in BootstrapConverter._get_sorted_conversions(): - if Version(src_version) < Version(version) <= Version(dst_version): - result.extend(conversions) - - if not result: - if Version(src_version) == Version(dst_version): - _logger.info("Source and destination versions are the same, no conversion needed.") - else: - raise NotImplementedError(f"Conversion from {src_version} to {dst_version} is not supported") - - return result - - -# TODO abt: remove this / usages -> replace with refactored converter classes -class BootstrapHTMLConverter: - def __init__(self, src, dst): - self.src = src - self.dst = dst - - def __call__(self, content): - if not content: - return False, content - converted_content = BootstrapConverter.convert_arch(content, self.src, self.dst, is_html=True, is_qweb=True) - return content != converted_content, converted_content diff --git a/src/util/domains.py b/src/util/domains.py index c7fa004f..72880962 100644 --- a/src/util/domains.py +++ b/src/util/domains.py @@ -37,7 +37,7 @@ from .inherit import for_each_inherit from .misc import SelfPrintEvalContext from .pg import column_exists, get_value_or_en_translation, table_exists -from .records import edit_view +from .views.records import edit_view # python3 shims try: diff --git a/src/util/models.py b/src/util/models.py index b593beff..2d47c4c0 100644 --- a/src/util/models.py +++ b/src/util/models.py @@ -31,8 +31,9 @@ # avoid namespace clash from .pg import rename_table as pg_rename_table -from .records import _rm_refs, remove_records, remove_view, replace_record_references_batch +from .records import _rm_refs, remove_records, replace_record_references_batch from .report import add_to_migration_reports +from .views.records import remove_view _logger = logging.getLogger(__name__) diff --git a/src/util/modules.py b/src/util/modules.py index 32c8ddff..248002ad 100644 --- a/src/util/modules.py +++ b/src/util/modules.py @@ -48,7 +48,8 @@ from .models import delete_model from .orm import env, flush from .pg import column_exists, table_exists, target_of -from .records import ref, remove_menus, remove_records, remove_view, replace_record_references_batch +from .records import ref, remove_menus, remove_records, replace_record_references_batch +from .views.records import remove_view INSTALLED_MODULE_STATES = ("installed", "to install", "to upgrade") NO_AUTOINSTALL = str2bool(os.getenv("UPG_NO_AUTOINSTALL", "0")) if version_gte("15.0") else False diff --git a/src/util/records.py b/src/util/records.py index 02d6a50d..ab05c82b 100644 --- a/src/util/records.py +++ b/src/util/records.py @@ -4,7 +4,6 @@ import logging import os import re -from contextlib import contextmanager from operator import itemgetter import lxml @@ -14,7 +13,6 @@ from odoo import release from odoo.tools.convert import xml_import from odoo.tools.misc import file_open - from odoo.tools.translate import xml_translate except ImportError: from openerp import release from openerp.tools.convert import xml_import @@ -44,7 +42,7 @@ table_exists, target_of, ) -from .report import add_to_migration_reports +from .views.records import add_view, edit_view, remove_view # noqa: F401 _logger = logging.getLogger(__name__) @@ -55,249 +53,6 @@ basestring = unicode = str -def remove_view(cr, xml_id=None, view_id=None, silent=False, key=None): - """ - Remove a view and all its descendants. - - This function recursively deletes the given view and its inherited views, as long as - they are part of a module. It will fail as soon as a custom view exists anywhere in - the hierarchy. It also removes multi-website COWed views. - - :param str xml_id: optional, the xml_id of the view to remove - :param int view_id: optional, the ID of the view to remove - :param bool silent: whether to show in the logs disabled custom views - :param str or None key: key used to detect multi-website COWed views, if `None` then - set to `xml_id` if provided, otherwise set to the xml_id - referencing the view with ID `view_id` if any - - .. warning:: - Either `xml_id` or `view_id` must be set. Specifying both will raise an error. - """ - assert bool(xml_id) ^ bool(view_id) - if xml_id: - view_id = ref(cr, xml_id) - if view_id: - module, _, name = xml_id.partition(".") - cr.execute("SELECT model FROM ir_model_data WHERE module=%s AND name=%s", [module, name]) - - [model] = cr.fetchone() - if model != "ir.ui.view": - raise ValueError("%r should point to a 'ir.ui.view', not a %r" % (xml_id, model)) - else: - # search matching xmlid for logging or renaming of custom views - xml_id = "?" - if not key: - cr.execute("SELECT module, name FROM ir_model_data WHERE model='ir.ui.view' AND res_id=%s", [view_id]) - if cr.rowcount: - xml_id = "%s.%s" % cr.fetchone() - - # From given or determined xml_id, the views duplicated in a multi-website - # context are to be found and removed. - if xml_id != "?" and column_exists(cr, "ir_ui_view", "key"): - cr.execute("SELECT id FROM ir_ui_view WHERE key = %s AND id != %s", [xml_id, view_id]) - for [v_id] in cr.fetchall(): - remove_view(cr, view_id=v_id, silent=silent, key=xml_id) - - if not view_id: - return - - cr.execute( - """ - SELECT v.id, x.module || '.' || x.name, v.name - FROM ir_ui_view v LEFT JOIN - ir_model_data x ON (v.id = x.res_id AND x.model = 'ir.ui.view' AND x.module !~ '^_') - WHERE v.inherit_id = %s; - """, - [view_id], - ) - for child_id, child_xml_id, child_name in cr.fetchall(): - if child_xml_id: - if not silent: - _logger.info( - "remove deprecated built-in view %s (ID %s) as parent view %s (ID %s) is going to be removed", - child_xml_id, - child_id, - xml_id, - view_id, - ) - remove_view(cr, child_xml_id, silent=True) - else: - if not silent: - _logger.warning( - "deactivate deprecated custom view with ID %s as parent view %s (ID %s) is going to be removed", - child_id, - xml_id, - view_id, - ) - disable_view_query = """ - UPDATE ir_ui_view - SET name = (name || ' - old view, inherited from ' || %%s), - inherit_id = NULL - %s - WHERE id = %%s - """ - # In 8.0, disabling requires setting mode to 'primary' - extra_set_sql = "" - if column_exists(cr, "ir_ui_view", "mode"): - extra_set_sql = ", mode = 'primary' " - - # Column was not present in v7 and it's older version - if column_exists(cr, "ir_ui_view", "active"): - extra_set_sql += ", active = false " - - disable_view_query = disable_view_query % extra_set_sql - cr.execute(disable_view_query, (key or xml_id, child_id)) - add_to_migration_reports( - {"id": child_id, "name": child_name}, - "Disabled views", - ) - if not silent: - _logger.info("remove deprecated %s view %s (ID %s)", key and "COWed" or "built-in", key or xml_id, view_id) - - remove_records(cr, "ir.ui.view", [view_id]) - - -@contextmanager -def edit_view(cr, xmlid=None, view_id=None, skip_if_not_noupdate=True, active=True): - """ - Context manager to edit a view's arch. - - This function returns a context manager that may yield a parsed arch of a view as an - `etree Element `_. Any changes done - in the returned object will be written back to the database upon exit of the context - manager, updating also the translated versions of the arch. Since the function may not - yield, use :func:`~odoo.upgrade.util.misc.skippable_cm` to avoid errors. - - .. code-block:: python - - with util.skippable_cm(), util.edit_view(cr, "xml.id") as arch: - arch.attrib["string"] = "My Form" - - To select the target view to edit use either `xmlid` or `view_id`, not both. - - When the view is identified by `view_id`, the arch is always yielded if the view - exists, with disregard to any `noupdate` flag it may have associated. When `xmlid` is - set, if the view `noupdate` flag is `True` then the arch will not be yielded *unless* - `skip_if_not_noupdate` is set to `False`. If `noupdate` is `False`, the view will be - yielded for edit. - - If the `active` argument is not `None`, the `active` flag of the view will be set - accordingly. - - .. warning:: - The default value of `active` is `True`, therefore views are always *activated* by - default. To avoid inadvertently activating views, pass `None` as `active` parameter. - - :param str xmlid: optional, xml_id of the view edit - :param int view_id: optional, ID of the view to edit - :param bool skip_if_not_noupdate: whether to force the edit of views requested via - `xmlid` parameter even if they are flagged as - `noupdate=True`, ignored if `view_id` is set - :param bool or None active: active flag value to set, nothing is set when `None` - :return: a context manager that yields the parsed arch, upon exit the context manager - writes back the changes. - """ - assert bool(xmlid) ^ bool(view_id), "You Must specify either xmlid or view_id" - noupdate = True - if xmlid: - if "." not in xmlid: - raise ValueError("Please use fully qualified name .") - - module, _, name = xmlid.partition(".") - cr.execute( - """ - SELECT res_id, noupdate - FROM ir_model_data - WHERE module = %s - AND name = %s - """, - [module, name], - ) - data = cr.fetchone() - if data: - view_id, noupdate = data - - if view_id and not (skip_if_not_noupdate and not noupdate): - arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" - jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" - cr.execute( - """ - SELECT {arch} - FROM ir_ui_view - WHERE id=%s - """.format( - arch=arch_col, - ), - [view_id], - ) - [arch] = cr.fetchone() or [None] - if arch: - - def parse(arch): - arch = arch.encode("utf-8") if isinstance(arch, unicode) else arch - return lxml.etree.fromstring(arch.replace(b" \n", b"\n").strip()) - - if jsonb_column: - - def get_trans_terms(value): - terms = [] - xml_translate(terms.append, value) - return terms - - translation_terms = {lang: get_trans_terms(value) for lang, value in arch.items()} - arch_etree = parse(arch["en_US"]) - yield arch_etree - new_arch = lxml.etree.tostring(arch_etree, encoding="unicode") - terms_en = translation_terms["en_US"] - arch_column_value = Json( - { - lang: xml_translate(dict(zip(terms_en, terms)).get, new_arch) - for lang, terms in translation_terms.items() - } - ) - else: - arch_etree = parse(arch) - yield arch_etree - arch_column_value = lxml.etree.tostring(arch_etree, encoding="unicode") - - set_active = ", active={}".format(bool(active)) if active is not None else "" - cr.execute( - "UPDATE ir_ui_view SET {arch}=%s{set_active} WHERE id=%s".format(arch=arch_col, set_active=set_active), - [arch_column_value, view_id], - ) - - -def add_view(cr, name, model, view_type, arch_db, inherit_xml_id=None, priority=16): - inherit_id = None - if inherit_xml_id: - inherit_id = ref(cr, inherit_xml_id) - if not inherit_id: - raise ValueError( - "Unable to add view '%s' because its inherited view '%s' cannot be found!" % (name, inherit_xml_id) - ) - arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" - jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" - arch_column_value = Json({"en_US": arch_db}) if jsonb_column else arch_db - cr.execute( - """ - INSERT INTO ir_ui_view(name, "type", model, inherit_id, mode, active, priority, %s) - VALUES(%%(name)s, %%(view_type)s, %%(model)s, %%(inherit_id)s, %%(mode)s, 't', %%(priority)s, %%(arch_db)s) - RETURNING id - """ - % arch_col, - { - "name": name, - "view_type": view_type, - "model": model, - "inherit_id": inherit_id, - "mode": "extension" if inherit_id else "primary", - "priority": priority, - "arch_db": arch_column_value, - }, - ) - return cr.fetchone()[0] - - # fmt:off if version_gte("saas~14.3"): def remove_asset(cr, name): diff --git a/src/util/views/__init__.py b/src/util/views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/util/views/bootstrap.py b/src/util/views/bootstrap.py new file mode 100644 index 00000000..dc6d5d4d --- /dev/null +++ b/src/util/views/bootstrap.py @@ -0,0 +1,491 @@ +"""Convert an XML/HTML document Bootstrap code from an older version to a newer one.""" + +import logging +from functools import lru_cache + +from lxml import etree + +try: + from packaging.version import Version +except ImportError: + from distutils.version import StrictVersion as Version # N.B. deprecated, will be removed in py3.12 + +from .convert import ( + ALL, + BE, + BS, + CSS, + AddClasses, + ElementOperation, + EtreeConverter, + PullUp, + RegexReplaceClass, + RemoveClasses, + RemoveElement, + RenameAttribute, + ReplaceClasses, + edit_element_classes, + get_classes, + regex_xpath, +) + +_logger = logging.getLogger(__name__) + + +class BS3to4ConvertBlockquote(ElementOperation): + """ + Convert a BS3 ``
`` element to a BS4 ``
`` element with the ``blockquote`` class. + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + blockquote = converter.copy_element(element, tag="div", add_classes="blockquote", copy_attrs=False) + element.addnext(blockquote) + element.getparent().remove(element) + return blockquote + + +# TODO abt: merge MakeCard and ConvertCard into one operation class +class BS3to4MakeCard(ElementOperation): + """ + Pre-processe a BS3 panel, thumbnail, or well element to be converted to a BS4 card. + + Card components conversion is then handled by the ``ConvertCard`` operation class. + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + card = converter.element_factory("
") + card_body = converter.copy_element( + element, tag="div", add_classes="card-body", remove_classes=ALL, copy_attrs=False + ) + card.append(card_body) + element.addnext(card) + element.getparent().remove(element) + return card + + +# TODO abt: refactor code +class BS3to4ConvertCard(ElementOperation): + """ + Fully convert a BS3 panel, thumbnail, or well element and their contents to a BS4 card. + + :meta private: exclude from online docs + """ + + POST_CONVERSIONS = { + "title": ["card-title"], + "description": ["card-description"], + "category": ["card-category"], + "panel-danger": ["card", "bg-danger", "text-white"], + "panel-warning": ["card", "bg-warning"], + "panel-info": ["card", "bg-info", "text-white"], + "panel-success": ["card", "bg-success", "text-white"], + "panel-primary": ["card", "bg-primary", "text-white"], + "panel-footer": ["card-footer"], + "panel-body": ["card-body"], + "panel-title": ["card-title"], + "panel-heading": ["card-header"], + "panel-default": [], + "panel": ["card"], + } + + def _convert_child(self, child, old_card, new_card, converter): + old_card_classes = get_classes(old_card) + + classes = get_classes(child) + + if "header" in classes or ("image" in classes and len(child)): + add_classes = "card-header" + remove_classes = ["header", "image"] + elif "content" in classes: + add_classes = "card-img-overlay" if "card-background" in old_card_classes else "card-body" + remove_classes = "content" + elif {"card-footer", "footer", "text-center"} & set(classes): + add_classes = "card-footer" + remove_classes = "footer" + else: + new_card.append(child) + return + + new_child = converter.copy_element( + child, "div", add_classes=add_classes, remove_classes=remove_classes, copy_attrs=True + ) + + if "image" in classes: + [img_el] = new_child.xpath("./img")[:1] or [None] + if img_el is not None and "src" in img_el: + new_child.attrib["style"] = ( + f'background-image: url("{img_el.attrib["src"]}"); ' + "background-position: center center; " + "background-size: cover;" + ) + new_child.remove(img_el) + + new_card.append(new_child) + + if "content" in classes: # TODO abt: consider skipping for .card-background + [footer] = new_child.xpath(converter.adapt_xpath("./*[hasclass('footer')]"))[:1] or [None] + if footer is not None: + self._convert_child(footer, old_card, new_card, converter) + new_child.remove(footer) + + def _postprocess(self, new_card, converter): + for old_class, new_classes in self.POST_CONVERSIONS.items(): + for element in new_card.xpath(converter.adapt_xpath(f"(.|.//*)[hasclass('{old_class}')]")): + edit_element_classes(element, add=new_classes, remove=old_class) + + def __call__(self, element, converter): + classes = get_classes(element) + new_card = converter.copy_element(element, tag="div", copy_attrs=True, copy_contents=False) + wrapper = new_card + if "card-horizontal" in classes: + wrapper = etree.SubElement(new_card, "div", {"class": "row"}) + + for child in element: + self._convert_child(child, element, wrapper, converter) + + self._postprocess(new_card, converter) + element.addnext(new_card) + element.getparent().remove(element) + return new_card + + +class BS4to5ConvertCloseButton(ElementOperation): + """ + Convert BS4 ``button.close`` elements to BS5 ``button.btn-close``. + + Also fixes the ``data-dismiss`` attribute to ``data-bs-dismiss``, and removes any inner contents. + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + new_btn = converter.copy_element(element, remove_classes="close", add_classes="btn-close", copy_contents=False) + + if "data-dismiss" in element.attrib: + new_btn.attrib["data-bs-dismiss"] = element.attrib["data-dismiss"] + del new_btn.attrib["data-dismiss"] + + element.addnext(new_btn) + element.getparent().remove(element) + + return new_btn + + +class BS4to5ConvertCardDeck(ElementOperation): + """ + Convert BS4 ``.card-deck`` elements to grid components (``.row``, ``.col``, etc.). + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + cards = element.xpath(converter.adapt_xpath("./*[hasclass('card')]")) + + cols_class = f"row-cols-{len(cards)}" if len(cards) in range(1, 7) else "row-cols-auto" + edit_element_classes(element, add=["row", cols_class], remove="card-deck", is_qweb=converter.is_qweb) + + for card in cards: + new_col = converter.build_element("div", classes=["col"]) + card.addprevious(new_col) + new_col.append(card) + + return element + + +class BS4to5ConvertFormInline(ElementOperation): + """ + Convert BS4 ``.form-inline`` elements to grid components (``.row``, ``.col``, etc.). + + :meta private: exclude from online docs + """ + + def __call__(self, element, converter): + edit_element_classes(element, add="row row-cols-lg-auto", remove="form-inline", is_qweb=converter.is_qweb) + + children_selector = converter.adapt_xpath( + CSS(".form-control,.form-group,.form-check,.input-group,.custom-select,button", prefix="./") + ) + indexed_children = sorted([(element.index(c), c) for c in element.xpath(children_selector)], key=lambda x: x[0]) + + nest_groups = [] + last_idx = -1 + for idx, child in indexed_children: + nest_start, nest_end = idx, idx + labels = [label for label in child.xpath("preceding-sibling::label") if element.index(label) > last_idx] + labels = [ + label + for label in labels + if "for" in label.attrib and child.xpath(f"descendant-or-self::*[@id='{label.attrib['for']}']") + ] or labels[-1:] + if labels: + first_label = labels[0] + assert last_idx < element.index(first_label) < idx, "label must be between last group and current" + nest_start = element.index(first_label) + + assert nest_start <= nest_end, f"expected start {nest_start} to be <= end {nest_end}" + nest_groups.append(element[nest_start : nest_end + 1]) + last_idx = nest_end + + for els in nest_groups: + wrapper = converter.build_element("div", classes=["col-12"]) + els[0].addprevious(wrapper) + for el in els: + wrapper.append(el) + assert el not in element, f"expected {el!r} to be removed from {element!r}" + + return element + + +class BootstrapConverter(EtreeConverter): + """ + Class for converting XML or HTML Bootstrap code across versions. + + :param str src_version: the source Bootstrap version to convert from. + :param str dst_version: the destination Bootstrap version to convert to. + :param bool is_html: whether the tree is an HTML document. + :param bool is_qweb: whether the tree contains QWeb directives. See :class:`EtreeConverter` for more info. + + :meta private: exclude from online docs + """ + + MIN_VERSION = "3.0" + """Minimum supported Bootstrap version.""" + + # Conversions definitions by destination version. + # It's a dictionary of version strings to a conversions list. + # Each item in the conversions list must either be an :class:`ElementOperation`, + # that can provide its own XPath, or a tuple of ``(xpath, operation(s))`` with the XPath + # and a single operation or a list of operations to apply to the nodes matching the XPath. + CONVERSIONS = { + "4.0": [ + # inputs + (CSS(".form-group .control-label"), ReplaceClasses("control-label", "form-control-label")), + (CSS(".form-group .text-help"), ReplaceClasses("text-help", "form-control-feedback")), + (CSS(".control-group .help-block"), ReplaceClasses("help-block", "form-text")), + ReplaceClasses("form-group-sm", "form-control-sm"), + ReplaceClasses("form-group-lg", "form-control-lg"), + (CSS(".form-control .input-sm"), ReplaceClasses("input-sm", "form-control-sm")), + (CSS(".form-control .input-lg"), ReplaceClasses("input-lg", "form-control-lg")), + # hide + ReplaceClasses("hidden-xs", "d-none"), + ReplaceClasses("hidden-sm", "d-sm-none"), + ReplaceClasses("hidden-md", "d-md-none"), + ReplaceClasses("hidden-lg", "d-lg-none"), + ReplaceClasses("visible-xs", "d-block d-sm-none"), + ReplaceClasses("visible-sm", "d-block d-md-none"), + ReplaceClasses("visible-md", "d-block d-lg-none"), + ReplaceClasses("visible-lg", "d-block d-xl-none"), + # image + ReplaceClasses("img-rounded", "rounded"), + ReplaceClasses("img-circle", "rounded-circle"), + ReplaceClasses("img-responsive", ("d-block", "img-fluid")), + # buttons + ReplaceClasses("btn-default", "btn-secondary"), + ReplaceClasses("btn-xs", "btn-sm"), + (CSS(".btn-group.btn-group-xs"), ReplaceClasses("btn-group-xs", "btn-group-sm")), + (CSS(".dropdown .divider"), ReplaceClasses("divider", "dropdown-divider")), + ReplaceClasses("badge", "badge badge-pill"), + ReplaceClasses("label", "badge"), + RegexReplaceClass(rf"{BS}label-(default|primary|success|info|warning|danger){BE}", r"badge-\1"), + (CSS(".breadcrumb > li"), ReplaceClasses("breadcrumb", "breadcrumb-item")), + # li + (CSS(".list-inline > li"), AddClasses("list-inline-item")), + # pagination + (CSS(".pagination > li"), AddClasses("page-item")), + (CSS(".pagination > li > a"), AddClasses("page-link")), + # carousel + (CSS(".carousel .carousel-inner > .item"), ReplaceClasses("item", "carousel-item")), + # pull + ReplaceClasses("pull-right", "float-right"), + ReplaceClasses("pull-left", "float-left"), + ReplaceClasses("center-block", "mx-auto"), + # well + (CSS(".well"), BS3to4MakeCard()), + (CSS(".thumbnail"), BS3to4MakeCard()), + # blockquote + (CSS("blockquote"), BS3to4ConvertBlockquote()), + (CSS(".blockquote.blockquote-reverse"), ReplaceClasses("blockquote-reverse", "text-right")), + # dropdown + (CSS(".dropdown-menu > li > a"), AddClasses("dropdown-item")), + (CSS(".dropdown-menu > li"), PullUp()), + # in + ReplaceClasses("in", "show"), + # table + (CSS("tr.active, td.active"), ReplaceClasses("active", "table-active")), + (CSS("tr.success, td.success"), ReplaceClasses("success", "table-success")), + (CSS("tr.info, td.info"), ReplaceClasses("info", "table-info")), + (CSS("tr.warning, td.warning"), ReplaceClasses("warning", "table-warning")), + (CSS("tr.danger, td.danger"), ReplaceClasses("danger", "table-danger")), + (CSS("table.table-condesed"), ReplaceClasses("table-condesed", "table-sm")), + # navbar + (CSS(".nav.navbar > li > a"), AddClasses("nav-link")), + (CSS(".nav.navbar > li"), AddClasses("nav-intem")), + ReplaceClasses("navbar-btn", "nav-item"), + (CSS(".navbar-nav"), ReplaceClasses("navbar-right nav", "ml-auto")), + ReplaceClasses("navbar-toggler-right", "ml-auto"), + (CSS(".navbar-nav > li > a"), AddClasses("nav-link")), + (CSS(".navbar-nav > li"), AddClasses("nav-item")), + (CSS(".navbar-nav > a"), AddClasses("navbar-brand")), + ReplaceClasses("navbar-fixed-top", "fixed-top"), + ReplaceClasses("navbar-toggle", "navbar-toggler"), + ReplaceClasses("nav-stacked", "flex-column"), + (CSS("nav.navbar"), AddClasses("navbar-expand-lg")), + (CSS("button.navbar-toggle"), ReplaceClasses("navbar-toggle", "navbar-expand-md")), + # card + (CSS(".panel"), BS3to4ConvertCard()), + (CSS(".card"), BS3to4ConvertCard()), + # grid + RegexReplaceClass(rf"{BS}col((?:-\w{{2}})?)-offset-(\d{{1,2}}){BE}", r"offset\1-\2"), + ], + "5.0": [ + # links + RegexReplaceClass(rf"{BS}text-(?!o-)", "link-", xpath=CSS("a")), + (CSS(".nav-item.active > .nav-link"), AddClasses("active")), + (CSS(".nav-link.active") + CSS(".nav-item.active", prefix="/parent::"), RemoveClasses("active")), + # badges + ReplaceClasses("badge-pill", "rounded-pill"), + RegexReplaceClass(rf"{BS}badge-", r"text-bg-"), + # buttons + ("//*[hasclass('btn-block')]/parent::div", AddClasses("d-grid gap-2")), + ("//*[hasclass('btn-block')]/parent::p[count(./*)=1]", AddClasses("d-grid gap-2")), + RemoveClasses("btn-block"), + (CSS("button.close"), BS4to5ConvertCloseButton()), + # card + # TODO abt: .card-columns (unused in odoo) + (CSS(".card-deck"), BS4to5ConvertCardDeck()), + # jumbotron + ReplaceClasses("jumbotron", "container-fluid py-5"), + # new data-bs- attributes + RenameAttribute("data-display", "data-bs-display", xpath="//*[not(@data-snippet='s_countdown')]"), + *[ + RenameAttribute(f"data-{attr}", f"data-bs-{attr}") + for attr in ( + "animation attributes autohide backdrop body container content delay dismiss focus" + " interval margin-right no-jquery offset original-title padding-right parent placement" + " ride sanitize show slide slide-to spy target toggle touch trigger whatever" + ).split(" ") + ], + # popover + (CSS(".popover .arrow"), ReplaceClasses("arrow", "popover-arrow")), + # form + ReplaceClasses("form-row", "row"), + ("//*[hasclass('form-group')]/parent::form", AddClasses("row")), + ReplaceClasses("form-group", "col-12 py-2"), + (CSS(".form-inline"), BS4to5ConvertFormInline()), + ReplaceClasses("custom-checkbox", "form-check"), + RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), + RegexReplaceClass(rf"{BS}custom-control-(input|label){BE}", r"form-check-\1"), + RegexReplaceClass(rf"{BS}custom-(check|select|range){BE}", r"form-\1"), + (CSS(".custom-switch"), ReplaceClasses("custom-switch", "form-check form-switch")), + ReplaceClasses("custom-radio", "form-check"), + RemoveClasses("custom-control"), + (CSS(".custom-file"), PullUp()), + RegexReplaceClass(rf"{BS}custom-file-", r"form-file-"), + RegexReplaceClass(rf"{BS}form-file(?:-input)?{BE}", r"form-control"), + (CSS("label.form-file-label"), RemoveElement()), + (regex_xpath(rf"{BS}input-group-(prepend|append){BE}", "class"), PullUp()), + ("//label[not(hasclass('form-check-label'))]", AddClasses("form-label")), + ReplaceClasses("form-control-file", "form-control"), + ReplaceClasses("form-control-range", "form-range"), + # TODO abt: .form-text no loger sets display, add some class? + # table + RegexReplaceClass(rf"{BS}thead-(light|dark){BE}", r"table-\1"), + # grid + RegexReplaceClass(rf"{BS}col-((?:\w{{2}}-)?)offset-(\d{{1,2}}){BE}", r"offset-\1\2"), # from BS4 + # gutters + ReplaceClasses("no-gutters", "g-0"), + # logical properties + RegexReplaceClass(rf"{BS}left-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"start-\1"), + RegexReplaceClass(rf"{BS}right-((?:\w{{2,3}}-)?[0-9]+|auto){BE}", r"end-\1"), + RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-left{BE}", r"\1-start"), + RegexReplaceClass(rf"{BS}((?:float|border|rounded|text)(?:-\w+)?)-right{BE}", r"\1-end"), + RegexReplaceClass(rf"{BS}rounded-sm(-(?:start|end|top|bottom))?", r"rounded\1-1"), + RegexReplaceClass(rf"{BS}rounded-lg(-(?:start|end|top|bottom))?", r"rounded\1-3"), + RegexReplaceClass(rf"{BS}([mp])l-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1s-\2"), + RegexReplaceClass(rf"{BS}([mp])r-((?:\w{{2,3}}-)?(?:[0-9]+|auto)){BE}", r"\1e-\2"), + ReplaceClasses("dropdown-menu-left", "dropdown-menu-start"), + ReplaceClasses("dropdown-menu-right", "dropdown-menu-end"), + ReplaceClasses("dropleft", "dropstart"), + ReplaceClasses("dropright", "dropend"), + # tooltips + ( + "//*[hasclass('tooltip') or @role='tooltip']//*[hasclass('arrow')]", + ReplaceClasses("arrow", "tooltip-arrow"), + ), + # utilities + ReplaceClasses("text-monospace", "font-monospace"), + RegexReplaceClass(rf"{BS}font-weight-", r"fw-"), + RegexReplaceClass(rf"{BS}font-style-", r"fst-"), + ReplaceClasses("font-italic", "fst-italic"), + # helpers + RegexReplaceClass(rf"{BS}embed-responsive-(\d+)by(\d+)", r"ratio-\1x\2"), + RegexReplaceClass(rf"{BS}ratio-(\d+)by(\d+)", r"ratio-\1x\2"), + RegexReplaceClass(rf"{BS}embed-responsive(?!-)", r"ratio"), + RegexReplaceClass(rf"{BS}sr-only(-focusable)?", r"visually-hidden\1"), + # media + ReplaceClasses("media-body", "flex-grow-1"), + ReplaceClasses("media", "d-flex"), + ], + } + + def __init__(self, src_version, dst_version, *, is_html=False, is_qweb=False): + self.src_version = src_version + self.dst_version = dst_version + conversions = self._get_conversions(src_version, dst_version) + super().__init__(conversions, is_html=is_html, is_qweb=is_qweb) + + @classmethod + def _get_sorted_conversions(cls): + """ + Return the conversions dict sorted by version, from oldest to newest. + + :meta private: exclude from online docs + """ + return sorted(cls.CONVERSIONS.items(), key=lambda kv: Version(kv[0])) + + @classmethod + @lru_cache(maxsize=8) + def _get_conversions(cls, src_version, dst_version): + """ + Return the list of conversions to convert Bootstrap from ``src_version`` to ``dst_version``. + + :param str src_version: the source Bootstrap version. + :param str dst_version: the destination Bootstrap version. + :rtype: list[ElementOperation | (str, ElementOperation | list[ElementOperation])] + + :meta private: exclude from online docs + """ + if Version(dst_version) < Version(src_version): + raise NotImplementedError("Downgrading Bootstrap versions is not supported.") + if Version(src_version) < Version(cls.MIN_VERSION): + raise NotImplementedError(f"Conversion from Bootstrap version {src_version} is not supported") + + result = [] + for version, conversions in BootstrapConverter._get_sorted_conversions(): + if Version(src_version) < Version(version) <= Version(dst_version): + result.extend(conversions) + + if not result: + if Version(src_version) == Version(dst_version): + _logger.info("Source and destination versions are the same, no conversion needed.") + else: + raise NotImplementedError(f"Conversion from {src_version} to {dst_version} is not supported") + + return result + + +# TODO abt: remove this / usages -> replace with refactored converter classes +class BootstrapHTMLConverter: + def __init__(self, src, dst): + self.src = src + self.dst = dst + + def __call__(self, content): + if not content: + return False, content + converted_content = BootstrapConverter.convert_arch(content, self.src, self.dst, is_html=True, is_qweb=True) + return content != converted_content, converted_content diff --git a/src/util/views_convert.py b/src/util/views/convert.py similarity index 88% rename from src/util/views_convert.py rename to src/util/views/convert.py index d705c385..142f4f92 100644 --- a/src/util/views_convert.py +++ b/src/util/views/convert.py @@ -9,12 +9,10 @@ from lxml import etree -from odoo import modules +from odoo.modules.module import get_modules -from . import snippets -from .misc import log_progress -from .pg import get_value_or_en_translation -from .records import edit_view +from .. import misc, pg, snippets +from . import records _logger = logging.getLogger(__name__) @@ -1112,110 +1110,128 @@ def adapt_xpath(self, xpath): return adapt_xpath_for_qweb(xpath) if self.is_qweb else xpath -def convert_views(cr, views_ids, converter): - """ - Convert the specified views xml arch using the provided converter. +if misc.version_gte("13.0"): - :param psycopg2.cursor cr: the database cursor. - :param typing.Collection[int] views_ids: the ids of the views to convert. - :param EtreeConverter converter: the converter to use. - :rtype: None + def convert_views(cr, views_ids, converter): + """ + Convert the specified views xml arch using the provided converter. - :meta private: exclude from online docs - """ - if converter.is_html: - raise TypeError(f"Cannot convert xml views with provided ``is_html`` converter {converter!r}") + :param psycopg2.cursor cr: the database cursor. + :param typing.Collection[int] views_ids: the ids of the views to convert. + :param EtreeConverter converter: the converter to use. + :rtype: None - _logger.info("Converting %s views/templates using %s", len(views_ids), repr(converter)) - for view_id in views_ids: - with edit_view(cr, view_id=view_id, active=None) as tree: - converter.convert_tree(tree) - # TODO abt: maybe notify in the log or report that custom views with noupdate=False were converted? + :meta private: exclude from online docs + """ + if converter.is_html: + raise TypeError("Cannot convert xml views with provided ``is_html`` converter %s" % (repr(converter),)) + _logger.info("Converting %d views/templates using %s", len(views_ids), repr(converter)) + for view_id in views_ids: + with records.edit_view(cr, view_id=view_id, active=None) as tree: + converter.convert_tree(tree) + # TODO abt: maybe notify in the log or report that custom views with noupdate=False were converted? -def convert_qweb_views(cr, converter): - """ - Convert QWeb views / templates using the provided converter. + def convert_qweb_views(cr, converter): + """ + Convert QWeb views / templates using the provided converter. - :param psycopg2.cursor cr: the database cursor. - :param EtreeConverter converter: the converter to use. - :rtype: None + :param psycopg2.cursor cr: the database cursor. + :param EtreeConverter converter: the converter to use. + :rtype: None - :meta private: exclude from online docs - """ - if not converter.is_qweb: - raise TypeError("Converter for xml views must be ``is_qweb``, got %s", repr(converter)) - - # views to convert must have `website_id` set and not come from standard modules - standard_modules = set(modules.get_modules()) - {"studio_customization", "__export__", "__cloc_exclude__"} - converter_where = converter.build_where_clause(cr, "v.arch_db") - - # Search for custom/cow'ed views (they have no external ID)... but also - # search for views with external ID that have a related COW'ed view. Indeed, - # when updating a generic view after this script, the archs are compared to - # know if the related COW'ed views must be updated too or not: if we only - # convert COW'ed views they won't get the generic view update as they will be - # judged different from them (user customization) because of the changes - # that were made. - # E.g. - # - In 15.0, install website_sale - # - Enable eCommerce categories: a COW'ed view is created to enable the - # feature (it leaves the generic disabled and creates an exact copy but - # enabled) - # - Migrate to 16.0: you expect your enabled COW'ed view to get the new 16.0 - # version of eCommerce categories... but if the COW'ed view was converted - # while the generic was not, they won't be considered the same - # anymore and only the generic view will get the 16.0 update. - cr.execute( - f""" - WITH keys AS ( - SELECT key - FROM ir_ui_view - GROUP BY key - HAVING COUNT(*) > 1 - ) - SELECT v.id - FROM ir_ui_view v - LEFT JOIN ir_model_data imd - ON imd.model = 'ir.ui.view' - AND imd.module IN %s - AND imd.res_id = v.id - LEFT JOIN keys - ON v.key = keys.key - WHERE v.type = 'qweb' - AND ({converter_where}) - AND ( - imd.id IS NULL - OR ( - keys.key IS NOT NULL - AND imd.noupdate = FALSE + :meta private: exclude from online docs + """ + if not converter.is_qweb: + raise TypeError("Converter for xml views must be ``is_qweb``, got %s" % (repr(converter),)) + + # views to convert must have `website_id` set and not come from standard modules + standard_modules = set(get_modules()) - {"studio_customization", "__export__", "__cloc_exclude__"} + converter_where = converter.build_where_clause(cr, "v.arch_db") + + # Search for custom/cow'ed views (they have no external ID)... but also + # search for views with external ID that have a related COW'ed view. Indeed, + # when updating a generic view after this script, the archs are compared to + # know if the related COW'ed views must be updated too or not: if we only + # convert COW'ed views they won't get the generic view update as they will be + # judged different from them (user customization) because of the changes + # that were made. + # E.g. + # - In 15.0, install website_sale + # - Enable eCommerce categories: a COW'ed view is created to enable the + # feature (it leaves the generic disabled and creates an exact copy but + # enabled) + # - Migrate to 16.0: you expect your enabled COW'ed view to get the new 16.0 + # version of eCommerce categories... but if the COW'ed view was converted + # while the generic was not, they won't be considered the same + # anymore and only the generic view will get the 16.0 update. + cr.execute( + """ + WITH keys AS ( + SELECT key + FROM ir_ui_view + GROUP BY key + HAVING COUNT(*) > 1 + ) + SELECT v.id + FROM ir_ui_view v + LEFT JOIN ir_model_data imd + ON imd.model = 'ir.ui.view' + AND imd.module IN %%s + AND imd.res_id = v.id + LEFT JOIN keys + ON v.key = keys.key + WHERE v.type = 'qweb' + AND (%s) + AND ( + imd.id IS NULL + OR ( + keys.key IS NOT NULL + AND imd.noupdate = FALSE + ) ) - ) - """, - [tuple(standard_modules)], - ) - views_ids = [view_id for (view_id,) in cr.fetchall()] - if views_ids: - convert_views(cr, views_ids, converter) - + """ + % converter_where, + [tuple(standard_modules)], + ) + views_ids = [view_id for (view_id,) in cr.fetchall()] + if views_ids: + convert_views(cr, views_ids, converter) -def convert_html_fields(cr, converter): - """ - Convert all html fields data in the database using the provided converter. + def convert_html_fields(cr, converter): + """ + Convert all html fields data in the database using the provided converter. - :param psycopg2.cursor cr: the database cursor. - :param EtreeConverter converter: the converter to use. - :rtype: None + :param psycopg2.cursor cr: the database cursor. + :param EtreeConverter converter: the converter to use. + :rtype: None - :meta private: exclude from online docs - """ - _logger.info("Converting html fields data using %s", repr(converter)) - - html_fields = list(snippets.html_fields(cr)) - for table, columns in log_progress(html_fields, _logger, "tables", log_hundred_percent=True): - if table not in ("mail_message", "mail_activity"): - extra_where = " OR ".join( - f"({converter.build_where_clause(cr, get_value_or_en_translation(cr, table, column))})" - for column in columns - ) + :meta private: exclude from online docs + """ + _logger.info("Converting html fields data using %s", repr(converter)) + + html_fields = list(snippets.html_fields(cr)) + for table, columns in misc.log_progress(html_fields, _logger, "tables", log_hundred_percent=True): + if table not in ("mail_message", "mail_activity"): + extra_where = " OR ".join( + "(%s)" % converter.build_where_clause(cr, pg.get_value_or_en_translation(cr, table, column)) + for column in columns + ) snippets.convert_html_columns(cr, table, columns, converter.convert_callback, extra_where=extra_where) + +else: + + def convert_views(*args, **kwargs): + raise NotImplementedError( + "This helper function is only available for Odoo 13.0 and above: %s", convert_views.__qualname__ + ) + + def convert_qweb_views(*args, **kwargs): + raise NotImplementedError( + "This helper function is only available for Odoo 13.0 and above: %s", convert_qweb_views.__qualname__ + ) + + def convert_html_fields(*args, **kwargs): + raise NotImplementedError( + "This helper function is only available for Odoo 13.0 and above: %s", convert_html_fields.__qualname__ + ) diff --git a/src/util/views/records.py b/src/util/views/records.py new file mode 100644 index 00000000..70869a2f --- /dev/null +++ b/src/util/views/records.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +import logging +from contextlib import contextmanager + +# python3 shims +try: + unicode # noqa: B018 +except NameError: + unicode = str + +import lxml +from psycopg2.extras import Json + +try: + from odoo.tools.translate import xml_translate +except ImportError: + xml_translate = lambda callback, value: value + +from ..pg import column_exists, column_type +from ..report import add_to_migration_reports + +_logger = logging.getLogger(__name__) + + +__all__ = [ + "remove_view", + "edit_view", + "add_view", +] + + +def remove_view(cr, xml_id=None, view_id=None, silent=False, key=None): + """ + Remove a view and all its descendants. + + This function recursively deletes the given view and its inherited views, as long as + they are part of a module. It will fail as soon as a custom view exists anywhere in + the hierarchy. It also removes multi-website COWed views. + + :param str xml_id: optional, the xml_id of the view to remove + :param int view_id: optional, the ID of the view to remove + :param bool silent: whether to show in the logs disabled custom views + :param str or None key: key used to detect multi-website COWed views, if `None` then + set to `xml_id` if provided, otherwise set to the xml_id + referencing the view with ID `view_id` if any + + .. warning:: + Either `xml_id` or `view_id` must be set. Specifying both will raise an error. + """ + from ..records import ref, remove_records + + assert bool(xml_id) ^ bool(view_id) + if xml_id: + view_id = ref(cr, xml_id) + if view_id: + module, _, name = xml_id.partition(".") + cr.execute("SELECT model FROM ir_model_data WHERE module=%s AND name=%s", [module, name]) + + [model] = cr.fetchone() + if model != "ir.ui.view": + raise ValueError("%r should point to a 'ir.ui.view', not a %r" % (xml_id, model)) + else: + # search matching xmlid for logging or renaming of custom views + xml_id = "?" + if not key: + cr.execute("SELECT module, name FROM ir_model_data WHERE model='ir.ui.view' AND res_id=%s", [view_id]) + if cr.rowcount: + xml_id = "%s.%s" % cr.fetchone() + + # From given or determined xml_id, the views duplicated in a multi-website + # context are to be found and removed. + if xml_id != "?" and column_exists(cr, "ir_ui_view", "key"): + cr.execute("SELECT id FROM ir_ui_view WHERE key = %s AND id != %s", [xml_id, view_id]) + for [v_id] in cr.fetchall(): + remove_view(cr, view_id=v_id, silent=silent, key=xml_id) + + if not view_id: + return + + cr.execute( + """ + SELECT v.id, x.module || '.' || x.name, v.name + FROM ir_ui_view v LEFT JOIN + ir_model_data x ON (v.id = x.res_id AND x.model = 'ir.ui.view' AND x.module !~ '^_') + WHERE v.inherit_id = %s; + """, + [view_id], + ) + for child_id, child_xml_id, child_name in cr.fetchall(): + if child_xml_id: + if not silent: + _logger.info( + "remove deprecated built-in view %s (ID %s) as parent view %s (ID %s) is going to be removed", + child_xml_id, + child_id, + xml_id, + view_id, + ) + remove_view(cr, child_xml_id, silent=True) + else: + if not silent: + _logger.warning( + "deactivate deprecated custom view with ID %s as parent view %s (ID %s) is going to be removed", + child_id, + xml_id, + view_id, + ) + disable_view_query = """ + UPDATE ir_ui_view + SET name = (name || ' - old view, inherited from ' || %%s), + inherit_id = NULL + %s + WHERE id = %%s + """ + # In 8.0, disabling requires setting mode to 'primary' + extra_set_sql = "" + if column_exists(cr, "ir_ui_view", "mode"): + extra_set_sql = ", mode = 'primary' " + + # Column was not present in v7 and it's older version + if column_exists(cr, "ir_ui_view", "active"): + extra_set_sql += ", active = false " + + disable_view_query = disable_view_query % extra_set_sql + cr.execute(disable_view_query, (key or xml_id, child_id)) + add_to_migration_reports( + {"id": child_id, "name": child_name}, + "Disabled views", + ) + if not silent: + _logger.info("remove deprecated %s view %s (ID %s)", key and "COWed" or "built-in", key or xml_id, view_id) + + remove_records(cr, "ir.ui.view", [view_id]) + + +@contextmanager +def edit_view(cr, xmlid=None, view_id=None, skip_if_not_noupdate=True, active=True): + """ + Context manager to edit a view's arch. + + This function returns a context manager that may yield a parsed arch of a view as an + `etree Element `_. Any changes done + in the returned object will be written back to the database upon exit of the context + manager, updating also the translated versions of the arch. Since the function may not + yield, use :func:`~odoo.upgrade.util.misc.skippable_cm` to avoid errors. + + .. code-block:: python + + with util.skippable_cm(), util.edit_view(cr, "xml.id") as arch: + arch.attrib["string"] = "My Form" + + To select the target view to edit use either `xmlid` or `view_id`, not both. + + When the view is identified by `view_id`, the arch is always yielded if the view + exists, with disregard to any `noupdate` flag it may have associated. When `xmlid` is + set, if the view `noupdate` flag is `True` then the arch will not be yielded *unless* + `skip_if_not_noupdate` is set to `False`. If `noupdate` is `False`, the view will be + yielded for edit. + + If the `active` argument is not `None`, the `active` flag of the view will be set + accordingly. + + .. warning:: + The default value of `active` is `True`, therefore views are always *activated* by + default. To avoid inadvertently activating views, pass `None` as `active` parameter. + + :param str xmlid: optional, xml_id of the view edit + :param int view_id: optional, ID of the view to edit + :param bool skip_if_not_noupdate: whether to force the edit of views requested via + `xmlid` parameter even if they are flagged as + `noupdate=True`, ignored if `view_id` is set + :param bool or None active: active flag value to set, nothing is set when `None` + :return: a context manager that yields the parsed arch, upon exit the context manager + writes back the changes. + """ + assert bool(xmlid) ^ bool(view_id), "You Must specify either xmlid or view_id" + noupdate = True + if xmlid: + if "." not in xmlid: + raise ValueError("Please use fully qualified name .") + + module, _, name = xmlid.partition(".") + cr.execute( + """ + SELECT res_id, noupdate + FROM ir_model_data + WHERE module = %s + AND name = %s + """, + [module, name], + ) + data = cr.fetchone() + if data: + view_id, noupdate = data + + if view_id and not (skip_if_not_noupdate and not noupdate): + arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" + jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" + cr.execute( + """ + SELECT {arch} + FROM ir_ui_view + WHERE id=%s + """.format( + arch=arch_col, + ), + [view_id], + ) + [arch] = cr.fetchone() or [None] + if arch: + + def parse(arch): + arch = arch.encode("utf-8") if isinstance(arch, unicode) else arch + return lxml.etree.fromstring(arch.replace(b" \n", b"\n").strip()) + + if jsonb_column: + + def get_trans_terms(value): + terms = [] + xml_translate(terms.append, value) + return terms + + translation_terms = {lang: get_trans_terms(value) for lang, value in arch.items()} + arch_etree = parse(arch["en_US"]) + yield arch_etree + new_arch = lxml.etree.tostring(arch_etree, encoding="unicode") + terms_en = translation_terms["en_US"] + arch_column_value = Json( + { + lang: xml_translate(dict(zip(terms_en, terms)).get, new_arch) + for lang, terms in translation_terms.items() + } + ) + else: + arch_etree = parse(arch) + yield arch_etree + arch_column_value = lxml.etree.tostring(arch_etree, encoding="unicode") + + set_active = ", active={}".format(bool(active)) if active is not None else "" + cr.execute( + "UPDATE ir_ui_view SET {arch}=%s{set_active} WHERE id=%s".format(arch=arch_col, set_active=set_active), + [arch_column_value, view_id], + ) + + +def add_view(cr, name, model, view_type, arch_db, inherit_xml_id=None, priority=16): + from ..records import ref + + inherit_id = None + if inherit_xml_id: + inherit_id = ref(cr, inherit_xml_id) + if not inherit_id: + raise ValueError( + "Unable to add view '%s' because its inherited view '%s' cannot be found!" % (name, inherit_xml_id) + ) + arch_col = "arch_db" if column_exists(cr, "ir_ui_view", "arch_db") else "arch" + jsonb_column = column_type(cr, "ir_ui_view", arch_col) == "jsonb" + arch_column_value = Json({"en_US": arch_db}) if jsonb_column else arch_db + cr.execute( + """ + INSERT INTO ir_ui_view(name, "type", model, inherit_id, mode, active, priority, %s) + VALUES(%%(name)s, %%(view_type)s, %%(model)s, %%(inherit_id)s, %%(mode)s, 't', %%(priority)s, %%(arch_db)s) + RETURNING id + """ + % arch_col, + { + "name": name, + "view_type": view_type, + "model": model, + "inherit_id": inherit_id, + "mode": "extension" if inherit_id else "primary", + "priority": priority, + "arch_db": arch_column_value, + }, + ) + return cr.fetchone()[0] diff --git a/tools/compile23.py b/tools/compile23.py index f39841c6..9594b4a1 100755 --- a/tools/compile23.py +++ b/tools/compile23.py @@ -13,8 +13,8 @@ "src/testing.py", "src/util/jinja_to_qweb.py", "src/util/snippets.py", - "src/util/views_convert.py", - "src/util/convert_bootstrap.py", + "src/util/views/convert.py", + "src/util/views/bootstrap.py", "src/*/tests/*.py", "src/*/17.0.*/*.py", ] From 0475cc5d98284c68cd5ad61b33ef5cdaf9f6a3ec Mon Sep 17 00:00:00 2001 From: abk16 Date: Fri, 17 Mar 2023 10:57:31 +0100 Subject: [PATCH 5/6] [IMP] util.snippets: add logging of minimal stats in html conversions Store and return the number of matched and converted record's values that are processed by `convert_html_columns()`. Add a `verbose=False` optional argument in `convert_html_content()` that collects these stats and logs them. Also, do the same in `util.views.records.convert_html_fields()`. --- src/util/snippets.py | 30 ++++++++++++++++++++++++++++-- src/util/views/convert.py | 22 +++++++++++++++++++--- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/util/snippets.py b/src/util/snippets.py index a12ed7b2..1575c5aa 100644 --- a/src/util/snippets.py +++ b/src/util/snippets.py @@ -275,13 +275,19 @@ def convert_html_columns(cr, table, columns, converter_callback, where_column="I update_sql = ", ".join(f'"{column}" = %({column})s' for column in columns) update_query = f"UPDATE {table} SET {update_sql} WHERE id = %(id)s" + matched_count = 0 + converted_count = 0 with ProcessPoolExecutor(max_workers=util.get_max_workers()) as executor: convert = Convertor(converters, converter_callback) for query in util.log_progress(split_queries, logger=_logger, qualifier=f"{table} updates"): cr.execute(query) for data in executor.map(convert, cr.fetchall()): + matched_count += 1 if "id" in data: cr.execute(update_query, data) + converted_count += 1 + + return matched_count, converted_count def determine_chunk_limit_ids(cr, table, column_arr, where): @@ -304,6 +310,7 @@ def convert_html_content( cr, converter_callback, where_column="IS NOT NULL", + verbose=False, **kwargs, ): r""" @@ -316,9 +323,16 @@ def convert_html_content( :param str where_column: filtering such as - "like '%abc%xyz%'" - "~* '\yabc.*xyz\y'" + :param bool verbose: print stats about the conversion :param dict kwargs: extra keyword arguments to pass to :func:`convert_html_column` """ - convert_html_columns( + if verbose: + _logger.info("Converting html fields data using %s", repr(converter_callback)) + + matched_count = 0 + converted_count = 0 + + matched, converted = convert_html_columns( cr, "ir_ui_view", ["arch_db"], @@ -326,6 +340,18 @@ def convert_html_content( where_column=where_column, **dict(kwargs, extra_where="type = 'qweb'"), ) + matched_count += matched + converted_count += converted for table, columns in html_fields(cr): - convert_html_columns(cr, table, columns, converter_callback, where_column=where_column, **kwargs) + matched, converted = convert_html_columns( + cr, table, columns, converter_callback, where_column=where_column, **kwargs + ) + matched_count += matched + converted_count += converted + + if verbose: + if matched_count: + _logger.info("Converted %d/%d matched html fields values", converted_count, matched_count) + else: + _logger.info("Did not match any html fields values to convert") diff --git a/src/util/views/convert.py b/src/util/views/convert.py index 142f4f92..c496c4e7 100644 --- a/src/util/views/convert.py +++ b/src/util/views/convert.py @@ -1198,17 +1198,22 @@ def convert_qweb_views(cr, converter): if views_ids: convert_views(cr, views_ids, converter) - def convert_html_fields(cr, converter): + def convert_html_fields(cr, converter, verbose=False): """ Convert all html fields data in the database using the provided converter. :param psycopg2.cursor cr: the database cursor. :param EtreeConverter converter: the converter to use. + :param bool verbose: whether to print stats about the conversion. :rtype: None :meta private: exclude from online docs """ - _logger.info("Converting html fields data using %s", repr(converter)) + if verbose: + _logger.info("Converting html fields data using %s", repr(converter)) + + matched_count = 0 + converted_count = 0 html_fields = list(snippets.html_fields(cr)) for table, columns in misc.log_progress(html_fields, _logger, "tables", log_hundred_percent=True): @@ -1217,7 +1222,18 @@ def convert_html_fields(cr, converter): "(%s)" % converter.build_where_clause(cr, pg.get_value_or_en_translation(cr, table, column)) for column in columns ) - snippets.convert_html_columns(cr, table, columns, converter.convert_callback, extra_where=extra_where) + # TODO abt: adapt to refactor, maybe make snippets compat w/ converter, instead of adapting? + matched, converted = snippets.convert_html_columns( + cr, table, columns, converter.convert_callback, extra_where=extra_where + ) + matched_count += matched + converted_count += converted + + if verbose: + if matched_count: + _logger.info("Converted %d/%d matched html fields values", converted_count, matched_count) + else: + _logger.info("Did not match any html fields values to convert") else: From cef81148d8961532b64c7453c689867978a04a6c Mon Sep 17 00:00:00 2001 From: abk16 Date: Wed, 5 Apr 2023 11:25:36 +0200 Subject: [PATCH 6/6] [FIX] util.snippets: replace odoo.upgrade.util import with relative `util.snippets` imports util from `odoo.upgrade` namespace package and that makes it incompatible with odoo versions <= 13.0. Replaced the import with direct package-relative ones. --- src/util/snippets.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/util/snippets.py b/src/util/snippets.py index 1575c5aa..ba349e0f 100644 --- a/src/util/snippets.py +++ b/src/util/snippets.py @@ -12,7 +12,9 @@ from psycopg2.extras import Json from .exceptions import MigrationError -from odoo.upgrade import util +from .helpers import table_of_model +from .misc import import_script, log_progress +from .pg import column_exists, column_type, get_max_workers, table_exists _logger = logging.getLogger(__name__) utf8_parser = html.HTMLParser(encoding="utf-8") @@ -38,7 +40,7 @@ def add_snippet_names(cr, table, column, snippets, select_query): _logger.info("Add snippet names on %s.%s", table, column) cr.execute(select_query) - it = util.log_progress(cr.fetchall(), _logger, qualifier="rows", size=cr.rowcount, log_hundred_percent=True) + it = log_progress(cr.fetchall(), _logger, qualifier="rows", size=cr.rowcount, log_hundred_percent=True) def quote(ident): return quote_ident(ident, cr._cnx) @@ -103,11 +105,11 @@ def html_fields(cr): """ ) for model, columns in cr.fetchall(): - table = util.table_of_model(cr, model) - if not util.table_exists(cr, table): + table = table_of_model(cr, model) + if not table_exists(cr, table): # an SQL VIEW continue - existing_columns = [column for column in columns if util.column_exists(cr, table, column)] + existing_columns = [column for column in columns if column_exists(cr, table, column)] if existing_columns: yield table, existing_columns @@ -169,7 +171,7 @@ def make_pickleable_callback(callback): """ callback_filepath = inspect.getfile(callback) name = f"_upgrade_{uuid.uuid4().hex}" - mod = sys.modules[name] = util.import_script(callback_filepath, name=name) + mod = sys.modules[name] = import_script(callback_filepath, name=name) try: return getattr(mod, callback.__name__) except AttributeError: @@ -257,7 +259,7 @@ def convert_html_columns(cr, table, columns, converter_callback, where_column="I """ assert "id" not in columns - converters = {column: "->>'en_US'" if util.column_type(cr, table, column) == "jsonb" else "" for column in columns} + converters = {column: "->>'en_US'" if column_type(cr, table, column) == "jsonb" else "" for column in columns} select = ", ".join(f'"{column}"' for column in columns) where = " OR ".join(f'"{column}"{converters[column]} {where_column}' for column in columns) @@ -277,9 +279,9 @@ def convert_html_columns(cr, table, columns, converter_callback, where_column="I matched_count = 0 converted_count = 0 - with ProcessPoolExecutor(max_workers=util.get_max_workers()) as executor: + with ProcessPoolExecutor(max_workers=get_max_workers()) as executor: convert = Convertor(converters, converter_callback) - for query in util.log_progress(split_queries, logger=_logger, qualifier=f"{table} updates"): + for query in log_progress(split_queries, logger=_logger, qualifier=f"{table} updates"): cr.execute(query) for data in executor.map(convert, cr.fetchall()): matched_count += 1