Gitiles
Code Review Sign In
gerrit.openbmc.org / stefanberger / openbmc / e1721f6c021ddb14c01a0a7c3e2035f0aaa2437c / . / poky / bitbake / lib / bs4 / builder / _html5lib.py
blob: 9e9216ef9c02dcf0191d3221ea3c7bbab3f0fd89 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1__all__ = [
2 'HTML5TreeBuilder',
3 ]
4
5import warnings
6from bs4.builder import (
7 PERMISSIVE,
8 HTML,
9 HTML_5,
10 HTMLTreeBuilder,
11 )
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]12from bs4.element import (
13 NamespacedAttribute,
14 whitespace_re,
15)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]16import html5lib
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]17try:
18 # html5lib >= 0.99999999/1.0b9
19 from html5lib.treebuilders import base as treebuildersbase
20except ImportError:
21 # html5lib <= 0.9999999/1.0b8
22 from html5lib.treebuilders import _base as treebuildersbase
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]23from html5lib.constants import namespaces
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]24
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]25from bs4.element import (
26 Comment,
27 Doctype,
28 NavigableString,
29 Tag,
30 )
31
32class HTML5TreeBuilder(HTMLTreeBuilder):
33 """Use html5lib to build a tree."""
34
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]35 NAME = "html5lib"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]36
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]37 features = [NAME, PERMISSIVE, HTML_5, HTML]
38
39 def prepare_markup(self, markup, user_specified_encoding,
40 document_declared_encoding=None, exclude_encodings=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]41 # Store the user-specified encoding for use later on.
42 self.user_specified_encoding = user_specified_encoding
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]43
44 # document_declared_encoding and exclude_encodings aren't used
45 # ATM because the html5lib TreeBuilder doesn't use
46 # UnicodeDammit.
47 if exclude_encodings:
48 warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]49 yield (markup, None, None, False)
50
51 # These methods are defined by Beautiful Soup.
52 def feed(self, markup):
53 if self.soup.parse_only is not None:
54 warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
55 parser = html5lib.HTMLParser(tree=self.create_treebuilder)
56 doc = parser.parse(markup, encoding=self.user_specified_encoding)
57
58 # Set the character encoding detected by the tokenizer.
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]59 if isinstance(markup, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]60 # We need to special-case this because html5lib sets
61 # charEncoding to UTF-8 if it gets Unicode input.
62 doc.original_encoding = None
63 else:
64 doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
65
66 def create_treebuilder(self, namespaceHTMLElements):
67 self.underlying_builder = TreeBuilderForHtml5lib(
68 self.soup, namespaceHTMLElements)
69 return self.underlying_builder
70
71 def test_fragment_to_document(self, fragment):
72 """See `TreeBuilder`."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]73 return '<html><head></head><body>%s</body></html>' % fragment
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]74
75
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]76class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]77
78 def __init__(self, soup, namespaceHTMLElements):
79 self.soup = soup
80 super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
81
82 def documentClass(self):
83 self.soup.reset()
84 return Element(self.soup, self.soup, None)
85
86 def insertDoctype(self, token):
87 name = token["name"]
88 publicId = token["publicId"]
89 systemId = token["systemId"]
90
91 doctype = Doctype.for_name_and_ids(name, publicId, systemId)
92 self.soup.object_was_parsed(doctype)
93
94 def elementClass(self, name, namespace):
95 tag = self.soup.new_tag(name, namespace)
96 return Element(tag, self.soup, namespace)
97
98 def commentClass(self, data):
99 return TextNode(Comment(data), self.soup)
100
101 def fragmentClass(self):
102 self.soup = BeautifulSoup("")
103 self.soup.name = "[document_fragment]"
104 return Element(self.soup, self.soup, None)
105
106 def appendChild(self, node):
107 # XXX This code is not covered by the BS4 tests.
108 self.soup.append(node.element)
109
110 def getDocument(self):
111 return self.soup
112
113 def getFragment(self):
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]114 return treebuildersbase.TreeBuilder.getFragment(self).element
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]115
116class AttrList(object):
117 def __init__(self, element):
118 self.element = element
119 self.attrs = dict(self.element.attrs)
120 def __iter__(self):
121 return list(self.attrs.items()).__iter__()
122 def __setitem__(self, name, value):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]123 # If this attribute is a multi-valued attribute for this element,
124 # turn its value into a list.
125 list_attr = HTML5TreeBuilder.cdata_list_attributes
126 if (name in list_attr['*']
127 or (self.element.name in list_attr
128 and name in list_attr[self.element.name])):
129 # A node that is being cloned may have already undergone
130 # this procedure.
131 if not isinstance(value, list):
132 value = whitespace_re.split(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]133 self.element[name] = value
134 def items(self):
135 return list(self.attrs.items())
136 def keys(self):
137 return list(self.attrs.keys())
138 def __len__(self):
139 return len(self.attrs)
140 def __getitem__(self, name):
141 return self.attrs[name]
142 def __contains__(self, name):
143 return name in list(self.attrs.keys())
144
145
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]146class Element(treebuildersbase.Node):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]147 def __init__(self, element, soup, namespace):
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]148 treebuildersbase.Node.__init__(self, element.name)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]149 self.element = element
150 self.soup = soup
151 self.namespace = namespace
152
153 def appendChild(self, node):
154 string_child = child = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]155 if isinstance(node, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]156 # Some other piece of code decided to pass in a string
157 # instead of creating a TextElement object to contain the
158 # string.
159 string_child = child = node
160 elif isinstance(node, Tag):
161 # Some other piece of code decided to pass in a Tag
162 # instead of creating an Element object to contain the
163 # Tag.
164 child = node
165 elif node.element.__class__ == NavigableString:
166 string_child = child = node.element
167 else:
168 child = node.element
169
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]170 if not isinstance(child, str) and child.parent is not None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]171 node.element.extract()
172
173 if (string_child and self.element.contents
174 and self.element.contents[-1].__class__ == NavigableString):
175 # We are appending a string onto another string.
176 # TODO This has O(n^2) performance, for input like
177 # "a</a>a</a>a</a>..."
178 old_element = self.element.contents[-1]
179 new_element = self.soup.new_string(old_element + string_child)
180 old_element.replace_with(new_element)
181 self.soup._most_recent_element = new_element
182 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]183 if isinstance(node, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]184 # Create a brand new NavigableString from this string.
185 child = self.soup.new_string(node)
186
187 # Tell Beautiful Soup to act as if it parsed this element
188 # immediately after the parent's last descendant. (Or
189 # immediately after the parent, if it has no children.)
190 if self.element.contents:
191 most_recent_element = self.element._last_descendant(False)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]192 elif self.element.next_element is not None:
193 # Something from further ahead in the parse tree is
194 # being inserted into this earlier element. This is
195 # very annoying because it means an expensive search
196 # for the last element in the tree.
197 most_recent_element = self.soup._last_descendant()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]198 else:
199 most_recent_element = self.element
200
201 self.soup.object_was_parsed(
202 child, parent=self.element,
203 most_recent_element=most_recent_element)
204
205 def getAttributes(self):
206 return AttrList(self.element)
207
208 def setAttributes(self, attributes):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]209
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]210 if attributes is not None and len(attributes) > 0:
211
212 converted_attributes = []
213 for name, value in list(attributes.items()):
214 if isinstance(name, tuple):
215 new_name = NamespacedAttribute(*name)
216 del attributes[name]
217 attributes[new_name] = value
218
219 self.soup.builder._replace_cdata_list_attribute_values(
220 self.name, attributes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]221 for name, value in list(attributes.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]222 self.element[name] = value
223
224 # The attributes may contain variables that need substitution.
225 # Call set_up_substitutions manually.
226 #
227 # The Tag constructor called this method when the Tag was created,
228 # but we just set/changed the attributes, so call it again.
229 self.soup.builder.set_up_substitutions(self.element)
230 attributes = property(getAttributes, setAttributes)
231
232 def insertText(self, data, insertBefore=None):
233 if insertBefore:
234 text = TextNode(self.soup.new_string(data), self.soup)
235 self.insertBefore(data, insertBefore)
236 else:
237 self.appendChild(data)
238
239 def insertBefore(self, node, refNode):
240 index = self.element.index(refNode.element)
241 if (node.element.__class__ == NavigableString and self.element.contents
242 and self.element.contents[index-1].__class__ == NavigableString):
243 # (See comments in appendChild)
244 old_node = self.element.contents[index-1]
245 new_str = self.soup.new_string(old_node + node.element)
246 old_node.replace_with(new_str)
247 else:
248 self.element.insert(index, node.element)
249 node.parent = self
250
251 def removeChild(self, node):
252 node.element.extract()
253
254 def reparentChildren(self, new_parent):
255 """Move all of this tag's children into another tag."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]256 # print "MOVE", self.element.contents
257 # print "FROM", self.element
258 # print "TO", new_parent.element
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]259 element = self.element
260 new_parent_element = new_parent.element
261 # Determine what this tag's next_element will be once all the children
262 # are removed.
263 final_next_element = element.next_sibling
264
265 new_parents_last_descendant = new_parent_element._last_descendant(False, False)
266 if len(new_parent_element.contents) > 0:
267 # The new parent already contains children. We will be
268 # appending this tag's children to the end.
269 new_parents_last_child = new_parent_element.contents[-1]
270 new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
271 else:
272 # The new parent contains no children.
273 new_parents_last_child = None
274 new_parents_last_descendant_next_element = new_parent_element.next_element
275
276 to_append = element.contents
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]277 append_after = new_parent_element.contents
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]278 if len(to_append) > 0:
279 # Set the first child's previous_element and previous_sibling
280 # to elements within the new parent
281 first_child = to_append[0]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]282 if new_parents_last_descendant:
283 first_child.previous_element = new_parents_last_descendant
284 else:
285 first_child.previous_element = new_parent_element
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]286 first_child.previous_sibling = new_parents_last_child
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]287 if new_parents_last_descendant:
288 new_parents_last_descendant.next_element = first_child
289 else:
290 new_parent_element.next_element = first_child
291 if new_parents_last_child:
292 new_parents_last_child.next_sibling = first_child
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]293
294 # Fix the last child's next_element and next_sibling
295 last_child = to_append[-1]
296 last_child.next_element = new_parents_last_descendant_next_element
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]297 if new_parents_last_descendant_next_element:
298 new_parents_last_descendant_next_element.previous_element = last_child
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]299 last_child.next_sibling = None
300
301 for child in to_append:
302 child.parent = new_parent_element
303 new_parent_element.contents.append(child)
304
305 # Now that this element has no children, change its .next_element.
306 element.contents = []
307 element.next_element = final_next_element
308
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]309 # print "DONE WITH MOVE"
310 # print "FROM", self.element
311 # print "TO", new_parent_element
312
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]313 def cloneNode(self):
314 tag = self.soup.new_tag(self.element.name, self.namespace)
315 node = Element(tag, self.soup, self.namespace)
316 for key,value in self.attributes:
317 node.attributes[key] = value
318 return node
319
320 def hasContent(self):
321 return self.element.contents
322
323 def getNameTuple(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500[diff] [blame]324 if self.namespace is None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]325 return namespaces["html"], self.name
326 else:
327 return self.namespace, self.name
328
329 nameTuple = property(getNameTuple)
330
331class TextNode(Element):
332 def __init__(self, element, soup):
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]333 treebuildersbase.Node.__init__(self, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]334 self.element = element
335 self.soup = soup
336
337 def cloneNode(self):
338 raise NotImplementedError