Gitiles
Code Review Sign In
gerrit.openbmc.org / stefanberger / openbmc / 8c9f27b086631f9c722c7a634bb8cc0ea0d4db24 / . / poky / bitbake / lib / bs4 / builder / _html5lib.py
blob: 2b7a70aa110e7447e8abbd29f1d2ca51b885b1cb [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1__all__ = [
2 'HTML5TreeBuilder',
3 ]
4
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]5from pdb import set_trace
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]6import warnings
7from bs4.builder import (
8 PERMISSIVE,
9 HTML,
10 HTML_5,
11 HTMLTreeBuilder,
12 )
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]13from bs4.element import (
14 NamespacedAttribute,
15 whitespace_re,
16)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]17import html5lib
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]18try:
19 # html5lib >= 0.99999999/1.0b9
20 from html5lib.treebuilders import base as treebuildersbase
21except ImportError:
22 # html5lib <= 0.9999999/1.0b8
23 from html5lib.treebuilders import _base as treebuildersbase
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]24from html5lib.constants import namespaces
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]25
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]26from bs4.element import (
27 Comment,
28 Doctype,
29 NavigableString,
30 Tag,
31 )
32
33class HTML5TreeBuilder(HTMLTreeBuilder):
34 """Use html5lib to build a tree."""
35
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]36 NAME = "html5lib"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]37
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]38 features = [NAME, PERMISSIVE, HTML_5, HTML]
39
40 def prepare_markup(self, markup, user_specified_encoding,
41 document_declared_encoding=None, exclude_encodings=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]42 # Store the user-specified encoding for use later on.
43 self.user_specified_encoding = user_specified_encoding
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]44
45 # document_declared_encoding and exclude_encodings aren't used
46 # ATM because the html5lib TreeBuilder doesn't use
47 # UnicodeDammit.
48 if exclude_encodings:
49 warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]50 yield (markup, None, None, False)
51
52 # These methods are defined by Beautiful Soup.
53 def feed(self, markup):
54 if self.soup.parse_only is not None:
55 warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
56 parser = html5lib.HTMLParser(tree=self.create_treebuilder)
57 doc = parser.parse(markup, encoding=self.user_specified_encoding)
58
59 # Set the character encoding detected by the tokenizer.
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]60 if isinstance(markup, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]61 # We need to special-case this because html5lib sets
62 # charEncoding to UTF-8 if it gets Unicode input.
63 doc.original_encoding = None
64 else:
65 doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
66
67 def create_treebuilder(self, namespaceHTMLElements):
68 self.underlying_builder = TreeBuilderForHtml5lib(
69 self.soup, namespaceHTMLElements)
70 return self.underlying_builder
71
72 def test_fragment_to_document(self, fragment):
73 """See `TreeBuilder`."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]74 return '<html><head></head><body>%s</body></html>' % fragment
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]75
76
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]77class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]78
79 def __init__(self, soup, namespaceHTMLElements):
80 self.soup = soup
81 super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
82
83 def documentClass(self):
84 self.soup.reset()
85 return Element(self.soup, self.soup, None)
86
87 def insertDoctype(self, token):
88 name = token["name"]
89 publicId = token["publicId"]
90 systemId = token["systemId"]
91
92 doctype = Doctype.for_name_and_ids(name, publicId, systemId)
93 self.soup.object_was_parsed(doctype)
94
95 def elementClass(self, name, namespace):
96 tag = self.soup.new_tag(name, namespace)
97 return Element(tag, self.soup, namespace)
98
99 def commentClass(self, data):
100 return TextNode(Comment(data), self.soup)
101
102 def fragmentClass(self):
103 self.soup = BeautifulSoup("")
104 self.soup.name = "[document_fragment]"
105 return Element(self.soup, self.soup, None)
106
107 def appendChild(self, node):
108 # XXX This code is not covered by the BS4 tests.
109 self.soup.append(node.element)
110
111 def getDocument(self):
112 return self.soup
113
114 def getFragment(self):
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]115 return treebuildersbase.TreeBuilder.getFragment(self).element
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]116
117class AttrList(object):
118 def __init__(self, element):
119 self.element = element
120 self.attrs = dict(self.element.attrs)
121 def __iter__(self):
122 return list(self.attrs.items()).__iter__()
123 def __setitem__(self, name, value):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]124 # If this attribute is a multi-valued attribute for this element,
125 # turn its value into a list.
126 list_attr = HTML5TreeBuilder.cdata_list_attributes
127 if (name in list_attr['*']
128 or (self.element.name in list_attr
129 and name in list_attr[self.element.name])):
130 # A node that is being cloned may have already undergone
131 # this procedure.
132 if not isinstance(value, list):
133 value = whitespace_re.split(value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]134 self.element[name] = value
135 def items(self):
136 return list(self.attrs.items())
137 def keys(self):
138 return list(self.attrs.keys())
139 def __len__(self):
140 return len(self.attrs)
141 def __getitem__(self, name):
142 return self.attrs[name]
143 def __contains__(self, name):
144 return name in list(self.attrs.keys())
145
146
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]147class Element(treebuildersbase.Node):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]148 def __init__(self, element, soup, namespace):
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]149 treebuildersbase.Node.__init__(self, element.name)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]150 self.element = element
151 self.soup = soup
152 self.namespace = namespace
153
154 def appendChild(self, node):
155 string_child = child = None
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]156 if isinstance(node, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]157 # Some other piece of code decided to pass in a string
158 # instead of creating a TextElement object to contain the
159 # string.
160 string_child = child = node
161 elif isinstance(node, Tag):
162 # Some other piece of code decided to pass in a Tag
163 # instead of creating an Element object to contain the
164 # Tag.
165 child = node
166 elif node.element.__class__ == NavigableString:
167 string_child = child = node.element
168 else:
169 child = node.element
170
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]171 if not isinstance(child, str) and child.parent is not None:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]172 node.element.extract()
173
174 if (string_child and self.element.contents
175 and self.element.contents[-1].__class__ == NavigableString):
176 # We are appending a string onto another string.
177 # TODO This has O(n^2) performance, for input like
178 # "a</a>a</a>a</a>..."
179 old_element = self.element.contents[-1]
180 new_element = self.soup.new_string(old_element + string_child)
181 old_element.replace_with(new_element)
182 self.soup._most_recent_element = new_element
183 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]184 if isinstance(node, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]185 # Create a brand new NavigableString from this string.
186 child = self.soup.new_string(node)
187
188 # Tell Beautiful Soup to act as if it parsed this element
189 # immediately after the parent's last descendant. (Or
190 # immediately after the parent, if it has no children.)
191 if self.element.contents:
192 most_recent_element = self.element._last_descendant(False)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]193 elif self.element.next_element is not None:
194 # Something from further ahead in the parse tree is
195 # being inserted into this earlier element. This is
196 # very annoying because it means an expensive search
197 # for the last element in the tree.
198 most_recent_element = self.soup._last_descendant()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]199 else:
200 most_recent_element = self.element
201
202 self.soup.object_was_parsed(
203 child, parent=self.element,
204 most_recent_element=most_recent_element)
205
206 def getAttributes(self):
207 return AttrList(self.element)
208
209 def setAttributes(self, attributes):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]210
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]211 if attributes is not None and len(attributes) > 0:
212
213 converted_attributes = []
214 for name, value in list(attributes.items()):
215 if isinstance(name, tuple):
216 new_name = NamespacedAttribute(*name)
217 del attributes[name]
218 attributes[new_name] = value
219
220 self.soup.builder._replace_cdata_list_attribute_values(
221 self.name, attributes)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]222 for name, value in list(attributes.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]223 self.element[name] = value
224
225 # The attributes may contain variables that need substitution.
226 # Call set_up_substitutions manually.
227 #
228 # The Tag constructor called this method when the Tag was created,
229 # but we just set/changed the attributes, so call it again.
230 self.soup.builder.set_up_substitutions(self.element)
231 attributes = property(getAttributes, setAttributes)
232
233 def insertText(self, data, insertBefore=None):
234 if insertBefore:
235 text = TextNode(self.soup.new_string(data), self.soup)
236 self.insertBefore(data, insertBefore)
237 else:
238 self.appendChild(data)
239
240 def insertBefore(self, node, refNode):
241 index = self.element.index(refNode.element)
242 if (node.element.__class__ == NavigableString and self.element.contents
243 and self.element.contents[index-1].__class__ == NavigableString):
244 # (See comments in appendChild)
245 old_node = self.element.contents[index-1]
246 new_str = self.soup.new_string(old_node + node.element)
247 old_node.replace_with(new_str)
248 else:
249 self.element.insert(index, node.element)
250 node.parent = self
251
252 def removeChild(self, node):
253 node.element.extract()
254
255 def reparentChildren(self, new_parent):
256 """Move all of this tag's children into another tag."""
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]257 # print "MOVE", self.element.contents
258 # print "FROM", self.element
259 # print "TO", new_parent.element
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]260 element = self.element
261 new_parent_element = new_parent.element
262 # Determine what this tag's next_element will be once all the children
263 # are removed.
264 final_next_element = element.next_sibling
265
266 new_parents_last_descendant = new_parent_element._last_descendant(False, False)
267 if len(new_parent_element.contents) > 0:
268 # The new parent already contains children. We will be
269 # appending this tag's children to the end.
270 new_parents_last_child = new_parent_element.contents[-1]
271 new_parents_last_descendant_next_element = new_parents_last_descendant.next_element
272 else:
273 # The new parent contains no children.
274 new_parents_last_child = None
275 new_parents_last_descendant_next_element = new_parent_element.next_element
276
277 to_append = element.contents
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]278 append_after = new_parent_element.contents
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]279 if len(to_append) > 0:
280 # Set the first child's previous_element and previous_sibling
281 # to elements within the new parent
282 first_child = to_append[0]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]283 if new_parents_last_descendant:
284 first_child.previous_element = new_parents_last_descendant
285 else:
286 first_child.previous_element = new_parent_element
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]287 first_child.previous_sibling = new_parents_last_child
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]288 if new_parents_last_descendant:
289 new_parents_last_descendant.next_element = first_child
290 else:
291 new_parent_element.next_element = first_child
292 if new_parents_last_child:
293 new_parents_last_child.next_sibling = first_child
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]294
295 # Fix the last child's next_element and next_sibling
296 last_child = to_append[-1]
297 last_child.next_element = new_parents_last_descendant_next_element
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]298 if new_parents_last_descendant_next_element:
299 new_parents_last_descendant_next_element.previous_element = last_child
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]300 last_child.next_sibling = None
301
302 for child in to_append:
303 child.parent = new_parent_element
304 new_parent_element.contents.append(child)
305
306 # Now that this element has no children, change its .next_element.
307 element.contents = []
308 element.next_element = final_next_element
309
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]310 # print "DONE WITH MOVE"
311 # print "FROM", self.element
312 # print "TO", new_parent_element
313
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]314 def cloneNode(self):
315 tag = self.soup.new_tag(self.element.name, self.namespace)
316 node = Element(tag, self.soup, self.namespace)
317 for key,value in self.attributes:
318 node.attributes[key] = value
319 return node
320
321 def hasContent(self):
322 return self.element.contents
323
324 def getNameTuple(self):
325 if self.namespace == None:
326 return namespaces["html"], self.name
327 else:
328 return self.namespace, self.name
329
330 nameTuple = property(getNameTuple)
331
332class TextNode(Element):
333 def __init__(self, element, soup):
Daniel Lublin33a391e2016-12-05 18:42:05 +0100[diff] [blame]334 treebuildersbase.Node.__init__(self, None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]335 self.element = element
336 self.soup = soup
337
338 def cloneNode(self):
339 raise NotImplementedError