python3 调用 beautifulSoup 进行简单的网页处理

1. from bs4 import BeautifulSoup
3. file = open('index.html','r',encoding='utf-16-le') #此处有坑！！！
4. soup = BeautifulSoup(file,'lxml')
6. print (soup) # 打印读出的内容
7. print ('\n ------------- \n')
8. print (soup.get_text()) # 取所有标签中的文字
9. print ('\n ------------- \n')
10. print (soup.prettify()) # 格式化输出

1. # 以标签的形式输出
2. print (soup.title)
3. print ('\n ------------- \n')
4. print (soup.body)
5. print ('\n ------------- \n')
6. print (soup.body.div)

1. import re
2. print (soup.find_all('br')) # 仅仅用来搜索标签
3. print ('\n ------------- \n')
4. print (soup.find_all(re.compile('^b')))#可以使用正则表达式 以b开头的标签
5. print ('\n ------------- \n')
6. print (soup.find_all(id='wiz_custom_css'))
7. print ('\n ------------- \n')
8. for strr in soup.strings: # 取所有下一级标签中的字符串 .stripped_strings可以去空白
9. print (strr)
10. print ('\n ------------- \n')

1. # 去除body中的标签，将结果保存于文件 待改进
2. # kill all script and style elements
3. for script in soup(["script", "style"]):
4. script.extract() # rip current tap
6. title_text = soup.title.get_text()
7. str_text = ''
8. for strr in soup.body.strings: # 取所有下一级标签中的字符串 .stripped_strings可以去空白
9. str_text = str_text + strr + '\n'
11. print (str_text)
12. if title_text == '':
13. md_file = open('index.md','w')
14. md_file.write(str_text)
16. else:
17. md_file = open(title_text+'.md','w')
18. md_file.write(str_text)

1. # 网上搜到的方式，<br/>标签没有转为换行，后面有另一种方式
3. #print soup
4. # kill all script and style elements
5. for script in soup(["script", "style"]):
6. script.extract() # rip current tap
8. # get text
9. text = soup.get_text()
11. #print text + '____________'
12. # break into lines and remove leading and trailing space on each
13. # splitlines 按\r \r\n \n三种标签分解为行
14. # strip()移除首尾字符，参数默认为空格
15. lines = (line.strip() for line in text.splitlines())
16. # break multi-headlines into a line each
17. chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
18. # drop blank lines
19. text = '\n'.join(chunk for chunk in chunks if chunk) # 这个循环…………
21. #wfile = open('aa.md','w')
22. #wfile.write(text)
23. print(text)